# Notebook from Thaslim/CMPE_255_Amzn
Path: Topic Modelling and Sentiment Analysis/Sentiment Analysis TextBlob/LDA_Sentiment_Similar_Products_Sheets.ipynb

# Topic Modelling By Grouping Multiple Similar Products_____no_output_____

import pandas as pd
import pickle
import numpy as np_____no_output_____# Read the preprocessed data file
df_huge = pd.read_pickle('../Clean_data')
df_huge.head()
df_huge.shape_____no_output_____# Selection of product- product with maximum review data
df_grouped = df_huge.groupby(['asin']).agg({'reviewText': ' ** '.join,'asin':'size'}).rename(columns={'asin':'count'}).reset_index()
df_grouped.sort_values(by = 'count',ascending=False,inplace=True)
df_grouped.head(30)
_____no_output_____df_grouped.head(50)_____no_output_____df_Sheets =  df_huge.loc[df_huge['asin'].isin(["B00LV4W8BI",'B00902X68W',"B00NLLUNSE", "B00635VODS"])]_____no_output_____df_Sheets.shape_____no_output_____df_Sheets['overall'].describe()_____no_output_____df_Sheets['overall'].value_counts()_____no_output_____

## Preparing the dataframe with review sentences
_____no_output_____

# Preapre a data frame by spliiting reviews into sentences
df_sentences = pd.DataFrame(df_Sheets.reviewText.str.split('.').tolist(), index=df_Sheets.reviewerID).stack()
df_sentences = df_sentences.reset_index([0, 'reviewerID'])
df_sentences.columns = ['reviewerID', 'sentences']

df_sentences
_____no_output_____# Merging the 50 reviews dataframe having all columns with new sentences dataframe
df_full = df_Sheets.merge(df_sentences, how='inner', left_on='reviewerID', right_on='reviewerID')
df_full

# #  Removed unneccesary columns
# df_50p_5800r = df_50p_5800r.drop(["reviewText", "rev_date", "product_ID"], axis=1)
# df_50p_5800r = df_50p_5800r

# df_B00ETP7D3E_full.head()_____no_output_____df_full.iloc[134079]['sentences']_____no_output_____

## LDA Modeling_____no_output_____

import time

import re
import numpy as np
import pandas as pd
from pprint import pprint

# Gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel

# spacy for lemmatization
import spacy

# Plotting tools
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
%matplotlib inline

# Enable logging for gensim - optional
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)

import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)unable to import 'smart_open.gcs', disabling that module
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
stop_words.extend(['from', 'subject', 're', 'edu', 'use'])_____no_output_____# Get sentences from the data frame
list_sentences = df_sentences.sentences.values.tolist()
print(list_sentences[:1], len(list_sentences))['This is a fine comforter, super soft and fluffy but not too hot at night'] 134078
def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations

list_sentences_tokenized = list(sent_to_words(list_sentences))
list_sentences_tokenized_copy = list(sent_to_words(list_sentences))
print(list_sentences_tokenized_copy[:1])[['this', 'is', 'fine', 'comforter', 'super', 'soft', 'and', 'fluffy', 'but', 'not', 'too', 'hot', 'at', 'night']]
# Build the bigram and trigram models

start_time = time.time()

bigram = gensim.models.Phrases(list_sentences_tokenized, min_count=5, threshold=100) # higher threshold fewer phrases.
trigram = gensim.models.Phrases(bigram[list_sentences_tokenized], threshold=100)

# Faster way to get a sentence clubbed as a trigram/bigram
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)

# See trigram example
print(trigram_mod[bigram_mod[list_sentences_tokenized[0]]])

print("--- %s seconds ---" % (time.time() - start_time))['this', 'is', 'fine', 'comforter', 'super', 'soft', 'and', 'fluffy', 'but', 'not', 'too', 'hot', 'at', 'night']
--- 12.095656156539917 seconds ---
# Define functions for stopwords, bigrams, trigrams and lemmatization
def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]

def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]

def make_trigrams(texts):
    return [trigram_mod[bigram_mod[doc]] for doc in texts]

def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent))
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    return texts_out_____no_output_____# Remove Stop Words
data_words_nostops = remove_stopwords(list_sentences_tokenized)
data_words_nostops[:1]_____no_output_____# Form Bigrams
data_words_bigrams = make_bigrams(data_words_nostops)
data_words_bigrams[:1]
_____no_output_____start_time = time.time()

nlp = spacy.load('en', disable=['parser', 'ner'])

# # Do lemmatization keeping only noun, adj, vb, adv
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

print(data_lemmatized[:1])
print("--- %s seconds ---" % (time.time() - start_time))[['fine', 'comforter', 'super', 'soft', 'fluffy', 'hot', 'night']]
--- 224.05514192581177 seconds ---
#  For Text Blob
from nltk.tokenize.treebank import TreebankWordDetokenizer
list_sentences_lemetised = []
for i in range(len(data_lemmatized)):
    lem_sen = TreebankWordDetokenizer().detokenize(data_lemmatized[i])
    list_sentences_lemetised.append(lem_sen)_____no_output_____list_sentences_lemetised_____no_output_____import pickle
with open('list_sentences_lemetised_Sheets.pkl', 'wb') as f:
    pickle.dump(list_sentences_lemetised, f)_____no_output_____# Create Dictionary
id2word = corpora.Dictionary(data_lemmatized)

# Create Corpus
texts = data_lemmatized

# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]

# View
print(corpus[:1])[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)]]
id2word[0]
_____no_output_____# Build LDA model
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=3,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha='auto',
                                           per_word_topics=True)_____no_output_____# Print the Keyword in the 10 topics
pprint(lda_model.print_topics())
doc_lda = lda_model[corpus][(0,
  '0.154*"soft" + 0.056*"good" + 0.044*"comfortable" + 0.039*"nice" + '
  '0.037*"fit" + 0.026*"well" + 0.024*"super" + 0.022*"purchase" + '
  '0.021*"really" + 0.019*"make"'),
 (1,
  '0.070*"great" + 0.049*"color" + 0.044*"price" + 0.044*"quality" + '
  '0.041*"star" + 0.033*"wash" + 0.026*"would" + 0.024*"get" + 0.022*"order" + '
  '0.021*"look"'),
 (2,
  '0.202*"sheet" + 0.045*"love" + 0.041*"buy" + 0.038*"feel" + 0.037*"set" + '
  '0.030*"bed" + 0.020*"thin" + 0.016*"mattress" + 0.014*"product" + '
  '0.013*"sleep"')]
# Compute Perplexity
print('\nPerplexity: ', lda_model.log_perplexity(corpus))  # a measure of how good the model is. lower the better.

# Compute Coherence Score
coherence_model_lda = CoherenceModel(model=lda_model, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)
Perplexity:  -6.105583198452926

Coherence Score:  0.2622397872066436
# Visualize the topics
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis
_____no_output_____

## LDA Mallet Model_____no_output_____

mallet_path = 'mallet-2.0.8/bin/mallet' # update this path
_____no_output_____ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=5, id2word=id2word)

# Show Topics
pprint(ldamallet.show_topics(formatted=False))

# Compute Coherence Score
coherence_model_ldamallet = CoherenceModel(model=ldamallet, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
coherence_ldamallet = coherence_model_ldamallet.get_coherence()
print('\nCoherence Score: ', coherence_ldamallet)
# 0- quality of cooking, 1-good appliance, 2-functionalities 3-product design, 5-how likely to recommend[(0,
  [('star', 0.14686502687813224),
   ('great', 0.14118603550583067),
   ('price', 0.09836304287153103),
   ('good', 0.09235641738159667),
   ('quality', 0.08411702605297965),
   ('color', 0.07007729738256743),
   ('nice', 0.046342025749614726),
   ('product', 0.03833319176303559),
   ('sheet', 0.019609508670169522),
   ('amazing', 0.018857163659309055)]),
 (1,
  [('soft', 0.23981194982130435),
   ('love', 0.12989614065385816),
   ('comfortable', 0.07073469094049076),
   ('warm', 0.04252716413614449),
   ('super', 0.038473531457628746),
   ('sleep', 0.03373629800196685),
   ('feel', 0.02958672135472884),
   ('perfect', 0.022462881677100574),
   ('light', 0.020555995298745532),
   ('night', 0.020196205416037033)]),
 (2,
  [('buy', 0.09898185251994528),
   ('set', 0.09047105967240106),
   ('wash', 0.05459982324669193),
   ('sheet', 0.04742073340516459),
   ('order', 0.04635537100035109),
   ('time', 0.030217551845619302),
   ('cover', 0.016125712763767994),
   ('hold', 0.016065180808949045),
   ('case', 0.014830328930642487),
   ('duvet', 0.013656009007154878)]),
 (3,
  [('sheet', 0.2391007763169015),
   ('bed', 0.07632776751542371),
   ('fit', 0.05924396633947867),
   ('mattress', 0.03404686877467492),
   ('recommend', 0.02648894684100595),
   ('deep', 0.022214978207589314),
   ('pillow', 0.021104229297209846),
   ('size', 0.020428121264804956),
   ('put', 0.017276974899489296),
   ('pocket', 0.014850229997464595)]),
 (4,
  [('sheet', 0.10078227811083491),
   ('feel', 0.04714383194424136),
   ('thin', 0.046802651459765074),
   ('purchase', 0.03850465467660964),
   ('comforter', 0.03433737875907784),
   ('make', 0.03159574986596481),
   ('expect', 0.022201101525564166),
   ('happy', 0.020275868791733685),
   ('review', 0.019045182044158504),
   ('cheap', 0.018228785884875958)])]

Coherence Score:  0.3841902362948132
# ldamallet_3topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=3, id2word=id2word)

# # Show Topics
# pprint(ldamallet_3topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_3topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____# ldamallet_4topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=4, id2word=id2word)

# # Show Topics
# pprint(ldamallet_4topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_4topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)
# # 0 - great product, 1- features and fucntions, 2-review after few days, 3-things to cook_____no_output_____# ldamallet_6topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=6, id2word=id2word)

# # Show Topics
# pprint(ldamallet_6topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_6topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____# ldamallet_10topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=10, id2word=id2word)

# # Show Topics
# pprint(ldamallet_10topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_10topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____# ldamallet_15topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=15, id2word=id2word)

# # Show Topics
# pprint(ldamallet_15topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_15topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____# ldamallet_20topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=20, id2word=id2word)

# # Show Topics
# pprint(ldamallet_20topics.show_topics(formatted=False))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_20topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____# ldamallet_25topics = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=25, id2word=id2word)

# # Show Topics
# pprint(ldamallet_25topics.show_topics(formatted=Tr))

# # Compute Coherence Score
# coherence_model_ldamallet = CoherenceModel(model=ldamallet_25topics, texts=data_lemmatized, dictionary=id2word, coherence='c_v')
# coherence_ldamallet = coherence_model_ldamallet.get_coherence()
# print('\nCoherence Score: ', coherence_ldamallet)_____no_output_____

#### Finding optimal number of topics_____no_output_____

def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    """
    Compute c_v coherence for various number of topics

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_values : Coherence values corresponding to the LDA model with respective number of topics
    """
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        model = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=num_topics, id2word=id2word)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values_____no_output_____
model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=data_lemmatized, start=3, limit=25, step=5)_____no_output_____# Show graph
limit=25; start=3; step=5;
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence score")
plt.legend(("coherence_values"), loc='best')
plt.show()_____no_output_____

#### Finding Dominant Topic in each Sentence_____no_output_____

def format_topics_sentences(ldamodel=lda_model, corpus=corpus, texts=data_lemmatized):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row in enumerate(ldamodel[corpus]):
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        # Get the Dominant topic, Perc Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
    return(sent_topics_df)_____no_output_____df_topic_sents_keywords = format_topics_sentences(ldamodel=ldamallet, corpus=corpus, texts=data_lemmatized)_____no_output_____
# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']_____no_output_____# Show
df_dominant_topic_____no_output_____

## Sentiment Analysis using Text Blob_____no_output_____

from textblob import TextBlob
# Get the polarity score using below function
def get_textBlob_score(sent):
    # This polarity score is between -1 to 1
    polarity = TextBlob(sent).sentiment.polarity
    return polarity_____no_output_____len(list_sentences_lemetised)_____no_output_____# Prepare sentiment scores for all sentences from text blob
texblog_senti_scores= []
for sentence in list_sentences_lemetised:
    texblog_score = get_textBlob_score(sentence)
    texblog_senti_scores.append(texblog_score)_____no_output_____len(texblog_senti_scores)_____no_output_____texblog_senti_scores[30000], list_sentences_lemetised[30000]_____no_output_____

## Combine Sentiment Analysis with LDA_____no_output_____

df_full['topics'] = df_dominant_topic['Dominant_Topic'].astype(float)_____no_output_____df_full['sentiment_scores'] = np.resize(texblog_senti_scores,len(df_full))

_____no_output_____df_full_____no_output_____df_full['sentiment_scores'].describe()_____no_output_____texblog_senti_scores
sentiments_textblob = []
for each in texblog_senti_scores:
    if(each >=0.4):
        a=1
    else:
        a=0
    sentiments_textblob.append(a)_____no_output_____df_full['sentiment'] = np.resize(sentiments_textblob,len(df_full))
_____no_output_____df_full_____no_output_____ df_full['topics'].value_counts()_____no_output_____topic_wise_sentiment_counts = df_full.groupby(['asin','topics'])['sentiment'].value_counts()
topic_wise_sentiment_counts_____no_output_____total_sentiment = df_full.groupby(['asin','topics'])['sentiment'].count()
total_sentiment_____no_output_____total_sentiment.loc['B00635VODS'][0], topic_wise_sentiment_counts['B00635VODS'][0][0]
# , topic_wise_sentiment_counts[0][1]_____no_output_____results = []
ratings= []
product_ids = ['B00635VODS', 'B00902X68W', 'B00LV4W8BI', 'B00NLLUNSE']
for i in range(len(product_ids)):
    for j in range(5):
        value = topic_wise_sentiment_counts[product_ids[i]][j][0]/total_sentiment.loc[product_ids[i]][j]
        results.append(value)
        if (value >0.75):
            ratings.append(5)
        elif (value >0.7):
            ratings.append(4)
        elif (value >0.5):
            ratings.append(3)
        elif (value >0.4):
            ratings.append(2)
        else: ratings.append(1)
results, ratings_____no_output_____product_ids_tmp =  [5, 5, 3, 4, 4, 5, 5, 2, 4, 4, 5, 5, 3, 4, 5, 5, 5, 3, 5, 5]
product_id_topics = [[],[],[],[],[]]
for i in range(len(product_ids_tmp)):
    product_id_topics[int(i%5)].append(product_ids_tmp[i])
product_id_topics_____no_output_____b = [[],[]]_____no_output_____b[0].append(1)
b_____no_output_____

# Results_____no_output_____### Topic wise product rating_____no_output_____

topics = ['Softnest and comfort', 'Fit', 'Quality', 'Recommendation', 'After wash quality']
products = {'comforter':'B00635VODS','sheets-1':'B00902X68W',
           'sheets-2':'B00LV4W8BI',
           'sheets-3': 'B00NLLUNSE'}_____no_output_____Product_Summary = pd.DataFrame({'productID':product_ids,
                                topics[0]: product_id_topics[0],
                                topics[1]: product_id_topics[1],
                                topics[2]: product_id_topics[2],
                                topics[3]: product_id_topics[3],
                                topics[4]: product_id_topics[4]})
Product_Summary_____no_output_____

## Product Wise Summary_____no_output_____

def get_n_sentimental_sentences(n, sentiment, sentences):
    print("\033[1m"+ "Most "+ sentiment+" sentences are: " + "\033[0m")
    print("")
    for i in range(len(sentences.index)):
        print(i,") ",df_full.iloc[sentences.index[i]]['sentences'])

def get_product_summary(product_name):
    product_id = products[product_name]
    product_df = df_full[df_full['asin']==product_id]
    print("\033[1m" + "Product summary for "+ product_name + "\033[0m")
    print("")
    print("\033[1m" + "Topic wise rating is..."+ "\033[0m")
    print("")
    product_rating = Product_Summary[Product_Summary['productID']==product_id]
    product_rating = product_rating.drop(['productID'], axis=1)
    print(product_rating.T)
    print("")
    Most_negative_sentences =  product_df.nsmallest(5, 'sentiment_scores')['sentences']
    Most_positive_sentences = product_df.nlargest(5, 'sentiment_scores')['sentences']
    get_n_sentimental_sentences(5, "positive", Most_positive_sentences )
    get_n_sentimental_sentences(5, "negative", Most_negative_sentences )
_____no_output_____# Product name can be given here which fetches productID from the dictionary created above
get_product_summary("comforter")Product summary for comforter

Topic wise rating is...

                      0
Softnest and comfort  5
Fit                   5
Quality               3
Recommendation        4
After wash quality    4

Most positive sentences are:

0 )   It would be perfect for a Queen sized bed
1 )  nothing overly wonderful
2 )  This is an excellent duvet for people allergic to feathers
3 )    It looks wonderful with my duvet cover, doesn't 'pill' and the stitching keeps it from bunching
4 )  This duvet was the perfect weight
Most negative sentences are:

0 )   Shipping issues aside (first one went AWOL, they shipped another without incident), I wish I'd never ordered it
1 )  Feels great even in the Texas weather
2 )    It barely covers the top of the bed
3 )   //great buy!
4 )  True size, I expected slightly thicker but it is a good price/ quality


# Notebook from banneker-aztlan/python-week-2
Path: Part 2/galaxy_phot.ipynb

# Simulating Galaxy Observations: Photometry_____no_output_____In this exercise, I want to focus on getting comfortable with a few core packages: `matplotlib` (the default plotting utility), `numpy` (useful for fast operations with a lot of numbers), and `scipy` (useful for a lot of built-in methods).

We will use these packages this week to do some basic science: simulating a galaxy observation. This will have two components: simulating a photometric observation (the intensity of light as a function of wavelength) and, if time permits, simulating a spectroscopic observation (the intensity of light as a function of position and wavelength)._____no_output_____# Preamble_____no_output_____We'll try and run the same block of code as before. This time, I want you to see if you can figure out what packages you'll need to install to get the `__builtin__` import to work before moving on. Once you do, try installing it with either `conda` or `pip` (depending on your installation version)._____no_output_____

# only necessary if you're running Python 2.7 or lower
from __future__ import print_function
from __builtin__ import range_____no_output_____

# Plotting_____no_output_____Before we get started, we need to initialize the environment. First, let's import `pyplot` from `matplotlib`, which we will use to plot things. Then, we can use one of the "magic commands" to enable *in-line plotting*, which ensures our plots will show up in our actual notebook (rather than externally)._____no_output_____

# import plotting utility and define our naming alias
from matplotlib import pyplot as plt

# plot figures within the notebook rather than externally
%matplotlib inline_____no_output_____

Let's make some quick plots to see how things look using some fake data. We will use `numpy` to help us with this. _____no_output_____

# import numpy
import numpy as np_____no_output_____

Let's quickly generate some data. We'll start with a "grid" of points $\mathbf{x}$ and compute the corresponding output $\mathbf{y}$._____no_output_____

# define a relationship: y = ax + b
a, b = 1., 0.  # the trailing decimal guarantees this is a "float" rather than "int"

# initialize our data
n = 1000  # number of data points
x = np.linspace(0., 100., n)  # our grid of `n` data points from 0. to 100.
y = a * x + b  # our output y_____no_output_____

Now let's add some noise to our results using `numpy`'s built-in `random` module._____no_output_____

# add in noise drawn from a normal distribution
ye = np.random.normal(loc=0., scale=5., size=n)  # jitter
yobs = y + ye  # observed result_____no_output_____

Let's see how our results look._____no_output_____

# plot our results
plt.plot(x, yobs)
plt.plot(x, y)
plt.xlabel('x')
plt.ylabel('y')
plt.title('A (Noisy) Line')_____no_output_____

Play around with the parameters above to get some more familiarity with plotting. If you have time, see if you can:
- Change the original relationship to a quadratic one.
- Change the type of random noise we are adding to the data.
- Change the colors used for plotting.
- Change the "linestyle" used for plotting from connected lines to unconnected dots.
- Change the x and y limits in the plot.
and any other changes you'd like to experiment with.

Feel free to use any resources you want to figure this out. For immediate results, try the `help` function (shown below) or **Shift-Tab** within a function for in-line documentation. There's also some official documentation [online](https://matplotlib.org/faq/usage_faq.html)._____no_output_____

help(plt.plot)_____no_output_____

I always find the default label and axes markers to be too small to easily read (especially when showing people plots). Luckily, it's pretty straightforward to change the plotting defaults for `matplotlib` to make things easier to read. We can override the defaults whenever we plot something (which we'll get to in a bit) or we can just update them all at once (as below)._____no_output_____

# re-defining plotting defaults
rcParams.update({'xtick.major.pad': '7.0'})
rcParams.update({'xtick.major.size': '7.5'})
rcParams.update({'xtick.major.width': '1.5'})
rcParams.update({'xtick.minor.pad': '7.0'})
rcParams.update({'xtick.minor.size': '3.5'})
rcParams.update({'xtick.minor.width': '1.0'})
rcParams.update({'ytick.major.pad': '7.0'})
rcParams.update({'ytick.major.size': '7.5'})
rcParams.update({'ytick.major.width': '1.5'})
rcParams.update({'ytick.minor.pad': '7.0'})
rcParams.update({'ytick.minor.size': '3.5'})
rcParams.update({'ytick.minor.width': '1.0'})
rcParams.update({'xtick.color': 'k'})
rcParams.update({'ytick.color': 'k'})
rcParams.update({'font.size': 30})_____no_output_____

The set of commands above probably didn't work on the first try. What gives? Looking at the error, it's telling us that the name `'rcParams'` is not defined in any capacity. This makes some sense: we never defined this variable anywhere. See if you can find out where to import it._____no_output_____Let's re-plot our results to see what updating our defaults has changed._____no_output_____

# plot our results
plt.figure(figsize=(10, 4))
plt.plot(x, yobs)
plt.plot(x, y)
plt.xlabel('x', fontsize=20, color='darkviolet')
plt.ylabel('y', fontsize=40, color='red')
plt.title('A (Noisy) Line with Larger Font', y=1.05, color='navy')_____no_output_____

Now that we've changed the defaults, we notice a number of issues with our plot to do with our font size. This is because all outputs from `pyplot` are intrinsically drawn on a `Figure` object. If one of these are not initialized explicitly at the beginning, a default one is created. With our new larger fonts, the default figure feels a little squished.

Play around with changing the size of our figure using the commented line. Feel free to also mess around with the arguments passed to the axes labels and titles._____no_output_____# Galaxies and Emission Lines_____no_output_____Now that we have some familiarity with plotting, let's move on to a bit more of the science. In the `seds` folder under `data`, there are a bunch of files containing galaxy **spectral energy distributions** (SEDs). Let's load in [UGCA 166](https://en.wikipedia.org/wiki/I_Zwicky_18), a nearby blue compact dwarf galaxy that's actively forming stars._____no_output_____

# load in text data
data = np.loadtxt('seds/brown_UGCA_166_spec.dat')
data_____no_output_____

The data we loaded in is a $N \times 2$-dimensional **`numpy` array**, where the first column is the wavelength (measured in **Angstroms**, $10^{-10}$ meters) while the second column is the relative **flux density** (energy per time per area per wavelength, i.e. the relative **intensity** of the light at that particular wavelength).

Arrays are fixed-size data structures in Python that allow you to quickly manipulate lots of numbers. We'll be exploiting them here and you'll definitely be using them if you code in Python more regularly._____no_output_____The way `data` is currently structured is a bit awkward: the relevant quantities are the columns here, which makes plotting a bit awkward. One possible way to get around this is to make some new variables by (1) iterating through the array (using an *implicit* `for` loop) or (2) **slicing** through the array. Both of these are deomnstrated below._____no_output_____

# iteration through the array using an implicit for loop (list comprehension)
print('Wavelength:', np.array([d[0] for d in data]))

# slice the array along the 0th entry in the 1st dimension (i.e. by column, not row)
print('Wavelength:', data[:, 0])_____no_output_____

To facilitate plotting and later manipulation, let's redefine our data to instead be 2 $N$-dimensional arrays called `wave` and `fgal_wave`. There are a bunch of ways to do this besides the two shown above, but the most direct way is to use `numpy`'s built-in **array manipulation** functions.

**Extra challenge: can you code up your own custom method to do this? How many lines of code does your method take compared with the one line implementations shown above?**_____no_output_____

# take the transpose of the array (N x M) -> (M x N)
print(data.T)
wave, fgal_wave = data.T_____no_output_____

Let's now plot the data to see what our galaxy looks like. Build on the bare-bones example below using the skills you've learned to make a better-looking plot. (At the minimum, please label the axes!)_____no_output_____

# flux density (per wavelength) vs wavelength (angstroms) for UGCA 166
#plt.plot(wave, fgal_wave)
#plt.semilogx(wave, fgal_wave)
#plt.semilogy(wave, fgal_wave)
#plt.loglog(wave, fgal_wave)_____no_output_____

Looking closely, we see that there are a number of very visible "spikes" on the plot. These are particular **emission lines** associated with atomic transitions. These specific **spectral features** are a direct result of the energetic photons emitted from all the new/young stars in the galaxy. Try and zoom in on the particular region on the plot where most of these are located. Can you identify any lines in particular based on [this list](http://classic.sdss.org/dr6/algorithms/linestable.html)? Some of the most common ones are also defined below.

**Extra challenge: Overplot the line wavelengths on the galaxy spectrum using the `plt.vlines` function.**_____no_output_____

# defining some common emission lines
ha = 6564.6  # H-alpha [A]
n2 = 6549.86  # NII [A]
o3_1 = 5008.240  # OIII doublet (1) [A]
o3_2 = 4960.295  # OIII doublet (2) [A]
hb = 4862.7  # H-beta [A]
o2 = 3728.4  # approximate center of (blended) OII doublet [A]

# consolidating results in an array (in order of decreasing wavelength)
emlines = np.array([ha, n2, o3_1, o3_2, hb, o2])
emline_names = np.array(['Ha', 'NII', 'OIII (1)', 'OIII (2)', 'Hb', 'OII (1,2)'])
Nlines = len(emlines)_____no_output_____

Before moving on, feel free to load in a few different galaxy spectra from the `seds/` folder (the `brown_` ones are particularly nice examples) to see how different galaxy spectra look._____no_output_____# Filter Transmission Curves_____no_output_____Getting spectra of a galaxy is actually quite expensive and time-consuming. Instead, many large upcoming surveys such as [*Euclid*](https://www.euclid-ec.org/) will be **photometric** surveys. This just means that instead of observing galaxies as a *function of* wavelength, they simply count how many (relative) photons they receive in a specific wavelength interval. In other words, they take pictures of the sky in a particular wavelength range!_____no_output_____UGCA 166 is kinda bizarre within our local neighborhood, but is probably a good example of what galaxies are like at higher [redshifts](https://en.wikipedia.org/wiki/Redshift) (i.e. earlier times). We would like to simulate what our galaxy would look like in the photometric **filters** that will be part of the *Euclid* and [LSST](https://www.lsst.org/) surveys._____no_output_____First we need to extract the relevant **filter transmission curves**, which tell us how much light is ultimately transmitted through the filter at a particular wavelength. I've stored these in the `filters/` folder, so we need to extract them now. See if you can the code below to point to the right place._____no_output_____

filt_path = '' # file path for the filters folder_____no_output_____

Our previous galaxy file was just a bunch of numbers, which we were able to load in using `np.loadtxt`. Our list of filters, however, is a bunch of names. This means we need to read in the file a bit differently. Let's do this line by line just to be very explicit about it._____no_output_____

# initialize lists
filt_names = []  # empty list
filt_files = []  # empty list

# read in our filter names and file paths line-by-line
filt_list = open(filt_path+'Euclid.list') # open list of filter files for Euclid+LSST
for line in filt_list:
    l = line.strip()  # strip out the end-carriage '\n' if present
    ls = l.split()  # split the line into component strings
    filt_names.append(ls[0]) # filter name
    filt_files.append(ls[1]) # filter file
filt_list.close()  # close file (**ALWAYS REMEMBER TO DO THIS**)
Nfilt = len(filt_files)  # number of filters_____no_output_____

Play around with the components above to get a sense of what we just did. What happens if you remove parts or don't close the file? What operations can you do with lists? How about with arrays? Some quick examples are below. We'll be coming back to some of these subtleties later, but please don't hesitate to ask if you have any questions._____no_output_____

#print(filt_names, filt_names * 2)
#print(emlines, emlines * 2)
#print(filt_list)_____no_output_____

Now let's load in our individual filters._____no_output_____

# initialize our lists
fw = []  # wavelengths
fnu = []  # frequencies
ft = []  # transmission
c = 2.998e18  # speed of light [A/s]
for filt in filt_files:
    fpath = filt_path + filt  # append filter name to filter path
    temp = np.loadtxt(fpath)  # load ASCII text file (wavelength, transmission)
    fw.append(temp[:, 0])  # wavelength ('lambda') [A]
    fnu.append(c / temp[:, 0])  # frequency ('nu') [Hz]
    ft.append(temp[:, 1])  # transmission (fraction from 0. to 1.)_____no_output_____

Try plotting some of these below building off of the basic layout. The basic `Figure` setup has been initialized along with two possible color schemes and a bunch of keyword arguments to the methods. Try to play around with the styles to see how they change things._____no_output_____

# initialize figure
plt.figure(figsize=(16, 5))

# define a sequence of colors
colors = ['blue', 'magenta', 'red', 'orange', 'brown',  # this works because python implicitly
          'green', 'teal', 'goldenrod', 'coral', 'black']  # continues bracketed statements

# define our colormap (see: https://matplotlib.org/examples/color/colormaps_reference.html)
color_scale = np.linspace(0, 1, Nfilt)
colors = plt.get_cmap('viridis')(color_scale)

# plot results
for i in range(Nfilt):
    plt.plot(fw[i], ft[i], ls='-', lw=2, label=filt_names[i], color=colors[i])
plt.ylim([0, 1.5])
plt.legend(loc=1, ncol=5, fontsize=16)
plt.tight_layout()_____no_output_____

What do you think is causing the different features in the $\lbrace U, G, R, I, Z, Y \rbrace$ transmission curves and the worse overall levels of transmission relative to the $\lbrace VIS, Y_w, J_w, H_w \rbrace$ curves? (Hint: it has something to do with one of the biggest differences between the LSST and *Euclid* surveys.)_____no_output_____Before moving on, let's take a second to examine how we've set up the for loop above in more detail above.
- The `range` argument initializes an **iterator** that goes from [0, `Nfilt`), where "[" signals *inclusive* (including zero) and ")" signals *exclusive* (up to but excluding `Nfilt`), respectively.
- We then step through the iterator using `i`, which takes on values 0, 1, 2, ... up to but excluding `Nfilt`.
- For each value of `i`, we plot the corresponding element of `fw`, `ft`, `filt_names`, and `colors`._____no_output_____Another way to do this is to step through all these values simultaneously. Python allows you to do this using the `zip` method, which iterates over all zipped quantities simultaneously. Using the example below, see if you can rewrite the `for` loop above to loop over all quantities (`fw`, `ft`, `filt_names`, and `colors`) simultaneously. Feel free to also play around with this formatting to get more comfortable with how `zip` works._____no_output_____

plt.figure(figsize=(16, 5))
for i, filt_x, filt_y in zip(range(Nfilt), fw, ft):
    plt.plot(filt_x, filt_y, ls='-', lw=2, label=filt_names[i], color=colors[i])_____no_output_____

In addition, there's also no need for us to generate a counter using `range`. Python natively allows us to "count" within our loop using the `enumerate` function, which wraps whatever we're looping over. See if you can use the example below to re-write the `for` loop above._____no_output_____

plt.figure(figsize=(16, 5))
for i, stuff in enumerate(zip(filt_names, colors)):
    plt.plot(fw[i], ft[i], ls='-', lw=2, label=stuff[0], color=stuff[1])_____no_output_____

# Photometry_____no_output_____We now want to compute some basic properties of our set of filters. We'll start with the **effective wavelength**. This can be seen as the approximate "mean" wavelength of the filter, accounting for the differing transmission as a function of wavelength. Defining our wavelengths as $\lambda$, our frequencies as $\nu$, and our transmission at a particular frequency as $T_\nu$, the "standard" defition for this effective wavelength $\lambda_{\textrm{eff}}$:

$$ \lambda_{\textrm{eff}} = \exp \left[ \frac{\int T_\nu \, \ln \lambda \, d(\ln \nu)}{\int T_\nu \, d(\ln \nu)} \right] \quad . $$

Although this definition might seem a bit weird, the basic idea is we want to compromise between averaging as a function of wavelength compared to as a function of frequency, which don't give the same result since $ \lambda = c / \nu \propto \nu^{-1}$._____no_output_____We can break down this computation into four steps:
1. compute the integral in the denominator,
2. compute the integral in the numerator,
3. exponentiate their ratio, and
4. iterate over all of our filters._____no_output_____Python has a bunch of [numerical integration packages](https://docs.scipy.org/doc/scipy/reference/tutorial/integrate.html) available as part of `scipy` for more general applications. There also is a basic numerical integration tool [`trapz`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.trapz.html) as part of `numpy`, which should suffice for our purposes here. An example is shown below._____no_output_____

# integrate our function from the beginning of the notebook
plt.plot(x, y)  # plot our original function
plt.fill_between(x, y, color='blue', alpha=0.3)  # fill between y and 0 over our x's
y_area = np.trapz(y, x)  # numerically integrate our function

print('Integral = {0}'.format(y_area))_____no_output_____

Using the example above, see if you can compute (1) the denominator, (2) numerator, and (3) effective wavelength for a particular filter in the style shown below. Then see if you can turn this into an array of effective wavelengths over all filters._____no_output_____

# denominator
denominator = ...

# numerator
numerator = ...

# effective wavelength (Schneider et al. 1983; Fukugita et al. 1996)
filt_cent = ..._____no_output_____

** Extra Challenge: Code up a simple numerical integration scheme by hand.**

**Extra Extra Challenge: Use `scipy.integrate` to do the integral instead.**_____no_output_____If you need to move on, here's a one-line solution:_____no_output_____

# array of effective wavelengths (compact solution)
filt_cent = np.array([np.exp(np.trapz(ft[i] * np.log(fw[i]), np.log(fnu[i])) /
                             np.trapz(ft[i], np.log(fnu[i])))
                      for i in range(Nfilt)])_____no_output_____

In addition to the effective wavelength, we also want some basic metric of how "wide" our filter is. There a lot of ways to define this, just as there are a lot of ways to define an "effective" wavelength. Here, we will use the 95% interval where most of the transmission is contained. In other words, our "width" will be determined by the wavelengths where there's a total of 2.5% transmission remaining on the left/right edges. This is computed below._____no_output_____

# initializes a Nfilt x 2 array of "empty" values
filt_bounds = np.empty((Nfilt, 2))

# fraction of total flux from filter (confidence interval)
fbound = 0.95
for i in range(Nfilt):
    cdf = np.cumsum(ft[i])  # compute the cumulative sum over the transmission curve
    cdf /= cdf[-1]  # normalize the transmission curve to sum to 1.
    fremain = (1 - fbound) / 2.  # amount remaining (fraction) on either end

    # compute left bound
    temp = np.abs(cdf - fremain)  # absolute value
    idx_left = np.argmin(temp)  # find the **index** of the minimum position

    # compute right bound
    temp = np.abs(cdf - (1. - fremain))  # absolute value
    idx_right = np.argmin(temp)  # find the **index** of the minimum position

    # assign bounds to our
    filt_bounds[i] = fw[i][idx_left], fw[i][idx_right]  # lower/upper bound [A]

# compute the "width" as the difference between the upper and lower 95% bounds
filt_width = filt_bounds[:,1] - filt_bounds[:,0] # filter width [A]

print(filt_width)_____no_output_____

Spend some time breaking down the above snippet of code so that you can (ideally) explicitly summarize what exactly each line is doing and why. One way I like to make sense of unfamiliar code is by pulling it apart and plotting some of the intermediate results. An example of this is shown below._____no_output_____

i = 3
cdf = np.cumsum(ft[i])
cdf /= cdf[-1]
fremain = (1 - fbound) / 2.

plt.figure(figsize=(16, 4))
plt.plot(fw[i], ft[i] / max(ft[i]), color='black')  # normalize to 1.
plt.plot(fw[i], cdf, color='blue')  # normalize to 1.

temp = np.abs(cdf - fremain)
idx_left = np.argmin(temp)
plt.plot(fw[i], temp, color='red', linestyle='--')  # function we will minimize
plt.vlines(fw[i][idx_left], 0., 1., color='red')

temp = np.abs(cdf - (1. - fremain))
idx_right = np.argmin(temp)
plt.plot(fw[i], temp, color='red', linestyle='--')  # function we will minimize
plt.vlines(fw[i][idx_right], 0., 1., color='red')

plt.xlabel('Wavelength')
plt.ylabel('Fraction')_____no_output_____

If this type of thing works for you, great! If not, definitely try and find out what general practices/strategies are effective for you since you'll probably be doing a lot of this as you become more involved in coding._____no_output_____Using this information, we can now compute the average **photon energy** within each filter (i.e. the minimum unit/quantum of energy). One of the seminal findings of quantum mechanics (and the thing that got Einstein the nobel prize) was that photon energy is *quantized* according to the relation

$$ E = h \nu = h c / \lambda  \quad . $$

Let's use this relation to compute the (average) energy of an individual photon in each of our filters._____no_output_____

c = 2.998e18  # speed of light [A/s]
h = 6.6260755e-27  # Planck constant in erg*s
ephot_cent = h * c / filt_cent # photon energies at effective wavelength [erg]
print(ephot_cent)_____no_output_____

Now that we have all these bits and pieces, see if you can plot the average photon energy in each filter as a function of wavelength. A short example is shown below using `plt.errorbar` along with a short printout summary that utilizes the `round` function, but you're welcome to be as creative as you like!_____no_output_____

# figure
plt.figure(figsize=(16, 6))
xlow, xhigh = filt_bounds.T
xe_low, xe_high = filt_cent - xlow, xhigh - filt_cent
plt.errorbar(filt_cent, ephot_cent, xerr=[xe_low, xe_high], linestyle='none', marker='o')

# printout
print('Filter', 'Center[A]', 'Width({0})[A]'.format(fbound), 'Low[A]', 'High[A]', 'E_phot[1e-12 erg]')
for i in xrange(Nfilt):
    print(filt_names[i], round(filt_cent[i], 1), filt_width[i], filt_bounds[i][0],
          filt_bounds[i][1], round(ephot_cent[i] * 1e12, 2))_____no_output_____

**Extra Challenge: Can you get the axes of the plot to be (semi-)logarithmic rather than linear?**_____no_output_____# Observing a Galaxy_____no_output_____We are now ready to simulate a basic observation of our original galaxy through our set of filters. First, we need to integrate our galaxy spectrum $S_\nu$ over each filter. This takes a similar form to our calculation of the effective wavelength:

$$ F_\nu = \frac{\int T_\nu \, S_\nu \, d(\ln \nu)}{\int T_\nu \, d(\ln \nu)} $$

where $S_\nu$ (flux density per *frequency*) and $S_\lambda$ (flux density per *wavelength*) are related via

$$ S_\nu = S_\lambda \frac{\lambda^2}{c}  $$

using the nifty result

$$ d\nu = d \left(\frac{c}{\lambda}\right) = - \frac{c}{\lambda^2} d\lambda \quad . $$_____no_output_____

# compute S_\nu
fgal_nu = fgal_wave * wave**2 / c  # this works because units are [A] and [A/s]

# plot our result
plt.loglog(wave, fgal_nu)
plt.xlabel('Wavelength')
plt.ylabel(r'$S_\nu$')  # LaTeX-style math; the preceding 'r' "protects" the string_____no_output_____

Why this conversion between $S_\nu$ and $S_\lambda$? A mix between historical reasons and plotting usefulness (sometimes it's easier to visualize trends as per unit wavelength instead of per unit frequency, and vice versa). Plotting things in $S_\nu$ tends to highlight behavior in the optical range (which becomes "flatter" the more star formation a galaxy tends to have)._____no_output_____Using everything we've covered up to this point, we're now ready to simulate our galaxy observation. First, let's integrate our galaxy over the filters to get the relative photometric flux density $F_\nu$._____no_output_____

freq = c / wave
phot = np.array([np.trapz(ft[i] * fgal_nu, np.log(freq)) /
                 np.trapz(ft[i], np.log(freq))
                 for i in range(Nfilt)])_____no_output_____

Oh no -- it seems we've hit an error! It turns out our galaxy is observed at a different number of wavelengths compared to our original filter. To integrate numerically, we need both our filter and the galaxy spectrum to be observed at the exact same wavelength values. This requires us to **interpolate** one of our values. Since the galaxy wavelength grid appears to be more precise, let's go with interpolating all of our filter results onto a galaxy grid._____no_output_____Interpolation in Python is super easy using functions like `np.interp`. See if you can interpolate the transmission from filter onto the wavelength grid spanned by our galaxy (`wave`), under the condition that values outside the boundaries of the filter are automatically set to zero. An incomplete solution is given below._____no_output_____

# interpolation (incomplete)
np.interp(wave, fw[i], ft[i])_____no_output_____

Using this, let's redo our integral from above. I've provided a one-statement solution below, but I would highly encourage everyone to try their hand at implementing something themselves._____no_output_____

# compute relative photometry
phot = np.array([np.trapz(np.interp(wave, fw[i], ft[i], left=0., right=0.) * fgal_nu, np.log(freq)) /
                 np.trapz(np.interp(wave, fw[i], ft[i], left=0., right=0.), np.log(freq))
                 for i in range(Nfilt)])

print(phot)_____no_output_____

Note that this result is unnormalized because we haven't compared this *relative* result to some standard value. To correct for this and make the final answer more realistic, we're just going to multiply this result by $10^{-23}$._____no_output_____

phot *= 1e-23_____no_output_____

Now we want to derive error bars on our photometry. Let's pretend for a (beautiful) second that there is no other source of noise other than the source itself (i.e. ignoring the sky, instrument, etc., and only counting the number of photons received from our observed galaxy). The uncertainty on our photometry is directly related to the uncertainty in the number of photons we expect to receive. This is an example of a [Poisson process](https://en.wikipedia.org/wiki/Poisson_point_process), and it turns out the standard deviation in the number of photons we receive is just

$$ \sigma_N = \sqrt{N} ~\Rightarrow~ \sigma_N / N = 1/\sqrt{N} \quad . $$

So our fractional uncertainty is just $1/\sqrt{N}$._____no_output_____**Using the photon energies we computed earlier, compute the fractional uncertainties we expect in each filter assuming a 4hr observation with a 8m telescope at the effective wavelength of each filter.** Remember that our photometric flux densities have units of erg/s/cm$^2$/Hz. Again, I've added in a one-statement solution so you can continue on, but please try and write your own code to compute this._____no_output_____

phot_ferr = np.empty(Nfilt)_____no_output_____phot_ferr = 1. / np.sqrt(phot * (c / filt_cent) * (8. * 100. * 100.) * (4. * 60. * 60.) / ephot_cent)
print(phot_ferr)_____no_output_____

Finally, let's plot everything together: the galaxy SED, the expected photometry, and the observed filter set. A bare-bones example is shown below, where I've input lots of values by hand using trial and error._____no_output_____

# plot our results
plt.figure(figsize=(16, 5))
plt.plot(wave, fgal_nu / np.median(fgal_nu) * 0.8, color='red', alpha=0.4)
for i in range(Nfilt):
    plt.plot(fw[i], ft[i], ls='-', lw=2, color='blue')
plt.xlim([2e3, 2e4])
plt.ylim([0, 1.5])
plt.errorbar(filt_cent, phot / max(phot) * 1.2, xerr=[xe_low, xe_high],
             yerr = phot * phot_ferr / max(phot) * 1.2,
             marker='o', markersize=10, linestyle='none', color='red')
plt.xlabel('Wavelength [A]')
plt.ylabel('Flux Density [per Hz]', fontsize=26)
plt.title('UGCA 166 Photometry')_____no_output_____

**Extra Challenge: Can you add in some Gaussian (i.e. Normal) white noise to the observation to better mimic an observed *realization* of our galaxy?**

**Extra Challenge: Assume that random sky noise creates 2 photons per hour per (projected) m$^2$. How does this change the noise calculations above?**

**Extra Extra Challenge: Our noise assumptions above assume that we can approximate a discrete counting process (Poisson) using a continuous function (Normal). This can lead to problems since we only observe a discrete number of photons and we can't observe negative photons. Simulate the actual expected photon counts from a Poisson distribution.**_____no_output_____And that's that!_____no_output_____


# Notebook from weiming9115/MSEplots
Path: Real_Parcel_Buoyancy.ipynb

Parcel theory: irreversible and reversible processes, CAPE/CINE, and entrainment methods.

Wei-Ming Tsai, Apr. 2019@RSMAS

This project stems from [Prof. Brian Mapes's notebook](https://github.com/ATMOcanes/ATM663_convection_meso/blob/master/Resources/Notebooks/Real_Parcel_Buoyancy.ipynb)

0. Introduction of this project
1. Define thermo functions
1. Undilute parcel buoyancy: entropy conserving
1. Pseudoadiabatic vs. reversible
1. Undilute parcel buoyancy: MSE conserving
1. Considering the latent heat of freezing
1. Entraining plume of MSE
1. CAPE of entraining plume of MSE
1. Sensitivity of CAPE of entraining plume of MSE

_____no_output_____
# Introduction

The concept of a parcel is fundamental to the understanding of convective processes. This notebook is applying a 1-D parcel model with different assumptions and entrainment treatments to understand the following topics:

(1) reversible and irreversible processes based on different assumptions

(2) the effect of considering latent heat of freezing on parcel buoyancy

(3) entraiment effects on parcel buoyancy using a constant entrainment rate and entrainment rates varying with height

The mean sounding profile of Miami 2018 summer (JJA) is applied here and variables are interpolated into pressure levels with $\Delta$p=10 hPa from 1020hPa to 20hPa for entrainment calculations._____no_output_____

%pylab inline
# makes plots appear inline in the notebook, cheaty/simple wayPopulating the interactive namespace from numpy and matplotlib
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime,timedelta
import MSEplots.plots as mpt

import metpy.calc as mpcalc
from metpy import units
from metpy.constants import Cp_d,Lv,Rd,g
from metpy.calc.thermo import *
from siphon.simplewebservice.wyoming import WyomingUpperAir_____no_output_____import warnings
warnings.filterwarnings('ignore')_____no_output_____

### Tropical sounding profile
interpolating into 100 levels with constant interval in pressure for further calculation of entrainment _____no_output_____

from siphon.simplewebservice.wyoming import WyomingUpperAir
from scipy.interpolate import interp1d

date = datetime(2018, 8, 22, 0)
station = 'MFL'

df = WyomingUpperAir.request_data(date, station)
pp = df['pressure'].values #hPa
TT = df['temperature'].values  #C
Tdd = df['dewpoint'].values #C
ZZ = df['height'].values   #m

# linear interpoaltion into pressure levels
p = np.linspace(pp[0],pp[-1],200) # 200 pressure levels
# interpolated variables T, Td, Z
f = interp1d(pp,TT); T = f(p)
f = interp1d(pp,Tdd); Td = f(p)
f = interp1d(pp,ZZ); Z = f(p)

r = mixing_ratio( saturation_vapor_pressure(Td*units.units('degC')), p*units.units('hPa') )
q = specific_humidity_from_mixing_ratio(r)
rs = mixing_ratio( saturation_vapor_pressure(T*units.units('degC')), p*units.units('hPa') )
qs = specific_humidity_from_mixing_ratio(rs)_____no_output_____ax = mpt.msed_plots(p,T,q,Z,ent_rate=np.arange(0,2,0.05),entrain=True) # altitude included
ax.set_title('Wyoming Upper Air, Majuro (7.07N, 171.29E), 2018/08/24', fontsize=14)_____no_output_____

-------------

### Functions needed for moist thermo calculations_____no_output_____

# Package of thermo functions

import numpy as np

def esat(T):
# Saturation vapor pressure IN HPA as a function of temperature IN CELSIUS
    if( np.max(np.abs(T)) > 100 ) :
        print, 'esat: Please provide T in CELSIUS ', np.max(np.abs(T))
        #return 0

# This one is from Emanuel page 116 (4.4.13), over liquid at all temperatures
    Tref = 273.15
    ESAT = np.exp( 53.67957 - 6743.769/(T+Tref) - 4.8451*np.log(T+Tref) )

    # Water only
    return ESAT

def esati(T):
# Saturation vapor pressure IN HPA as a function of temperature IN CELSIUS
    if( np.max(np.abs(T)) > 100 ) :
        print, 'esat: Please provide T in CELSIUS ', np.max(np.abs(T))
        return 0

    # This one is from Emanuel page 116 (4.4.13), over liquid at all temperatures
    Tref = 273.15
    ESAT = np.exp( 53.67957 - 6743.769/(T+Tref) - 4.8451*np.log(T+Tref) )
    # This one is over ice, but valid at subzero temperatures only
    ESATI = np.exp( 23.33086 - 6111.72784/(T+Tref) + 0.15215*np.log(T+Tref) )

    # Stitch them together into the optput array based on sign of T in C
    return np.where(T>0, ESAT,ESATI)

def mixrat(Td,p):
# Td in C# p in mb
    if( np.max(np.abs(Td)) > 100 or np.max(np.abs(p)) > 1050) :
        print, 'mixrat: provide Td in C, p in mb',np.max(np.abs(Td)),np.max(np.abs(p))
        #return 0

    # simple formula with 0.622 rato of molecular weight or R vapor vs. air
    es = esat(Td)
    return 0.622 * es / (p-es)

def entropy(T,p,qtot):

# T in C, p in mb, qtot in kg/kg
    if( np.max(T) > 100. or np.max(np.abs(p)) > 1050. or np.max(np.abs(qtot)) > 0.1 ):
        print( 'entropy: provide T in centigrade, p in mb, qtot in kg/kg')
        print( T,p,qtot )
        return 0

    # from Raymond & Blyth 1992
    Tref = 273.15
    pref = 1000.0
    Cpd = 1005.		# dry air heat capacity at const p
    Cpv = 1850.		# water vapor heat capacity at const p
    Rv = 461.51		# Gas constant for vapor
    Rd = 287.05		# Gas constant for dry air
    Lref = 2.501e6 		# vaporization heat at Tref
    Cl  =  4218.		# liquid water heat capacity
    eref = 6.106		# sat p at Tref (mb)

    L = Lref - (Cl - Cpv)*T		# T ARRAY IN CENTIGRADE

    qsat = mixrat(T,p)
    qv = np.where(qtot < qsat,qtot, qsat)
    e = esat(T) *qv/qsat

    s = (Cpd + qtot*Cl)*np.log( (T+Tref)/Tref ) - Rd*np.log( (p-e)/pref ) + \
        L*qv/(T+Tref) - qv*Rv*np.log(qv/qsat)

    return s

def invert_entropy(sm,qt,p):
# Inverts entropy sm & total water qt at pressures p (mb)
# Returns T & qv vapor mixing ratio
# Met units: mb, C, kg/kg

    LOOPMAX =  200 #  /* max number of iteration loops */

    #/* invert the entropy equation -- use Newton's method. Measure ds for dT=1K

    ts = 200.0 + p/10. - 273.16	 # crude first guess (C)
    for i in range(0, LOOPMAX+1):
        fs0 = entropy(ts,p,qt) - sm
        fs1 = entropy(ts - 1.,p,qt) - sm
        dts = fs0/(fs1 - fs0) # dT/ds * (entropy error of guess ts)
        ts  = ts+dts          # correct that guess, linearized way

        #print('loop iteration i',i,'largest error is:', np.max(np.abs(dts)))

        if (np.max(np.abs(dts)) < .01): break # this will end the loop
        if (i == LOOPMAX - 1):
            print( "s invert not converging" )
            return 0

# gather up result and return it
    T = ts   # latest guess from iteration
    qvsat = mixrat(T,p)		#/* saturation mixing ratio at computed temp */
    qv = np.where(qt < qvsat,qt, qvsat)      #/* check for saturation */

    return T,qv

def MSE(T,Z,qv): # units C, m, kg/kg
    Cp = 1005.		# dry air heat capacity at const p
    L = 2.501e6 		# vaporization heat at Tref
    g = 9.8
    Tref = 273.16

    return Cp*(T+Tref) + g*Z + L*qv

def invert_MSE(h,qt,Z,p):
# Inverts h value & total water qt at a set of heights Z (in m) and pressures p (in mb)
# Returns T & qv vapor mixing ratio of parcel
# Met units: m, C, kg/kg, mb

#/* invert the transitive MSE equation -- use Newton's method
    LOOPMAX =  400 #  /* max number of iteration loops */

    ts = h*0 +20 -6*Z/1000. # crude first guess at T (6 K/km lapse rate)

# Now iterate
    for i in range(0, LOOPMAX+1):

    # The MSE profile based on guess ts and min(qt,qvsat)
        qsat0 = mixrat(ts, p)
        qv0 = np.where(qt<qsat0,qt, qsat0)
        MSE0 = MSE(ts,   Z, qv0      )

    # now MSE profile for a +1K perturbation
        qsat1 = mixrat(ts+1, p)
        qv1 = np.where(qt<qsat1,qt, qsat1)
        MSE1 = MSE(ts+1,Z, qv1)

    # So we can estimate the derivative dMSE/dT
        dMSEdT = MSE1-MSE0      # J/kg per K

    # Use that derivative to estimate the correction to the guess ts
        dts = (h-MSE0)/dMSEdT # convert MSE error of first guess ts to K units
        ts  = ts+dts          # correct initial T guess using that slope

    # test if correction is small, and quit if so. Otherwise, repeat.
        if (np.max(np.abs(dts)) < .01): break # end loop
        if (i == LOOPMAX - 1) :
            print( "MSE invert not converging" )
            #return 0

    # After the corrections have converged, gather up result and return it
    qvsat = mixrat(ts,p)		#/* saturation mixing ratio at computed temp */
    qv = np.where(qt < qvsat,qt, qvsat)      #/* check for saturation */

    return ts,qv _____no_output_____

---------------

### Diagrams for static energy and entropy _____no_output_____

fig = plt.figure(figsize=(14,5))

s = MSE(T,Z,q*1e-9) /1000*units.units('kJ/kg')
h = MSE(T,Z,q)      /1000*units.units('kJ/kg')
hs = MSE(T,Z,qs)    /1000*units.units('kJ/kg')

ax1 = plt.subplot(1,2,1)
ax1.plot(s, p, 'k')
ax1.plot(h, p, 'royalblue')
ax1.plot(hs, p, 'darkblue')
ax1.set_xlim([300,360]); ax1.set_ylim([1020,100]); ax1.set_title('Static energies by mass (pressure)')
ax1.plot(h*0+h[0],p,'grey')

sd = entropy(T,p,q*1e-6) *units.units('J/kg/K')
sm = entropy(T,p,q)      *units.units('J/kg/K')
sms = entropy(T,p,qs)    *units.units('J/kg/K')
cwv = mpcalc.precipitable_water(Td*units.units('degC'),p*units.units('mbar'))

ax2 = plt.subplot(1,2,2)
ax2.plot(sd, p, 'k')
ax2.plot(sm, p, 'royalblue')
ax2.plot(sms, p, 'darkblue')
#for n in range(3):
#    ax2.plot(sm_mod[n,:], p,'--',color='royalblue')

ax2.set_ylim([1020,100]); ax2.set_xlim([80,300]); ax2.set_title('Specific entropies by mass (pressure)')
ax2.plot(sm*0+sm[0],p,'grey')_____no_output_____

## 1. Assumptions in the parcel thoery _____no_output_____### Reversible process
- During the reversible process, **the MSE /entropy and total water are conserved**
- Condensate is carried by the rising parcel all the way to upper levels, **0% precipitation efficiency!**
- There is no external exchange of heat and mass between the parcel and the environment!_____no_output_____

fig = plt.figure(figsize=(7,5))

sm = entropy(T,p,q)      #*units.units('J/kg/K')
qtot = q[0] # total water content
Tp,qvp = invert_entropy(sm[0],qtot, np.where(p>50,p,50))

plot(Tp,p,'b'); plot(T,p,'k')
ylim([1020,50]); xlim([-80,30]); title('Tenv and T of entropy and qtot conserving parcel')
plt.legend(['T_parcel','T_env'])
ylabel('[hPa]',fontsize=12);xlabel('T [degC]',fontsize=12)
plt.grid()_____no_output_____# http://glossary.ametsoc.org/wiki/Virtual_temperature
def Tv(T,qv):
    Tref = 273.16
    return (1 + 0.61*qv)*(T+Tref) - Tref # virtual temperature

def Trho(T,qv,qc):
    Tref = 273.16
    return (1 + 0.61*qv -qc)*(T+Tref) - Tref # density temperature, considering condensed water loading_____no_output_____qcp = qtot - qvp

fig = plt.figure(figsize=(14,5))

ax1 = plt.subplot(1,2,1)
ax1.plot(Tp-T,p,'k')
ax1.plot( Trho(Tp,qvp,0)-Tv(T,q) ,p,'m')
ax1.plot( Trho(Tp,qvp,qcp)-Tv(T,q),p ,'g'); plot(p*0,p,'--',color='grey') # less buoyant due to condensed water loading
ax1.set_ylim([1020,50]); ax1.set_xlim([-10,10]); ax1.set_title('Parcel Buoyancy, w/wo condensate loading')
ax1.legend(['Tp-T, reversible','Tvp-Tv, non-loading','Trho-Tv, loading'])
ax1.set_xlabel('$\Delta$T [K]',fontsize=12)
ax1.set_ylabel('[hPa]',fontsize=12)

ax2 = plt.subplot(1,2,2)
ax2.plot(qvp*1000,p,'royalblue')
ax2.plot(qcp*1000,p,'darkblue')
ax2.set_xlabel('[g/kg]',fontsize=12)
ax2.set_ylabel('[hPa]',fontsize=12)
ax2.invert_yaxis()
ax2.legend(['qvp','qcp'])

dTrho_entropy = Trho(Tp,qvp,qcp)-Tv(T,q)_____no_output_____

Green and purple lines overlap below 963hpa (LCL) where water vapor in the rising parcel is not saturated yet.
Total water content is conserved during the reversible process._____no_output_____---------------

### Irreversible process: pseudo-adiabatic process
- The MSE /entropy is nearly conserved, but **the total water is Not conserved**
- Liquid water is removed immediately from the rising parcel once it forms, **100% precipitation efficiency!**
- The mass of eliminated condensate can be negligible comparing to the mass of parcel_____no_output_____

Tp_precip,qvp_precip = invert_entropy(sm[0],qvp, np.where(p>100,p,100)) # conserving energy

fig = plt.figure(figsize=(7,5))
plot( Trho(Tp,       qvp,       qcp  )-Tv(T,q) ,p, 'g') # reversible
plot( Trho(Tp_precip,qvp_precip, 0)-Tv(T,q) ,p, 'b'); plot(p*0,p,'--',color='grey') # persudo-adiabatic
ylabel('[hPa]',fontsize=12); xlabel('$\Delta$T [K]',fontsize=12)
plt.legend(['reversible','pseudo-adiabatic'])

ylim([1020,50]); xlim([-10,10]); title('Reversible vs. pseudo-adiabatic Trho excesses')_____no_output_____

In the pseudo-adiabatic process, the liquid water is removed out immediately once it forms. This assumption eliminates the effect of condensate loading and heat capacity of condensate, so the density temperature is warmer than that in the reversible process. However, this assumption also implies a 100% precipitation efficiency, which apparently conflicts with observations._____no_output_____---------------

### Compute parcel b (as T excesses), conserving MSE and q (total water)_____no_output_____

s = MSE(T,Z,q*1e-9) #/1000*units.units('kJ/kg')
h = MSE(T,Z,q)      #/1000*units.units('kJ/kg')
hs = MSE(T,Z,qs)    #/1000*units.units('kJ/kg')

TpMSE,qvpMSE = invert_MSE(h[0],q[0], np.where(Z<18000.,Z,18000.), np.where(p>100.,p,100.))
qcpMSE = q[0]-qvpMSEMSE invert not converging
fig = plt.figure(figsize=(7,5))
Tp_precip,qvp_precip = invert_MSE(h[0],qvpMSE, np.where(Z<20000.,Z,20000.),np.where(p>100,p,100)) # conserving energy

plot( Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q) ,p, 'g') # MSE-conserved T
plot( Trho(Tp_precip,qvp_precip,0)-Tv(T,q) ,p, 'b') # persudo-adiabatic
plot(p*0,p,'--',color='grey')

ylim([1020,50]); xlim([-10,10])
ylabel('[hPa]',fontsize=12); xlabel('$\Delta$T [K]',fontsize=12)
plt.legend(['reversible','pseudo-adiabatic'],loc=1)

dTrho_MSE = Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q)MSE invert not converging
# Z coord plot as it becomes CAPE = integral (b dZ)
fig = plt.figure(figsize=(7,5))

plot(TpMSE-T,Z,'darkviolet')
plot( Tv(TpMSE,qvpMSE)-Tv(T,q) ,Z, 'violet')
plot( Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q) ,Z, 'r')

ylim([0,17000]); xlim([-10,10]); title('MSE and qtot conserving Tvp-Tvenv and Trhop-Tvenv')
plot( dTrho_entropy ,Z, 'k'); plot(p*0,Z,'--',color='grey')
plt.legend(['TpMSE-T','TvMSE-Tv','TrhoMSE-Tv','TrhoEnt-Tv'])
title('Reversible vs. pseudo-adiabatic Trho excesses')

dTrho_MSE = Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q)_____no_output_____

## 2. Considering the effect of latent heat of freezing
The pseudo-adiabatic process shows departure from observed tropical conditions, such as complete fallout of condensate (100% precipitaiton efficiency). The reversible process indicates that condensate is carried all the way to the upper level without precipitation or transfromation into the ice phase. Williams and Renno (1992) argued that a reversible process with ice is more accurate comparing to the previous two and shows consistent to observed characteristics of deep convection, such as enough CAPE, upper cirrus, and lightning activities. _____no_output_____

# esati, mixrat
def mixrat_i(T,p):
    es = esati(T)
    return 0.622*es/(p-es)

def delT4ice(TpMSE,p,qcpMSE,qvpMSE,Tf=0,simple=True):
# computes delta T, an increase in temperature due to latent heat of freezing and the deposition
# of water vapor on ice (Iribarne and Godson, 1981 eq.94)
    Tref = 273.15 # reference Temp.
    Ls = 2.83e6 # J/kg, latent heat of sublimation
    Lf = 3.34e5 # J/kg, latent heat of freezing
    Cpd = 1005  # J/kg
    Cpv = 1996  # J/kg
    Cpi = 2108  # J/kg
    Rv = 461.51 # J/K/kg

    if simple == True:

        delT = Lf*qcpMSE/Cpd
        Tdev = TpMSE + delT

        Tdev = np.where(TpMSE < Tf, Tdev, TpMSE)
        qcpdev = np.where(TpMSE < Tf, 0, qcpMSE) # 100% convert to ice phase below zero deg
        qipdev = np.where(TpMSE < Tf, qcpMSE, 0) # 100% convert to ice phase below zero deg
        return Tdev,qcpdev,qipdev

    else: # precise calculation based on Iribarne and Godson (1981)
        LOOPMAX = 200 # iteration number

        fg0 = -8*np.ones(len(Tp))   # first guess
        k = np.zeros(len(Tp))

        for i in range(0, LOOPMAX+1):

            rs = qcp + mixrat(Tp,p) - mixrat_i(fg0,p)
            Cpt = Cpd + mixrat(Tp,p)*Cpv + rs*Cpi
            a = ( Lf*qcp + Ls*qvp*(1-esati(Tp)/esat(Tp)) )
            b = (Cpt + mixrat_i(fg0,p)*Ls**2/(Rv*(Tp+Tref)**2))
            delT = a / b

            print(len(delT))
            if np.max(np.abs(delT-k)) < 1e-2:
                break
                fg0 = Tf + delT

            else:
                k = delT
                fg0 = Tf + delT

            Tdev = np.where(Tp < Tf, fg0, Tp)

            return Tdev_____no_output_____Tpice,qcpice,qipice = delT4ice(TpMSE,p,qcpMSE,qvpMSE,Tf=0,simple=True) # including latent heat of freezing in a simple way_____no_output_____fig = plt.figure(figsize=(14,5))

ax1 = plt.subplot(1,2,1)
ax1.plot(Tpice,Z/1000)
ax1.plot(TpMSE,Z/1000,'r')
ax1.plot(T,Z/1000,'k')
ylim([0,18])

ax2 = plt.subplot(1,2,2)
ax2.plot(qcpice,Z/1000,'k')
ax2.plot(qipice,Z/1000,'r')
ax2.plot(qcpMSE,Z/1000,'b')
ax2.plot(qcpice+qipice,Z/1000,'--',color='grey',linewidth=3)
ax2.plot(qvpMSE,Z/1000,'g')
ylim([0,18])
ax2.legend(['qcp$_{wi}$','qip$_{wi}$','qcp'])_____no_output_____# conservation of total water
plot(qvpMSE+qcpice+qipice,p)
ylim([1020,50])_____no_output_____# Trho-Tv based on different assumptions and w/wo ice
fig = plt.figure(figsize=(7,5))

plot( Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q) ,p, 'violet') #
plot( Trho(Tpice,qvp,qipice+qcpice)-Tv(T,q) ,p ,'r') # condensate becomes ice with the same mixing ratio
plot( Trho(Tp_precip,qvp_precip, 0)-Tv(T,q) ,p, '--r')
ylim([1020,50]); xlim([-10,10]); title('Trho diff, w/wo latent heat of freezing')
plot(p*0,p,'--',color='grey') # entropy
plt.legend(['without ice','with ice','pseudo'])
plt.xlabel('$\Delta$ T [K]',fontsize=12)
plt.ylabel('[hPa]')

dTrho_MSE = Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q)_____no_output_____

---------------

### Determination of convective available potential energy (CAPE) and convective inhibition energy (CINE)
The buoyancy acceleration is in the form of
\begin{equation*}
b = \frac{T_{vp}-T_v}{T_v}g
\end{equation*}
where g is gravity acceleration, 9.8m/s$^2$. T$_{vp}$ and T$_v$ are virtual temperature of
the parcel and its environment, respectively.

The definition of CAPE is the vertical integrated buoyancy between LFC and LNB
\begin{equation*}
CAPE = \int_{LFC}^{LNB}b \ \ dz
\end{equation*}
where LFC is the level of free convection (b>0), and LNB is the level of neutral buoyancy (b=0)

and the definition of CAPE is the vertical integrated buoyancy between the surface and LFC
\begin{equation*}
CIN = \int_{sfc}^{LFC}b \ \ dz
\end{equation*}

Besides, a basic meteorological concept is that updraft vertical velocities are closely related to CAPE, the maximum vertical velocity of a rising parcel can be estimated as
\begin{equation*}
W_{max} = \sqrt{2CAPE}
\end{equation*}_____no_output_____

dTrho = Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q) # no mixing, no ice
dTrho_ice = Trho(Tpice,qvp,qcpice+qipice)-Tv(T,q)
dTrho_pseu = Trho(Tp_precip,qvp_precip, 0)-Tv(T,q)

b = 9.8*dTrho/(Tv(T,q) + 273.15)
b_ice = 9.8*dTrho_ice/(Tv(T,q) + 273.15)
b_pseu = 9.8*dTrho_pseu/(Tv(T,q) + 273.15)

fig = plt.figure(figsize=(7,5))

plot(b , p,'violet'); plot(b_ice, p,'r');plot(b_pseu, p,'--r')
plot(p*0, p,'--',color='grey')
ylim([1020,50]); xlim([-0.1,0.4]);
title('Plume buoyancy (m/s$^2$)');xlabel('m/s$^2$',fontsize=12);ylabel('[hPa]',fontsize=12)
plt.legend(['without ice','with ice','pseudo'])_____no_output_____dZ = np.gradient(Z)

fig = plt.figure(figsize=(7,5))

cape = np.cumsum( np.where((b>0),b,0) *dZ)
cape_ice = np.cumsum( np.where((b_ice>0),b_ice,0) *dZ)
cape_pseu = np.cumsum( np.where((b_pseu>0),b_pseu,0) *dZ )
plot(cape,Z/1000,'violet')
plot(cape_ice,Z/1000,'r')
plot(cape_pseu,Z/1000,'--r')
plt.legend(['without ice','with ice','pesudo'],loc=4)
title('Cumulative bdZ'); xlabel('CAPE [J/kg]',fontsize=12); ylabel('Z [km]',fontsize=12)
ylim([0,18]); xlim([-100,3000]);

print('CAPE without ice:  ' + str(round(cape[-1])) + ' J/kg')
print('CAPE with ice:     ' + str(round(cape_ice[-1])) + ' J/kg')
print('CAPE pesudo:       '  + str(round(cape_pseu[-1])) + ' J/kg')CAPE without ice:  471.0 J/kg
CAPE with ice:     1969.0 J/kg
CAPE pesudo:       1702.0 J/kg

If we consider the latent heat of freezing in the reversible process, increasing buoyancy of the parcel due to the heating leads to much higher CAPE. On the other hand, the pseudo-adiabatic process removes condensate immediately, so there is no liquid water which can tranform into the ice phase above the freezing level (~5 km).

### The impact of latent heat of freezing on the CAPE is considerably huge._____no_output_____
## 3. Entrainment methods and applications
---------------
#### Detrainment is assumed zero in this discussion.
For the p-coordinate, The standard continuity expression for mass flux in the updraft is
\begin{equation*}
\frac {d{M}}{d{p}} = M(\epsilon - \delta), \   \
\end{equation*}

Considering entrainment rate is much larger than detrainment rate,
\begin{equation*}
M = M_0 exp(\epsilon (p_0-p))
\end{equation*}

In this form, the mass flux increases exponentially with height due to entraining air from the environment.

Becasue of the entrainment, the conserved variables (MSE or entropy) is not conserved but mixed with environmental values.

\begin{equation*}
\frac {d{r}}{d{p}} = X(\hat{r} - r)
\end{equation*}
where $r$ is the conserved variable, r_hat is its environmental value. X is the entrainment rate with a unit of hPa$^{-1}$

After reconstructing the entrainment formula, the conserved variable in the parcel at each level with entrained environmental air becomes

\begin{equation*}
r_k = (1 - X_{k-1}\Delta p)r_{k-1} + X_{k-1}\Delta p\hat{r}_{k-1}
\end{equation*}

### Three entrainment assumptions are applied as follows:_____no_output_____### (1) constant mixing rate (Stommel 1947; Siebesma et al. 2007)
The mixing coefficient, X$\Delta$p, is specified as a constant value at all levels
\begin{equation*}
    X = 0.001 \  \ hPa^{-1}
\end{equation*}
which means the originally undiluted parcel will mix with the environmental air with a fraction of 0.001 per hPa. _____no_output_____### (2) mixing rate with vertical dependence (Holloway and Neelin, 2009; Schiro and Neelin 2019)

#### 1. Deep inflow mixing A: corresponding to LES-based simulation
\begin{equation*}
X_k\Delta p = c_\epsilon z_k^{-1}\Delta z , \     \
c_\epsilon = 0.4
\end{equation*}

#### 2. Deep inflow mixing B: an linear increase in mass flux at low levels, tapering at midtroposphere
Observations of vertical velocity show that the mass flux increases nearly linearly through a deep lower-tropospheric layer, motivating the assumptions for the mass flux profile in deep-inflow-B here. The entrainment depends on mass flux (updraft) itself
\begin{equation*}
X_k\Delta p = -m^{-1}(\partial m/\partial p)\Delta p
\end{equation*}
\begin{equation*}
Below \  \ 900hPa,\   \, \ \ X_k\Delta p = (p_0 - p_k)^{-1}\Delta p
\end{equation*}
\begin{equation*}
Above \  \ 430hPa,\   \ X_k = 0
\end{equation*}_____no_output_____### (3) interactive entrainment (Sahany et al., 2012)
This assumption takes into account effects of
vertical acceleration of the plume that would enhance entrainment by a mass flux balance.Therefore,unlike previous formulations which are independent of the evolution of the parcel, the interactive entrainment is dynamically determined by the updraft vertical velocity.

\begin{equation*}
X_k = X_{min} + \frac{\omega_k - \omega_{k-1}}{\Delta p(\omega_k + \omega_{k-1})/2}
\end{equation*}

where $\omega$ is the pressure vertical velocity and $X_{min}$ is the minimum prescribed mixing rate, 0.001 hPa$^{-1}$.

The equation of vertical veolcity can be expressed as
\begin{equation*}
\frac{\partial W^2_u}{\partial z} = -\frac{1}{z}W^2_u + B
\end{equation*}

A solution of updraft vertical velocity is
\begin{equation*}
W^2_u = z^{-1}\int_{0}^{z}z' B dz' + W^2_0(\frac{z}{z_0})^{-1}, \  \ W_0 = 10 m/s, \  \ z_0 = 350 m
\end{equation*}
_____no_output_____

def entrain_const(c,p):
    dp = -np.append(0,np.diff(p))
    X = c
    X = np.where(p > 950,0.18,X) # In the PBL, X = 0.18 hPa^-1
    return X*dp

def entrain_coeffA(Z,p,ce=0.4):
# calculate entrainment coefficient from depp inflow mixing A
    dz = np.append(0,np.diff(Z))
    dp = -np.append(0,np.diff(p))
    X = ce*dz/Z/dp
    X[0] = 0
    return X*dp

def entrain_coeffB(p):
# calculate entrainment coefficient from deep inflow mixing B

    # specify a vertical profile of m that increases linearly at lower levels:
    # m=0 at surface and m reaches to the maximum at 430hPa, no entrainment above
    ps = p[0]
    b = pi/2/(1-430/ps); a = -b/ps
    m = sin( a*p + b ) # specified vertical profile of mass flux

    #dmdp = np.gradient(m,p)
    dmdp = np.append(0,np.diff(m))/np.append(0,np.diff(p))
    dp = -np.append(0,np.diff(p))
    X = -dmdp/m

    X = np.where(p > 950, 1/(ps-p),X)
    X = np.where(p > 430, X, 0) # no entrainment above 430 hPa
    X[0] = 0 # at surface, denominator is zero

    return X*dp, m

def conserved_dmix(s,q,X):
# calculate entrained conserved variable over the whole levels
# where sp is conserved variable of parcel, s is environmental value,
# and X is entrainment coefficient.

    s_mix = np.zeros(len(s)); q_mix = s_mix.copy()
    s_mix[0] = s[0]; q_mix[0] = q[0] # total water content
    for z in range(1,len(s)):
        s_mix[z] = (1-X[z-1])*s_mix[z-1] + X[z-1]*s[z-1]
        q_mix[z] = (1-X[z-1])*q_mix[z-1] + X[z-1]*q[z-1]

    return s_mix, q_mix_____no_output_____def entrain_dyn(TpMSE,qvpMSE,qcpMSE,T,q,Z,p,Xmin=1e-3):
# entrainment coeff for interactive/dynamic entrainment rate
# considering the dependence of buoyancy/vertical velocity
    Rd = 287.15
    Z0 = 350 # meter
    W0 = 10 # m/s
    ps = p[0]
    b = 9.8*(Trho(TpMSE,qvpMSE,qcpMSE)-Tv(T,q))/(Tv(T,q)+273.15) # buoyancy

    # diagnosis of vertical velocity
    W2 = np.zeros(len(Z))
    dz = np.append(0,np.diff(Z))
    for k in range(len(Z)):
        B = np.trapz(b[:k+1]*Z[:k+1],Z[:k+1])
        W2[k] = B/Z[k] + W0**2*((Z[k]/Z0)**-1)

    W = W2**(1/2) # vertical velocity [m/s]

    # convert W into omega: omega = -ro*g*w
    ro = p/(Rd*(Tv(TpMSE,qvpMSE)+273.15))*100
    omega = -ro*9.8*W

    X = W2.copy()
    dp = -np.append(0,np.diff(p))
    for k in range(1,len(p)):
        X[k] = Xmin + (omega[k]-omega[k-1])/(dp[k]*(omega[k]+omega[k-1])/2)

    X[X<0] = 0 # avoid negative entrainment [not documented implicitly in Sahany et al., 2012]
    X = np.where(p > 950, 1/(ps-p),X)
    X[0] = 0

    return X*dp, omega, B, W2_____no_output_____# calculate mixing coefficients for each method
X_c1 = entrain_const(1e-3,p)
hp_c1, qp_c1 = conserved_dmix(h,q,X_c1)

X_A = entrain_coeffA(Z,p,0.4)
hp_mixA, qp_mixA = conserved_dmix(h,q,X_A)

X_B, m = entrain_coeffB(p)
hp_mixB, qp_mixB = conserved_dmix(h,q,X_B)

X_dyn, omega_diag, B_diag, b = entrain_dyn(TpMSE,qvpMSE,qcpMSE,T,q,Z,p,Xmin=1e-3)
hp_mixdyn, qp_mixdyn = conserved_dmix(h,q,X_dyn)_____no_output_____plt.plot(X_dyn,p)
#xlim([0,100])
ylim([1020,200])_____no_output_____fig = plt.figure(figsize=(14,5))

ax1 = plt.subplot(1,2,1)

# mixing curves
ax1.plot(hp_c1/1000,p,'m',zorder=2) # const epsilon 1/15km
ax1.plot(hp_mixA/1000,p,'r',zorder=2) # deep flow entrainment A
ax1.plot(hp_mixB/1000,p,'--r',zorder=2) # deep flow entrainment B
ax1.plot(hp_mixdyn/1000,p,':r',zorder=2) # dynamic entrainment
ax1.plot(h*0+h[0]/1000,p,'grey',zorder=2)
xlabel('[kJ/kg]',fontsize=12);ylabel('[hPa]',fontsize=12)
xlim([300,360]); ylim([1020,100])
ax1.legend(['const, 0.001/hPa','deep mixing A','deep mixing B','dynamic mixing','no mixing'])

ax1.plot(s/1000, p, 'k')
ax1.plot(h/1000,p,'royalblue')
ax1.plot(hs/1000, p, 'darkblue')

dp = -np.gradient(p)
ax2 = plt.subplot(1,2,2)
ax2.plot(X_c1/dp,p,'m',zorder=2)
ax2.plot(X_A/dp,p,'r',zorder=2)
ax2.plot(X_B/dp,p,'--r',zorder=2)
ax2.plot(X_dyn/dp,p,':r',zorder=2)
ax2.plot(X_c1*0,p,color='k',zorder=1)
xlabel('Mixing coeff X [hPa$^{-1}$]',fontsize=12);ylabel('[hPa]',fontsize=12)
ylim([1020,100]);xlim([0,0.02])
ax2.legend(['const, 0.001/hPa','deep mixing A','deep mixing B','dynamic mixing'])_____no_output_____

Dynamic mixing may have negative values in this case, which may be associated with the acceleration of vertical velocity and the specified Xmin. X is assumed zero if it's negative._____no_output_____

# calculate Tp for each entrainment method
Tp_c1,qvp_c1 = invert_MSE(hp_c1,qp_c1, np.where(Z<20000.,Z,20000.), np.where(p>50.,p,50.))
qcp_c1 = qp_c1 - qvp_c1
Tpice_c1,qcpice_c1,qipice_c1  = delT4ice(Tp_c1, p, qcp_c1, qvp_c1, Tf=0, simple=True)

Tp_mixA,qvp_mixA = invert_MSE(hp_mixA,qp_mixA, np.where(Z<20000.,Z,20000.), np.where(p>50.,p,50.))
qcp_mixA = qp_mixA - qvp_mixA
Tpice_mixA,qcpice_mixA,qipice_mixA  = delT4ice(Tp_mixA, p, qcp_mixA, qvp_mixA, Tf=0, simple=True)

Tp_mixB,qvp_mixB = invert_MSE(hp_mixB,qp_mixB, np.where(Z<20000.,Z,20000.), np.where(p>50.,p,50.))
qcp_mixB = qp_mixB - qvp_mixB
Tpice_mixB,qcpice_mixB,qipice_mixB  = delT4ice(Tp_mixB, p, qcp_mixB, qvp_mixB, Tf=0, simple=True)

Tp_mixdyn,qvp_mixdyn = invert_MSE(hp_mixdyn,qp_mixdyn, np.where(Z<20000.,Z,20000.), np.where(p>50.,p,50.))
qcp_mixdyn = qp_mixdyn - qvp_mixdyn
Tpice_dyn,qcpice_dyn,qipice_dyn  = delT4ice(Tp_mixdyn, p, qcp_mixdyn, qvp_mixdyn, Tf=0, simple=True)MSE invert not converging
MSE invert not converging
MSE invert not converging
MSE invert not converging
fig = plt.figure(figsize=(7,5))

plot( Trho(Tp_mixA,qvp_mixA,qcp_mixA)-Tv(T,q) ,p, 'r',linewidth=2.5)
plot( Trho(Tp_mixB,qvp_mixB,qcp_mixB)-Tv(T,q) ,p, '--r',linewidth=2.5)
plot( Trho(Tp_c1,qvp_c1,qcp_c1)-Tv(T,q) ,p, '-.r')
plot( Trho(Tp_mixdyn,qvp_mixdyn,qcp_mixdyn)-Tv(T,q) ,p, ':r')
plot( dTrho_MSE ,p ,'violet') # condensate loading
plot(p*0,p,'--',color='grey')

ylim([1020,50]); xlim([-10,10]);
title('Plume buoyancy compared to undilute');xlabel('$\Delta$T [K]',fontsize=12);ylabel('[hPa]',fontsize=12)
plt.legend(['deep mixing A','deep mixing B','const, 0.001/hPa','dynamic mixing','no mixing'])_____no_output_____fig = plt.figure(figsize=(7,5))

plot( Trho(Tpice_mixA,qvp_mixA,qcpice_mixA+qipice_mixA)-Tv(T,q) ,p, 'r',linewidth=2.5)
plot( Trho(Tpice_mixB,qvp_mixB,qcpice_mixB+qipice_mixB)-Tv(T,q) ,p, '--r',linewidth=2.5)
plot( Trho(Tpice_c1,qvp_c1,qcpice_c1+qipice_c1)-Tv(T,q) ,p, 'violet')
plot( Trho(Tpice_dyn,qvp_mixdyn,qcpice_dyn+qipice_dyn)-Tv(T,q) ,p, ':r')
plot( Trho(Tpice,qvp,qcpice+qipice)-Tv(T,q) ,p ,'k') # no mixing
plot(p*0,p,'--',color='grey')

ylim([1020,50]); xlim([-10,10]);
title('Plume buoyancy compared to undilute'); xlabel('$\Delta$T [K]',fontsize=12);ylabel('[hPa]',fontsize=12)
plt.legend(['deep mixing A','deep mixing B','const, 0.001/hPa','dynamic mixing','no mixing'])_____no_output_____

### Sensitivities of CAPE and LNB to constant mixing rates_____no_output_____

mixing_rates = np.array([5e-4,1e-3,2e-3,3e-3,4e-3]) # entrainment rate [%/hpa]
cape_sum = np.zeros(len(mixing_rates))
LNB_sum = cape_sum.copy()
dTpice_sum = np.zeros((len(mixing_rates),len(p)))
Xdp_sum = dTpice_sum.copy()

for n,X in enumerate(mixing_rates):

    Xdp_cmix = entrain_const(X,p)
    hp_cmix, qp_cmix = conserved_dmix(h,q,Xdp_cmix) # vars after mixing
    Tp_cmix, qvp_cmix = invert_MSE(hp_cmix,qp_cmix, np.where(Z<20000.,Z,20000.), np.where(p>50.,p,50.))
    qcp_cmix = qp_cmix - qvp_cmix
    Tpice_cmix,qcpice_cmix,qipice_cmix  = delT4ice(Tp_cmix, p, qcp_cmix, qvp_cmix, Tf=0, simple=True)

    Xdp_sum[n] = Xdp_cmix
    dTpice_sum[n,:] = Trho(Tpice_cmix,qvp_cmix,qcpice_cmix+qipice_cmix)-Tv(T,q)

    b_mix = 9.8*(dTpice_sum[n,:])/(Tv(T,q)+273.15) # buoyancy
    b_mix = np.where(p>50,b_mix,0) # avoid noises at extremely high altitudes
    cape_sum[n] = np.cumsum( np.where((b_mix>0),b_mix,0) *dZ)[-1]
    LNB_idx = np.where(b_mix>0)[0][-1] # last point where b > 0
    LNB_sum[n] = p[LNB_idx]MSE invert not converging
MSE invert not converging
MSE invert not converging
MSE invert not converging
MSE invert not converging
fig = plt.figure(figsize=(14,5))

ax1 = plt.subplot(1,2,1)
ax1.plot(Trho(Tpice,qvp,qcpice+qipice)-Tv(T,q),p ,'k') # no mixing
for n in range(len(mixing_rates)):
    ax1.plot(dTpice_sum[n],p)
    ax1.set_ylim([1020,50]); ax1.set_xlim([-10,10]);
ax1.plot(p*0,p,'--',color='grey')
ax1.set_title('Const. mixing compared to undilute')
ax1.set_xlabel('$\Delta$T [K]',fontsize=12);ax1.set_ylabel('[hPa]',fontsize=12)
ax1.legend(['no mixing','5e-4/hPa','1e-3/hPa','2e-3/hPa','3e-3/hPa','4e-3/hPa'])

ax2 = plt.subplot(1,2,2)
for n in range(len(mixing_rates)):
    ax2.plot(Xdp_sum[n,:],p)
    ax2.set_ylim([1020,50]); ax2.set_xlim([0,0.05]);
ax2.set_xlabel('Mixing coefficient, $Xdp$',fontsize=12);ax2.set_ylabel('[hPa]',fontsize=12)
_____no_output_____fig = plt.figure(figsize=(14,5))

ax1 = plt.subplot(1,2,1)
ax1.plot(mixing_rates,cape_sum,'-o')
ax1.set_xlabel('mixing rate [hPa$^{-1}$]',fontsize=12)
ax1.set_ylabel('CAPE [J/kg]')

ax2 = plt.subplot(1,2,2)
ax2.plot(mixing_rates,LNB_sum,'-o')
ax2.set_xlabel('mixing rate [hPa$^{-1}$]',fontsize=12)
ax2.set_ylabel('LNB [hPa]')_____no_output_____

### Conclusions
- Both reversible and pseudo-adiabatic processed are discussed using a parcel model. Parcel buoyancy is strongly damped by condensate loading.
- Considering the existence of ice phase and associated latent heat allows the parcel rising to upper levels, becoming a deeper plume.
- Four types of entrainment rate assumptions are tested: (1) constant mixing, (2) inversely proportional to z, (3) mass-flux determined, and (4) interactive mixing depending on velocity. The plume buoyancy is quite sensitive to different entrainment assumptions.
- Unlike the free troposphere, buoyancy within boundary layer (~ below 950 hPa) is less subject to different entrainment treatments.
- How entrainment is represented in GCMs determines characteristics of moist convection, including the LNB, CAPE and associated precipitation processes. _____no_output_____Top

_____no_output_____