# Notebook from siddgood/podcast-recommendation-engine
Path: notebooks/podcast_recommender.ipynb

import re
import nltk
import random
from nltk.tokenize import RegexpTokenizer
from stop_words import get_stop_words
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer

import numpy as np
import pandas as pd
import heapq
import numpy as np
from collections import Counter_____no_output_____

# Preparing DataFrame_____no_output_____

podcasts_df = pd.read_pickle('../pickle_files/english_podcasts_detailed_cleaned.pkl')_____no_output_____podcasts_df['text'] = podcasts_df[['title', 'producer', 'genre', 'description', 'episode_titles', 'episode_descriptions']].apply(lambda x: ' '.join(x), axis=1)
podcasts_df = podcasts_df.drop(columns=['genre', 'description', 'num_episodes', 'rating', 'num_reviews', 'link', 'episode_titles', 'episode_descriptions'])
podcasts_df['ID'] = list(range(podcasts_df.shape[0]))_____no_output_____

# Preprocessing Text_____no_output_____

# create list of stop words
stop = get_stop_words('en')

# remove non-alphanumeric, non-space
stop = [re.sub(r'([^\s\w]|_)+', '', x) for x in stop]

# add in custom stop words
days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']

months = ['january', 'february', 'march', 'april', 'may', 'june',
          'july', 'august', 'september', 'october', 'november', 'december']

other = ['nan', 'podcast', 'podcasts', 'every', 'new', 'weekly',
         'stories', 'story', 'episode', 'episodes', 'listen',
         'host', 'hosted', 'join']

[stop.append(str(day)) for day in days]
[stop.append(str(month)) for month in months]
[stop.append(str(x)) for x in other]

def topKFrequent(tokenized_text, k):

    count = Counter(tokenized_text)

    return heapq.nlargest(k, count.keys(), key=count.get)

def remove_stop(text, stop):
    custom_stop = stop
#     top5 = topKFrequent(text, 5)
#     custom_stop = custom_stop + top5

    new_text = []
    for word in text:
        if word not in custom_stop:
            new_text.append(word)
    return new_text

# create tokenizer
tokenizer = RegexpTokenizer(r'\w+')

# create stemmer
p_stemmer = PorterStemmer()
l_stemmer = WordNetLemmatizer()

def stem_list(text, p_stemmer):
    new_list = []
    for word in text:
        new_list.append(p_stemmer.stem(word))
    return new_list

def lem_list(text, l_stemmer):
    new_list = []
    for word in text:
        new_list.append(l_stemmer.lemmatize(word))
    return new_list

def preprocess_text(text):
    # remove mixed alphanumeric
    text = re.sub(r"""(?x) # verbose regex
                            \b    # Start of word
                            (?=   # Look ahead to ensure that this word contains...
                             \w*  # (after any number of alphanumeric characters)
                             \d   # ...at least one digit.
                            )     # End of lookahead
                            \w+   # Match the alphanumeric word
                            \s*   # Match any following whitespace""",
                             "", text)

    # remove urls (will check and remove http and www later)
    text = re.sub(r'\s([\S]*.com[\S]*)\b', '', text)
    text = re.sub(r'\s([\S]*.org[\S]*)\b', '', text)
    text = re.sub(r'\s([\S]*.net[\S]*)\b', '', text)
    text = re.sub(r'\s([\S]*.edu[\S]*)\b', '', text)
    text = re.sub(r'\s([\S]*.gov[\S]*)\b', '', text)

    # remove non-alphanumeric, non-space
    text = re.sub(r'([^\s\w]|_)+', '', text)

    # tokenize text
    text = tokenizer.tokenize(text.lower())

    # remove stop words
    text = remove_stop(text, stop)

    # stem
    text = lem_list(text, l_stemmer)

    # remove instances of http or www
    new_text_list = []
    for word in text:
        if re.search(r'http', word):
            continue
        if re.search(r'www', word):
            continue
        new_text_list.append(word)

    new_text = ' '.join(new_text_list)

    return new_text_____no_output_____podcasts_df['text'] = podcasts_df['text'].map(preprocess_text)
podcasts_df = podcasts_df[podcasts_df.text != '']_____no_output_____podcasts_df.head()_____no_output_____

# Podcast Recommendation Models_____no_output_____

# helper functions
def get_title_from_index(index):
    return podcasts_df[podcasts_df.ID == index]["title"].values[0]

def get_index_from_title(title):
    return podcasts_df[podcasts_df.title == title]["ID"].values[0]

def get_recommendations(podcast_id, sim_matrix):
    recommendations = list()

    podcast_title = get_title_from_index(podcast_id)
    similar_podcasts =  list(enumerate(sim_matrix[podcast_id]))
    sorted_similar_podcast = sorted(similar_podcasts,key=lambda x:x[1],reverse=True)

    for i in range(11):
        title = get_title_from_index(sorted_similar_podcast[i][0])
        recommendations.append(title)

    return recommendations

test_podcasts = ['The Daily', 'Up First', 'VIEWS with David Dobrik and Jason Nash', 'Impaulsive with Logan Paul',
                 'The Bill Simmons Podcast', 'My Favorite Murder with Karen Kilgariff and Georgia Hardstark',
                 'This American Life', 'Joel Osteen Podcast', 'TED Radio Hour', 'Call Her Daddy',
                 'Skip and Shannon: Undisputed'
                ]_____no_output_____

### Cosine Similarity + CountVectorizer (Bag of Words) Method_____no_output_____

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

cv = CountVectorizer()
cv_matrix = cv.fit_transform(podcasts_df["text"])
cv_cosine_sim = cosine_similarity(cv_matrix)_____no_output_____for i in test_podcasts:
    print(get_recommendations(get_index_from_title(i), cv_cosine_sim))
    print()['The Daily', 'Impeachment Inquiry: Updates from The Washington Post', 'Impeachment: A Daily Podcast', 'The Takeaway', 'Article II: Inside Impeachment', "The Daily 202's Big Idea", 'The 11th Hour with Brian Williams', 'Bill O’Reilly’s No Spin News and Analysis', 'The Last Word with Lawrence O’Donnell', 'Up First', 'Impeachment Today']

['Up First', 'Impeachment: A Daily Podcast', 'Impeachment Inquiry: Updates from The Washington Post', 'The Daily', 'Article II: Inside Impeachment', 'The 11th Hour with Brian Williams', 'Bill O’Reilly’s No Spin News and Analysis', "The Daily 202's Big Idea", 'The Takeaway', 'Can He Do That?', 'The Last Word with Lawrence O’Donnell']

['VIEWS with David Dobrik and Jason Nash', 'Instant Message', 'I Am In Eskew', 'Jalen & Jacoby', 'The Axe Files with David Axelrod', 'Getting Things Done', 'The Tower', 'The Permaculture Podcast', 'Making It With Jimmy Diresta, Bob Clagett and David Picciuto', 'Psychology of Eating', 'Blank Check with Griffin & David']

['Impaulsive with Logan Paul', 'Sexology', 'BEHIND THE SCENES', 'MOONFACE', 'The Tom Woods Show', 'The Flop House', 'The Long Run with Luke Timmerman', 'Heartland Radio 2.0', 'Another Kingdom', 'Curious with Josh Peck', 'Just Roll With It - A Dungeons and Dragons Podcast']

['The Bill Simmons Podcast', 'The Rewatchables', 'Book of Basketball 2.0', 'Real Time with Bill Maher', 'The Ringer NFL Show', 'NFL: The Dave Dameshek Football Program', 'Monday Morning Podcast', 'Chris Simms Unbuttoned', 'The Ryen Russillo Podcast', 'The Bill Bert Podcast', 'The GM Shuffle with Michael Lombardi and Adnan Virk']

['My Favorite Murder with Karen Kilgariff and Georgia Hardstark', "Don't Talk to Strangers", 'Murder Minute', 'Wine & Crime', 'Fresh Hell Podcast', 'Unsolved Murders: True Crime Stories', 'Murder, Myth & Mystery', 'Criminology', 'Jensen and Holes: The Murder Squad', 'Murder, etc.', 'Murder In The Rain']

['This American Life', 'History of Japan', 'ED MYLETT SHOW', 'The Toxic People Detox | Self-Care & Difficult People Survival Strategies', 'Meaningful Conversations with Maria Shriver', 'This is the Gospel Podcast', 'The Hidden People', 'Locked Up Abroad', 'For The Love With Jen Hatmaker Podcast', 'The Incomparable Radio Theater', 'The Stacking Benjamins Show']

['Joel Osteen Podcast', 'Joel Osteen Podcast', 'Saddleback Church Weekend Messages', 'Daily Grace', 'Marriage After God', 'Love Worth Finding on Oneplace.com', 'All Things Catholic with Dr. Edward Sri', 'Bethel Church Sermon of the Week', 'The Porch', 'Rush: Holy Spirit in Modern Life | A Practical & Prophetic Podcast for Men and Women', "Don't Mom Alone Podcast"]

['TED Radio Hour', 'The Metaphysical Hour hosted by Julia Cannon', 'TED Talks Art', 'TED Talks Business', 'CarProUSA Radio Show', 'TED Talks Science and Medicine', 'The TED Interview', 'Freakonomics Radio', 'The Creeping Hour', 'TED Talks Society and Culture', 'Zero Hours']

['Call Her Daddy', 'Stiff Socks', 'Two Judgey Girls', 'NAKED with Catt Sadler', 'Becoming Something with Jonathan Pokluda', 'Slay Girl Slay', 'Hot Marriage. Cool Parents.', 'Safe For Work', 'The Walynn Collins Show', 'Coffee Convos Podcast with Kail Lowry & Lindsie Chrisley', 'Girl Were You Alone? An *NSYNC Podcast']

['Skip and Shannon: Undisputed', 'First Things First', 'Golic and Wingo', 'Speak For Yourself with Whitlock & Wiley', 'The Herd with Colin Cowherd', 'The Odd Couple with Chris Broussard & Rob Parker', 'The Michael Kay Show', 'Joe Benigno and Evan Roberts', 'Pick Six NFL Podcast', 'Pardon My Take', 'The Ringer NFL Show']

### Cosine Similarity + TFIDF_____no_output_____

from sklearn.feature_extraction.text import TfidfVectorizer

tf = TfidfVectorizer()
tf_matrix = tf.fit_transform(podcasts_df["text"])
tf_cosine_sim = cosine_similarity(tf_matrix)_____no_output_____for i in test_podcasts:
    print(get_recommendations(get_index_from_title(i), tf_cosine_sim))
    print()['The Daily', 'Impeachment Inquiry: Updates from The Washington Post', "The Daily 202's Big Idea", 'The 11th Hour with Brian Williams', 'Article II: Inside Impeachment', 'Impeachment: A Daily Podcast', 'The Takeaway', 'The Last Word with Lawrence O’Donnell', 'Bill O’Reilly’s No Spin News and Analysis', 'The Situation Room with Wolf Blitzer', 'The Rachel Maddow Show']

['Up First', 'Impeachment Inquiry: Updates from The Washington Post', 'The Daily', 'Article II: Inside Impeachment', 'The 11th Hour with Brian Williams', 'Impeachment: A Daily Podcast', "The Daily 202's Big Idea", 'Can He Do That?', 'The Last Word with Lawrence O’Donnell', 'Bill O’Reilly’s No Spin News and Analysis', 'The Takeaway']

['VIEWS with David Dobrik and Jason Nash', 'Instant Message', 'How Did This Get Made?', 'Light The Fight- Parenting Podcast', 'Flow Sessions with Jason Silva', 'This Week in Startups - Audio', 'The Tower', 'True Cold Case Files with Jason and Daisy', 'Horror Hill: A Horror Anthology and Scary Stories Series Podcast', 'Kids Bible Stories', 'Psychology of Eating']

['Impaulsive with Logan Paul', 'MOONFACE', 'The Tom Woods Show', 'Sexology', 'The Tim Dillon Show', 'The Long Run with Luke Timmerman', 'Ripper Magoos with Bob Menery', 'Come Follow Me Daily', 'The Flop House', 'Sex Talk With My Mom', 'Below the Belt']

['The Bill Simmons Podcast', 'The Rewatchables', 'Book of Basketball 2.0', 'Headlong: Missing Richard Simmons', 'The Ringer NFL Show', 'Real Time with Bill Maher', 'Chris Simms Unbuttoned', 'Speak For Yourself with Whitlock & Wiley', 'The Pat McAfee Show 2.0', 'NFL: The Dave Dameshek Football Program', 'The GM Shuffle with Michael Lombardi and Adnan Virk']

['My Favorite Murder with Karen Kilgariff and Georgia Hardstark', 'Do You Need A Ride?', 'Wire Talk with Karen Stubbs', 'The Murder In My Family', 'Murder Minute', 'The Walking Dead ‘Cast', 'Murderous Minors: killer kids', "Don't Talk to Strangers", 'Murder, Myth & Mystery', 'Jensen and Holes: The Murder Squad', 'Criminology']

['This American Life', 'Experimental Brewing', '1A', 'Through the Looking Glass: A LOST Retrospective', 'BeerSmith Home and Beer Brewing Podcast', 'Talking Beat - from the Portland Police Bureau', 'Haunted Hell House of Horror', 'The Grave Talks | Haunted, Paranormal & Supernatural', 'Sinisterhood', 'Slow Burn', 'Haunted Places']

['Joel Osteen Podcast', 'Joel Osteen Podcast', 'Joyce Meyer Ministries TV Podcast', 'Joyce Meyer Enjoying Everyday Life® TV Audio Podcast', 'Marriage After God', 'Daily Grace', 'Fresh Life Church', 'Brilliant Perspectives', 'Bethel Church Sermon of the Week', 'Saddleback Church Weekend Messages', 'Pastor Robert Morris Ministries on Oneplace.com']

['TED Radio Hour', 'The TED Interview', 'TED Talks Art', 'TED Talks Science and Medicine', 'TED Talks Business', "Hunted: Inside Ted Bundy's Trail of Terror", 'TED Talks Society and Culture', 'TED Talks Education', 'TED Talks Daily (HD video)', 'TED Talks Daily (SD video)', 'TED Talks Music']

['Call Her Daddy', 'hey, girl.', 'Becoming Something with Jonathan Pokluda', 'Fierce Girls', 'Stiff Socks', 'Girls Night with Stephanie May Wilson', 'Grammar Girl Quick and Dirty Tips for Better Writing', 'Two Judgey Girls', 'The Clever Girls Know Podcast', 'Quarter Life Crisis', 'Slay Girl Slay']

['Skip and Shannon: Undisputed', 'First Things First', 'Golic and Wingo', 'Speak For Yourself with Whitlock & Wiley', 'The Herd with Colin Cowherd', 'The Odd Couple with Chris Broussard & Rob Parker', 'Pick Six NFL Podcast', 'First Take', 'Pardon My Take', 'Chris Simms Unbuttoned', 'Pro Football Talk Live with Mike Florio']

### Cosine Similarity + Custom Word2Vec Embedding_____no_output_____

from gensim.models import Word2Vec
from sklearn.decomposition import PCA
from matplotlib import pyplot_____no_output_____class MyTokenizer:
    def __init__(self):
        pass

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        transformed_X = []
        for document in X:
            tokenized_doc = []
            for sent in nltk.sent_tokenize(document):
                tokenized_doc += nltk.word_tokenize(sent)
            transformed_X.append(np.array(tokenized_doc))
        return np.array(transformed_X)

    def fit_transform(self, X, y=None):
        return self.transform(X)

class MeanEmbeddingVectorizer(object):
    def __init__(self, word2vec):
        self.word2vec = word2vec
        # if a text is empty we should return a vector of zeros
        # with the same dimensionality as all the other vectors
        self.dim = len(word2vec.wv.syn0[0])

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X = MyTokenizer().fit_transform(X)

        return np.array([
            np.mean([self.word2vec.wv[w] for w in words if w in self.word2vec.wv]
                    or [np.zeros(self.dim)], axis=0)
            for words in X
        ])

    def fit_transform(self, X, y=None):
        return self.transform(X)_____no_output_____text_list = list(podcasts_df.text)
tokenized_text = [tokenizer.tokenize(i) for i in text_list]_____no_output_____w2v_model = Word2Vec(tokenized_text, sg=1)_____no_output_____mean_embedding_vectorizer = MeanEmbeddingVectorizer(w2v_model)
mean_embedded = mean_embedding_vectorizer.fit_transform(podcasts_df['text'])
w2v_cosine_sim = cosine_similarity(mean_embedded)/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:25: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.vectors instead).
for i in test_podcasts:
    print(get_recommendations(get_index_from_title(i), w2v_cosine_sim))
    print()['The Daily', 'Impeachment: A Daily Podcast', 'Up First', 'The Takeaway', 'Can He Do That?', 'Global News Podcast', 'Skimm This', 'Mark Levin Podcast', 'Mark Levin Podcast', 'Post Reports', 'The Asset']

['Up First', 'Impeachment: A Daily Podcast', 'The Daily', 'The Takeaway', 'Mark Levin Podcast', 'Mark Levin Podcast', 'Post Reports', 'Can He Do That?', 'PBS NewsHour - Full Show', 'Skimm This', 'Global News Podcast']

['VIEWS with David Dobrik and Jason Nash', 'Amy Schumer Presents: 3 Girls, 1 Keith', 'Not Skinny But Not Fat', 'The Archers', 'The Fighter & The Kid', 'Adulting', 'Ringer Dish', 'Drew and Mike Show', 'Andrea Savage: A Grown-Up Woman #buttholes', 'The Three Questions with Andy Richter', 'Dead Pilots Society']

['Impaulsive with Logan Paul', 'Comments by Celebs', 'Drew and Mike Show', 'Heartland Radio 2.0', 'Curious with Josh Peck', 'The MeatEater Podcast', 'KFC Radio', 'Not Skinny But Not Fat', 'Who? Weekly', '#GetSome with Gary Owen', 'Ya Neva Know: you know what I mean?']

['The Bill Simmons Podcast', 'The Ryen Russillo Podcast', 'ESPN Podcasts', 'The Peter King Podcast', 'Pardon My Take', 'The Jump', 'NFL: Move the Sticks with Daniel Jeremiah & Bucky Brooks', 'The Mina Kimes Show featuring Lenny', 'The Ringer NBA Show', 'The Woj Pod', 'ESPN Daily']

['My Favorite Murder with Karen Kilgariff and Georgia Hardstark', 'Fresh Hell Podcast', 'Wine & Crime', 'Murderous Minors: killer kids', 'Jensen and Holes: The Murder Squad', 'Murder, Myth & Mystery', "Don't Talk to Strangers", "Let's Taco 'Bout True Crime", 'Cult Liter with Spencer Henry', 'Housewives of True Crime', 'True Crime Chronicles']

['This American Life', 'Radio Diaries', 'The Incomparable Radio Theater', 'Undiscovered', 'Locked Up Abroad', 'Brought to you by...', 'Our Strange Skies', 'CuriosiD', 'The Dirtbag Diaries', 'Travel with Rick Steves', 'The Brooklyn Blast Furnace']

['Joel Osteen Podcast', 'Joel Osteen Podcast', 'Daily Grace', 'Saddleback Church Weekend Messages', 'The Porch', 'VOUS Church', 'The Fr. Mike Schmitz Catholic Podcast', 'North Point Community Church', 'Marriage After God', 'Living on the Edge with Chip Ingram Daily Podcast', 'Passion City Church Podcast']

['TED Radio Hour', 'TED Talks Society and Culture', 'WGRL NYC', 'Deepak Chopra’s Infinite Potential', 'Clear+Vivid with Alan Alda', 'The goop Podcast', 'The Kevin Rose Show', 'The After On Podcast', 'SAPIENS: A Podcast for Everything Human', 'The Psychology Podcast', 'Experts on Expert with Dax Shepard']

['Call Her Daddy', 'Stiff Socks', 'Zane and Heath: Unfiltered', 'Bully and the Beast', 'Pretty Basic with Alisha Marie and Remi Cruz', 'Coffee Convos Podcast with Kail Lowry & Lindsie Chrisley', 'Berning In Hell', 'Schnitt Talk', 'Woman Evolve with Sarah Jakes Roberts', 'DELETE THIS', 'Again With This: Beverly Hills, 90210 & Melrose Place']

['Skip and Shannon: Undisputed', 'First Things First', 'High Noon', 'Speak For Yourself with Whitlock & Wiley', 'Jalen & Jacoby', 'Golic and Wingo', 'The Herd with Colin Cowherd', 'PTI', 'First Take', 'Pro Football Talk Live with Mike Florio', 'The Ryen Russillo Podcast']

### Cosine Similarity + GloVe Embedding_____no_output_____

from gensim.models import KeyedVectors_____no_output_____glove_model = KeyedVectors.load_word2vec_format("../word2vec/glove.6B.50d.txt.word2vec")_____no_output_____glove_mean_embedding_vectorizer = MeanEmbeddingVectorizer(glove_model)
glove_mean_embedded = glove_mean_embedding_vectorizer.fit_transform(podcasts_df['text'])
glove_cosine_sim = cosine_similarity(glove_mean_embedded)/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:25: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).
/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:25: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.vectors instead).
/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:34: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).
for i in test_podcasts:
    print(get_recommendations(get_index_from_title(i), glove_cosine_sim))
    print()['The Daily', 'Up First', 'Impeachment: A Daily Podcast', 'The Lawfare Podcast', 'Bag Man', 'Can He Do That?', 'CBS This Morning', 'Impeachment Inquiry: Updates from The Washington Post', 'All In with Chris Hayes', 'The Takeaway', 'The New Yorker: Politics and More']

['Up First', 'The Daily', 'The Lawfare Podcast', 'Impeachment: A Daily Podcast', 'Bag Man', 'What Next | Daily News and Analysis', 'Impeachment Inquiry: Updates from The Washington Post', 'The Situation Room with Wolf Blitzer', 'PBS NewsHour - Segments', 'All In with Chris Hayes', 'The Lead with Jake Tapper']

['VIEWS with David Dobrik and Jason Nash', 'Help! I Suck at Dating with Dean, Vanessa and Jared', 'Amy Schumer Presents: 3 Girls, 1 Keith', 'Dead Pilots Society', "So Bad It's Good with Ryan Bailey", 'The Ben and Ashley I Almost Famous Podcast', 'Out in the Wild', 'Your 2 Dads w/ Sean & Julian', 'Good Morning From Hell', 'The Bobby Bones Show', 'Scrubbing In with Becca Tilley & Tanya Rad']

['Impaulsive with Logan Paul', 'The Kirk Minihane Show', 'Reality Steve Podcast', 'Hollywood Unlocked [UNCENSORED]', 'Two Judgey Girls', 'The HoneyDew with Ryan Sickler', 'The Last Laugh: A Daily Beast Podcast', 'The Adam and Dr. Drew Show', 'QAnon Anonymous', 'Hot Marriage. Cool Parents.', 'The Candace Owens Show']

['The Bill Simmons Podcast', 'The Peter King Podcast', 'First Things First', 'ESPN Podcasts', 'NFL: Move the Sticks with Daniel Jeremiah & Bucky Brooks', 'Cleveland Browns Daily & More', 'Joe Benigno and Evan Roberts', 'Book of Basketball 2.0', 'Chris Simms Unbuttoned', 'Footballguys.com - The Audible - Fantasy Football Info for Serious Fans', 'The Herd with Colin Cowherd']

['My Favorite Murder with Karen Kilgariff and Georgia Hardstark', 'Murderous Minors: killer kids', 'You Must Remember Manson', '...These Are Their Stories: The Law & Order Podcast', 'Fresh Hell Podcast', "Let's Taco 'Bout True Crime", 'Hollywood & Crime', 'Atlanta Monster', 'Red Scare', 'Reality Life with Kate Casey', 'Once Upon A Crime | True Crime']

['This American Life', 'UK True Crime Podcast', 'A Waste Of Time with ItsTheReal', "Heaven's Gate", 'Locked Up Abroad', 'The David Banner Podcast', 'The Brooklyn Blast Furnace', 'The Incomparable Radio Theater', 'Astonishing Legends', 'Outside Podcast', 'Upzoned']

['Joel Osteen Podcast', 'Joel Osteen Podcast', 'North Point Community Church', 'VOUS Church', 'Pathway to Victory on Oneplace.com', 'Passion City Church Podcast', 'The Fr. Mike Schmitz Catholic Podcast', 'As For Me And My House', '4:13 Podcast', 'All In', 'Daily Grace']

['TED Radio Hour', 'Under The Skin with Russell Brand', 'Creative Processing with Joseph Gordon-Levitt', 'Love Your Voice with Roger Love', 'The Ezra Klein Show', 'The Kevin Rose Show', 'Ten Percent Happier with Dan Harris', 'Conversations with Tyler', 'Being Well with Dr. Rick Hanson', 'Freakonomics Radio', 'The Enneagram Journey']

['Call Her Daddy', 'A Date With Dateline', "Couldn't Help But Wonder: A Sex and the City Podcast with Jamie Lee and Rose Surnow", 'Derek Sivers', "The On-Call Room: A Grey's Anatomy Podcast", 'Get Off My Lawn Podcast w/ Gavin McInnes', 'The Papaya Podcast', 'The Podfathers', 'DELETE THIS', 'Crime in Sports', 'Take it or Leave it']

['Skip and Shannon: Undisputed', 'First Things First', 'Golic and Wingo', 'Speak For Yourself with Whitlock & Wiley', 'Joe Benigno and Evan Roberts', 'The Odd Couple with Chris Broussard & Rob Parker', 'Scal and Pals', 'The Lowe Post', 'The Herd with Colin Cowherd', 'The Bill Simmons Podcast', 'Jalen & Jacoby']

### Cosine Similarity + Word2Vec + Smooth Inverse Frequency_____no_output_____

from scipy.sparse.linalg import svds
from sklearn.decomposition import TruncatedSVD_____no_output_____def remove_first_principal_component(X):
    svd = TruncatedSVD(n_components=1, n_iter=7, random_state=0)
    svd.fit(X)
    pc = svd.components_
    XX = X - X.dot(pc.transpose()) * pc
    return XX

def smooth_inverse_frequency(sent, a=0.001, word2vec_model=w2v_model):
    word_counter = {}
    sentences = []
    total_count = 0
    no_of_sentences = 0

    for s in sent:
        for w in s:
            if w in word_counter:
                word_counter[w] = word_counter[w] + 1
            else:
                word_counter[w] = 1
        total_count = total_count + len(s)
        no_of_sentences = no_of_sentences + 1

    sents_emd = []
    for s in sent:
        sent_emd = []
        for word in s:
            if word in word2vec_model:
                emd = (a/(a + (word_counter[word]/total_count)))*word2vec_model[word]
                sent_emd.append(emd)
        sum_ = np.array(sent_emd).sum(axis=0)
        sentence_emd = sum_/float(no_of_sentences)
        sents_emd.append(sentence_emd)

    new_sents_emb = remove_first_principal_component(np.array(sents_emd))
    return new_sents_emb_____no_output_____sif_text_emd = smooth_inverse_frequency(text_list)
sif_cosine_sim = cosine_similarity(sif_text_emd)/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:27: DeprecationWarning: Call to deprecated `__contains__` (Method will be removed in 4.0.0, use self.wv.__contains__() instead).
/Users/sidd/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:28: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).
for i in test_podcasts:
    print(get_recommendations(get_index_from_title(i), sif_cosine_sim))
    print()['The Daily', 'Impeachment Inquiry: Updates from The Washington Post', 'The Drum Show', "Official Prime Minister's Questions (PMQs) Podcast", 'Criminal', 'Robb Wolf - The Paleo Solution Podcast - Paleo diet, nutrition, fitness, and health', 'Queens Podcast', 'Aquarium Co-Op Podcast', 'The Beauty Brains', 'Story Time', 'Plenary Session']

['Up First', 'The Paul Tripp Podcast', "The Daily 202's Big Idea", 'Stay Tuned with Preet', 'The General Hospital Podcast', 'Post Reports', 'Article II: Inside Impeachment', 'Impeachment Inquiry: Updates from The Washington Post', 'Intelligence Squared', 'WGRL NYC', 'The Drum Show']

['VIEWS with David Dobrik and Jason Nash', 'The Man With A Thousand Faces', 'Solid Joys Daily Devotional', 'The Axe Files with David Axelrod', "DJ Private Ryan's Podcast", 'Beach Too Sandy, Water Too Wet', 'Kid Friendly Joke Of The Day', 'AI: Hype vs. Reality', 'Just The Sip', 'Tumble Science Podcast for Kids', 'Binge Mode: Star Wars']

['Impaulsive with Logan Paul', 'Leveling Up: Creating Everything From Nothing with Natalie Jill', 'The Tom Ferry Podcast Experience', 'PAVE: Professionals Against Violence Podcast', 'The Trail Went Cold', 'The Overwhelmed Brain', 'The American War', 'The Fraudcast', 'All In The Mind - ABC RN', 'Airplane Geeks Podcast', 'Building a StoryBrand with Donald Miller']

['The Bill Simmons Podcast', 'Bible Stories My Kids Love', 'Bloomberg Businessweek', 'The Mandolins and Beer Podcast', 'College Basketball Talk on NBC Sports Podcast', 'The GM Shuffle with Michael Lombardi and Adnan Virk', 'Not Skinny But Not Fat', 'The Michael Brooks Show', 'Footballguys.com - The Audible - Fantasy Football Info for Serious Fans', 'The CONAN Podcast', 'The Glass Appeal']

['My Favorite Murder with Karen Kilgariff and Georgia Hardstark', 'Food Talk with Dani Nierenberg', 'Dave & Chuck the Freak Podcast', 'The Lefkoe Show', 'Deck The Hallmark', 'Car Talk', 'Call Your Girlfriend', 'Rock N Roll Manifesto (mp3)', 'Through the Wire', 'Mindful Mama - Parenting with Mindfulness', 'Nightmare On Film Street - A Horror Movie Podcast']

['This American Life', 'The Bridge', 'RULES FOR RETROGRADES', 'Bizarre Albums', 'Mysterious Radio: Paranormal, UFO & Lore Interviews', 'Surprisingly Awesome', 'Witness History: World War 2 Collection', 'Hey Riddle Riddle', 'Twenty Thousand Hertz', 'Haunted Hell House of Horror', 'Hard Factor']

['Joel Osteen Podcast', 'Joel Osteen Podcast', "WHOA That's Good Podcast", 'My Dad Wrote A Porno', 'Sibling Revelry with Kate Hudson and Oliver Hudson', 'Start Today Morning Show', 'Astonishing Legends', 'That Sounds Fun with Annie F. Downs', 'The Endless Honeymoon Podcast', 'MarriageToday Audio Podcast', 'Detective']

['TED Radio Hour', 'The History of Exploration', 'How to Be a Girl', 'Sincerely, X: Season 1', 'The Research Like a Pro Genealogy Podcast', 'ONE Extraordinary Marriage Show', 'Shameless Sex', 'Bedtime Explorers', 'Elon Musk Interviews', 'Deep Energy 2.0 - Music for Sleep, Meditation, Relaxation, Massage and Yoga', 'StartUp Podcast']

['Call Her Daddy', 'Clapcast from Claptone', 'The Martin Garrix Show', 'Frame Trap', 'W&W Rave Culture Radio', 'Nora En Pure - Purified Radio', 'Box-Trick: A Retro Gaming Podcast', 'Perfecto Podcast: featuring Paul Oakenfold', 'Read-Aloud Revival', 'X-Men: The Audio Drama', 'The Box of Oddities']

['Skip and Shannon: Undisputed', 'All The Smoke', 'Pull Up with CJ McCollum', 'Chicks in the Office', 'The Sports Junkies', 'The Great Books', 'To L And Back: An L Word Podcast', 'Pete and Sebastian Show', 'SModcast', "Van Lathan's The Red Pill", 'Comedy Bang Bang: The Podcast']


# Notebook from PizzaMyHeart/python-bioimage-analysis-tutorial
Path: image_analysis_tutorial_solutions.ipynb

# Image Analysis with Python - Tutorial Pipeline Solutions

*originally created in 2016*

*updated and converted to a Jupyter notebook in 2017*

*updated and converted to python 3 in 2018*

*by Jonas Hartmann (Gilmour group, EMBL Heidelberg)*
_____no_output_____##  Table of Contents

1. [About this Tutorial](#about)
2. [Importing Modules & Packages](#import)
3. [Loading & Handling Image Data](#load)
4. [Preprocessing](#prepro)
5. [Manual Thresholding & Threshold Detection](#thresh)
6. [Adaptive Thresholding](#adaptive)
7. [Improving Masks with Binary Morphology](#morpho)
8. [Connected Components Labeling](#label)
9. [Cell Segmentation by Seeding & Expansion](#seg)
10. [Postprocessing: Removing Cells at the Image Border](#postpro)
11. [Identifying Cell Edges](#edges)
12. [Extracting Quantitative Measurements](#measure)
13. [Simple Analysis & Visualization](#analysis)
14. [Writing Output to Files](#write)
15. [Batch Processing](#batch)_____no_output_____##  About this Tutorial -  Notes on the Solutions

These are the solutions to the tutorial pipeline (see `image_analysis_tutorial.py`). *For the best learning experience, it is highly recommended that you first try to implement a solution yourself!* Only come here if you are totally stuck or if you have a working solution and would like to double-check it.

Note that there are multiple ways of implementing any particular step in the pipeline, so if your solution is different from the solution here, it is not necessarily wrong. However, *some solutions are better than others* because they are...

- **...more readable:**
    - When reading the code, it is obvious and clear what the code is doing
    - The code is clearly commented to help others (and your future self) understand it

- **...more general:**
    - The code still works if there are minor changes to the data (e.g. size of the image being processed)
    - The code can easily be transformed into a solution for a similar problem

- **...more computationally efficient:**
    - No unnecessary copies of large datasets are made (memory efficiency)
    - Faster algorithms are used, e.g. array operations instead of loops (CPU efficiency)
    - No unnecessary loading and writing of data (io efficiency)

It is up to you to decide if your solution is better, equally good, or not as good as the solution presented here. Either way, we hope you can learn something by looking at our solutions. **:)**_____no_output_____## Importing Modules & Packages _____no_output_____

# The numerical array package numpy as np
import numpy as np_____no_output_____

#### Exercise Solution

Using the import command as above, follow the instructions in the comments below to import two additional modules that we will be using frequently in this pipeline._____no_output_____

# The plotting module matplotlib.pyplot as plt
import matplotlib.pyplot as plt

# The image processing package scipy.ndimage as ndi
import scipy.ndimage as ndi_____no_output_____

#### Side Note for Jupyter Notebook Users

You can configure how the figures made by matplotlib are displayed._____no_output_____

# Set matplotlib backend
%matplotlib inline
#%matplotlib notebook
#%matplotlib qt _____no_output_____

## Loading & Handling Image Data _____no_output_____#### Exercise Solution

We will now proceed to load one of the example images and verify that we get what we expect._____no_output_____

# (i) Specify the directory path and file name

# Create a string variable with the name of the file you'd like to load (here: 'example_cells_1.tif').
# Suggested name for the variable: filename
# Note: Paths and filenames can contain slashes, empty spaces and other special symbols, which can cause
#       trouble for programming languages under certain circumstances. To circumvent such trouble, add
#       the letter r before your string definition to create a so-called 'raw string', which is not
#       affected by these problems (e.g. `my_raw_string = r"some string with funny symbols: \\\!/~***!"`).
filename = r'example_cells_1.tif'

# If the file is not in the current working directory, you must also have a way of specifying the path
# to the directory where the file is stored. Most likely, your example images are stored in a directory
# called 'example_data' in the same folder as this notebook. Note that you can use either the full path
#  - something like r"/home/jack/data/python_image_analysis/example_data/example_cells_1.tif"
# or the relative path, starting from the current working directory
#   - here that would just be r"example_data"
# Create a string variable with the path to the directory that contains the file you'd like to load.
# Suggested name for the variable: dirpath
dirpath = r'example_data'  # Relative path
#dirpath = r'/home/jack/data/python_image_analysis/example_data/example_cells_1.tif'  # Absolute path _____no_output_____# (ii) Combine the directory path and file name into one variable, the file path

# Import the function 'join' from the module 'os.path'
# This function automatically takes care of the slashes that need to be added when combining two paths.
from os.path import join

# Use the 'join' function to combine the directory path with the file name and create a new variable.
# Print the result to see that everything is correct (this is always a good idea!)
# Suggested name for the variable: filepath
filepath = join(dirpath, filename)
print(filepath)_____no_output_____# (iii) Load the image

# Import the function 'imread' from the module 'skimage.io'.
from skimage.io import imread

# Load 'example_cells_1.tif' and store it in a variable.
# Suggested name for the variable: img
img = imread(filepath)_____no_output_____# (iv) Check that everything is in order

# Check that 'img' is a variable of type 'ndarray' - use Python's built-in function 'type'.
print("Loaded array is of type:", type(img))

# Print the shape of the array using the numpy-function 'shape'.
# Make sure you understand the output!
print("Loaded array has shape:", img.shape)

# Check the datatype of the individual numbers in the array. You can use the array attribute 'dtype' to do so.
# Make sure you understand the output!
print("Loaded values are of type:", img.dtype)

# SOLUTION NOTE: The dtype should be 'uint8', because these are unsigned 8-bit integer images.
#                This means that the intensity values range from 0 to 255 in steps of 1._____no_output_____# (v) Look at the image to confirm that everything worked as intended

# To plot the array as an image, use pyplot's functions 'plt.imshow' followed by 'plt.show'.
# Check the documentation for 'plt.imshow' and note the parameters that can be specified, such as colormap (cmap)
# and interpolation. Since you are working with scientific data, interpolation is unwelcome, so you should set it
# to "none". The most common cmap for grayscale images is naturally "gray".
# You may also want to adjust the size of the figure. You can do this by preparing the figure canvas with
# the function 'plt.figure' before calling 'plt.imshow'. The canvas size is adjusted using the keyword argument
# 'figsize' when calling 'plt.figure'.
plt.figure(figsize=(7,7))
plt.imshow(img, interpolation='none', cmap='gray')
plt.show()_____no_output_____

## Preprocessing _____no_output_____#### Exercise Solution

Perform Gaussian smoothing and visualize the result._____no_output_____

# (i) Create a variable for the smoothing factor sigma, which should be an integer value

sigma = 3

# After implementing the Gaussian smoothing function below, you can modify this variable
# to find the ideal value of sigma._____no_output_____# (ii) Perform the smoothing on the image

# To do so, use the Gaussian filter function 'ndi.filters.gaussian_filter' from the
# image processing module 'scipy.ndimage', which was imported at the start of the tutorial.
# Check out the documentation of scipy to see how to use this function.
# Allocate the output to a new variable.
img_smooth = ndi.filters.gaussian_filter(img, sigma)_____no_output_____# (iii) Visualize the result using 'plt.imshow'

# Compare with the original image visualized above.
# Does the output make sense? Is this what you expected?
# Can you optimize sigma such that the image looks smooth without blurring the membranes too much?
plt.figure(figsize=(7,7))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.show()

# To have a closer look at a specific region of the image, crop that region out and show it in a
# separate plot. Remember that you can crop arrays by "indexing" or "slicing" them similar to lists.
# Use such "zoomed-in" views throughout this tutorial to take a closer look at your intermediate
# results when necessary.
plt.figure(figsize=(6,6))
plt.imshow(img_smooth[400:600, 200:400], interpolation='none', cmap='gray')
plt.show()_____no_output_____# (iv) BONUS: Show the raw and smoothed images side by side using 'plt.subplots'

fig, ax = plt.subplots(1, 2, figsize=(10,7))
ax[0].imshow(img, interpolation='none', cmap='gray')
ax[1].imshow(img_smooth, interpolation='none', cmap='gray')
ax[0].set_title('Raw Image')
ax[1].set_title('Smoothed Image')
plt.show()_____no_output_____

## Manual Thresholding & Threshold Detection _____no_output_____#### Exercise Solution

Try out manual thresholding and automated threshold detection._____no_output_____

# (i) Create a variable for a manually set threshold, which should be an integer

# This can be changed later to find a suitable value.
thresh = 70_____no_output_____# (ii) Perform thresholding on the smoothed image

# Remember that you can use relational (Boolean) expressions such as 'smaller' (<), 'equal' (==)
# or 'greater or equal' (>=) with numpy arrays - and you can directly assign the result to a new
# variable.
mem = img_smooth > thresh

# Check the dtype of your thresholded image
# You should see that the dtype is 'np.bool', which stands for 'Boolean' and means the array
# is now simply filled with 'True' and 'False', where 'True' is the foreground (the regions
# above the threshold) and 'False' is the background.
print(mem.dtype)_____no_output_____# (iii) Visualize the result

fig, ax = plt.subplots(1, 2, figsize=(10,7))
ax[0].imshow(img_smooth, interpolation='none', cmap='gray')
ax[1].imshow(mem, interpolation='none', cmap='gray')
ax[0].set_title('Smoothed Image')
ax[1].set_title('Thresholded Membranes')
plt.show()_____no_output_____# (iv) Try out different thresholds to find the best one

# If you are using jupyter notebook, you can adapt the code below to
# interactively change the threshold and look for the best one. These
# kinds of interactive functions are called 'widgets' and are very
# useful in exploratory data analysis to create greatly simplified
# 'User Interfaces' (UIs) on the fly.
# As a bonus exercise, try to understand or look up how the widget works
# and play around with it a bit!

# Prepare widget
from ipywidgets import interact
@interact(thresh=(10,250,10))
def select_threshold(thresh=100):

    # Thresholding
    mem = img_smooth > thresh

    # Visualization
    plt.figure(figsize=(7,7))
    plt.imshow(mem, interpolation='none', cmap='gray')
    plt.show()_____no_output_____# (v) Perfom automated threshold detection with Otsu's method

# The scikit-image module 'skimage.filters.thresholding' provides
# several threshold detection algorithms. The most popular one
# among them is Otsu's method. Using what you've learned so far,
# import the 'threshold_otsu' function, use it to automatically
# determine a threshold for the smoothed image, apply the threshold,
# and visualize the result.

# Import
from skimage.filters.thresholding import threshold_otsu

# Calculate and apply threshold
thresh = threshold_otsu(img_smooth)
mem = img_smooth > thresh

# Visualization
plt.figure(figsize=(7,7))
plt.imshow(mem, interpolation='none', cmap='gray')
plt.show()_____no_output_____# (vi) BONUS: Did you notice the 'try_all_threshold' function?

# That's convenient! Use it to automatically test the threshold detection
# functions in 'skimage.filters.thresholding'. Don't forget to adjust the
# 'figsize' parameter so the resulting images are clearly visible.
from skimage.filters.thresholding import try_all_threshold
fig = try_all_threshold(img_smooth, figsize=(10,10), verbose=False)_____no_output_____

## Adaptive Thresholding _____no_output_____#### Exercise Solution

Implement the two steps of adaptive background subtraction:

1. Use a strong "mean filter" with a circular `SE` to create the background image.

2. Use the background image for thresholding._____no_output_____

# Step 1
# ------

# The expression below creates circular structuring elements.
# It is an elegant but complicated piece of code and at the moment it is not
# necessary for you to understand it in detail. Use it to create structuring
# elements of different sizes (by changing 'i') and find a way to visualize
# the result (remember that the SE is just a small 'image').
#
# Try to answer the following questions:
#
#   - Is the resulting SE really circular?
#       >>> It is close enough for large i, not so much for small i.
#
#   - Can certain values of 'i' cause problems? If so, why?
#       >>> Even values create a slight asymmetry! It is best to use only odd values!
#
#   - What value of 'i' should be used for the SE?
#       >>> My first guess was i=30, which is about 3x the membrane diameter.
#           I tried out some other values but ultimately stuck with this.

# Create SE
i = 31
SE = (np.mgrid[:i,:i][0] - np.floor(i/2))**2 + (np.mgrid[:i,:i][1] - np.floor(i/2))**2 <= np.floor(i/2)**2

# Visualize the result
plt.imshow(SE, interpolation='none')
plt.show()_____no_output_____# (ii) Create the background

# Run a mean filter over the image using the disc SE and assign the output to a new variable.
# Use the function 'skimage.filters.rank.mean'.
from skimage.filters import rank
bg = rank.mean(img_smooth, selem=SE)_____no_output_____# (iii) Visualize the resulting background image. Does what you get make sense?

plt.figure(figsize=(7,7))
plt.imshow(bg, interpolation='none', cmap='gray')
plt.show()_____no_output_____# Step 2
# ------

# (iv) Threshold the Gaussian-smoothed original image against the background image created in step 1
#      using a relational expression

mem = img_smooth > bg_____no_output_____# (v) Visualize and understand the output.

plt.figure(figsize=(7,7))
plt.imshow(mem, interpolation='none', cmap='gray')
plt.show()

# Are you happy with this result as a membrane segmentation?
# ->> It's pretty good, except for the speckles that litter the inside of the cells!_____no_output_____

## Improving Masks with Binary Morphology _____no_output_____#### Exercise Solution

Improve the membrane segmentation from above with morphological operations._____no_output_____

# (i) Get rid of speckles using binary hole filling

# Use the function 'ndi.binary_fill_holes' for this. Be sure to check the docs to
# understand exactly what it does. For this to work as intended, you will have to
# invert the mask, which you can do using the function `np.logical_not` or the
# corresponding operator '~'. Again, be sure to understand why this has to be done
# and don't forget to revert the result back.

#mem_holefilled = np.logical_not(ndi.binary_fill_holes(np.logical_not(mem)))  # Long form
mem_holefilled = ~ndi.binary_fill_holes(~mem)  # Short form_____no_output_____# (ii) Try out other morphological operations to further improve the membrane mask

# The various operations are available in the ndimage module, for example 'ndi.binary_closing'.
# Play around and see how the different functions affect the mask. Can you optimize the mask,
# for example by closing gaps?
# Note that the default SE for these functions is a square. Feel free to create another disc-
# shaped SE and see how that changes the outcome.
# BONUS: If you pay close attention, you will notice that some of these operations introduce
# artifacts at the image boundaries. Can you come up with a way of solving this?

# New circular SE of appropriate size (determined by trial and error)
i = 15
SE = (np.mgrid[:i,:i][0] - np.floor(i/2))**2 + (np.mgrid[:i,:i][1] - np.floor(i/2))**2 <= np.floor(i/2)**2

# One solution to the boundary artefact issue is padding with the reflection.
# 'Padding' refers to the extension of the image at the boundaries, in this case using
# a 'reflection' of the pixel values next to the boundary. If morphological operations
# are done on the padded image, the boundary artefacts will occur in the padded region
# outside the original image, which can simply be cropped out again at the end.
pad_size = i+1
mem_padded = np.pad(mem_holefilled, pad_size, mode='reflect')

# Binary closing works well to round off the membranes and close gaps
mem_final = ndi.binary_closing(mem_padded, structure=SE)

# This slicing operation crops the padded image back to the original size
mem_final = mem_final[pad_size:-pad_size, pad_size:-pad_size]_____no_output_____# (iii) Visualize the final result

fig, ax = plt.subplots(1, 2, figsize=(10,7))
ax[0].imshow(img_smooth, interpolation='none', cmap='gray')
ax[1].imshow(mem_final, interpolation='none', cmap='gray')
ax[0].set_title('Smoothed Image')
ax[1].set_title('Final Membrane Mask')
plt.show()_____no_output_____

## Connected Components Labeling _____no_output_____#### Exercise Solution

Use your membrane segmentation for connected components labeling._____no_output_____

# (i) Label connected components

# Use the function 'ndi.label' from the 'ndimage' module.
# Note that this function labels foreground pixels (1s, not 0s), so you may need
# to invert your membrane mask just as for hole filling above.
# Also, note that 'ndi.label' returns another result in addition to the labeled
# image. Read up on this in the function's documention and make sure you don't
# mix up the two outputs!

cell_labels, _ = ndi.label(~mem_final)

# Solution note: For functions with multiple outputs (here the labeled image on
# the one hand and the number of detected objects on the other), it is a python
# convention to unpack those outputs that will not be used in the remainder of
# code into the variable '_' (underscore). This makes it clear to those reading
# the code that the function returns multiple things but some of them are not
# important in this case._____no_output_____# (ii) Visualize the output

# Here, it is no longer ideal to use a 'gray' colormap, since we want to visualize that each
# cell has a unique ID. Play around with different colormaps (check the docs to see what
# types of colormaps are available) and choose one that you are happy with.

plt.figure(figsize=(7,7))
plt.imshow(cell_labels, interpolation='none', cmap='inferno')
plt.show()_____no_output_____

## Cell Segmentation by Seeding & Expansion _____no_output_____#### Exercise Solution

Find seeds using the distance transform approach._____no_output_____

# (i) Run a distance transform on the membrane mask

# Use the function 'ndi.distance_transform_edt'.
# You may need to invert your membrane mask so the distances are computed on
# the cells, not on the membranes.
dist_trans = ndi.distance_transform_edt(~mem_final)_____no_output_____# (ii) Visualize the output and understand what you are seeing.

plt.figure(figsize=(7,7))
plt.imshow(dist_trans, interpolation='none', cmap='viridis')
plt.show()_____no_output_____# (iii) Smoothen the distance transform

# Use 'ndi.filters.gaussian_filter' to do so.
# You will have to optimize your choice of 'sigma' based on the outcome below.

# Applying the filter
dist_trans_smooth = ndi.filters.gaussian_filter(dist_trans, sigma=5)

# Visualizing
plt.figure(figsize=(7,7))
plt.imshow(dist_trans_smooth, interpolation='none', cmap='viridis')
plt.show()_____no_output_____# (iv) Retrieve the local maxima (the 'peaks') from the distance transform

# Use the function 'peak_local_max' from the module 'skimage.feature'. By default, this function will return the
# indices of the pixels where the local maxima are. However, we instead need a boolean mask of the same shape
# as the original image, where all the local maximum pixels are labeled as `1` and everything else as `0`.
# This can be achieved by setting the keyword argument 'indices' to False.
from skimage.feature import peak_local_max
seeds = peak_local_max(dist_trans_smooth, indices=False, min_distance=10)_____no_output_____# (v) Visualize the output as an overlay on the raw (or smoothed) image

# If you just look at the local maxima image, it will simply look like a bunch of distributed dots.
# To get an idea if the seeds are well-placed, you will need to overlay these dots onto the original image.

# To do this, it is important to first understand a key point about how the 'pyplot' module works:
# every plotting command is slapped on top of the previous plotting commands, until everything is ultimately
# shown when 'plt.show' is called. Hence, you can first plot the raw (or smoothed) input image and then
# plot the seeds on top of it before showing both with 'plt.show'.

# As you can see if you try this, you will not get the desired result because the zero values in seed array
# are painted in black over the image you want in the background. To solve this problem, you need to mask
# these zero values before plotting the seeds. You can do this by creating an appropriately masked array
# using the function 'np.ma.array' with the keyword argument 'mask'.
# Check the docs or Stack Overflow to figure out how to do this.

# BONUS: As an additional improvement to the visualization, use 'ndi.filters.maximum_filter' to dilate the
# seeds a little bit, making them bigger and thus better visible.

# Dilate seeds
seeds_dil = ndi.filters.maximum_filter(seeds, size=10)

# Create plot
plt.figure(figsize=(7,7))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.imshow(np.ma.array(seeds_dil, mask=seeds_dil==0), interpolation='none', cmap='autumn')
plt.show()_____no_output_____# (vii) Label the seeds

# Use connected component labeling to give each cell seed a unique ID number.
seeds_labeled = ndi.label(seeds)[0]

# Visualize the final result (the labeled seeds) as an overlay on the raw (or smoothed) image
seeds_labeled_dil = ndi.filters.maximum_filter(seeds_labeled, size=10)  # Expand a bit for visualization
plt.figure(figsize=(10,10))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.imshow(np.ma.array(seeds_labeled_dil, mask=seeds_labeled_dil==0), interpolation='none', cmap='prism')
plt.show()_____no_output_____

### Expansion by Watershed_____no_output_____#### Exercise Solution

Expand your seeds by means of a watershed expansion._____no_output_____

# (i) Perform watershed

# Use the function 'watershed' from the module 'skimage.morphology'.
# Use the labeled cell seeds and the smoothed membrane image as input.
from skimage.morphology import watershed
ws = watershed(img_smooth, seeds_labeled)_____no_output_____# (ii) Show the result as transparent overlay over the smoothed input image

# Like the masked overlay of the seeds, this can be achieved by making two calls to 'imshow',
# one for the background image and one for the segmentation. Instead of masking away background,
# this time you simply make the segmentation image semi-transparent by adjusting the keyword
# argument 'alpha' of the 'imshow' function, which specifies opacity.
# Be sure to choose an appropriate colormap that allows you to distinguish the segmented cells
# even if cells with a very similar ID are next to each other (I would recommend 'prism').
plt.figure(figsize=(10,10))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.imshow(ws, interpolation='none', cmap='prism', alpha=0.4)
plt.show()_____no_output_____

#### *A Note on Segmentation Quality*

This concludes the segmentation of the cells in the example image. Depending on the quality you achieved in each step along the way, the final segmentation may be of greater or lesser quality (in terms of over-/under-segmentation errors).

It should be noted that the segmentation will likely *never* be perfect, as there is usually a trade-off between over- and undersegmentation.

This raises an important question: ***When should I stop trying to optimize my segmentation?***

There is no absolute answer to this question but the best answer is probably this: ***When you can use it to address your biological questions!***

*Importantly, this implies that you should already have relatively clear questions in mind when you are working on the segmentation!*_____no_output_____## Postprocessing: Removing Cells at the Image Border _____no_output_____####  Exercise Solution

Iterate through all the cells in your segmentation and remove those touching the image border._____no_output_____

# (i) Create an image border mask

border_mask = np.zeros(ws.shape, dtype=np.bool)
border_mask = ndi.binary_dilation(border_mask, border_value=1)_____no_output_____# (ii) 'Delete' the cells at the border

# When modifying a segmentation (in this case by deleting some cells), it makes sense
# to work on a copy of the array, not on the original. This avoids unexpected behaviors,
# especially within jupyter notebooks. Use the function 'np.copy' to copy an array.
clean_ws = np.copy(ws)

# Iterate over the IDs of all the cells in the segmentation. Use a for-loop and the
# function 'np.unique' (remember that each cell in our segmentation is labeled with a
# different integer value).
for cell_ID in np.unique(ws):

    # Create a mask that contains only the 'current' cell of the iteration
    # Hint: Remember that the comparison of an array with some number (array==number)
    #       returns a Boolean mask of the pixels in 'array' whose value is 'number'.
    cell_mask = ws==cell_ID

    # Using the cell mask and the border mask from above, test if the cell has pixels touching
    # the image border or not.
    # Hint: 'np.logical_and'
    cell_border_overlap = np.logical_and(cell_mask, border_mask)  # Overlap of cell mask and boundary mask
    total_overlap_pixels = np.sum(cell_border_overlap)            # Sum overlapping pixels

    # If a cell touches the image boundary, delete it by setting its pixels in the segmentation to 0.
    if total_overlap_pixels > 0:
        clean_ws[cell_mask] = 0_____no_output_____# OPTIONAL: re-label the remaining cells to keep the numbering consistent from 1 to N (with 0 as background).

for new_ID, cell_ID in enumerate(np.unique(clean_ws)[1:]):  # The [1:] excludes 0 from the list (background)!
    clean_ws[clean_ws==cell_ID] = new_ID+1                  # The same here for the +1_____no_output_____# (iii) Visualize the result

# Show the result as transparent overlay over the raw or smoothed image.
# Here you have to combine alpha (to make cells transparent) and 'np.ma.array'
# (to hide empty space where the border cells were deleted).

plt.figure(figsize=(7,7))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.imshow(np.ma.array(clean_ws, mask=clean_ws==0), interpolation='none', cmap='prism', alpha=0.4)
plt.show()_____no_output_____

## Identifying Cell Edges _____no_output_____#### Exercise Solution

Create a labeled mask of cell edges._____no_output_____

# (i) Create an array of the same size and data type as the segmentation but filled with only zeros

edges = np.zeros_like(clean_ws)_____no_output_____# (ii) Iterate over the cell IDs
for cell_ID in np.unique(clean_ws)[1:]:

    # (iii) Erode the cell's mask by 1 pixel
    # Hint: 'ndi.binary_erode'
    cell_mask = clean_ws==cell_ID
    eroded_cell_mask = ndi.binary_erosion(cell_mask, iterations=1) # Increase iterations to make boundary wider!

    # (iv) Create the cell edge mask
    # Hint: 'np.logical_xor'
    edge_mask = np.logical_xor(cell_mask, eroded_cell_mask)

    # (v) Add the cell edge mask to the empty array generated above, labeling it with the cell's ID
    edges[edge_mask] = cell_ID_____no_output_____# (vi) Visualize the result

# Note: Because the lines are so thin (1pxl wide), they may not be displayed correctly in small figures.
#       You can 'zoom in' by showing a sub-region of the image which is then rendered bigger. You can
#       also go back to the edge identification code and make the edges multiple pixels wide (but keep
#       in mind that this will have an effect on your quantification results!).

plt.figure(figsize=(7,7))
plt.imshow(np.zeros_like(edges)[300:500, 300:500], cmap='gray', vmin=0, vmax=1)  # Simple black background
plt.imshow(np.ma.array(edges, mask=edges==0)[300:500, 300:500], interpolation='none', cmap='prism')
plt.show()_____no_output_____

## Extracting Quantitative Measurements _____no_output_____#### Exercise Solution

Extract the measurements listed above for each cell and collect them in a dictionary.

Note: The ideal data structure for data like this is the `DataFrame` offered by the module `Pandas`. However, for the sake of simplicty, we will here stick with a dictionary of lists._____no_output_____

# (i) Create a dictionary that contains a key-value pairing for each measurement

# The keys should be strings describing the type of measurement (e.g. 'intensity_mean') and
# the values should be empty lists. These empty lists will be filled with the results of the
# measurements

results = {"cell_id"      : [],
           "int_mean"     : [],
           "int_mem_mean" : [],
           "cell_area"    : [],
           "cell_edge"    : []}

# Solution note: the spacing between the strings and colons doesn't matter for the code's
# execution. It is used solely to make the code more readable!_____no_output_____# (ii) Record the measurements for each cell

# Iterate over the segmented cells ('np.unique').
# Inside the loop, create a mask for the current cell and use it to extract the measurements listed above.
# Add them to the appropriate list in the dictionary using the 'append' method.
# Hint: Remember that you can get out all the values within a masked area by indexing the image
#       with the mask. For example, 'np.mean(image[cell_mask])' will return the mean of all the
#       intensity values of 'image' that are masked by 'cell_mask'!

# Iterate over cell IDs
for cell_id in np.unique(clean_ws)[1:]:

    # Mask the current cell and cell edge
    cell_mask = clean_ws==cell_id
    edge_mask = edges==cell_id

    # Get the measurements
    results["cell_id"].append(cell_id)
    results["int_mean"].append(np.mean(img[cell_mask]))
    results["int_mem_mean"].append(np.mean(img[edge_mask]))
    results["cell_area"].append(np.sum(cell_mask))
    results["cell_edge"].append(np.sum(edge_mask))_____no_output_____# (iii) Print the results and check that they make sense

for key in results.keys():
    print(key + ":", results[key][:5], '\n')_____no_output_____

## Simple Analysis & Visualisation _____no_output_____#### Exercise Solution

Analyze and plot the extracted data in a variety of ways._____no_output_____

# (i) Familiarize yourself with the data structure of the results dict and summarize the results

# Recall that dictionaries are unordered; a dataset of interest is accessed through its key.
# In our case, the datasets inside the dict are lists of values, ordered in the same order
# as the cell IDs.

# For each dataset in the results dict, print its name (the key) along with its mean, standard
# deviation, maximum, minimum, and median. The appropriate numpy methods (e.g. 'np.median') work
# with lists just as well as with arrays.

# Custom function for nice printing of summary statistics.
# Note the use of format strings for nice number padding.
def print_summary(data):
    print( "  Mean:    {:7.2f}".format(np.mean(data))   )
    print( "  Stdev:   {:7.2f}".format(np.std(data))    )
    print( "  Max:     {:7.2f}".format(np.max(data))    )
    print( "  Min:     {:7.2f}".format(np.min(data))    )
    print( "  Median:  {:7.2f}".format(np.median(data)) )

# Calling the custom function for each dataset
for key in results.keys():
    print( '\n'+key )
    print_summary(results[key])

# There are also pre-made functions to get summary statistics,
# for example 'scipy.stats.describe'.
from scipy.stats import describe
stat_summary = describe(results['int_mean'])

print( '\nscipy.stats.describe of int_mean' )
for key in stat_summary._asdict().keys():
    print( ' ', key+': ', stat_summary._asdict()[key] )_____no_output_____# (ii) Create a box plot showing the mean cell and mean membrane intensities for both channels.

# Use the function 'plt.boxplot'. Use the 'label' keyword of 'plt.boxplot' to label the x axis with
# the corresponding key names. Feel free to play around with the various options of the boxplot
# function to make your plot look nicer. Remember that you can first call 'plt.figure' to adjust
# settings such as the size of the plot.

plt.figure(figsize=(3,6))
plt.boxplot([results['int_mean'], results['int_mem_mean']],
            labels=['int_mean', 'int_mem_mean'],
            widths=0.6, notch=True)
plt.show()_____no_output_____# (iii) Create a scatter plot of cell outline length over cell area

# Use the function 'plt.scatter' for this. Be sure to properly label the
# plot using 'plt.xlabel' and 'plt.ylabel'.

plt.figure(figsize=(8,5))
plt.scatter(results["cell_area"], results["cell_edge"],
            edgecolor='k', s=30, alpha=0.5)
plt.xlabel("cell area [pxl^2]")
plt.ylabel("cell edge length [pxl]")

# BONUS: Do you understand why you are seeing the pattern this produces?
# ->> The curve reflects how circumference scales with area!

# Can you generate a 'null model' curve that assumes all cells to be circular?
cell_area_range = np.linspace(min(results["cell_area"]), max(results["cell_area"]), num=100)
circle_circumference = 2 * np.pi * np.sqrt( cell_area_range / np.pi )
plt.plot(cell_area_range, circle_circumference, color='r', alpha=0.8)
plt.legend(['circles', 'data'], loc=2, fontsize=10)
plt.show()

# What is the result? Do you notice something odd about it? What could be the reason for
# this and how could it be fixed?
# ->> In general, the cells don't deviate all that much from the circular case.
# ->> Strangely, some cells have a smaller outline than the circumference of a circle
#     of equivalent area. This is mathematically impossible.
# ->> A possible reason could be that the measures are taken in pixels, which leads
#     to a so-called discretization error. It could be fixed by "meshing" the cell
#     outline and interpolating a more accurate measurement of circumference._____no_output_____# (iv) Perform a linear fit of membrane intensity over cell area

# Use the function 'linregress' from the module 'scipy.stats'. Be sure to read the docs to
# understand the output of this function. Print the output.

# Compute linear fit
from scipy.stats import linregress
linfit = linregress(results["cell_area"], results["int_mem_mean"])

# Print all the results
linprops = ['slope','interc','rval','pval','stderr']
for index,prop in enumerate(linprops):
    print( prop, '\t', '{:4.2e}'.format(linfit[index]) )_____no_output_____# (v) Think about the result

# Note that the fit seems to return a highly significant p-value but a very low correlation
# coefficient (r-value). Based on prior knowledge, we would not expect a linear correlation of
# this sort to be present in our data.
#
# This should prompt several questions:
#   1) What does this p-value actually mean? Check the docs of 'linregress'!
#       ->> This p-value only means that, given a linear fit through this data, the slope of the
#           fit is very unlikely to be zero. However, it does not make a statement on whether or
#           not it makes sense to use a linear fit in the first place. Looking at the scatterplot
#           below or at the correlation coefficient r, it is clear that a linear fit on this data
#           is not meaningful.
#       ->> Note also: With single-cell approaches, we quickly get to a large number of data points.
#           This makes hypothesis testing in general less useful, as p-values tend to become very
#           small even if the null hypothesis holds. It makes sense to instead report effect sizes.
#           This is a tricky topic but well worth reading up on.
#
#   2) Could there be artifacts in our segmentation that bias this analysis?
#       ->> Oversegmentation is an important source of bias here. If a cell is oversegmented,
#           it will be considered as two or three cells. These will naturally have a lower
#           cell area and will naturally have a lower membrane intensity because some of their
#           edges are actually not on membranes. In other words, they will fall into the bottom
#           left of the plot, distorting the data.
#
# In general, it's always good to be very careful when doing any kind of data analysis. Make sure you
# understand the functions you are using and always check for possible errors or sources of bias!_____no_output_____# (vi) Overlay the linear fit onto a scatter plot

# Recall that a linear function is defined by `y = slope * x + intercept`.

# To define the line you'd like to plot, you need two values of x (the starting point and
# and the end point of the line). What values of x make sense? Can you get them automatically?
#   ->> The max and min values in the data are a good choice.
x_vals = [min(results["cell_area"]), max(results["cell_area"])]

# When you have the x-values for the starting point and end point, get the corresponding y
# values from the fit through the equation above.
y_vals = [linfit[0] * x_vals[0] + linfit[1], linfit[0] * x_vals[1] + linfit[1]]

# Plot the line with 'plt.plot'. Adjust the line's properties so it is well visible.
# Note: Remember that you have to create the scatterplot before plotting the line so that
#       the line will be placed on top of the scatterplot.
plt.figure(figsize=(8,5))
plt.scatter(results["cell_area"], results["int_mem_mean"],
            edgecolor='k', s=30, alpha=0.5)
plt.plot(x_vals, y_vals, color='red', lw=2, alpha=0.8)

# Use 'plt.legend' to add information about the line to the plot.
plt.legend(["linear fit, Rsq={:4.2e}".format(linfit[2]**2.0)], frameon=False, loc=4)

# Label the plot and finally show it with 'plt.show'.
plt.xlabel("cell area [pxl]")
plt.ylabel("Mean membrane intensity [a.u.]")
plt.title("Scatterplot with linear fit")
plt.show()_____no_output_____# (vii) Map the cell area back onto the image as a 'heatmap'

# Scale the cell area data to 8bit so that it can be used as pixel intensity values.
# Hint: if the largest cell area should correspond to the value 255 in uint8, then
#       the other cell areas correspond to 'cell_area * 255 / largest_cell_area'.
# Hint: To perform an operation on all cell area values at once, convert the list
#       of cell areas to a numpy array.
areas_8bit = np.array(results["cell_area"]) / max(results["cell_area"]) * 255

# Initialize a new image array; all values should be zeros, the shape should be identical
# to the images we worked with before and the dtype should be uint8.
area_map = np.zeros_like(clean_ws, dtype=np.uint8)

# Iterate over the segmented cells. In addition to the cell IDs, the for-loop should
# also include a simple counter (starting from 0) with which the area measurement can be
# accessed by indexing.
for index, cell_id in enumerate(results["cell_id"]):

    # Mask the current cell and assign the cell's (re-scaled) area value to the cell's pixels.
    area_map[clean_ws==cell_id] = areas_8bit[index]

# Visualize the result as a colored semi-transparent overlay over the raw/smoothed original input image.
# BONUS: See if you can exclude outliers to make the color mapping more informative!

# Mask of outliers (the largest and smallest 5% of all cells)
outlier_mask = np.logical_or(area_map > np.percentile(areas_8bit, 95),
                             area_map < np.percentile(areas_8bit, 5))

# Mask of all regions to leave blank (outliers + image boundary cells)
full_mask = np.logical_or(area_map==0, outlier_mask)

# Create the plot
plt.figure(figsize=(10,10))
plt.imshow(img_smooth, interpolation='none', cmap='gray')
plt.imshow(np.ma.array(area_map, mask=full_mask),
           interpolation='none', cmap='viridis', alpha=0.6)
plt.show()_____no_output_____

## Writing Output to Files _____no_output_____#### Exercise Solution

Write the generated data into a variety of different output files._____no_output_____

# (i) Write one or more of the images you produced to a tif file

# Use the function 'imsave' from the 'skimage.io' module. Make sure that the array you are
# writing is of integer type. If necessary, you can use the method 'astype' for conversions,
# e.g. 'some_array.astype(np.uint8)' or 'some_array.astype(np.uint16)'. Careful when
# converting a segmentation to uint8; if there are more than 255 cells, the 8bit format
# doesn't have sufficient bit-depth to represent all cell IDs!
#
# You can also try adding the segmentation to the original image, creating an image with
# two channels, one of them being the segmentation.
#
# After writing the file, load it into Fiji and check that everything worked as intended.

from skimage.io import imsave
imsave("example_cells_1_edges.tif", edges.astype(np.uint16))_____no_output_____# (ii) Write a figure to a png or pdf

# Recreate the scatter plot from above (with or without the regression line), then save the figure
# as a png using 'plt.savefig'. Alternatively, you can also save it to a pdf, which will create a
# vector graphic that can be imported into programs like Adobe Illustrator.

# Create plot (but don't show)
plt.scatter(results["cell_area"], results["int_mem_mean"],
            edgecolor='k', s=30, alpha=0.5)
plt.plot(x_vals, y_vals, color='red', lw=2, alpha=0.8)
plt.legend(["linear fit, Rsq={:4.2e}".format(linfit[2]**2.0)], frameon=False, loc=4)
plt.xlabel("cell area [pxl]")
plt.ylabel("Mean membrane intensity [a.u.]")
plt.title("Scatterplot with linear fit")

# Save as png and pdf
plt.savefig('example_cells_1_scatterFit.png')
plt.savefig('example_cells_1_scatterFit.pdf')
plt.clf()  # Clear the figure buffer_____no_output_____# (iii) Save the segmentation as a numpy file

# Numpy files allow fast storage and reloading of numpy arrays. Use the function 'np.save'
# to save the array and reload it using 'np.load'.

np.save("example_cells_1_seg", clean_ws)  # Save
seg = np.load("example_cells_1_seg.npy")  # Load
print(clean_ws.shape, seg.shape)_____no_output_____# (iv) Save the result dictionary as a pickle file

# Pickling is a way of generating generic files from almost any python object, which can easily
# be reloaded into python at a later point in time.
# You will need to open an empty file object using 'open' in write mode ('w'). It's best to do
# so using the 'with'-statement (context manager) to make sure that the file object will be closed
# automatically when you are done with it.
# Use the function 'pickle.dump' from the 'pickle' module to write the results to the file.
# Hint: Refer to the python documention for input and output to understand how file objects are
#       handled in python in general.

import pickle
with open('example_cells_1_results.pkl','wb') as outfile:
    pickle.dump(results, outfile)

# Note: Pickled files can be re-loaded again as follows:
with open('example_cells_1_results.pkl', 'rb') as infile:
    results_reloaded = pickle.load(infile)
    print(results_reloaded.keys())_____no_output_____# (v) Write a tab-separated text file of the results dict

# The most generic way of saving numeric results is a simple text file. It can be imported into
# pretty much any other program.

# To write normal text files, open an empty file object in write mode ('w') using the 'with'-statement.
with open('example_cells_1_results.txt','w') as outfile:

    # Use the 'file_object.write(string)' method to write strings to the file, one line at a time,
    # First, write the header of the data (the result dict keys), separated by tabs ('\t').
    # It makes sense to first generate a complete string with all the headers and then write this
    # string to the file as one line. Note that you will need to explicitly write 'newline' characters
    # ('\n') at the end of the line to switch to the next line.
    # Hint: the string method 'join' is very useful here!
    header_string = '\t'.join(results.keys()) + '\n'
    outfile.write(header_string)

    # After writing the headers, iterate over all the cells and write the result data to the file line
    # by line, by creating strings similar to the header string.
    for index in range(len(results['cell_id'])):
        data_string = '\t'.join([str(results[key][index]) for key in results.keys()]) + '\n'
        outfile.write(data_string)

# After writing the data, have a look at the output file in a text editor or in a spreadsheet
# program like Excel._____no_output_____

## Batch Processing _____no_output_____#### Exercise Solution

Convert the pipeline to a single function in a python script.

Import that function and write the code necessary to iterate over all input files in a folder, apply the function to each file, and collect the results._____no_output_____#### Solution Note

The converted version of the pipeline can be found in `batch_processing_solution.py`.

Note that the most of the exercise comments have been removed so it doesn't look too cluttered. However, this level of clean-up is probably a bit extreme; it is generally recommended to retain at least basic comments on the purpose of each code block.

Also note that a doc string was added to the function definition (the string designated with three `"""` at the start and end, directly under the function definition). This is a very useful reference, since it is automatically recognized as a help message by Jupyter notebook (and other IDEs), so you can easily double-check what an imported function does, for example by typing `run_pipeline?` in a code cell._____no_output_____

# (i) Test if your pipeline function actually works

# Import your function using the normal python syntax for imports, like this:
#   from your_module import your_function
# Run the function and visualize the resulting segmentation. Make sure everything works as intended.

from batch_processing_solution import run_pipeline
pip_seg, pip_results = run_pipeline(r"example_data", r'example_cells_1.tif')

plt.figure(figsize=(7,7))
plt.imshow(np.zeros_like(pip_seg), interpolation='none', cmap='gray', vmax=1)  # Simple black background
plt.imshow(np.ma.array(pip_seg, mask=pip_seg==0), interpolation='none', cmap='prism')
plt.show()_____no_output_____# (ii) Get all relevant filenames from the input directory

# Use the function 'listdir' from the module 'os' to get a list of all the files
# in a directory. Find a way to filter out only the relevant input files, namely
# "example_cells_1.tif" and "example_cells_2.tif". Of course, one would usually
# do this for many more images, otherwise it's not worth the effort.
# Hint: Loop over the filenames and use if statements to decide which ones to
#       keep and which ones to throw away.

# Get all files
dirpath = r"example_data"
from os import listdir
filelist = listdir(dirpath)

# Filter for target files: simple option
# Note that this will use ALL files with a .tif ending, which in some circumstances
# may include files that are not supposed to be used!
target_files = []
for fname in filelist:
    if fname.endswith('.tif'):
        target_files.append(fname)
print(target_files)

# Filter for target files: advanced option using regex and a list comprehension
import re
target_pattern = re.compile("^example_cells_\d+\.tif$")
target_files = [fname for fname in filelist if target_pattern.match(fname)]
print(target_files)_____no_output_____# (iii) Iterate over the relevant input filenames and run the pipeline function

# Be sure to collect the output of the pipeline function in a way that allows
# you to trace it back to the file it came from. You could for example use a
# dictionary with the filenames as keys.

# Initialize empty dictionaries
all_seg = {}
all_results = {}

# Iterate over files and run pipeline for each
for fname in target_files:
    pip_seg, pip_results = run_pipeline(dirpath, fname)
    all_seg[fname] = pip_seg
    all_results[fname] = pip_results_____no_output_____# (iv) Recreate one of the scatterplots from above, but this time with all the cells

# You can color-code the dots to indicate which file they came from. Don't forget to
# add a corresponding legend.

plt.figure(figsize=(8,5))

colors = ['blue','red']
for key, color in zip(sorted(all_results.keys()), colors):
    plt.scatter(all_results[key]["cell_area"], all_results[key]["cell_edge"],
                edgecolor='k', c=color, s=30, alpha=0.5, label=key)

plt.legend()
plt.xlabel("cell area [pxl^2]")
plt.ylabel("cell edge length [pxl]")
plt.show()_____no_output_____

## *This is the end of the tutorial!*

**We hope you enjoyed the ride and learned a lot!**_____no_output_____### Concluding Remarks

It's important to remember that the phrase ***"Use it or lose it!"*** fully applies for the skills taught in this tutorial.

If you now just go back to the lab and don't touch python or image analysis for the next half year, most of the things you have learned here will be lost.

So, what can you do?

- If possible, start applying what you have learned to your own work right away

- Even if your current work doesn't absolutely *need* coding / image analysis (which to be honest is hard to believe! ;p), you can still use it at least to make some nice plots!

- Another very good approach is to find yourself an interesting little side project you can play around with

- Of course, there is still much more to learn and the internet happens to be full of excellent tutorials!
    - As a starting point, have a look at [Bio-IT's curated list of tutorials](https://bio-it.embl.de/coding-club/curated-tutorials/)_____no_output_____***We wish you the best of luck for all your coding endeavors!***_____no_output_____


# Notebook from bgraille/pylbm
Path: notebooks/05_Poiseuille.ipynb

# Poiseuille flow

$$
\renewcommand{\DdQq}[2]{{\mathrm D}_{#1}{\mathrm Q}_{#2}}
\renewcommand{\drondt}{\partial_t}
\renewcommand{\drondx}{\partial_x}
\renewcommand{\drondy}{\partial_y}
\renewcommand{\drondtt}{\partial_{tt}}
\renewcommand{\drondxx}{\partial_{xx}}
\renewcommand{\drondyy}{\partial_{yy}}
\renewcommand{\dx}{\Delta x}
\renewcommand{\dt}{\Delta t}
\renewcommand{\grandO}{{\mathcal O}}
\renewcommand{\density}[2]{\,f_{#1}^{#2}}
\renewcommand{\fk}[1]{\density{#1}{\vphantom{\star}}}
\renewcommand{\fks}[1]{\density{#1}{\star}}
\renewcommand{\moment}[2]{\,m_{#1}^{#2}}
\renewcommand{\mk}[1]{\moment{#1}{\vphantom{\star}}}
\renewcommand{\mke}[1]{\moment{#1}{e}}
\renewcommand{\mks}[1]{\moment{#1}{\star}}
$$

In this tutorial, we consider the classical $\DdQq{2}{9}$ to simulate a Poiseuille flow modeling by the Navier-Stokes equations._____no_output_____

%matplotlib inline_____no_output_____

## The $\DdQq{2}{9}$ **for Navier-Stokes**

The $\DdQq{2}{9}$ is defined by:

* a space step $\dx$ and a time step $\dt$ related to the scheme velocity $\lambda$ by the relation $\lambda=\dx/\dt$,

* nine velocities $\{(0,0), (\pm1,0), (0,\pm1), (\pm1, \pm1)\}$, identified in pylbm by the
numbers $0$ to $8$,

* nine polynomials used to build the moments

$$ \{1, \lambda X, \lambda Y, 3E-4, (9E^2-21E+8)/2, 3XE-5X, 3YE-5Y,X^2-Y^2, XY\},$$

where $E = X^2+Y^2$.

* three conserved moments $\rho$, $q_x$, and $q_y$,

* nine relaxation parameters (three are $0$ corresponding to conserved moments): $\{0,0,0,s_\mu,s_\mu,s_\eta,s_\eta,s_\eta,s_\eta\}$, where $s_\mu$ and $s_\eta$ are in $(0,2)$,

* equilibrium value of the non conserved moments
$$
\begin{aligned}\mke{3} &= -2\rho + 3(q_x^2+q_y^2)/(\rho_0\lambda^2), \\ \mke{4} &= \rho-3(q_x^2+q_y^2)/(\rho_0\lambda^2), \\ \mke{5} &= -q_x/\lambda, \\ \mke{6} &= -q_y/\lambda, \\ \mke{7} &= (q_x^2-q_y^2)/(\rho_0\lambda^2), \\ \mke{8} &= q_xq_y/(\rho_0\lambda^2),\end{aligned}
$$

where $\rho_0$ is a given scalar.

This scheme is consistant at second order with the following equations (taken $\rho_0=1$)
$$
\begin{gathered} \drondt\rho + \drondx q_x + \drondy q_y = 0,\\ \drondt q_x + \drondx (q_x^2+p) + \drondy (q_xq_y) = \mu \drondx (\drondx q_x + \drondy q_y ) + \eta (\drondxx+\drondyy)q_x, \\ \drondt q_y + \drondx (q_xq_y) + \drondy (q_y^2+p) = \mu \drondy (\drondx q_x + \drondy q_y ) + \eta (\drondxx+\drondyy)q_y,\end{gathered}
$$
with $p=\rho\lambda^2/3$._____no_output_____## Build the simulation with pylbm
In the following, we build the dictionary of the simulation step by step.

### The geometry

The simulation is done on a rectangle of length $L$ and width $W$. We can use $L=W=1$.

We propose a dictionary that build the geometry of the domain. The labels of the bounds can be specified to different values for the moment._____no_output_____

import numpy as np
import matplotlib.pyplot as plt

import pylbm

L, W = 1., 1.
dico_geom = {
    'box': {'x': [0, L],
            'y': [-.5*W, .5*W],
            'label':list(range(4))
           }
}
geom = pylbm.Geometry(dico_geom)
print(geom)
geom.visualize(viewlabel=True);+----------------------+
| Geometry information |
+----------------------+
    - spatial dimension: 2
    - bounds of the box: [0. 1.] x [-0.5  0.5]
    - labels: [0, 1, 2, 3]

### The stencil

The stencil of the $\DdQq{2}{9}$ is composed by the nine following velocities in 2D:
$$
\begin{gathered}v_0=(0,0),\\ v_1=(1,0), \quad v_2=(0,1), \quad v_3=(-1,0), \quad v_4=(0,-1), \\ v_5=(1,1), \quad v_6=(-1,1), \quad v_7=(-1,-1), \quad v_8=(1,-1).\end{gathered}
$$_____no_output_____

dico_sten = {
    'dim': 2,
    'schemes': [
        {'velocities': list(range(9))}
    ],
}
sten = pylbm.Stencil(dico_sten)
print(sten)
sten.visualize();+---------------------+
| Stencil information |
+---------------------+
    - spatial dimension: 2
    - minimal velocity in each direction: [-1 -1]
    - maximal velocity in each direction: [1 1]
    - information for each elementary stencil:
        stencil 0
            - number of velocities: 9
            - velocities
                (0: 0, 0)
                (1: 1, 0)
                (2: 0, 1)
                (3: -1, 0)
                (4: 0, -1)
                (5: 1, 1)
                (6: -1, 1)
                (7: -1, -1)
                (8: 1, -1)

### The domain

In order to build the domain of the simulation, the dictionary should contain the space step $\dx$ and the stencils of the velocities (one for each scheme). _____no_output_____

dico_dom = {
    'space_step': .1,
    'box': {'x': [0, L],
            'y': [-.5*W, .5*W],
            'label': list(range(4))
           },
    'schemes': [
        {'velocities': list(range(9))}
    ],
}
dom = pylbm.Domain(dico_dom)
print(dom)
dom.visualize(view_distance=True);+--------------------+
| Domain information |
+--------------------+
    - spatial dimension: 2
    - space step: 0.1
    - with halo:
        bounds of the box: [-0.05 -0.55] x [1.05 0.55]
        number of points: [12, 12]
    - without halo:
        bounds of the box: [ 0.05 -0.45] x [0.95 0.45]
        number of points: [10, 10]

    +----------------------+
    | Geometry information |
    +----------------------+
        - spatial dimension: 2
        - bounds of the box: [0. 1.] x [-0.5  0.5]
        - labels: [0, 1, 2, 3]

### The scheme

In pylbm, a simulation can be performed by using several coupled schemes. In this example, a single scheme is used and defined through a list of one single dictionary. This dictionary should contain:

* 'velocities': a list of the velocities
* 'conserved_moments': a list of the conserved moments as sympy variables
* 'polynomials': a list of the polynomials that define the moments
* 'equilibrium': a list of the equilibrium value of all the moments
* 'relaxation_parameters': a list of the relaxation parameters ($0$ for the conserved moments)
* 'init': a dictionary to initialize the conserved moments

(see the documentation for more details)

In order to fix the bulk ($\mu$) and the shear ($\eta$) viscosities, we impose
$$ s_\eta = \frac{2}{1+\eta d}, \qquad s_\mu = \frac{2}{1+\mu d}, \qquad d = \frac{6}{\lambda\rho_0\dx}.$$

The scheme velocity could be taken to $1$ and the inital value of $\rho$ to $\rho_0=1$, $q_x$ and $q_y$ to $0$.

In order to accelerate the simulation, we can use another generator. The default generator is Numpy (pure python). We can use for instance Cython that generates a more efficient code. This generator can be activated by using  'generator': pylbm.generator.CythonGenerator in the dictionary._____no_output_____

import sympy as sp
X, Y, rho, qx, qy, LA = sp.symbols('X, Y, rho, qx, qy, LA')

# parameters
dx = 1./128  # spatial step
la = 1.      # velocity of the scheme
L = 1        # length of the domain
W = 1        # width of the domain
rhoo = 1.    # mean value of the density
mu   = 1.e-3 # shear viscosity
eta = 1.e-1 # bulk viscosity
# initialization
xmin, xmax, ymin, ymax = 0.0, L, -0.5*W, 0.5*W
dummy = 3.0/(la*rhoo*dx)
s_mu = 1.0/(0.5+mu*dummy)
s_eta = 1.0/(0.5+eta*dummy)
s_q = s_eta
s_es = s_mu
s  = [0.,0.,0.,s_mu,s_es,s_q,s_q,s_eta,s_eta]
dummy = 1./(LA**2*rhoo)
qx2 = dummy*qx**2
qy2 = dummy*qy**2
q2  = qx2+qy2
qxy = dummy*qx*qy

dico_sch = {
    'box': {'x': [xmin, xmax],
            'y': [ymin, ymax],
            'label':0
           },
    'space_step': dx,
    'scheme_velocity': la,
    'parameters': {LA: la},
    'schemes': [
        {
            'velocities': list(range(9)),
            'conserved_moments': [rho, qx, qy],
            'polynomials': [
                1, LA*X, LA*Y,
                3*(X**2+Y**2)-4,
                (9*(X**2+Y**2)**2-21*(X**2+Y**2)+8)/2,
                3*X*(X**2+Y**2)-5*X, 3*Y*(X**2+Y**2)-5*Y,
                X**2-Y**2, X*Y
            ],
            'relaxation_parameters': s,
            'equilibrium': [
                rho, qx, qy,
                -2*rho + 3*q2,
                rho-3*q2,
                -qx/LA, -qy/LA,
                qx2-qy2, qxy
            ],
        },
    ],
}
sch = pylbm.Scheme(dico_sch)
print(sch)+--------------------+
| Scheme information |
+--------------------+
    - spatial dimension: 2
    - number of schemes: 1
    - number of velocities: 9
    - conserved moments: [ρ, qx, qy]

    +----------+
    | Scheme 0 |
    +----------+
        - velocities
            (0: 0, 0)
            (1: 1, 0)
            (2: 0, 1)
            (3: -1, 0)
            (4: 0, -1)
            (5: 1, 1)
            (6: -1, 1)
            (7: -1, -1)
            (8: 1, -1)

        - polynomials

            ⎡                1                 ⎤
            ⎢                                  ⎥
            ⎢               LA⋅X               ⎥
            ⎢                                  ⎥
            ⎢               LA⋅Y               ⎥
            ⎢                                  ⎥
            ⎢            2      2              ⎥
            ⎢         3⋅X  + 3⋅Y  - 4          ⎥
            ⎢                                  ⎥
            ⎢                             2    ⎥
            ⎢      2       2     ⎛ 2    2⎞     ⎥
            ⎢  21⋅X    21⋅Y    9⋅⎝X  + Y ⎠     ⎥
            ⎢- ───── - ───── + ──────────── + 4⎥
            ⎢    2       2          2          ⎥
            ⎢                                  ⎥
            ⎢           ⎛ 2    2⎞              ⎥
            ⎢       3⋅X⋅⎝X  + Y ⎠ - 5⋅X        ⎥
            ⎢                                  ⎥
            ⎢           ⎛ 2    2⎞              ⎥
            ⎢       3⋅Y⋅⎝X  + Y ⎠ - 5⋅Y        ⎥
            ⎢                                  ⎥
            ⎢              2    2              ⎥
            ⎢             X  - Y               ⎥
            ⎢                                  ⎥
            ⎣               X⋅Y                ⎦

        - equilibria

            ⎡           ρ            ⎤
            ⎢                        ⎥
            ⎢           qx           ⎥
            ⎢                        ⎥
            ⎢           qy           ⎥
            ⎢                        ⎥
            ⎢             2         2⎥
            ⎢       3.0⋅qx    3.0⋅qy ⎥
            ⎢-2⋅ρ + ─────── + ───────⎥
            ⎢           2         2  ⎥
            ⎢         LA        LA   ⎥
            ⎢                        ⎥
            ⎢           2         2  ⎥
            ⎢     3.0⋅qx    3.0⋅qy   ⎥
            ⎢ ρ - ─────── - ───────  ⎥
            ⎢         2         2    ⎥
            ⎢       LA        LA     ⎥
            ⎢                        ⎥
            ⎢          -qx           ⎥
            ⎢          ────          ⎥
            ⎢           LA           ⎥
            ⎢                        ⎥
            ⎢          -qy           ⎥
            ⎢          ────          ⎥
            ⎢           LA           ⎥
            ⎢                        ⎥
            ⎢         2         2    ⎥
            ⎢   1.0⋅qx    1.0⋅qy     ⎥
            ⎢   ─────── - ───────    ⎥
            ⎢       2         2      ⎥
            ⎢     LA        LA       ⎥
            ⎢                        ⎥
            ⎢       1.0⋅qx⋅qy        ⎥
            ⎢       ─────────        ⎥
            ⎢            2           ⎥
            ⎣          LA            ⎦

        - relaxation parameters

            ⎡       0.0       ⎤
            ⎢                 ⎥
            ⎢       0.0       ⎥
            ⎢                 ⎥
            ⎢       0.0       ⎥
            ⎢                 ⎥
            ⎢1.13122171945701 ⎥
            ⎢                 ⎥
            ⎢1.13122171945701 ⎥
            ⎢                 ⎥
            ⎢0.025706940874036⎥
            ⎢                 ⎥
            ⎢0.025706940874036⎥
            ⎢                 ⎥
            ⎢0.025706940874036⎥
            ⎢                 ⎥
            ⎣0.025706940874036⎦

    - moments matrices

        ⎡1   1   1    1    1   1    1    1    1 ⎤
        ⎢                                       ⎥
        ⎢0   LA  0   -LA   0   LA  -LA  -LA  LA ⎥
        ⎢                                       ⎥
        ⎢0   0   LA   0   -LA  LA  LA   -LA  -LA⎥
        ⎢                                       ⎥
        ⎢-4  -1  -1  -1   -1   2    2    2    2 ⎥
        ⎢                                       ⎥
        ⎢4   -2  -2  -2   -2   1    1    1    1 ⎥
        ⎢                                       ⎥
        ⎢0   -2  0    2    0   1   -1   -1    1 ⎥
        ⎢                                       ⎥
        ⎢0   0   -2   0    2   1    1   -1   -1 ⎥
        ⎢                                       ⎥
        ⎢0   1   -1   1   -1   0    0    0    0 ⎥
        ⎢                                       ⎥
        ⎣0   0   0    0    0   1   -1    1   -1 ⎦

    - inverse of moments matrices

        ⎡1/9   0     0    -1/9    1/9     0      0     0     0  ⎤
        ⎢                                                       ⎥
        ⎢      1                                                ⎥
        ⎢1/9  ────   0    -1/36  -1/18  -1/6     0    1/4    0  ⎥
        ⎢     6⋅LA                                              ⎥
        ⎢                                                       ⎥
        ⎢            1                                          ⎥
        ⎢1/9   0    ────  -1/36  -1/18    0    -1/6   -1/4   0  ⎥
        ⎢           6⋅LA                                        ⎥
        ⎢                                                       ⎥
        ⎢     -1                                                ⎥
        ⎢1/9  ────   0    -1/36  -1/18   1/6     0    1/4    0  ⎥
        ⎢     6⋅LA                                              ⎥
        ⎢                                                       ⎥
        ⎢           -1                                          ⎥
        ⎢1/9   0    ────  -1/36  -1/18    0     1/6   -1/4   0  ⎥
        ⎢           6⋅LA                                        ⎥
        ⎢                                                       ⎥
        ⎢      1     1                                          ⎥
        ⎢1/9  ────  ────  1/18   1/36   1/12   1/12    0    1/4 ⎥
        ⎢     6⋅LA  6⋅LA                                        ⎥
        ⎢                                                       ⎥
        ⎢     -1     1                                          ⎥
        ⎢1/9  ────  ────  1/18   1/36   -1/12  1/12    0    -1/4⎥
        ⎢     6⋅LA  6⋅LA                                        ⎥
        ⎢                                                       ⎥
        ⎢     -1    -1                                          ⎥
        ⎢1/9  ────  ────  1/18   1/36   -1/12  -1/12   0    1/4 ⎥
        ⎢     6⋅LA  6⋅LA                                        ⎥
        ⎢                                                       ⎥
        ⎢      1    -1                                          ⎥
        ⎢1/9  ────  ────  1/18   1/36   1/12   -1/12   0    -1/4⎥
        ⎣     6⋅LA  6⋅LA                                        ⎦

### Run the simulation

For the simulation, we take

* The domain $x\in(0, L)$ and $y\in(-W/2,W/2)$, $L=2$, $W=1$,
* the viscosities $\mu=10^{-2}=\eta=10^{-2}$,
* the space step $\dx=1/128$, the scheme velocity $\lambda=1$,
* the mean density $\rho_0=1$.

Concerning the boundary conditions, we impose the velocity on all the edges by a bounce-back condition with a source term that reads
$$q_x(x, y) = \rho_0 v_{\text{max}} \Bigl( 1 - \frac{4y^2}{W^2} \Bigr), \qquad q_y(x, y) = 0,$$
with $v_{\text{max}}=0.1$.

We compute the solution for $t\in(0,50)$ and we plot several slices of the solution during the simulation.

This problem has an exact solution given by
$$ q_x = \rho_0 v_{\text{max}} \Bigl( 1 - \frac{4y^2}{W^2} \Bigr), \qquad q_y = 0, \qquad p = p_0 + K x, $$
where the pressure gradient $K$ reads
$$K = -\frac{8 v_{\text{max}} \eta}{W^2}.$$

We compute the exact and the numerical gradients of the pressure._____no_output_____

X, Y, LA = sp.symbols('X, Y, LA')
rho, qx, qy = sp.symbols('rho, qx, qy')

def bc(f, m, x, y):
    m[qx] = rhoo * vmax * (1.-4.*y**2/W**2)
    m[qy] = 0.

def plot_coupe(sol):
    fig, ax1 = plt.subplots()
    ax2 = ax1.twinx()
    ax1.cla()
    ax2.cla()
    mx = int(sol.domain.shape_in[0]/2)
    my = int(sol.domain.shape_in[1]/2)
    x = sol.domain.x
    y = sol.domain.y
    u = sol.m[qx] / rhoo
    for i in [0,mx,-1]:
        ax1.plot(y+x[i], u[i, :], 'b')
    for j in [0,my,-1]:
        ax1.plot(x+y[j], u[:,j], 'b')
    ax1.set_ylabel('velocity', color='b')
    for tl in ax1.get_yticklabels():
        tl.set_color('b')
    ax1.set_ylim(-.5*rhoo*vmax, 1.5*rhoo*vmax)
    p = sol.m[rho][:,my] * la**2 / 3.0
    p -= np.average(p)
    ax2.plot(x, p, 'r')
    ax2.set_ylabel('pressure', color='r')
    for tl in ax2.get_yticklabels():
        tl.set_color('r')
    ax2.set_ylim(pressure_gradient*L, -pressure_gradient*L)
    plt.title('Poiseuille flow at t = {0:f}'.format(sol.t))
    plt.draw()
    plt.pause(1.e-3)

# parameters
dx = 1./16  # spatial step
la = 1.      # velocity of the scheme
Tf = 50      # final time of the simulation
L = 2        # length of the domain
W = 1        # width of the domain
vmax = 0.1   # maximal velocity obtained in the middle of the channel
rhoo = 1.    # mean value of the density
mu = 1.e-2   # bulk viscosity
eta = 1.e-2  # shear viscosity
pressure_gradient = - vmax * 8.0 / W**2 * eta
# initialization
xmin, xmax, ymin, ymax = 0.0, L, -0.5*W, 0.5*W
dummy = 3.0/(la*rhoo*dx)
s_mu = 1.0/(0.5+mu*dummy)
s_eta = 1.0/(0.5+eta*dummy)
s_q = s_eta
s_es = s_mu
s  = [0.,0.,0.,s_mu,s_es,s_q,s_q,s_eta,s_eta]
dummy = 1./(LA**2*rhoo)
qx2 = dummy*qx**2
qy2 = dummy*qy**2
q2  = qx2+qy2
qxy = dummy*qx*qy

dico = {
    'box': {'x': [xmin, xmax],
            'y': [ymin, ymax],
            'label': 0
           },
    'space_step': dx,
    'scheme_velocity': la,
    'parameters': {LA: la},
    'schemes': [
        {
            'velocities': list(range(9)),
            'conserved_moments': [rho, qx, qy],
            'polynomials': [
                1, LA*X, LA*Y,
                3*(X**2+Y**2)-4,
                (9*(X**2+Y**2)**2-21*(X**2+Y**2)+8)/2,
                3*X*(X**2+Y**2)-5*X, 3*Y*(X**2+Y**2)-5*Y,
                X**2-Y**2, X*Y
            ],
            'relaxation_parameters': s,
            'equilibrium': [
                rho, qx, qy,
                -2*rho + 3*q2,
                rho-3*q2,
                -qx/LA, -qy/LA,
                qx2-qy2, qxy
            ],
        },
    ],
    'init': {rho: rhoo,
             qx:0.,
             qy:0.
    },
    'boundary_conditions': {
        0: {'method': {0: pylbm.bc.BouzidiBounceBack}, 'value': bc}
    },
    'generator': 'cython',
}

sol = pylbm.Simulation(dico)
while (sol.t<Tf):
    sol.one_time_step()
plot_coupe(sol)
ny = int(sol.domain.shape_in[1]/2)
num_pressure_gradient = (sol.m[rho][-2,ny] - sol.m[rho][1,ny]) / (xmax-xmin) * la**2/ 3.0
print("Exact pressure gradient    : {0:10.3e}".format(pressure_gradient))
print("Numerical pressure gradient: {0:10.3e}".format(num_pressure_gradient))_____no_output_____