# Notebook from gesiscss/IWAAN
Path: 1_General_Metadata.ipynb

# Welcome to IWAAN*
\**Interactive Wikipedia Article Analysis Notebooks*

This  collection of notebooks lets you analyze the evolution of the revision history of a Wikipedia article with **live data**. Among other things, it allows you to inspect **article- or editor-specific writing dynamics, productivity & conflicts.**
The main notebooks are the current one and notebook two, with notebook three offering some additional information.

Each Notebook can be **explored like a Web app, without interacting with the code behind it**. But you can also - if you choose to - click on **"Show solution"** after each block and work with the code directly (green buttons, comment the first line of code to use).

**Please choose a small or medium-sized article for your first exploration to reduce computation times**. All data is fetched live, so please be patient and wait for the kernel to finish before you interact with the notebook.

E.g., for your very first try you should simply go with our default choice, "The camp of the saints".

---

#### Troubleshooting:

- Allow some time for the notebook to run fully before interacting with the interface controls. For articles with a long revision history, this could take minutes and the interaction with the controls will be slow.
- Very large articles ("Donald Trump", "World War 2") may lead to memory overload, depending on which Binder platform you are using. You could get more memory by shutting down some previous running notebooks.
- **All cells should run automatically** when you open the notebook and **you should not see any code**. If that is not the case, please just reload the tab in your browser.
- After chosing a new article, please rerun the cells/modules you want to use.
- You should not see any code when you run it for the first time. If so, please let us know by posting an issue in our Github repository: https://github.com/gesiscss/IWAAN/issues.
_____no_output_____

from IPython.display import display, Javascript, HTML, clear_output
from ipywidgets import widgets, Output, interact, Layout
from ipywidgets.widgets import Dropdown
from datetime import datetime

## SOME EXTENTIONS ##
#%load_ext autoreload
%reload_ext autoreload
%autoreload 2
if 'the_page' not in locals():
    import pickle
    print("Loading default data...")
    the_page = pickle.load(open("data/the_page.p",'rb'))

display(Javascript('IPython.notebook.execute_cells_below()'))

display(Javascript('Jupyter.notebook.get_cells()'))

re_hide = """

"""
display(HTML(re_hide))

scroll_to_top = """

"""
display(HTML(scroll_to_top))Loading default data...
%%capture
%store -r the_page_____no_output_____

###  Import all modules _____no_output_____

## Modules Imported ##

# Display
from IPython.display import display, Markdown as md, clear_output
from datetime import date
import urllib

# APIs
from wikiwho_wrapper import WikiWho
from external.wikipedia import WikipediaDV, WikipediaAPI
from external.wikimedia import WikiMediaDV, WikiMediaAPI
from external.xtools import XtoolsAPI, XtoolsDV

# Data Processing
import pickle
import pandas as pd

# Visualization tools
import qgrid
import matplotlib.pyplot as plt

# Page views timeline
from visualization.views_listener import ViewsListener

# Change actions timeline
from visualization.actions_listener import ActionsListener

# Conflicts visualization
from visualization.conflicts_listener import ConflictsListener, ConflictsActionListener
from visualization.calculator_listener import ConflictCalculatorListener

# Word cloud visualization
from visualization.wordcloud_listener import WCListener, WCActionsListener
from visualization.wordclouder import WordClouder

# Wikipedia talk pages visualization
from visualization.talks_listener import TalksListener
from visualization.topics_listener import TopicsListener

# Tokens ownership visualization
from visualization.owned_listener import OwnedListener

# Templates visualization
from visualization.templates_listener import ProtectListener, TemplateListener

# Metrics management
from metrics.conflict import ConflictManager
from metrics.token import TokensManager

# For language selection
from utils.lngselection import abbreviation, lng_listener_____no_output_____

---

## A. Selecting a Wikipedia article

You can enter/search a Wikipedia article of your choice, or use our default example: "The Camp of the Saints" (a novel).
_____no_output_____

# the method that listens to the click event
def on_button_clicked(b):
    global wikipedia_dv
    global the_page
    # use the out widget so the output is overwritten when two or more
    # searches are performed
    with out:
        try:
            # query wikipedia
            wikipedia_dv = WikipediaDV(WikipediaAPI(lng=abbreviation(languageSelection.value)))
            search_result = wikipedia_dv.search_page(searchTerm.value)
            the_page = wikipedia_dv.get_page(search_result)
            %store the_page
            clear_output()
            display(md(f"The following page was found: **{the_page['title']}**"))
            display(md(f"Page id: **{the_page['page_id']}**"))
            url = f"{wikipedia_dv.api.base}action=query&titles={urllib.parse.quote_plus(the_page['title'])}&format=json"
            display(md(f"Metadata can be found in:"))
            print(url)
            #display(Javascript('Jupyter.notebook.execute_cells([8])'))

        except:
            clear_output()
            display(md(f'The page title *"{searchTerm.value}"* was not found'))
            #display(Javascript('Jupyter.notebook.execute_cells([8])'))

# Language selection.
languageSelection = Dropdown(options=['English', 'Deutsch', 'Español', 'Türkçe', 'Euskara'], value='English', description='Language:')

out_main = Output()
display(out_main)
with out_main:
    # by default display the last search
    try:
        searchTerm = widgets.Text(the_page['title'], description='Page title:')
    except:
        searchTerm = widgets.Text("The Camp of the Saints", description='Page title:')

    # Update selected language
    initial_select = widgets.interactive(lng_listener, lng=languageSelection, search_term=searchTerm)
    display(initial_select.children[0])
    display(initial_select.children[1])

    # create and display the button
    button = widgets.Button(description="Search")
    example = md("e.g. *The Camp of the Saints*")
    display(example, button)

# the output widget is used to remove the output after the search field
out = Output()
display(out)

# set the event
button.on_click(on_button_clicked)

# trigger the event with the default value
on_button_clicked(button)_____no_output_____

 Try yourself! This is the code executed when  you click the 'Search' button: _____no_output_____

%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### --------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Search' BUTTON ###
### --------------------------------------------------------------------- ###

## This is the default data and used for initialization ##
the_page = pickle.load(open("data/the_page.p",'rb'))  # global
title_default = the_page['title']
print('The pre-filled value for the title:', title_default)

## The search term you have input ##

# this was extracted from the previous search box, searchTerm, which is built by
# e.g. searchTerm = widgets.Text(the_page['title'], description='Page title:'), you could
# know more about it through https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html
search_language = languageSelection.value
language_for_api = abbreviation(search_language)

search_value = searchTerm.value
print('The language you use now:', search_language)
print('The value you input in search box:', search_value)

# of course you could also update the value here, like in the "Page title:" box above.
#search_value = 'Matrix Completion'

## query wikipedia using WikiWho API, more details please see:                ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/wikipedia.py ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/api.py       ##
wikipedia_dv = WikipediaDV(WikipediaAPI(lng=language_for_api)) # create an instance
result_after_search = wikipedia_dv.search_page(search_value)

print("The following page was found:", result_after_search)

## Get page id through get_page() method ##
the_page = wikipedia_dv.get_page(result_after_search)  # global
page_id = the_page['page_id']
print('Page id:', page_id)

## Metadata ##
url = f"{wikipedia_dv.api.base}action=query&titles={urllib.parse.quote_plus(result_after_search)}&format=json"
print('Metadata can be found in:', url)_____no_output_____

---

**Click on "Refresh All Variables" to reload all following cells with the selected article. (may take a some time)**

Alternatively, reload each module individually via their respective buttons._____no_output_____

def restart_and_run():
    display(HTML(
        '''

            <button onclick="restart_run_all()">Refresh All Variables</button>
        '''
    ))
restart_and_run()_____no_output_____

---

## B. General Statistics

Thanks to the Xtools API **(1)**, we can obtain general statistics related to the articlee. This information is relevant to select an appropiate article to analyze.

Our default example contains less than 1000 revisions (as of April, 2021), making it a good first case for our analysis. Articles such as *"Chicago"*, *"Barack Obama"* and *"Climate Change"* can have over 20K revisions, and therefore the processing will be slow and even fail due to lack of memory on the system running the notebooks (e.g.[notebooks.gesis.org](https://notebooks.gesis.org)).

Wikipedia offers a [list of the pages with most revisions](https://en.wikipedia.org/wiki/Wikipedia:Database_reports/Pages_with_the_most_revisions); **we recommend to avoid such articles for a smooth first experience with IWAAN**. Once you are confortable with our notebooks, you can tackle more complex articles._____no_output_____

def xtools_on_click(b):
    with out_xtools:
        clear_output()
        xtools_api = XtoolsAPI(lng=abbreviation(languageSelection.value))
        xtools_dv = XtoolsDV(xtools_api)
        page_info = xtools_dv.get_page_info(the_page['title'])
        page_info['assessment'] = page_info['assessment']['value'] if type(page_info['assessment']) != bool else page_info['assessment']

        page_info = page_info.to_frame('value').rename(index={
            'project': 'Project name',
            'page': 'Page name',
            'watchers': 'Watchers (2)',    'pageviews': f"Page Views (per {page_info['pageviews_offset']} days)",
            'revisions': 'Revisions',
            'editors': 'Editors',
            'author': 'Creator of the page',
            'created_at': 'Creation Date',
            'created_rev_id': 'Creation revision id',
            'modified_at': 'Last modified',
            'last_edit_id': 'Last revision id',
            'assessment': 'Content Assessment (3)',
        }).drop(index = ['pageviews_offset', 'author_editcount', 'secs_since_last_edit','elapsed_time'])

        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
        url = f"{xtools_dv.api.base}page/articleinfo/{xtools_dv.api.project}/" + urllib.parse.quote(the_page['title'])
        display(md(f"Metadata can be found in"))
        print(url)
        display(page_info)
        #display(Javascript('Jupyter.notebook.execute_cells([14])'))
        #display(Javascript('Jupyter.notebook.execute_cells([18])'))

# create and display the button
button = widgets.Button(description="Get Page Info")
display(button)

# the output widget is used to remove the output after the search field
out_xtools = Output()
display(out_xtools)

# set the event
button.on_click(xtools_on_click)

# trigger the event with the default value
xtools_on_click(button)_____no_output_____

**(1)** *A community-built service for article statistics at xtools.wmflabs.org*

**(2)** *Users that added this page to their watchlist*

**(3)** *See [Wikipedia Content Assessment](https://en.wikipedia.org/wiki/Wikipedia:Content_assessment)*
_____no_output_____ Try yourself! This is the code executed when  you click the 'Get Page Info' button: _____no_output_____

%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### ----------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Get Page Info' BUTTON  ###
### ----------------------------------------------------------------------------- ###

## Define a Xtools instance, more details see:                             ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/xtools.py ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/api.py    ##
xtools_api = XtoolsAPI(lng=abbreviation(languageSelection.value))
xtools_dv = XtoolsDV(xtools_api)
print('Provided through the Xtools API (1)')

## The page you are insterested in ##
print('The page that was found:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

## Get the page info through Xtools method get_page_info() ##
page_info = xtools_dv.get_page_info(the_page['title'])

## Metadata ##
url = f"{xtools_dv.api.base}page/articleinfo/{xtools_dv.api.project}/" + urllib.parse.quote(the_page['title'])
print("Metadata can be found in:", url)

## Use a dictionary to construct a pd.DataFrame to present the general info from Xtools ##
dict_for_df = {
    'Project name': page_info['project'], 'Page name': page_info['page'], 'Watchers(2)': page_info['watchers'],
    'Page Views (per 30 days)': page_info['pageviews'], 'Revisions': page_info['revisions'],
    'Editors': page_info['editors'], 'minor_edits': page_info['minor_edits'], 'Creator of the page': page_info['author'],
    'Creation Date': page_info['created_at'], 'Creation revision id': page_info['created_at'],
    'Last modified': page_info['modified_at'], 'Last revision id': page_info['last_edit_id'],
    'Content Assessment (3)': page_info['assessment']['value'] if type(page_info['assessment']) != bool else page_info['assessment']
}

df_info = pd.DataFrame.from_dict(dict_for_df, orient='index', columns=['value'])
display(df_info)

## Some footnotes ##
display(md('**(1)** *A community-built service for article statistics at xtools.wmflabs.org*
'
           '**(2)** *Users that added this page to their watchlist*
'
           '**(3)** *See [Wikipedia Content Assessment](https://en.wikipedia.org/wiki/Wikipedia:Content_assessment)*'))_____no_output_____

---

## C. Page Views

We use the Wikimedia Pageviews API to display the page views of an article, which can be helpful when assesing the impact of changes in Wikipedia articles. For example, it is possible that a period with a lot of changes might due to internal politics of Wikipedia rather than an increase of interest on the topic by the general population._____no_output_____

def pageviews_button(b):
    with out_pageviews:
        clear_output()

        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))
        # Query request
        wikimedia_api = WikiMediaAPI(lng=abbreviation(languageSelection.value))
        wikimedia_dv = WikiMediaDV(wikimedia_api)
        views = wikimedia_dv.get_pageviews(the_page['title'], 'daily')

        # Visualization
        listener = ViewsListener(views)
        inter_func = interact(listener.listen,
                         begin=Dropdown(options=views.timestamp),
                         end=Dropdown(options=views.timestamp.sort_values(ascending=False)),
                         granularity=Dropdown(options=['Yearly', 'Monthly', 'Weekly', 'Daily'], value='Monthly'))

        # raw data url
        start = 19900101
        today = datetime.today().strftime("%Y%m%d")
        end = int(today)
        article_name = urllib.parse.quote(the_page['title'])
        granularity = 'daily'

        url = (f'{wikimedia_dv.api.base}metrics/pageviews/per-article/{wikimedia_dv.api.project}/'
                f'all-access/all-agents/{article_name}/{granularity}/{start}/{end}')
        display(md(f"Metadata can be found in:"))
        print(url)

        # The df_plotted keeps a reference to the plotted data above
        pageviews_agg = listener.df_plotted['views'].agg({
                            'Total views': sum,
                            'Max views period': max,
                            'Min views period': min,
                            'Average views': min,}).to_frame('Value')
        display(pageviews_agg)

# create and display the button
button = widgets.Button(description="Get Pageviews", layout=Layout(width='150px'))
display(button)

# the output widget is used to remove the output after the search field
out_pageviews = Output()
display(out_pageviews)

# set the event
button.on_click(pageviews_button)

# trigger the event with the default value
pageviews_button(button)_____no_output_____

 Try yourself! This is the code executed when  you click the  'Get Pageviews' button: _____no_output_____

%%script false --no-raise-error

### IMPORTANT NOTE: COMMENT THE ABOVE LINE TO EXECUTE THE CELL ###

### ----------------------------------------------------------------------------- ###
### TRY YOURSELF! THIS IS WHAT WILL HAPPEN WHEN YOU CLICK 'Get Pageviews' BUTTON  ###
### ----------------------------------------------------------------------------- ###

## define a WikiMediaAPI instance, more details see:                          ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/wikimedia.py ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/api.py       ##

wikimedia_api = WikiMediaAPI(lng=abbreviation(languageSelection.value))
wikimedia_dv = WikiMediaDV(wikimedia_api)

## Page of insterest ##
print('The page that was found:', the_page['title'], f'({abbreviation(languageSelection.value).upper()})')

## get pageview counts for the article, more details see:                     ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/external/wikimedia.py ##

views = wikimedia_dv.get_pageviews(the_page['title'], 'daily')

## Visualization, core visual code lies in ViewsListener, then the interact function    ##
## make it interactive, mode details see:                                               ##
## https://github.com/gesiscss/wikiwho_demo/blob/master/visualization/views_listener.py ##
## https://ipywidgets.readthedocs.io/en/latest/examples/Using%20Interact.html           ##

# Create a ViewListener instance with the page views counts
listener = ViewsListener(views)

# You could customize begin/end dates and granularity to generate different graphs
# e.g. begin='20160809', end='20191020', granularity='Monthly'
begin='20150701'
end='20210314'
granularity='Monthly' # 'Yearly', 'Monthly', 'Weekly' ,'Daily'

# Metadata
url = (f'{wikimedia_dv.api.base}metrics/pageviews/per-article/'
       f"{wikimedia_dv.api.project}/all-access/all-agents/{urllib.parse.quote(the_page['title'])}/{granularity.lower()}"
       f"/{int(begin)}/{int(end)}")
print('Metadata can be found in:', url)
print('(Note that: the time unit for the response data only supports granularity for "daily" and "monthly")\n')

# Time range you have selected
print('Time range you have selected:')
print('Start date:', begin)
print('End date:', end)
print('Granularity:', granularity)

# Visulization
listener.listen(begin, end, granularity)

# Pageviews aggregation data. Use the attribute "df_plotted".
pageviews_agg = listener.df_plotted['views'].agg({
                    'Total views': sum,
                    'Max views period': max,
                    'Min views period': min,
                    'Average views': min,}).to_frame('Value')

print('Total views of this page:', pageviews_agg['Value']['Total views'])
print('Max views during the selected period:', pageviews_agg['Value']['Max views period'])
print('Min views during the selected period:', pageviews_agg['Value']['Min views period'])
print('Average views during the selected period:', pageviews_agg['Value']['Average views'])_____no_output_____

---

# D. Templates and Protection

In this step, all the [WikiWho tokens](https://www.wikiwho.net/) and associated revisions are downloaded from the [WikiWho API](https://www.wikiwho.net/). Depending on the article, this might take some time, but, once finished, it will display the period of times in which the article was on any of the following statuses:

- **Semi-page edit protection**: all phases in the article’s life where edits or moves where only allowed for registered users
- **Full-page edit protection**: all phases in the article’s life where edits or moves where only allowed for rusers with special rights, such  as  admins
- **Good article**: as described by Wikipedia guidelines for [good articles](https://en.wikipedia.org/wiki/Wikipedia:Good_articles)
- **Featured article**: as described by Wikipedia guidelines for [featured articles](https://en.wikipedia.org/wiki/Wikipedia:Featured_articles)
- **Special templates**: special templates posted by editors indicating disputes, as well as bias and neutrality issues

Similar to the page views, this information puts the edits, conflicts and discussions in the the following notebook in   context._____no_output_____

def template_button(b):
    global token_source
    global new_template
    global new_protect
    global token_source
    with template_out:
        clear_output()

        # WikiWho API.
        wikiwho = WikiWho(lng=abbreviation(languageSelection.value))
        display(md("Downloading all_content from the WikiWhoApi..."))
        content = wikiwho.dv.all_content(the_page['page_id'])
        display(md("Downloading revisions from the WikiWhoApi..."))
        revisions = wikiwho.dv.rev_ids_of_article(the_page['page_id'])
        clear_output()

        # Wikipedia API
        pp_log = wikipedia_dv.get_protection(the_page['title'])

        # Use ConflictManager to join content and revision tables.
        cm = ConflictManager(content,
                             revisions,
                             lng=abbreviation(languageSelection.value),
                             include_stopwords=True)
        cm.calculate()
        clear_output()
        token_source = cm.all_actions.copy()

        display(md(f"***Page: {the_page['title']} ({abbreviation(languageSelection.value).upper()})***"))

        # Templateslistener
        new_protect = ProtectListener(pp_log, lng=abbreviation(languageSelection.value))

        display(md("Analysing protection data..."))
        plot_protect = [new_protect.get_protect(level)[1] for level in ["semi_edit", "semi_move", "fully_edit", "fully_move", "unknown"]]
        plot_protect = pd.concat(plot_protect)

        new_template = TemplateListener(token_source, plot_protect, lng=abbreviation(languageSelection.value),
                                       wikipediadv_api=wikipedia_dv, page=the_page)
        new_template.listen()

# create and display the button
button = widgets.Button(description="Get Templates & Protection", layout=Layout(width='200px'))
display(button)

# the output widget is used to remove the output after the search field
template_out = Output()
display(template_out)

# set the event
button.on_click(template_button)

# trigger the event with the default value
template_button(button)_____no_output_____

Now that we have a base understanding of the historical context of the article, we will explore  specific kinds of changes that have affected the articles, and the editors involved in its evolution.

Click below to go to the next notebook . This notebook stays open in its own tab and you can always come back to it. Or simply enter another article name to start the process over with a new article. _____no_output_____[Go to next workbook](./2_Page_analysis.ipynb)_____no_output_____

# from utils.notebooks import get_next_notebook
# display(HTML(f'Go to next workbook'))

scroll_to_top = """

"""
display(HTML(scroll_to_top))_____no_output_____


# Notebook from NCBI-Hackathons/ncbi-cloud-tutorials
Path: BLAST tutorials/notebooks/Searching for coding sequences in genomes using BLAST and Python.ipynb

![NCBI](NCBI.png)_____no_output_____# Searching for coding sequences in genomes using BLAST and Python

This notebook takes a sequence of a budding yeast gene and identifies the orthologs in genome sequences. In the process, it'll collect both the coding sequences and the encoded protein sequences for the orthologs.

This notebook builds on the previous ones. See those for more information and acknowledgements.  It also collects subsequences from a collection of PacBio sequenced yeast genomes from [Yue et al 2017](https://www.ncbi.nlm.nih.gov/pubmed/28416820). This tutorial is further expanded to included using PatMatch on the mined ortholog protein sequences [here](GSD/GSD%20Rpb1_orthologs_in_PB_genomes.ipynb#GSD:-Rpb1-orthologs-in-PB-genomes) and [here](GSD/GSD%20Rpb1_orthologs_in_1011_genomes.ipynb).

Reference for sequence data:
[Contrasting evolutionary genome dynamics between domesticated and wild yeasts.
Yue JX, Li J, Aigrain L, Hallin J, Persson K, Oliver K, Bergström A, Coupland P, Warringer J, Lagomarsino MC, Fischer G, Durbin R, Liti G. Nat Genet. 2017 Jun;49(6):913-924. doi: 10.1038/ng.3847. Epub 2017 Apr 17. PMID: 28416820](https://www.ncbi.nlm.nih.gov/pubmed/28416820)

-----_____no_output_____## Overview
![overview of steps](../images/ortholog_mining_summarized.png)_____no_output_____## Preparation

The next cell allows you to examine a gene of interest. (If there were a lot of genes to examine, we could use [papermill](https://github.com/nteract/papermill) to provide the necessary information.)

If you don't want to get the gene sequence from elsewhere on the internet, set `get_seq_from_link` to `False` and ignore the value for `link_to_FASTA_of_gene`. You'll be prompted later to edit a file in the session.

(Caveat: this is written for genes with no introns; their presence would only become important when trying to translate in late stages of this workflow.)_____no_output_____

gene_name = "VPH1" # can be what you want to use in this notebook; not used to match anything external
size_expected = 2523 #use bp length of coding sequence at SGD
get_seq_from_link = True #Change what is between quotes on next line if `True`
link_to_FASTA_of_gene = "https://gist.githubusercontent.com/fomightez/f46b0624f1d8e3abb6ff908fc447e63b/raw/625eaba76bb54e16032f90c8812350441b753a0c/uz_S288C_YOR270C_VPH1_coding.fsa"
#**Possible future enhancement would be to add getting the FASTA of the gene from Yeastmine with just systematic id using get_gene_genomic_seq_as_FASTA.py**_____no_output_____

Get the genome data, the `blast_to_df` script, and target sequence by running these commands._____no_output_____

import os
file_needed = "blast_to_df.py"
if not os.path.isfile(file_needed):
    !curl -O https://raw.githubusercontent.com/fomightez/sequencework/master/blast-utilities/blast_to_df.py
import pandas as pd
# Prepare for getting PacBio (Yue et al 2017 sequences)
#make a list of the strain designations
yue_et_al_strains = ["S288C","DBVPG6044","DBVPG6765","SK1","Y12",
                     "YPS128","UWOPS034614","CBS432","N44","YPS138",
                     "UFRJ50816","UWOPS919171"]
# Get & unpack the genome sequences from strains
for s in yue_et_al_strains:
    !curl -LO http://yjx1217.github.io/Yeast_PacBio_2016/data/Nuclear_Genome/{s}.genome.fa.gz
    !gunzip -f {s}.genome.fa.gz  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 13759  100 13759    0     0  32604      0 --:--:-- --:--:-- --:--:-- 32527
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   1745      0 --:--:-- --:--:-- --:--:--  1745
100 3687k  100 3687k    0     0  7346k      0 --:--:-- --:--:-- --:--:-- 21.5M
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   1648      0 --:--:-- --:--:-- --:--:--  1648
100 3387k  100 3387k    0     0  8797k      0 --:--:-- --:--:-- --:--:-- 18.3M
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0    579      0 --:--:-- --:--:-- --:--:--   577
100 3348k  100 3348k    0     0  4975k      0 --:--:-- --:--:-- --:--:-- 4975k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2342      0 --:--:-- --:--:-- --:--:--  2342
100 3406k  100 3406k    0     0  7638k      0 --:--:-- --:--:-- --:--:-- 12.9M
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2617      0 --:--:-- --:--:-- --:--:--  2617
100 3357k  100 3357k    0     0  11.5M      0 --:--:-- --:--:-- --:--:-- 11.5M
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2542      0 --:--:-- --:--:-- --:--:--  2542
100 3375k  100 3375k    0     0  9221k      0 --:--:-- --:--:-- --:--:-- 9221k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2373      0 --:--:-- --:--:-- --:--:--  2373
100 3332k  100 3332k    0     0  8332k      0 --:--:-- --:--:-- --:--:-- 8332k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   3122      0 --:--:-- --:--:-- --:--:--  3068
100 3658k  100 3658k    0     0  9032k      0 --:--:-- --:--:-- --:--:-- 9032k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   1745      0 --:--:-- --:--:-- --:--:--  1728
100 3350k  100 3350k    0     0  7939k      0 --:--:-- --:--:-- --:--:-- 7939k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2311      0 --:--:-- --:--:-- --:--:--  2282
100 3349k  100 3349k    0     0  9202k      0 --:--:-- --:--:-- --:--:-- 9202k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   1780      0 --:--:-- --:--:-- --:--:--  1780
100 3701k  100 3701k    0     0  8985k      0 --:--:-- --:--:-- --:--:-- 14.4M
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   178  100   178    0     0   2438      0 --:--:-- --:--:-- --:--:--  2507
100 3362k  100 3362k    0     0  8241k      0 --:--:-- --:--:-- --:--:-- 12.5M
# add identifiers to each `chr` so results for each strain clear later
chromosome_id_prefix = "chr"
def add_strain_id_to_description_line(file,strain_id):
    '''
    Takes a file and edits every description line to add
    strain_id after the caret.

    Saves the fixed file
    '''
    import sys
    output_file_name = "temp.txt"
    # prepare output file for saving so it will be open and ready
    with open(output_file_name, 'w') as output_file:

        # read in the input file
        with open(file, 'r') as input_handler:
            # prepare to give feeback later or allow skipping to certain start
            lines_processed = 0

            for line in input_handler:
                lines_processed += 1
                if line.startswith(">"):
                    rest_o_line = line.split(">")
                    new_line = ">"+strain_id + rest_o_line[1]
                else:
                    new_line = line

                # Send text to output
                output_file.write(new_line)

    # replace the original file with edited
    !mv temp.txt {file}
    # Feedback
    sys.stderr.write("\n{} chromosome identifiers tagged.".format(file))

for s in yue_et_al_strains:
    add_strain_id_to_description_line(s+".genome.fa",s)
S288C.genome.fa chromosome identifiers tagged.
DBVPG6044.genome.fa chromosome identifiers tagged.
DBVPG6765.genome.fa chromosome identifiers tagged.
SK1.genome.fa chromosome identifiers tagged.
Y12.genome.fa chromosome identifiers tagged.
YPS128.genome.fa chromosome identifiers tagged.
UWOPS034614.genome.fa chromosome identifiers tagged.
CBS432.genome.fa chromosome identifiers tagged.
N44.genome.fa chromosome identifiers tagged.
YPS138.genome.fa chromosome identifiers tagged.
UFRJ50816.genome.fa chromosome identifiers tagged.
UWOPS919171.genome.fa chromosome identifiers tagged.# Get SGD gene sequence in FASTA format to search for best matches in the genomes
import sys
gene_filen = gene_name + ".fsa"
if get_seq_from_link:
    !curl -o {gene_filen} {link_to_FASTA_of_gene}
else:
    !touch {gene_filen}
    sys.stderr.write("\nEDIT THE FILE '{}' TO CONTAIN "
        "YOUR GENE OF INTEREST (FASTA-FORMATTED)"
        ".".format(gene_filen))  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  2596  100  2596    0     0  11746      0 --:--:-- --:--:-- --:--:-- 11746

Now you are prepared to run BLAST to search each PacBio-sequenced genomes for the best match to a gene from the Saccharomyces cerevisiae strain S288C reference sequence._____no_output_____## Use BLAST to search for matches to gene in reference genome at SGD

We are going to search the [Saccharomyces cerevisiae Genome Database](http:yeastgenome.org) (SGD) with a reference genome from S288C. This will go through each genome, make a database, and search for matches. The information on the best match will be collected._____no_output_____

from blast_to_df import blast_to_df_____no_output_____# Make a list of all `genome.fa` files, excluding `genome.fa.nhr` and `genome.fa.nin` and `genome.fansq`
# The excluding was only necessary because I had run some queries preiminarily in development. Normally, it would just be the `.re.fa` at the outset.
fn_to_check = "genome.fa"
genomes = []
import os
import fnmatch
for file in os.listdir('.'):
    if fnmatch.fnmatch(file, '*'+fn_to_check):
        if not file.endswith(".nhr") and not file.endswith(".nin") and not file.endswith(".nsq") :
            genomes.append(file)
genomes_____no_output_____SGD_gene = gene_filen
dfs = []
for genome in genomes:
    !makeblastdb -in {genome} -dbtype nucl
    result = !blastn -query {SGD_gene} -db {genome} -outfmt "6 qseqid sseqid stitle pident qcovs length mismatch gapopen qstart qend sstart send qframe sframe frames evalue bitscore qseq sseq" -task blastn
    from blast_to_df import blast_to_df
    blast_df = blast_to_df(result.n)
    dfs.append(blast_df.head(1))

Building a new DB, current time: 04/14/2019 20:44:37
New DB name:   /home/jovyan/notebooks/Y12.genome.fa
New DB title:  Y12.genome.fa
Sequence type: Nucleotide
Keep MBits: T
Maximum file size: 1000000000B
Adding sequences from FASTA; added 16 sequences in 0.353903 seconds.
# merge the dataframes in the list `dfs` into one dataframe
df = pd.concat(dfs)_____no_output_____#Save the df
filen_prefix = gene_name + "_orthologBLASTdf"
df.to_pickle(filen_prefix+".pkl")
df.to_csv(filen_prefix+'.tsv', sep='\t',index = False) _____no_output_____df_____no_output_____

Computationally check if any genomes are missing from the BLAST results list:_____no_output_____

subjids = df.sseqid.tolist()
#print (subjids)
#print (subjids[0:10])
subjids = [x.split(chromosome_id_prefix)[0] for x in subjids]
#print (subjids)
#print (subjids[0:10])
len_genome_fn_end = len(fn_to_check) + 1 # plus one to accound for the period that will be
# between `fn_to_check` and strain_id`, such as `SK1.genome.fa`
genome_ids = [x[:-len_genome_fn_end] for x in genomes]
#print (genome_ids[0:10])

a = set(genome_ids)
#print (a)
print ("initial:",len(a))
r = set(subjids)
print("results:",len(r))
print ("missing:",len(a-r))
#a - rinitial: 12
results: 12
missing: 0

Let's perform a quick sanity check. How does expected size compare to max size seen?_____no_output_____

size_seen = df.length.max(0)
print ("Expected size of gene:", size_expected)
print ("Most frequent size of matches:", df.length.mode()[0])
print ("Maximum size of matches:", df.length.max(0))Expected size of gene: 2523
Most frequent size of matches: 2523
Maximum size of matches: 2523

## Collect the identified, raw sequences

Get the expected size centered on the best match, plus a little flanking each because they might not exactly cover the entire open reading frame. (Although, the example here all look to be full size.)_____no_output_____

# Get the script for extracting based on position (and install dependency pyfaidx)
import os
file_needed = "extract_subsequence_from_FASTA.py"
if not os.path.isfile(file_needed):
    !curl -O https://raw.githubusercontent.com/fomightez/sequencework/master/Extract_from_FASTA/extract_subsequence_from_FASTA.py
!pip install pyfaidx  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 16964  100 16964    0     0  75395      0 --:--:-- --:--:-- --:--:-- 75395
Collecting pyfaidx
  Downloading https://files.pythonhosted.org/packages/75/a5/7e2569527b3849ea28d79b4f70d7cf46a47d36459bc59e0efa4e10e8c8b2/pyfaidx-0.5.5.2.tar.gz
Requirement already satisfied: six in /srv/conda/lib/python3.7/site-packages (from pyfaidx) (1.12.0)
Requirement already satisfied: setuptools>=0.7 in /srv/conda/lib/python3.7/site-packages (from pyfaidx) (40.8.0)
Building wheels for collected packages: pyfaidx
  Building wheel for pyfaidx (setup.py) ... done
  Stored in directory: /home/jovyan/.cache/pip/wheels/54/a2/b4/e242e58d23b2808e191b214067880faa46cd2341f363886e0b
Successfully built pyfaidx
Installing collected packages: pyfaidx
Successfully installed pyfaidx-0.5.5.2
size_expected = size_expected # use value from above, or alter at this point.
#size_expected = df.length.max(0) #bp length of SGD coding sequence; should be equivalent and that way not hardcoded?
extra_add_to_start = 51 #to allow for 'fuzziness' at starting end
extra_add_to_end = 51 #to allow for 'fuzziness' at far end
genome_fn_end = "genome.fa"

def midpoint(items):
    '''
    takes a iterable of items and returns the midpoint (integer) of the first
    and second values
    '''
    return int((int(items[0])+int(items[1]))/2)

#midpoint((1,100))

def determine_pos_to_get(match_start,match_end):
    '''
    Take the start and end of the matched region.

    Calculate midpoint between those and then
    center expected size on that to determine
    preliminary start and preliminary end to get.
    Add the extra basepairs to get at each end
    to allow for fuzziness/differences of actual
    gene ends for orthologs.
    Return the final start and end positions to get.

    '''
    center_of_match = midpoint((match_start,match_end))
    half_size_expected = int(size_expected/2.0)
    if size_expected % 2 != 0:
        half_size_expected += 1
    start_pos = center_of_match - half_size_expected
    end_pos = center_of_match + half_size_expected
    start_pos -= extra_add_to_start
    end_pos += extra_add_to_end

    # Because of getting some flanking sequences to account for 'fuzziness', it
    # is possible the start and end can exceed possible. 'End' is not a problem
    # because the `extract_subsequence_from_FASTA.py` script will get as much as
    # it from the indicated sequence if a larger than possible number is
    # provided. However,'start' can become negative and because the region to
    # extract is provided as a string the dash can become a problem. Dealing
    # with it here by making sequence positive only.
    if start_pos < 0:
        raw_amount_missing_at_start = abs(start_pos)# for counterbalancing; needs
        # to be collected before `start_pos` adjusted
        start_pos = 1
        end_pos += 2 * raw_amount_missing_at_start

    return start_pos, end_pos

# go through the dataframe using information on each to come up with sequence file,
# specific indentifier within sequence file, and the start and end to extract
# store these valaues as a list in a dictionary with the strain identifier as the key.
extracted_info = {}
start,end = 0,0
for row in df.itertuples():
    #print (row.length)
    start_to_get, end_to_get = determine_pos_to_get(row.sstart, row.send)
    posns_to_get = "{}-{}".format(start_to_get, end_to_get)
    record_id = row.sseqid
    strain_id = row.sseqid.split(chromosome_id_prefix)[0]
    seq_fn = strain_id + "." + genome_fn_end
    extracted_info[strain_id] = [seq_fn, record_id, posns_to_get]
# Use the dictionary to get the sequences
for id_ in extracted_info:
    #%run extract_subsequence_from_FASTA.py {*extracted_info[id_]} #unpacking doesn't seem to work here in `%run`
    %run extract_subsequence_from_FASTA.py {extracted_info[id_][0]} {extracted_info[id_][1]} {extracted_info[id_][2]}

#package up the retrieved sequences
archive_file_name = gene_name+"_raw_ortholog_seqs.tar.gz"
# make list of extracted files using fnmatch
fn_part_to_match = "seq_extracted"
collected_seq_files_list = []
import os
import sys
import fnmatch
for file in os.listdir('.'):
    if fnmatch.fnmatch(file, fn_part_to_match+'*'):
        #print (file)
        collected_seq_files_list.append(file)
!tar czf {archive_file_name} {" ".join(collected_seq_files_list)} # use the list for archiving command
sys.stderr.write("\n\nCollected RAW sequences gathered and saved as "
                 "`{}`.".format(archive_file_name))
# move the collected raw sequences to a folder in preparation for
# extracting encoding sequence from original source below
!mkdir raw
!mv seq_extracted*.fa raw

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedY12chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUWOPS034614chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedN44chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUFRJ50816chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedDBVPG6044chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedS288CchrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedYPS128chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUWOPS919171chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedCBS432chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedYPS138chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedSK1chrXV.fa'.
*****************DONE**************************

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedDBVPG6765chrXV.fa'.
*****************DONE**************************

Collected RAW sequences gathered and saved as `VPH1_raw_ortholog_seqs.tar.gz`.

This archive should contain the "raw" sequence for each gene, even if the ends are a little different for each. At minimum, the entire gene sequence is there at this point, with extra on each end being preferable.

You should inspect them as soon as possible and adjust the extra sequence to add higher or lower depending on whether the ortholog genes vary more or less respectively. They don't need to be perfect yet because next we are going to extract the longest open reading frame, which presumably demarcates the entire gene. Then we can return to  clean up the collected sequences into just the coding sequence._____no_output_____## Collect protein translations of the genes and then clean up "raw" sequences to just be coding

We'll assume the longest translatable frame in the collected "raw" sequences encodes the protein sequence for the gene orthologs of interest. These steps are based on [section '20.1.13  Identifying open reading frames'](http://biopython.org/DIST/docs/tutorial/Tutorial.html#htoc299) in the present version of the [Biopython Tutorial and Cookbook](http://biopython.org/DIST/docs/tutorial/Tutorial.html)._____no_output_____Run the next cell to get a script needed for dealing with the strand during the translation and gathering of thge encoding sequence._____no_output_____

import os
file_needed = "convert_fasta_to_reverse_complement.py"
if not os.path.isfile(file_needed):
    !curl -O https://raw.githubusercontent.com/fomightez/sequencework/master/ConvertSeq/convert_fasta_to_reverse_complement.py  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  8851  100  8851    0     0  47586      0 --:--:-- --:--:-- --:--:-- 47331

Now to collect protein translations of the genes, followed by cleaning up the "raw" sequences:_____no_output_____

# find the featured open reading frame and collect presumed protein sequences
# Collect the corresponding encoding sequence from the original source
def len_ORF(items):
    # orf is fourth item in the tuples
    return len(items[3])
def find_orfs_with_trans(seq, trans_table, min_protein_length):
    '''
    adapted from the present section '20.1.13  Identifying open reading frames'
    http://biopython.org/DIST/docs/tutorial/Tutorial.html#htoc299 in the
    present version of the [Biopython Tutorial and Cookbook at
    http://biopython.org/DIST/docs/tutorial/Tutorial.html
    (Last Update – 18 December 2018 (Biopython 1.73)
    Same as there except altered to sort on the length of the
    open reading frame.
    '''
    answer = []
    seq_len = len(seq)
    for strand, nuc in [(+1, seq), (-1, seq.reverse_complement())]:
        for frame in range(3):
            trans = str(nuc[frame:].translate(trans_table))
            trans_len = len(trans)
            aa_start = 0
            aa_end = 0
            while aa_start < trans_len:
                aa_end = trans.find("*", aa_start)
                if aa_end == -1:
                    aa_end = trans_len
                if aa_end-aa_start >= min_protein_length:
                    if strand == 1:
                        start = frame+aa_start*3
                        end = min(seq_len,frame+aa_end*3+3)
                    else:
                        start = seq_len-frame-aa_end*3-3
                        end = seq_len-frame-aa_start*3
                    answer.append((start, end, strand,
                                   trans[aa_start:aa_end]))
                aa_start = aa_end+1
    answer.sort(key=len_ORF, reverse = True)
    return answer

def generate_rcoutput_file_name(file_name,suffix_for_saving = "_rc"):
    '''
    from https://github.com/fomightez/sequencework/blob/master/ConvertSeq/convert_fasta_to_reverse_complement.py
    Takes a file name as an argument and returns string for the name of the
    output file. The generated name is based on the original file
    name.
    Specific example
    =================
    Calling function with
        ("sequence.fa", "_rc")
    returns
        "sequence_rc.fa"
    '''
    main_part_of_name, file_extension = os.path.splitext(
        file_name) #from
    #http://stackoverflow.com/questions/541390/extracting-extension-from-filename-in-python
    if '.' in file_name:  #I don't know if this is needed with the os.path.splitext method but I had it before so left it
        return main_part_of_name + suffix_for_saving  + file_extension
    else:
        return file_name + suffix_for_saving + ".fa"

def add_strand_to_description_line(file,strand="-1"):
    '''
    Takes a file and edits description line to add
    strand info at end.

    Saves the fixed file
    '''
    import sys
    output_file_name = "temp.txt"
    # prepare output file for saving so it will be open and ready
    with open(output_file_name, 'w') as output_file:

        # read in the input file
        with open(file, 'r') as input_handler:
            # prepare to give feeback later or allow skipping to certain start
            lines_processed = 0

            for line in input_handler:
                lines_processed += 1
                if line.startswith(">"):
                    new_line = line.strip() + "; {} strand\n".format(strand)
                else:
                    new_line = line

                # Send text to output
                output_file.write(new_line)

    # replace the original file with edited
    !mv temp.txt {file}
    # Feedback
    sys.stderr.write("\nIn {}, strand noted.".format(file))

table = 1 #sets translation table to standard nuclear, see
# https://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi
min_pro_len = 80 #cookbook had the standard `100`. Feel free to adjust.
prot_seqs_info = {} #collect as dictionary with strain_id as key. Values to
# be list with source id as first item and protein length as second and
# strand in source seq as third item, and start and end in source sequence as fourth and fifth,
# and file name of protein and gene as sixth and seventh.
# Example key and value pair: 'YPS138':['<source id>','<protein length>',-1,52,2626,'<gene file name>','<protein file name>']
gene_seqs_fn_list = []
prot_seqs_fn_list = []
from Bio import SeqIO
for raw_seq_filen in collected_seq_files_list:
    #strain_id = raw_seq_filen[:-len_genome_fn_end] #if was dealing with source seq
    strain_id = raw_seq_filen.split(chromosome_id_prefix)[0].split("seq_extracted")[1]
    record = SeqIO.read("raw/"+raw_seq_filen,"fasta")
    raw_seq_source_fn = strain_id + "." + genome_fn_end
    raw_seq_source_id = record.description.split(":")[0]
    orf_list = find_orfs_with_trans(record.seq, table, min_pro_len)
    orf_start, orf_end, strand, prot_seq = orf_list[0] #longest ORF seq for protein coding

    location_raw_seq = record.description.rsplit(":",1)[1] #get to use in calculating
    # the start and end position in original genome sequence.
    raw_loc_parts = location_raw_seq.split("-")
    start_from_raw_seq = int(raw_loc_parts[0])
    end_from_raw_seq = int(raw_loc_parts[1])
    length_extracted = len(record) #also to use in calculating relative original

    #Fix negative value. (Somehow Biopython can report negative value when hitting
    # end of sequence without encountering stop codon and negatives messes up
    # indexing later it seems.)
    if orf_start < 0:
        orf_start = 0

    # Trim back to the first Methionine, assumed to be the initiating MET.
    # (THIS MIGHT BE A SOURCE OF EXTRA 'LEADING' RESIDUES IN SOME CASES & ARGUES
    # FOR LIMITING THE AMOUNT OF FLANKING SEQUENCE ADDED TO ALLOW FOR FUZINESS.)
    try:
        amt_resi_to_trim = prot_seq.index("M")
    except ValueError:
        sys.stderr.write("**ERROR**When searching for initiating methionine,\n"
                         "no Methionine found in the traslated protein sequence.**ERROR**")
        sys.exit(1)
    prot_seq = prot_seq[amt_resi_to_trim:]
    len_seq_trimmed = amt_resi_to_trim * 3

    # Calculate the adjusted start and end values for the untrimmed ORF
    adj_start = start_from_raw_seq + orf_start
    adj_end = end_from_raw_seq - (length_extracted - orf_end)

    # Adjust for trimming for appropriate strand.
    if strand == 1:
        adj_start += len_seq_trimmed
        #adj_end += 3 # turns out stop codon is part of numbering biopython returns
    elif strand == -1:
        adj_end -= len_seq_trimmed
        #adj_start -= 3 # turns out stop codon is part of numbering biopython returns
    else:
        sys.stderr.write("**ERROR**No strand match option detected!**ERROR**")
        sys.exit(1)

    # Collect the sequence for the actual gene encoding region from
    # the original sequence. This way the original numbers will
    # be put in the file.
    start_n_end_str = "{}-{}".format(adj_start,adj_end)
    %run extract_subsequence_from_FASTA.py {raw_seq_source_fn} {raw_seq_source_id} {start_n_end_str}

    # rename the extracted subsequence a more distinguishing name and notify
    g_output_file_name = strain_id +"_" + gene_name + "_ortholog_gene.fa"
    !mv {raw_seq_filen} {g_output_file_name} # because the sequence saved happens to
    # be same as raw sequence file saved previously, that name can be used to
    # rename new file.
    gene_seqs_fn_list.append(g_output_file_name)
    sys.stderr.write("\n\nRenamed gene file to "
                     "`{}`.".format(g_output_file_name))

    # Convert extracted sequence to reverse complement if translation was on negative strand.
    if strand == -1:
        %run convert_fasta_to_reverse_complement.py {g_output_file_name}
        # replace original sequence file with the produced file
        produced_fn = generate_rcoutput_file_name(g_output_file_name)
        !mv {produced_fn} {g_output_file_name}
        # add (after saved) onto the end of the description line for that `-1 strand`
        # No way to do this in my current version of convert sequence. So editing descr line.
        add_strand_to_description_line(g_output_file_name)

    #When settled on actual protein encoding sequence, fill out
    # description to use for saving the protein sequence.
    prot_descr =  (record.description.rsplit(":",1)[0]+ " "+ gene_name
                   + "_ortholog"+ "| " +str(len(prot_seq)) + " aas | from "
                   + raw_seq_source_id + " "
                   + str(adj_start) + "-"+str(adj_end))
    if strand == -1:
        prot_descr += "; {} strand".format(strand)

    # save the protein sequence as FASTA
    chunk_size = 70 #<---amino acids per line to have in FASTA
    prot_seq_chunks = [prot_seq[i:i+chunk_size] for i in range(
        0, len(prot_seq),chunk_size)]
    prot_seq_fa = ">" + prot_descr + "\n"+ "\n".join(prot_seq_chunks)
    p_output_file_name = strain_id +"_" + gene_name + "_protein_ortholog.fa"
    with open(p_output_file_name, 'w') as output:
        output.write(prot_seq_fa)
    prot_seqs_fn_list.append(p_output_file_name)
    sys.stderr.write("\n\nProtein sequence saved as "
                     "`{}`.".format(p_output_file_name))

    # at end store information in `prot_seqs_info` for later making a dataframe
    # and then text table for saving summary
    #'YPS138':['<source id>',<protein length>,-1,52,2626,'<gene file name>','<protein file name>']
    prot_seqs_info[strain_id] = [raw_seq_source_id,len(prot_seq),strand,adj_start,adj_end,
                                 g_output_file_name,p_output_file_name]

    sys.stderr.write("\n******END OF A SET OF PROTEIN ORTHOLOG "
                     "AND ENCODING GENE********")/srv/conda/lib/python3.7/site-packages/Bio/Seq.py:2609: BiopythonWarning: Partial codon, len(sequence) not a multiple of three. Explicitly trim the sequence or add trailing N before translation. This may become an error in future.
  BiopythonWarning)

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedYPS138chrXV.fa'.
*****************DONE**************************

Renamed gene file to `YPS138_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'YPS138_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'YPS138_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In YPS138_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `YPS138_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUWOPS919171chrXV.fa'.
*****************DONE**************************

Renamed gene file to `UWOPS919171_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'UWOPS919171_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'UWOPS919171_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In UWOPS919171_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `UWOPS919171_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedN44chrXV.fa'.
*****************DONE**************************

Renamed gene file to `N44_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'N44_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'N44_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In N44_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `N44_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedSK1chrXV.fa'.
*****************DONE**************************

Renamed gene file to `SK1_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'SK1_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'SK1_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In SK1_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `SK1_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedY12chrXV.fa'.
*****************DONE**************************

Renamed gene file to `Y12_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'Y12_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'Y12_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In Y12_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `Y12_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUFRJ50816chrXV.fa'.
*****************DONE**************************

Renamed gene file to `UFRJ50816_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'UFRJ50816_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'UFRJ50816_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In UFRJ50816_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `UFRJ50816_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedS288CchrXV.fa'.
*****************DONE**************************

Renamed gene file to `S288C_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'S288C_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'S288C_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In S288C_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `S288C_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedDBVPG6044chrXV.fa'.
*****************DONE**************************

Renamed gene file to `DBVPG6044_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'DBVPG6044_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'DBVPG6044_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In DBVPG6044_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `DBVPG6044_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedDBVPG6765chrXV.fa'.
*****************DONE**************************

Renamed gene file to `DBVPG6765_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'DBVPG6765_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'DBVPG6765_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In DBVPG6765_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `DBVPG6765_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedUWOPS034614chrXV.fa'.
*****************DONE**************************

Renamed gene file to `UWOPS034614_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'UWOPS034614_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'UWOPS034614_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In UWOPS034614_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `UWOPS034614_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedYPS128chrXV.fa'.
*****************DONE**************************

Renamed gene file to `YPS128_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'YPS128_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'YPS128_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In YPS128_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `YPS128_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********

*****************DONE**************************
Extracted sequence saved in FASTA format as 'seq_extractedCBS432chrXV.fa'.
*****************DONE**************************

Renamed gene file to `CBS432_VPH1_ortholog_gene.fa`.

*****************DONE**************************
Sequences in FASTA file 'CBS432_VPH1_ortholog_gene.fa' converted to reverse complement
and saved as 'CBS432_VPH1_ortholog_gene_rc.fa'.
*****************DONE**************************

In CBS432_VPH1_ortholog_gene.fa, strand noted.

Protein sequence saved as `CBS432_VPH1_protein_ortholog.fa`.
******END OF A SET OF PROTEIN ORTHOLOG AND ENCODING GENE********# use `prot_seqs_info` for saving a summary text table (first convert to dataframe?)
table_fn_prefix = gene_name + "_orthologs_table"
table_fn = table_fn_prefix + ".tsv"
pkl_table_fn = table_fn_prefix + ".pkl"
import pandas as pd
info_df = pd.DataFrame.from_dict(prot_seqs_info, orient='index',
    columns=['descr_id', 'length', 'strand', 'start','end','gene_file','prot_file']) # based on
# https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.from_dict.html and
# note from Python 3.6 that `pd.DataFrame.from_items` is deprecated;
#"Please use DataFrame.from_dict"
info_df.to_pickle(pkl_table_fn)
info_df.to_csv(table_fn, sep='\t') # keep index is default
sys.stderr.write("Text file of associated details saved as '{}'.".format(table_fn))Text file of associated details saved as 'VPH1_orthologs_table.tsv'.# pack up archive of gene and protein sequences plus the table
seqs_list = gene_seqs_fn_list + prot_seqs_fn_list + [table_fn,pkl_table_fn]
archive_file_name = gene_name+"_ortholog_seqs.tar.gz"
!tar czf {archive_file_name} {" ".join(seqs_list)} # use the list for archiving command
sys.stderr.write("\nCollected gene and protein sequences"
                 " (plus table of details) gathered and saved as "
                 "`{}`.".format(archive_file_name))
Collected gene and protein sequences (plus table of details) gathered and saved as `VPH1_ortholog_seqs.tar.gz`.

Save the tarballed archive to your local machine._____no_output_____-----_____no_output_____


# Notebook from Gendo90/Hots-Soundboard
Path: data/transform_audio/Audio_Splitter.ipynb

<code>
# Import the AudioSegment class for processing audio and the
# split_on_silence function for separating out silent chunks.
from pydub import AudioSegment
from pydub.silence import split_on_silence

import speech_recognition as sr
# obtain path to "english.wav" in the same folder as this script
from os import path

#set the GOOGLE_APPLICATION_CREDENTIALS env variable for this notebook to use the text-to-speech API
import os

#import pandas to export the quotes to a file
import pandas as pd

os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "../../../Google Speech-to-Text key/HotS Soundboards-1ca9e2e5d0be.json"

#set current hero
curr_hero = "Kerrigan"_____no_output_____# Define a function to normalize a chunk to a target amplitude.
def match_target_amplitude(aChunk, target_dBFS):
    ''' Normalize given audio chunk '''
    change_in_dBFS = target_dBFS - aChunk.dBFS
    return aChunk.apply_gain(change_in_dBFS)

def split_apart_recording(audio):
    print("starting function")
    # Split track where the silence is 2 seconds or more and get chunks using
    # the imported function.
    chunks = split_on_silence (
        # Use the loaded audio.
        audio,
        # Specify that a silent chunk must be at least 2 seconds or 2000 ms long.
        min_silence_len = 400,
        # Consider a chunk silent if it's quieter than -16 dBFS.
        # (You may want to adjust this parameter.)
        silence_thresh = -44
    )
    print("split on silence already")
    num_chunks = len(chunks)
    print(num_chunks)

    # Process each chunk with your parameters
    for i, chunk in enumerate(chunks):
        print("on chunk {}".format(i+1))
        # Create a silence chunk that's 0.5 seconds (or 500 ms) long for padding.
        silence_chunk = AudioSegment.silent(duration=500)

        # Add the padding chunk to beginning and end of the entire chunk.
        audio_chunk = silence_chunk + chunk + silence_chunk
#         audio_chunk=chunk

        # Normalize the entire chunk.
        #NOTE: Add back later if necessary
        normalized_chunk = match_target_amplitude(audio_chunk, -20.0)

        # Export the audio chunk with new bitrate.
        print("Exporting chunk{}.wav.".format(i+1))
        #could also use normalized chunk if necessary and that feature is used
        normalized_chunk.export(
            "../output/{}/chunk{}.wav".format(curr_hero, i+1),
            bitrate = "192k",
            format = "wav"
        )
        print("Exported chunk {}".format(i+1))

    return num_chunks_____no_output_____# Load your audio.
audio = AudioSegment.from_mp3("../source_audio/{}/{} FULL Quotes - Heroes of the Storm.mp3".format(curr_hero, curr_hero))
print(audio)<pydub.audio_segment.AudioSegment object at 0x0000019EDC010048>
num_clips = split_apart_recording(audio)starting function
split on silence already
215
on chunk 1
Exporting chunk1.wav.
Exported chunk 1
on chunk 2
Exporting chunk2.wav.
Exported chunk 2
on chunk 3
Exporting chunk3.wav.
Exported chunk 3
on chunk 4
Exporting chunk4.wav.
Exported chunk 4
on chunk 5
Exporting chunk5.wav.
Exported chunk 5
on chunk 6
Exporting chunk6.wav.
Exported chunk 6
on chunk 7
Exporting chunk7.wav.
Exported chunk 7
on chunk 8
Exporting chunk8.wav.
Exported chunk 8
on chunk 9
Exporting chunk9.wav.
Exported chunk 9
on chunk 10
Exporting chunk10.wav.
Exported chunk 10
on chunk 11
Exporting chunk11.wav.
Exported chunk 11
on chunk 12
Exporting chunk12.wav.
Exported chunk 12
on chunk 13
Exporting chunk13.wav.
Exported chunk 13
on chunk 14
Exporting chunk14.wav.
Exported chunk 14
on chunk 15
Exporting chunk15.wav.
Exported chunk 15
on chunk 16
Exporting chunk16.wav.
Exported chunk 16
on chunk 17
Exporting chunk17.wav.
Exported chunk 17
on chunk 18
Exporting chunk18.wav.
Exported chunk 18
on chunk 19
Exporting chunk19.wav.
Exported chunk 19
on chunk 20
Exporting chunk20.wav.
Exported chunk 20
on chunk 21
Exporting chunk21.wav.
Exported chunk 21
on chunk 22
Exporting chunk22.wav.
Exported chunk 22
on chunk 23
Exporting chunk23.wav.
Exported chunk 23
on chunk 24
Exporting chunk24.wav.
Exported chunk 24
on chunk 25
Exporting chunk25.wav.
Exported chunk 25
on chunk 26
Exporting chunk26.wav.
Exported chunk 26
on chunk 27
Exporting chunk27.wav.
Exported chunk 27
on chunk 28
Exporting chunk28.wav.
Exported chunk 28
on chunk 29
Exporting chunk29.wav.
Exported chunk 29
on chunk 30
Exporting chunk30.wav.
Exported chunk 30
on chunk 31
Exporting chunk31.wav.
Exported chunk 31
on chunk 32
Exporting chunk32.wav.
Exported chunk 32
on chunk 33
Exporting chunk33.wav.
Exported chunk 33
on chunk 34
Exporting chunk34.wav.
Exported chunk 34
on chunk 35
Exporting chunk35.wav.
Exported chunk 35
on chunk 36
Exporting chunk36.wav.
Exported chunk 36
on chunk 37
Exporting chunk37.wav.
Exported chunk 37
on chunk 38
Exporting chunk38.wav.
Exported chunk 38
on chunk 39
Exporting chunk39.wav.
Exported chunk 39
on chunk 40
Exporting chunk40.wav.
Exported chunk 40
on chunk 41
Exporting chunk41.wav.
Exported chunk 41
on chunk 42
Exporting chunk42.wav.
Exported chunk 42
on chunk 43
Exporting chunk43.wav.
Exported chunk 43
on chunk 44
Exporting chunk44.wav.
Exported chunk 44
on chunk 45
Exporting chunk45.wav.
Exported chunk 45
on chunk 46
Exporting chunk46.wav.
Exported chunk 46
on chunk 47
Exporting chunk47.wav.
Exported chunk 47
on chunk 48
Exporting chunk48.wav.
Exported chunk 48
on chunk 49
Exporting chunk49.wav.
Exported chunk 49
on chunk 50
Exporting chunk50.wav.
Exported chunk 50
on chunk 51
Exporting chunk51.wav.
Exported chunk 51
on chunk 52
Exporting chunk52.wav.
Exported chunk 52
on chunk 53
Exporting chunk53.wav.
Exported chunk 53
on chunk 54
Exporting chunk54.wav.
Exported chunk 54
on chunk 55
Exporting chunk55.wav.
Exported chunk 55
on chunk 56
Exporting chunk56.wav.
Exported chunk 56
on chunk 57
Exporting chunk57.wav.
Exported chunk 57
on chunk 58
Exporting chunk58.wav.
Exported chunk 58
on chunk 59
Exporting chunk59.wav.
Exported chunk 59
on chunk 60
Exporting chunk60.wav.
Exported chunk 60
on chunk 61
Exporting chunk61.wav.
Exported chunk 61
on chunk 62
Exporting chunk62.wav.
Exported chunk 62
on chunk 63
Exporting chunk63.wav.
Exported chunk 63
on chunk 64
Exporting chunk64.wav.
Exported chunk 64
on chunk 65
Exporting chunk65.wav.
Exported chunk 65
on chunk 66
Exporting chunk66.wav.
Exported chunk 66
on chunk 67
Exporting chunk67.wav.
Exported chunk 67
on chunk 68
Exporting chunk68.wav.
Exported chunk 68
on chunk 69
Exporting chunk69.wav.
Exported chunk 69
on chunk 70
Exporting chunk70.wav.
Exported chunk 70
on chunk 71
Exporting chunk71.wav.
Exported chunk 71
on chunk 72
Exporting chunk72.wav.
Exported chunk 72
on chunk 73
Exporting chunk73.wav.
Exported chunk 73
on chunk 74
Exporting chunk74.wav.
Exported chunk 74
on chunk 75
Exporting chunk75.wav.
Exported chunk 75
on chunk 76
Exporting chunk76.wav.
Exported chunk 76
on chunk 77
Exporting chunk77.wav.
Exported chunk 77
on chunk 78
Exporting chunk78.wav.
Exported chunk 78
on chunk 79
Exporting chunk79.wav.
Exported chunk 79
on chunk 80
Exporting chunk80.wav.
Exported chunk 80
on chunk 81
Exporting chunk81.wav.
Exported chunk 81
on chunk 82
Exporting chunk82.wav.
Exported chunk 82
on chunk 83
Exporting chunk83.wav.
Exported chunk 83
on chunk 84
Exporting chunk84.wav.
Exported chunk 84
on chunk 85
Exporting chunk85.wav.
Exported chunk 85
on chunk 86
Exporting chunk86.wav.
Exported chunk 86
on chunk 87
Exporting chunk87.wav.
Exported chunk 87
on chunk 88
Exporting chunk88.wav.
Exported chunk 88
on chunk 89
Exporting chunk89.wav.
Exported chunk 89
on chunk 90
Exporting chunk90.wav.
Exported chunk 90
on chunk 91
Exporting chunk91.wav.
Exported chunk 91
on chunk 92
Exporting chunk92.wav.
Exported chunk 92
on chunk 93
Exporting chunk93.wav.
Exported chunk 93
on chunk 94
Exporting chunk94.wav.
Exported chunk 94
on chunk 95
Exporting chunk95.wav.
Exported chunk 95
on chunk 96
Exporting chunk96.wav.
Exported chunk 96
on chunk 97
Exporting chunk97.wav.
Exported chunk 97
on chunk 98
Exporting chunk98.wav.
Exported chunk 98
on chunk 99
Exporting chunk99.wav.
Exported chunk 99
on chunk 100
Exporting chunk100.wav.
Exported chunk 100
on chunk 101
Exporting chunk101.wav.
Exported chunk 101
on chunk 102
Exporting chunk102.wav.
Exported chunk 102
on chunk 103
Exporting chunk103.wav.
Exported chunk 103
on chunk 104
Exporting chunk104.wav.
Exported chunk 104
on chunk 105
Exporting chunk105.wav.
Exported chunk 105
on chunk 106
Exporting chunk106.wav.
Exported chunk 106
on chunk 107
Exporting chunk107.wav.
Exported chunk 107
on chunk 108
Exporting chunk108.wav.
Exported chunk 108
on chunk 109
Exporting chunk109.wav.
Exported chunk 109
on chunk 110
Exporting chunk110.wav.
Exported chunk 110
on chunk 111
Exporting chunk111.wav.
Exported chunk 111
on chunk 112
Exporting chunk112.wav.
Exported chunk 112
on chunk 113
Exporting chunk113.wav.
Exported chunk 113
on chunk 114
Exporting chunk114.wav.
Exported chunk 114
on chunk 115
Exporting chunk115.wav.
Exported chunk 115
on chunk 116
Exporting chunk116.wav.
Exported chunk 116
on chunk 117
Exporting chunk117.wav.
Exported chunk 117
on chunk 118
Exporting chunk118.wav.
Exported chunk 118
on chunk 119
Exporting chunk119.wav.
Exported chunk 119
on chunk 120
Exporting chunk120.wav.
Exported chunk 120
on chunk 121
Exporting chunk121.wav.
Exported chunk 121
on chunk 122
Exporting chunk122.wav.
Exported chunk 122
on chunk 123
Exporting chunk123.wav.
Exported chunk 123
on chunk 124
Exporting chunk124.wav.
Exported chunk 124
on chunk 125
Exporting chunk125.wav.
Exported chunk 125
on chunk 126
Exporting chunk126.wav.
Exported chunk 126
on chunk 127
Exporting chunk127.wav.
Exported chunk 127
on chunk 128
Exporting chunk128.wav.
Exported chunk 128
on chunk 129
Exporting chunk129.wav.
Exported chunk 129
on chunk 130
Exporting chunk130.wav.
Exported chunk 130
on chunk 131
Exporting chunk131.wav.
Exported chunk 131
on chunk 132
Exporting chunk132.wav.
Exported chunk 132
on chunk 133
Exporting chunk133.wav.
Exported chunk 133
on chunk 134
Exporting chunk134.wav.
Exported chunk 134
on chunk 135
Exporting chunk135.wav.
Exported chunk 135
on chunk 136
Exporting chunk136.wav.
Exported chunk 136
on chunk 137
Exporting chunk137.wav.
Exported chunk 137
on chunk 138
Exporting chunk138.wav.
Exported chunk 138
on chunk 139
Exporting chunk139.wav.
Exported chunk 139
on chunk 140
Exporting chunk140.wav.
Exported chunk 140
on chunk 141
Exporting chunk141.wav.
Exported chunk 141
on chunk 142
Exporting chunk142.wav.
Exported chunk 142
on chunk 143
Exporting chunk143.wav.
Exported chunk 143
on chunk 144
Exporting chunk144.wav.
Exported chunk 144
on chunk 145
Exporting chunk145.wav.
Exported chunk 145
on chunk 146
Exporting chunk146.wav.
Exported chunk 146
on chunk 147
Exporting chunk147.wav.
Exported chunk 147
on chunk 148
Exporting chunk148.wav.
Exported chunk 148
on chunk 149
Exporting chunk149.wav.
Exported chunk 149
on chunk 150
Exporting chunk150.wav.
Exported chunk 150
on chunk 151
Exporting chunk151.wav.
Exported chunk 151
on chunk 152
Exporting chunk152.wav.
Exported chunk 152
on chunk 153
Exporting chunk153.wav.
Exported chunk 153
on chunk 154
Exporting chunk154.wav.
Exported chunk 154
on chunk 155
Exporting chunk155.wav.
Exported chunk 155
on chunk 156
Exporting chunk156.wav.
Exported chunk 156
on chunk 157
Exporting chunk157.wav.
Exported chunk 157
on chunk 158
Exporting chunk158.wav.
Exported chunk 158
on chunk 159
Exporting chunk159.wav.
Exported chunk 159
on chunk 160
Exporting chunk160.wav.
Exported chunk 160
on chunk 161
Exporting chunk161.wav.
Exported chunk 161
on chunk 162
Exporting chunk162.wav.
Exported chunk 162
on chunk 163
Exporting chunk163.wav.
Exported chunk 163
on chunk 164
Exporting chunk164.wav.
Exported chunk 164
on chunk 165
Exporting chunk165.wav.
Exported chunk 165
on chunk 166
Exporting chunk166.wav.
Exported chunk 166
on chunk 167
Exporting chunk167.wav.
Exported chunk 167
on chunk 168
Exporting chunk168.wav.
Exported chunk 168
on chunk 169
Exporting chunk169.wav.
Exported chunk 169
on chunk 170
Exporting chunk170.wav.
def audio_transcription(num_chunks):
    # use the audio file as the audio source
    r = sr.Recognizer()

    for i in range(1, num_chunks+1):
        with sr.AudioFile("../output/{}/chunk{}.wav".format(curr_hero, i)) as source:
            chunk = r.record(source)  # read the entire audio file

        # recognize speech using Sphinx
        try:
            print("Sphinx thinks you said -> " + r.recognize_sphinx(chunk))
        except sr.UnknownValueError:
            print("Sphinx could not understand audio")
        except sr.RequestError as e:
            print("Sphinx error; {0}".format(e))_____no_output_____audio_transcription(num_clips)Sphinx thinks you said -> and it's mortal
Sphinx thinks you said -> what do you think
Sphinx thinks you said -> i'm sorry did you say something i was busy thinking about justice
Sphinx thinks you said -> you want more hope you never know what your curiosity likes of which
Sphinx thinks you said -> saint louis just to sort of do love this war and you'll get lost to be doomed to imagine what to visit the bullshit these days and ha-ha ha ha ha ha ha ha ha and where are we do and where does it didn't work
Sphinx thinks you said -> just because i'm mortal bosnia one to wait around forever
Sphinx thinks you said -> the person's body glaring at you right now
Sphinx thinks you said -> in heaven everything is fine
Sphinx thinks you said -> you can always sell and angels have big when their wings locker
Sphinx thinks you said -> perfect gemma activated
Sphinx thinks you said -> you're quite proficient at likud do ya bills or you might be right up your alley
Sphinx thinks you said -> ever taco bell rings basulto r. manifests as the physical representation of an angel's wings
Sphinx thinks you said -> of course is that what i felt from heaven winds are you trying to draw me
Sphinx thinks you said -> just to susan blind eye to face but i'm laing
Sphinx thinks you said -> there is no wisdom in the high heavens and no safety wooden sanctuary po on to the president's state of concerns
Sphinx thinks you said -> on news lines of the one bill out of humanity saving your people from being erased from creation i'm beginning to regret that decision
Sphinx thinks you said -> blessings to you and mortal
Sphinx thinks you said -> my eternal gratitude bureau
Sphinx thinks you said -> so it is gone judy i'm not warn you
Sphinx thinks you said -> my wrath has been stated retribution job in mind
Sphinx thinks you said -> the plot and call it demands justice
Sphinx thinks you said -> who taught you reverence george you want to
Sphinx thinks you said -> justice dogs all these whew
Sphinx thinks you said -> the pursuit of justice is about as big
Sphinx thinks you said -> i have reforms
Sphinx thinks you said -> it's not unusual to see you fly
Sphinx thinks you said -> now these moves modern art
Sphinx thinks you said -> i must leave you
Sphinx thinks you said -> i'm telling you to get
Sphinx thinks you said -> grandsons
Sphinx thinks you said -> had you either do or forgiveness my apologies
Sphinx thinks you said -> you have my gratitude
Sphinx thinks you said -> thank you
Sphinx thinks you said -> well played a clever tactic
Sphinx thinks you said -> i'll agree
Sphinx thinks you said -> and how you get
Sphinx thinks you said -> i'd love a tactic
Sphinx thinks you said -> this is ho hum
Sphinx thinks you said -> affliction
Sphinx thinks you said -> you cannot out wrong justice
Sphinx thinks you said -> i can smell your sins from a year of war two
Sphinx thinks you said -> status do and face your judgment
Sphinx thinks you said -> let's hope it doesn't take twenty years for me to reform this time
Sphinx thinks you said -> hey room mortal
Sphinx thinks you said -> your presence bombers me
Sphinx thinks you said -> dude freely without compulsion
Sphinx thinks you said -> the diminutive than half
Sphinx thinks you said -> we will dispense justice on this world
Sphinx thinks you said -> your tests were folded foreman
Sphinx thinks you said -> i'm always a court challenge mortal
Sphinx thinks you said -> there is much too soon to be purged from this world war two
Sphinx thinks you said -> the on my sword to join you and the fight against evil
Sphinx thinks you said -> i see you up a great wealth of spare time perhaps we would make better use of its fighting
Sphinx thinks you said -> that no one ever instruct you that it is deemed a lot tuesday at
Sphinx thinks you said -> and the latest judgment
Sphinx thinks you said -> spam to produce runs
Sphinx thinks you said -> not him directed pete's justice
Sphinx thinks you said -> for two by two of the boulder understood
Sphinx thinks you said -> without delay
Sphinx thinks you said -> our time is short
Sphinx thinks you said -> like the wind
Sphinx thinks you said -> remain steadfast
Sphinx thinks you said -> finally
Sphinx thinks you said -> i must act indeed
Sphinx thinks you said -> eternity await socks attack now the kennedy is weak
Sphinx thinks you said -> this is all won't strike now heroes
Sphinx thinks you said -> the indians caught in an instant friends is answer that
Sphinx thinks you said -> watch out on that in the most to you by
Sphinx thinks you said -> maximum strike
Sphinx thinks you said -> i'm good at it
Sphinx thinks you said -> where would fault
Sphinx thinks you said -> heal myself and
Sphinx thinks you said -> you look and served head toward healing bouncing quickly
Sphinx thinks you said -> i'm down to remembers america
Sphinx thinks you said -> i shall live july and
Sphinx thinks you said -> bread not myers says it is yours
Sphinx thinks you said -> i must retreat
Sphinx thinks you said -> i'm going to pull back from the hall
Sphinx thinks you said -> but grieve malls
Sphinx thinks you said -> that is disappointing
Sphinx thinks you said -> no
Sphinx thinks you said -> diodes ruins moonlight
Sphinx thinks you said -> cooled off the unit
Sphinx thinks you said -> the room and we'll be rooted out
Sphinx thinks you said -> just as beat down
Sphinx thinks you said -> to that an
Sphinx thinks you said -> you cannot stand is loyal to ruin
Sphinx thinks you said -> i will bring order to cut offs for cannot move
Sphinx thinks you said -> welcome friends to evolve of light we have donuts
Sphinx thinks you said -> we are closing in on them do not move easily
Sphinx thinks you said -> so
Sphinx thinks you said -> my thanks to you humor
Sphinx thinks you said -> excellent in the bird out for him
Sphinx thinks you said -> my health is removed
Sphinx thinks you said -> i stand with you
Sphinx thinks you said -> judgment has gone through this world
Sphinx thinks you said -> our emily is a whole
Sphinx thinks you said -> but we do job coldest day
Sphinx thinks you said -> move the rule jealous because i don't
Sphinx thinks you said -> well let's methadone is going to fight by your side towards again
Sphinx thinks you said -> can you put aside our differences and fight the ban
Sphinx thinks you said -> you are of course you gotta do
Sphinx thinks you said -> the u.
Sphinx thinks you said -> well man's friend are you ready to meet about justice
Sphinx thinks you said -> piper and you are ready for him to
Sphinx thinks you said -> oh workers
Sphinx thinks you said -> the light of justice shots on israel
Sphinx thinks you said -> hi i'm for him
Sphinx thinks you said -> we all know is a lot i'm for him
Sphinx thinks you said -> you will saw him do that not to use only the press
Sphinx thinks you said -> havana eloquent do and my young through it
Sphinx thinks you said -> make one mistake and i hope will and you
Sphinx thinks you said -> i'm glad to see you as well for him
Sphinx thinks you said -> justice has been met
Sphinx thinks you said -> i stabbed victorious
Sphinx thinks you said -> the battle with these blind
Sphinx thinks you said -> you've been saddam wanted
Sphinx thinks you said -> meet your hand to roll back to the right it gets want you were aroused know that
Sphinx thinks you said -> i had such high hopes for you
Sphinx thinks you said -> then lord of terror has fallen
Sphinx thinks you said -> thank you to adjust
Sphinx thinks you said -> and of impotent
Sphinx thinks you said -> my held these low
Sphinx thinks you said -> you have saved my life threatened by him even your external debt
Sphinx thinks you said -> thank you the darkness was close in upon me
Sphinx thinks you said -> i move on them
Sphinx thinks you said -> lima is low
Sphinx thinks you said -> no the men are for
Sphinx thinks you said -> i am a little too good law
Sphinx thinks you said -> i'm sorry for him to vote no
Sphinx thinks you said -> that is not possible
Sphinx thinks you said -> that i i hide requires to
Sphinx thinks you said -> to live the arose
Sphinx thinks you said -> i need of assistance
Sphinx thinks you said -> we should make got to ask you this
def audio_transcription_google(num_chunks):
    r = sr.Recognizer()
    output_quotes = []

    for i in range(1, num_chunks+1):
        with sr.AudioFile("../output/{}/chunk{}.wav".format(curr_hero, i)) as source:
                    chunk = r.record(source)

        #setup google speech-to-text transcription
        # recognize speech using Google Speech Recognition
        try:
            # for testing purposes, we're just using the default API key
            # to use another API key, use `r.recognize_google(audio, key="GOOGLE_SPEECH_RECOGNITION_API_KEY")`
            # instead of `r.recognize_google(audio)`
            print("Currently on quote #{}".format(i))
            curr_quote = r.recognize_google(chunk)
            print("Google Speech Recognition thinks you said -> " + curr_quote)
            output_quotes.append(curr_quote)
        except sr.UnknownValueError:
            print("Google Speech Recognition could not understand audio")
            output_quotes.append("")
        except sr.RequestError as e:
            print("Could not request results from Google Speech Recognition service; {0}".format(e))
            output_quotes.append("")

    return output_quotes_____no_output_____transcribed_quotes = audio_transcription_google(num_clips)Currently on quote #1
Google Speech Recognition thinks you said -> what is it
Currently on quote #2
Google Speech Recognition thinks you said -> speak
Currently on quote #3
Google Speech Recognition thinks you said -> I like you that's why I'm going to kill you
Currently on quote #4
Google Speech Recognition could not understand audio
Currently on quote #5
Google Speech Recognition thinks you said -> it's not easy being Queen
Currently on quote #6
Google Speech Recognition thinks you said -> yeah I'm still the queen b**** of the universe
Currently on quote #7
Google Speech Recognition thinks you said -> really are you staring at my thorax
Currently on quote #8
Google Speech Recognition thinks you said -> still easily amused I see
Currently on quote #9
Google Speech Recognition thinks you said -> looks like those energy drinks actually do give you wings
Currently on quote #10
Google Speech Recognition thinks you said -> I just supposed to be human now make up your mind blizzard humans or humans
Currently on quote #11
Google Speech Recognition thinks you said -> pick one
Currently on quote #12
Google Speech Recognition thinks you said -> the worst part about having a hive mind is knowing exactly how many of your zerglings are licking themselves at any given moment
Currently on quote #13
Google Speech Recognition thinks you said -> are we really supposed to believe that I involved high heels
Currently on quote #14
Google Speech Recognition thinks you said -> Hell hath no fury like a woman scorned
Currently on quote #15
Google Speech Recognition thinks you said -> it takes forever to comb your hair when it keeps biting you
Currently on quote #16
Google Speech Recognition thinks you said -> literally
Currently on quote #17
Google Speech Recognition thinks you said -> it's never wise to poke the hive mind
Currently on quote #18
Google Speech Recognition thinks you said -> phenomenal psionic powers
Currently on quote #19
Google Speech Recognition thinks you said -> icky sticky living space
Currently on quote #20
Google Speech Recognition thinks you said -> you will be assimilated resistance is futile
Currently on quote #21
Google Speech Recognition thinks you said -> I'm a Killer Queen
Currently on quote #22
Google Speech Recognition thinks you said -> guaranteed to blow your mind
Currently on quote #23
Google Speech Recognition thinks you said -> nice save
Currently on quote #24
Google Speech Recognition thinks you said -> I'm back
Currently on quote #25
Google Speech Recognition thinks you said -> there was only one way for this to end
Currently on quote #26
Google Speech Recognition thinks you said -> I'm not done with you yet
Currently on quote #27
Google Speech Recognition thinks you said -> vengeance
Currently on quote #28
Google Speech Recognition thinks you said -> how delicious
Currently on quote #29
Google Speech Recognition thinks you said -> you forced my hand
Currently on quote #30
Google Speech Recognition thinks you said -> that's it
Currently on quote #31
Google Speech Recognition thinks you said -> someone for you
Currently on quote #32
Google Speech Recognition thinks you said -> you will pay for this
Currently on quote #33
Google Speech Recognition thinks you said -> the Swarm is eternal
Currently on quote #34
Google Speech Recognition thinks you said -> ready or not here I come
Currently on quote #35
Google Speech Recognition thinks you said -> I'm waiting on you
Currently on quote #36
Google Speech Recognition thinks you said -> when I dance I dance to wear
Currently on quote #37
Google Speech Recognition thinks you said -> Tsongas Center Hollow ones
Currently on quote #38
Google Speech Recognition thinks you said -> don't wait for me
Currently on quote #39
Google Speech Recognition thinks you said -> goodbye
Currently on quote #40
Google Speech Recognition thinks you said -> hello
Currently on quote #41
Google Speech Recognition thinks you said -> yes
Currently on quote #42
Google Speech Recognition thinks you said -> I'm sorry
Currently on quote #43
Google Speech Recognition thinks you said -> my bad
Currently on quote #44
Google Speech Recognition could not understand audio
Currently on quote #45
Google Speech Recognition thinks you said -> thank you
Currently on quote #46
Google Speech Recognition could not understand audio
Currently on quote #47
Google Speech Recognition thinks you said -> I like it
Currently on quote #48
Google Speech Recognition thinks you said -> I have to admit
Currently on quote #49
Google Speech Recognition thinks you said -> that was well done
Currently on quote #50
Google Speech Recognition thinks you said -> of course
Currently on quote #51
Google Speech Recognition thinks you said -> all right
Currently on quote #52
Google Speech Recognition thinks you said -> I feel stronger
Currently on quote #53
Google Speech Recognition thinks you said -> I like it
Currently on quote #54
Google Speech Recognition thinks you said -> sounds like fun
Currently on quote #55
Google Speech Recognition thinks you said -> what's wrong scared
Currently on quote #56
Google Speech Recognition thinks you said -> come closer I don't bite
Currently on quote #57
Google Speech Recognition thinks you said -> I have zerglings for that
Currently on quote #58
Google Speech Recognition thinks you said -> face me coward
Currently on quote #59
Google Speech Recognition thinks you said -> I am not happy about this
Currently on quote #60
Google Speech Recognition thinks you said -> we need to evolve
Currently on quote #61
Google Speech Recognition thinks you said -> I am the Swarm
Currently on quote #62
Google Speech Recognition thinks you said -> the swollen is the instruments of my will
Currently on quote #63
Google Speech Recognition thinks you said -> I will bring death to their enemies
Currently on quote #64
Google Speech Recognition thinks you said -> wait till they get a load of me
Currently on quote #65
Google Speech Recognition thinks you said -> perfect
Currently on quote #66
Google Speech Recognition thinks you said -> looking forward to it
Currently on quote #67
Google Speech Recognition thinks you said -> a demonstration of power
Currently on quote #68
Google Speech Recognition thinks you said -> amusing
Currently on quote #69
Google Speech Recognition thinks you said -> Let's Go destroy something already
Currently on quote #70
Google Speech Recognition thinks you said -> if I wanted to stand around doing nothing I would have stayed in my Leviathan
Currently on quote #71
Google Speech Recognition thinks you said -> let's go I feel like ripping a world apart
Currently on quote #72
Google Speech Recognition thinks you said -> you realize evolution is an active process right
Currently on quote #73
Google Speech Recognition thinks you said -> your time's up
Currently on quote #74
Google Speech Recognition thinks you said -> Sikkim
Currently on quote #75
Google Speech Recognition thinks you said -> unbridled Glory
Currently on quote #76
Google Speech Recognition thinks you said -> now that
Currently on quote #77
Google Speech Recognition thinks you said -> I always get what I want
Currently on quote #78
Google Speech Recognition thinks you said -> I am the Swarm
Currently on quote #79
Google Speech Recognition could not understand audio
Currently on quote #80
Google Speech Recognition thinks you said -> no time like the present
Currently on quote #81
Google Speech Recognition thinks you said -> this world is mine
Currently on quote #82
Google Speech Recognition thinks you said -> we are without limit
Currently on quote #83
Google Speech Recognition could not understand audio
Currently on quote #84
Google Speech Recognition thinks you said -> all right
Currently on quote #85
Google Speech Recognition thinks you said -> on my way
Currently on quote #86
Google Speech Recognition thinks you said -> my Conquest continues
Currently on quote #87
Google Speech Recognition thinks you said -> attack now our enemy is weak
Currently on quote #88
Google Speech Recognition thinks you said -> our enemy has been weakened strike now
Currently on quote #89
Google Speech Recognition thinks you said -> this is it hit them with all you got
Currently on quote #90
curr_quotes = pd.Series(transcribed_quotes)
curr_quotes.to_csv("../output/{}/transcription_arr.csv".format(curr_hero))_____no_output_____#change the file names for the hero to the transcribed quotes

for i, quote in enumerate(transcribed_quotes):
    chunk_num = i+1
    try:
        os.rename("../output/{}/chunk{}.wav".format(curr_hero, chunk_num),
                  "../output/{}/{:0>3d}_{}.wav".format(curr_hero, chunk_num, quote))
    except:
        print("Chunk #{} could not be renamed!".format(i))

print("{} output quotes renamed!".format(curr_hero))
    Chunk #0 could not be renamed!
Chunk #1 could not be renamed!
Chunk #2 could not be renamed!
Chunk #3 could not be renamed!
Chunk #4 could not be renamed!
Chunk #5 could not be renamed!
Chunk #184 could not be renamed!
Kerrigan output quotes renamed!

</code>