# Notebook from precidiag-inc/ASD_16S_metaanalysis
Path: Figure 5/screening_ages.ipynb

# Optimal Age Range

Our preliminary assumption is that there is sufficient information in the microbiome to predict Autism status. But are prediction accuracies uniformly spread across ages, or are there subsets of ages in which the samples are more separable? To test this, we first perform a sweep of different age ranges: subsetting data and training different models._____no_output_____

<code>
%matplotlib inline
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import os, sys
import numpy as np
import pickle
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import RepeatedStratifiedKFold

sys.path.insert(0, '..')

from utils import UnityScaler
_____no_output_____reads_threshold = 6000

data_directory = '../data/'
data_filename = 'metaanalysis_data.pickle'
metadata_filename = 'metaanalysis_metadata.pickle'

data = pickle.load(open(os.path.join(data_directory, data_filename), 'rb'))
metadata = pickle.load(open(os.path.join(data_directory, metadata_filename), 'rb'))

# remove samples with fewer than reads_threshold
data = data.loc[data.sum(axis = 1) >= reads_threshold]
metadata = metadata.loc[data.index]

# remove samples with valid ages
metadata = metadata.loc[~metadata['Age'].isna()]
data = data.loc[metadata.index]

# label encoder
label_encoder = LabelEncoder()
metadata['Status'] = label_encoder.fit_transform(metadata['Status'])

# normalize each sample
data = UnityScaler().fit_transform(data)/home/erxw/PycharmProjects/ASD_16S_metaanalysis/ASD_16S_metaanalysis/Figure 5/../utils/_unity.py:15: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version.  Convert to a numpy array before indexing instead.
  df /= np.sum(df, axis = self.axis)[:, None]
start_range = [1, 16]
end_range = [3, 18]
model = GradientBoostingClassifier()
results = {(start,end): [] for start in range(start_range[0], start_range[1]) for end in range(end_range[0], end_range[1])}
iterations = 30_____no_output_____for start in range(start_range[0], start_range[1]):
    for end in range(end_range[0], end_range[1]):
        print(start, end)
        if start >= end-1:
            continue
        inclusion = (metadata['Age'] >= start) & (metadata['Age'] < end)
        iteration = 0
        while iteration < iterations:
            try:
                xtrain, xtest, ytrain, ytest = train_test_split(data.loc[inclusion], metadata.loc[inclusion, 'Status'].values)
                model.fit(xtrain, ytrain)
                results[start,end].append(roc_auc_score(model.predict(xtest), ytest))
            except Exception as e:
                print(iteration, e)
                iteration -= 1
                pass
            iteration += 11 3
1 4
1 5
1 6
1 7
1 8
1 9
1 10
1 11
1 12
1 13
1 14
1 15
1 16
1 17
2 3
2 4
2 5
2 6
2 7
2 8
2 9
2 10
2 11
2 12
2 13
2 14
2 15
2 16
2 17
3 3
3 4
3 5
3 6
3 7
3 8
3 9
3 10
3 11
3 12
3 13
3 14
3 15
3 16
3 17
4 3
4 4
4 5
4 6
4 7
4 8
4 9
4 10
4 11
4 12
4 13
4 14
4 15
4 16
4 17
5 3
5 4
5 5
5 6
5 7
5 8
5 9
5 10
5 11
5 12
5 13
5 14
5 15
5 16
5 17
6 3
6 4
6 5
6 6
6 7
6 8
6 9
6 10
6 11
6 12
6 13
6 14
6 15
6 16
6 17
7 3
7 4
7 5
7 6
7 7
7 8
7 9
7 10
7 11
7 12
7 13
7 14
7 15
7 16
7 17
8 3
8 4
8 5
8 6
8 7
8 8
8 9
8 10
8 11
8 12
8 13
8 14
8 15
8 16
8 17
9 3
9 4
9 5
9 6
9 7
9 8
9 9
9 10
9 11
9 12
9 13
9 14
9 15
9 16
9 17
10 3
10 4
10 5
10 6
10 7
10 8
10 9
10 10
10 11
10 12
10 13
10 14
10 15
10 16
10 17
11 3
11 4
11 5
11 6
11 7
11 8
11 9
11 10
11 11
11 12
11 13
11 14
11 15
11 16
11 17
12 3
12 4
12 5
12 6
12 7
12 8
12 9
12 10
12 11
12 12
12 13
12 14
7 Only one class present in y_true. ROC AUC score is not defined in that case.
13 Only one class present in y_true. ROC AUC score is not defined in that case.
12 15
23 Only one class present in y_true. ROC AUC score is not defined in that case.
12 16
18 Only one class present in y_true. ROC AUC score is not defined in that case.
12 17
13 3
13 4
13 5
13 6
13 7
13 8
13 9
13 10
13 11
13 12
13 13
13 14
13 15
2 Only one class present in y_true. ROC AUC score is not defined in that case.
13 16
13 17
14 3
14 4
14 5
14 6
14 7
14 8
14 9
14 10
14 11
14 12
14 13
14 14
14 15
14 16
10 Only one class present in y_true. ROC AUC score is not defined in that case.
18 Only one class present in y_true. ROC AUC score is not defined in that case.
14 17
13 Only one class present in y_true. ROC AUC score is not defined in that case.
15 3
15 4
15 5
15 6
15 7
15 8
15 9
15 10
15 11
15 12
15 13
15 14
15 15
15 16
15 17
23 Only one class present in y_true. ROC AUC score is not defined in that case.
pickle.dump(results, open('optimal_ages.pickle', 'wb'))_____no_output_____results = pickle.load(open('optimal_ages.pickle', 'rb'))_____no_output_____def get_grid(results):
    grid = np.zeros((start_range[1] - start_range[0], end_range[1] - end_range[0], iterations))
    for (start, end), value in results.items():
        temp = value.copy()
        try:
            grid[start - start_range[0], end - end_range[0]] = temp
        except Exception as e:
            continue
    grid = np.nanmean(grid,axis = 2)[:-1, 1:]

    for i in range(1, 14):
        grid[i:,i-1] = np.nan
    return grid

def display_grid(grid, title, cmap = 'jet'):

    plt.imshow(grid, cmap = 'jet', )
    plt.ylabel('Lower bound (years)')
    plt.xlabel('Upper bound (years)')
    plt.yticks(np.arange(grid.shape[0]), np.arange(grid.shape[0]) + start_range[0])
    plt.xticks(np.arange(grid.shape[1]), np.arange(grid.shape[1]) + end_range[0])
    plt.colorbar()
    plt.savefig(f'{title}.pdf', dpi = 1200)_____no_output_____np.nanmax(get_grid(results))_____no_output_____display_grid(get_grid(results),  title = 'OptimalAge')_____no_output_____# get sample size grid
sample_size = {}
for start in range(start_range[0], start_range[1]):
    for end in range(end_range[0], end_range[1]):
        if start >= end:
            continue
        inclusion = (metadata['Age'] >= start) & (metadata['Age'] < end)
        sample_size[start,end] = [inclusion.sum()]
display_grid(get_grid(sample_size),  title = 'SampleSize')_____no_output_____df = pd.DataFrame([get_grid(sample_size).flatten(), get_grid(results).flatten()], index = ['Sample Size', 'AUC']).T
df = df.dropna()

sns.lmplot(data = df, x = 'Sample Size', y = 'AUC', order = 2)
plt.savefig('Relationship between sample size and AUC.pdf', dpi = 1200)_____no_output_____
</code>


# Notebook from haoxusci/starfish
Path: notebooks/osmFISH.ipynb

## Reproduce Published osmFISH results with Starfish

osmFISH is an image based transcriptomics technique that can spatially resolve tens of RNA transcripts and their expression levels in-situ. The protocol anddata analysis are described in this [publication](https://www.nature.com/articles/s41592-018-0175-z). This notebook walks through how to use Starfish to process the raw images from an osmFISH experiment into a spatially resolved gene expression image. We verify taht Starfish can accurately reproduce the results from the authors' original Python [pipeline](http://linnarssonlab.org/osmFISH/image_analysis/)

Please see [documentation](https://spacetx-starfish.readthedocs.io/en/stable/index.html) for detailed descriptions of all the data structures and methods used here._____no_output_____

import os
import pickle

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from starfish import FieldOfView, data
from starfish.types import Axes

%matplotlib inline
%load_ext autoreload
%autoreload 2_____no_output_____

## Load Data into Starfish from the Cloud

The data from an osmFISH experiment are similar in form to a standard smFISH experiment. For each round, each color channel corresponds to presence of a particular gene. Across rounds, the color channels index different genes. Here, we analyze one FOV from the first round (r) channel (c) which consists of 45 z-planes (z). Each image in this image stack is of dimensions 2048x2048 (y X x). The data are taken from mouse somatosensory cortex, and the gene in this channel is Adloc_____no_output_____

experiment = data.osmFISH(use_test_data=True)
imgs = experiment["fov_000"].get_image(FieldOfView.PRIMARY_IMAGES)
print(imgs)_____no_output_____

## Filter and Visualize Data_____no_output_____First, we remove background signal using a gaussian high-pass filter_____no_output_____

from starfish.image import Filter

filter_ghp = Filter.GaussianHighPass(sigma=(1,8,8), is_volume=True)
imgs_ghp = filter_ghp.run(imgs, in_place=False)_____no_output_____

Next, we enhance the spots by filtering with a Laplace filter_____no_output_____

filter_laplace = Filter.Laplace(sigma=(0.2, 0.5, 0.5), is_volume=True)
imgs_ghp_laplace = filter_laplace.run(imgs_ghp, in_place=False)_____no_output_____

Finally, we take a maximum projection over z, which effectively mitigates effects of out of focus z-planes_____no_output_____

mp = imgs_ghp_laplace.reduce({Axes.ZPLANE}, func="max")_____no_output_____

We can now visualize our data before and after filtering_____no_output_____

single_plane = imgs.reduce({Axes.ZPLANE}, func="max").xarray.sel({Axes.CH:0}).squeeze()
single_plane_filtered = mp.xarray.sel({Axes.CH: 0}).squeeze()

plt.figure(figsize=(10,10))
plt.subplot(121)
plt.imshow(single_plane, cmap='gray', clim = list(np.percentile(single_plane.data, [1, 99.9])))
plt.axis('off')
plt.title('Original data, Round:0, Channel: 0')
plt.subplot(122)
plt.imshow(single_plane_filtered, cmap='gray', clim = list(np.percentile(single_plane_filtered.data, [1, 99.9])))
plt.title('Filtered data, Round:0, Channel: 0')
plt.axis('off');_____no_output_____

## Decode the processed data into spatially resolved gene expression_____no_output_____Decoding in a non-multiplexed image based transcriptomics method is equivalent to simple spot finding, since each spot in each color channel and round corresponds to a different gene. To find spots in osmFISH data, the authors employ a peak finder that distinguishes local maxima from their surroundings whose absolute intensities exceed a threshold value. It tests a number of different thresholds, building a curve from the number of peaks detected at each threshold. A threshold in the _stable region_ or _knee_ of the curve is selected, and final peaks are called with that threshold.

This process is repeated independently for each round and channel. Here we show this process on a single round and channel to demonstrate the procedure. See the documentation for a precise description of the parameters._____no_output_____

from starfish.spots import DecodeSpots, FindSpots
from starfish.types import TraceBuildingStrategies

lmp = FindSpots.LocalMaxPeakFinder(
    min_distance=6,
    stringency=0,
    min_obj_area=6,
    max_obj_area=600,
    is_volume=True
)
spots = lmp.run(mp)

decoder = DecodeSpots.PerRoundMaxChannel(codebook=experiment.codebook,
                                         trace_building_strategy=TraceBuildingStrategies.SEQUENTIAL)
decoded_intensities = decoder.run(spots=spots)_____no_output_____

## Compare to pySMFISH peak calls_____no_output_____The Field of view that we've used for the test data corresponds to Aldoc, imaged in round one, in position 33. We've also packaged the results from the osmFISH publication for this target to demonstrate that starfish is capable of recovering the same results._____no_output_____

def load_results(pickle_file):
    with open(pickle_file, "rb") as f:
        return pickle.load(f)

def get_benchmark_peaks(loaded_results, redo_flag=False):

    if not redo_flag:
        sp = pd.DataFrame(
            {
                "y":loaded_results["selected_peaks"][:,0],
                "x":loaded_results["selected_peaks"][:,1],
                "selected_peaks_int": loaded_results["selected_peaks_int"],
            }
        )
    else:
        p = peaks(loaded_results)
        coords = p[p.thr_array==loaded_results["selected_thr"]].peaks_coords
        coords = coords.values[0]
        sp = pd.DataFrame({"x":coords[:,0], "y":coords[:,1]})

    return sp

try:
    module_path = __file__
except NameError:
    # this is probably being run from jupyter
    cwd = "."
else:
    cwd = os.path.dirname(module_path)
benchmark_results = load_results(os.path.join(
    cwd, "data", "EXP-17-BP3597_hyb1_Aldoc_pos_33.pkl"))
benchmark_peaks = get_benchmark_peaks(benchmark_results, redo_flag=False)_____no_output_____

Plot spots detected in the benchmark as blue spots, and overlay spots from starfish as orange x's. Starfish detects the same spot positions, but 41 fewer spots in total._____no_output_____

benchmark_spot_count = len(benchmark_peaks)
starfish_spot_count = len(decoded_intensities)

plt.figure(figsize=(10,10))
plt.plot(benchmark_peaks.x, -benchmark_peaks.y, "o")
plt.plot(decoded_intensities[Axes.X.value], -decoded_intensities[Axes.Y.value], "x")

plt.legend(["Benchmark: {} spots".format(benchmark_spot_count),
            "Starfish: {} spots".format(starfish_spot_count)])
plt.title("Starfish x osmFISH Benchmark Comparison");_____no_output_____spot_difference = benchmark_spot_count - starfish_spot_count
print(f"Starfish finds {spot_difference} fewer spots")
assert spot_difference == 41  # for starfish testing purposes_____no_output_____


# Notebook from bionetslab/python-intro
Path: 6_sequence_analysis_with_biopython.ipynb

# Sequence Analysis with Biopython

*David B. Blumenthal, Suryadipto Sarkar*_____no_output_____## Biopython

- A very comprehensive collection of tools for biological computation in Python.
- Today, we'll cover only a very small subset of the available functionality.
- For more information, read the [documentation](https://biopython.org/) or have a look at Peter Cock's [Introduction to Biopython](https://github.com/peterjc/biopython_workshop) course._____no_output_____### Biopython imports_____no_output_____

import Bio
from Bio import SeqIO, SearchIO, Entrez
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.SeqUtils import GC
from Bio.Blast import NCBIWWW_____no_output_____

---
## Very first steps

### Reading FASTA files via `SeqIO.parse()` and `SeqIO.read()`

- `SeqIO.parse()` returns iterator over all sequences contained in the record.
- `SeqIO.read()` can be used to return the unique sequence contained in FASTA files that contain exactly one record. _____no_output_____

fasta_file = 'input/unknown_sequence.fasta'
file_format = 'fasta'
records = SeqIO.parse(fasta_file, file_format)
for record in records:
    print(record)
record = SeqIO.read(fasta_file, file_format)
print(record)ID: Unknown_sequence
Name: Unknown_sequence
Description: Unknown_sequence
Number of features: 0
Seq('ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGT...AAA')
ID: Unknown_sequence
Name: Unknown_sequence
Description: Unknown_sequence
Number of features: 0
Seq('ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGT...AAA')

### Assessing the lenght of the sequence_____no_output_____

print(f'The length of the sequence is {len(record)} BP.')The length of the sequence is 29903 BP.

### Assessing the GC content of the sequence_____no_output_____

print(f'The GC content of the sequence is {GC(record.seq):.2f} %.')The GC content of the sequence is 37.97 %.

---
## Sequence alignment with BLAST

- Use `Bio.Blast.NCBIWWW.qblast()` to run BLAST via the online API provided by NCBI.

### Use `help(FUNCTIONNAME)` to show information about usage

- This works for all well-written Python functions (and also with Python classes).
- If you want to learn more, google for **docstring**._____no_output_____

help(NCBIWWW.qblast)Help on function qblast in module Bio.Blast.NCBIWWW:

qblast(program, database, sequence, url_base='https://blast.ncbi.nlm.nih.gov/Blast.cgi', auto_format=None, composition_based_statistics=None, db_genetic_code=None, endpoints=None, entrez_query='(none)', expect=10.0, filter=None, gapcosts=None, genetic_code=None, hitlist_size=50, i_thresh=None, layout=None, lcase_mask=None, matrix_name=None, nucl_penalty=None, nucl_reward=None, other_advanced=None, perc_ident=None, phi_pattern=None, query_file=None, query_believe_defline=None, query_from=None, query_to=None, searchsp_eff=None, service=None, threshold=None, ungapped_alignment=None, word_size=None, short_query=None, alignments=500, alignment_view=None, descriptions=500, entrez_links_new_window=None, expect_low=None, expect_high=None, format_entrez_query=None, format_object=None, format_type='XML', ncbi_gi=None, results_file=None, show_overview=None, megablast=None, template_type=None, template_length=None)
    BLAST search using NCBI's QBLAST server or a cloud service provider.

    Supports all parameters of the old qblast API for Put and Get.

    Please note that NCBI uses the new Common URL API for BLAST searches
    on the internet (http://ncbi.github.io/blast-cloud/dev/api.html). Thus,
    some of the parameters used by this function are not (or are no longer)
    officially supported by NCBI. Although they are still functioning, this
    may change in the future.

    The Common URL API (http://ncbi.github.io/blast-cloud/dev/api.html) allows
    doing BLAST searches on cloud servers. To use this feature, please set
    ``url_base='http://host.my.cloud.service.provider.com/cgi-bin/blast.cgi'``
    and ``format_object='Alignment'``. For more details, please see
    https://blast.ncbi.nlm.nih.gov/Blast.cgi?PAGE_TYPE=BlastDocs&DOC_TYPE=CloudBlast

    Some useful parameters:

     - program        blastn, blastp, blastx, tblastn, or tblastx (lower case)
     - database       Which database to search against (e.g. "nr").
     - sequence       The sequence to search.
     - ncbi_gi        TRUE/FALSE whether to give 'gi' identifier.
     - descriptions   Number of descriptions to show.  Def 500.
     - alignments     Number of alignments to show.  Def 500.
     - expect         An expect value cutoff.  Def 10.0.
     - matrix_name    Specify an alt. matrix (PAM30, PAM70, BLOSUM80, BLOSUM45).
     - filter         "none" turns off filtering.  Default no filtering
     - format_type    "HTML", "Text", "ASN.1", or "XML".  Def. "XML".
     - entrez_query   Entrez query to limit Blast search
     - hitlist_size   Number of hits to return. Default 50
     - megablast      TRUE/FALSE whether to use MEga BLAST algorithm (blastn only)
     - short_query    TRUE/FALSE whether to adjust the search parameters for a
                      short query sequence. Note that this will override
                      manually set parameters like word size and e value. Turns
                      off when sequence length is > 30 residues. Default: None.
     - service        plain, psi, phi, rpsblast, megablast (lower case)

    This function does no checking of the validity of the parameters
    and passes the values to the server as is.  More help is available at:
    https://ncbi.github.io/blast-cloud/dev/api.html

### Run the query and write the result in an XML file (might take a while, we are doing an online search)_____no_output_____

%%time
result_handle = NCBIWWW.qblast(program='blastn', database='nt', sequence=record.seq)
with open('input/blast_unknown_sequence.xml', 'w') as fp:
    fp.write(result_handle.read())
    result_handle.close()_____no_output_____

### Read the BLAST result and print the first high-scoring pair_____no_output_____

blast_qresult = SearchIO.read('input/blast_unknown_sequence.xml', 'blast-xml')
first_hit = blast_qresult[0]
print(f'Description: {first_hit.description}')
first_hsp = first_hit[0]
print(first_hsp)Description: Severe acute respiratory syndrome coronavirus 2 isolate Wuhan-Hu-1, complete genome
      Query: No definition line
        Hit: gi|1798174254|ref|NC_045512.2| Severe acute respiratory syndrome...
Query range: [0:29903] (1)
  Hit range: [0:29903] (1)
Quick stats: evalue 0; bitscore 53927.40
  Fragments: 1 (29903 columns)
     Query - ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATC~~~AAAAA
             |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||~~~|||||
       Hit - ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATC~~~AAAAA

---
## Extract meta-data from NCBI Entrez

### Extract NCBI ID from result and tell NCBI who you are_____no_output_____

NCBI_id = first_hit.id.split('|')[3]
Entrez.email = 'david.b.blumenthal@fau.de'_____no_output_____

### Query NCBI Entrez_____no_output_____

genbank_handle = Entrez.efetch(db='nucleotide', id=NCBI_id, retmode='text', rettype='gb')
genbank_record = SeqIO.read(genbank_handle, 'genbank')_____no_output_____print(genbank_record)ID: NC_045512.2
Name: NC_045512
Description: Severe acute respiratory syndrome coronavirus 2 isolate Wuhan-Hu-1, complete genome
Database cross-references: BioProject:PRJNA485481
Number of features: 57
/molecule_type=ss-RNA
/topology=linear
/data_file_division=VRL
/date=18-JUL-2020
/accessions=['NC_045512']
/sequence_version=2
/keywords=['RefSeq']
/source=Severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2)
/organism=Severe acute respiratory syndrome coronavirus 2
/taxonomy=['Viruses', 'Riboviria', 'Orthornavirae', 'Pisuviricota', 'Pisoniviricetes', 'Nidovirales', 'Cornidovirineae', 'Coronaviridae', 'Orthocoronavirinae', 'Betacoronavirus', 'Sarbecovirus']
/references=[Reference(title='A new coronavirus associated with human respiratory disease in China', ...), Reference(title='Programmed ribosomal frameshifting in decoding the SARS-CoV genome', ...), Reference(title='The structure of a rigorously conserved RNA element within the SARS virus genome', ...), Reference(title="A phylogenetically conserved hairpin-type 3' untranslated region pseudoknot functions in coronavirus RNA replication", ...), Reference(title='Direct Submission', ...), Reference(title='Direct Submission', ...)]
/comment=REVIEWED REFSEQ: This record has been curated by NCBI staff. The
reference sequence is identical to MN908947.
On Jan 17, 2020 this sequence version replaced NC_045512.1.
Annotation was added using homology to SARSr-CoV NC_004718.3. ###
Formerly called 'Wuhan seafood market pneumonia virus.' If you have
questions or suggestions, please email us at info@ncbi.nlm.nih.gov
and include the accession number NC_045512.### Protein structures
can be found at
https://www.ncbi.nlm.nih.gov/structure/?term=sars-cov-2.### Find
all other Severe acute respiratory syndrome coronavirus 2
(SARS-CoV-2) sequences at
https://www.ncbi.nlm.nih.gov/genbank/sars-cov-2-seqs/
COMPLETENESS: full length.
/structured_comment=OrderedDict([('Assembly-Data', OrderedDict([('Assembly Method', 'Megahit v. V1.1.3'), ('Sequencing Technology', 'Illumina')]))])
Seq('ATTAAAGGTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGT...AAA')

### What kind of virus are we looking at?_____no_output_____

print(f"Virus type: {genbank_record.annotations['molecule_type']} virus.")Virus type: ss-RNA virus.

### What is its taxonomy?_____no_output_____

print(f'Virus taxonomy: {" >> ".join(genbank_record.annotations["taxonomy"])}')Virus taxonomy: Viruses >> Riboviria >> Orthornavirae >> Pisuviricota >> Pisoniviricetes >> Nidovirales >> Cornidovirineae >> Coronaviridae >> Orthocoronavirinae >> Betacoronavirus >> Sarbecovirus

### Relevant publications or labs_____no_output_____

for reference in genbank_record.annotations['references']:
    print(reference)location: [0:29903]
authors: Wu,F., Zhao,S., Yu,B., Chen,Y.M., Wang,W., Song,Z.G., Hu,Y., Tao,Z.W., Tian,J.H., Pei,Y.Y., Yuan,M.L., Zhang,Y.L., Dai,F.H., Liu,Y., Wang,Q.M., Zheng,J.J., Xu,L., Holmes,E.C. and Zhang,Y.Z.
title: A new coronavirus associated with human respiratory disease in China
journal: Nature 579 (7798), 265-269 (2020)
medline id:
pubmed id: 32015508
comment: Erratum:[Nature. 2020 Apr;580(7803):E7. PMID: 32296181]

location: [13475:13503]
authors: Baranov,P.V., Henderson,C.M., Anderson,C.B., Gesteland,R.F., Atkins,J.F. and Howard,M.T.
title: Programmed ribosomal frameshifting in decoding the SARS-CoV genome
journal: Virology 332 (2), 498-510 (2005)
medline id:
pubmed id: 15680415
comment:

location: [29727:29768]
authors: Robertson,M.P., Igel,H., Baertsch,R., Haussler,D., Ares,M. Jr. and Scott,W.G.
title: The structure of a rigorously conserved RNA element within the SARS virus genome
journal: PLoS Biol. 3 (1), e5 (2005)
medline id:
pubmed id: 15630477
comment:

location: [29608:29657]
authors: Williams,G.D., Chang,R.Y. and Brian,D.A.
title: A phylogenetically conserved hairpin-type 3' untranslated region pseudoknot functions in coronavirus RNA replication
journal: J. Virol. 73 (10), 8349-8355 (1999)
medline id:
pubmed id: 10482585
comment:

location: [0:29903]
authors:
consrtm: NCBI Genome Project
title: Direct Submission
journal: Submitted (17-JAN-2020) National Center for Biotechnology Information, NIH, Bethesda, MD 20894, USA
medline id:
pubmed id:
comment:

location: [0:29903]
authors: Wu,F., Zhao,S., Yu,B., Chen,Y.-M., Wang,W., Hu,Y., Song,Z.-G., Tao,Z.-W., Tian,J.-H., Pei,Y.-Y., Yuan,M.L., Zhang,Y.-L., Dai,F.-H., Liu,Y., Wang,Q.-M., Zheng,J.-J., Xu,L., Holmes,E.C. and Zhang,Y.-Z.
title: Direct Submission
journal: Submitted (05-JAN-2020) Shanghai Public Health Clinical Center & School of Public Health, Fudan University, Shanghai, China
medline id:
pubmed id:
comment:

---
## Protein level analysis

### Get the protein-coding sequences (CDSs)_____no_output_____

CDSs = [feature for feature in genbank_record.features if feature.type == "CDS"]

protein_seq = Seq(CDSs[0].qualifiers['translation'][0])
protein_seq_____no_output_____

### Have a closer look at the first CDS_____no_output_____

first_cds = CDSs[0]
print(f'Gene name of first CDS: {first_cds.qualifiers["gene"][0]}.') # Gene name of first CDS.
protein_seq = Seq(first_cds.qualifiers['translation'][0])            # Get amino acid sequence of first CDS.
print(first_cds)                                                     # Print first CDS.Gene name of first CDS: ORF1ab.
type: CDS
location: join{[265:13468](+), [13467:21555](+)}
qualifiers:
    Key: codon_start, Value: ['1']
    Key: db_xref, Value: ['GeneID:43740578']
    Key: gene, Value: ['ORF1ab']
    Key: locus_tag, Value: ['GU280_gp01']
    Key: note, Value: ['pp1ab; translated by -1 ribosomal frameshift']
    Key: product, Value: ['ORF1ab polyprotein']
    Key: protein_id, Value: ['YP_009724389.1']
    Key: ribosomal_slippage, Value: ['']
    Key: translation, Value: ['MESLVPGFNEKTHVQLSLPVLQVRDVLVRGFGDSVEEVLSEARQHLKDGTCGLVEVEKGVLPQLEQPYVFIKRSDARTAPHGHVMVELVAELEGIQYGRSGETLGVLVPHVGEIPVAYRKVLLRKNGNKGAGGHSYGADLKSFDLGDELGTDPYEDFQENWNTKHSSGVTRELMRELNGGAYTRYVDNNFCGPDGYPLECIKDLLARAGKASCTLSEQLDFIDTKRGVYCCREHEHEIAWYTERSEKSYELQTPFEIKLAKKFDTFNGECPNFVFPLNSIIKTIQPRVEKKKLDGFMGRIRSVYPVASPNECNQMCLSTLMKCDHCGETSWQTGDFVKATCEFCGTENLTKEGATTCGYLPQNAVVKIYCPACHNSEVGPEHSLAEYHNESGLKTILRKGGRTIAFGGCVFSYVGCHNKCAYWVPRASANIGCNHTGVVGEGSEGLNDNLLEILQKEKVNINIVGDFKLNEEIAIILASFSASTSAFVETVKGLDYKAFKQIVESCGNFKVTKGKAKKGAWNIGEQKSILSPLYAFASEAARVVRSIFSRTLETAQNSVRVLQKAAITILDGISQYSLRLIDAMMFTSDLATNNLVVMAYITGGVVQLTSQWLTNIFGTVYEKLKPVLDWLEEKFKEGVEFLRDGWEIVKFISTCACEIVGGQIVTCAKEIKESVQTFFKLVNKFLALCADSIIIGGAKLKALNLGETFVTHSKGLYRKCVKSREETGLLMPLKAPKEIIFLEGETLPTEVLTEEVVLKTGDLQPLEQPTSEAVEAPLVGTPVCINGLMLLEIKDTEKYCALAPNMMVTNNTFTLKGGAPTKVTFGDDTVIEVQGYKSVNITFELDERIDKVLNEKCSAYTVELGTEVNEFACVVADAVIKTLQPVSELLTPLGIDLDEWSMATYYLFDESGEFKLASHMYCSFYPPDEDEEEGDCEEEEFEPSTQYEYGTEDDYQGKPLEFGATSAALQPEEEQEEDWLDDDSQQTVGQQDGSEDNQTTTIQTIVEVQPQLEMELTPVVQTIEVNSFSGYLKLTDNVYIKNADIVEEAKKVKPTVVVNAANVYLKHGGGVAGALNKATNNAMQVESDDYIATNGPLKVGGSCVLSGHNLAKHCLHVVGPNVNKGEDIQLLKSAYENFNQHEVLLAPLLSAGIFGADPIHSLRVCVDTVRTNVYLAVFDKNLYDKLVSSFLEMKSEKQVEQKIAEIPKEEVKPFITESKPSVEQRKQDDKKIKACVEEVTTTLEETKFLTENLLLYIDINGNLHPDSATLVSDIDITFLKKDAPYIVGDVVQEGVLTAVVIPTKKAGGTTEMLAKALRKVPTDNYITTYPGQGLNGYTVEEAKTVLKKCKSAFYILPSIISNEKQEILGTVSWNLREMLAHAEETRKLMPVCVETKAIVSTIQRKYKGIKIQEGVVDYGARFYFYTSKTTVASLINTLNDLNETLVTMPLGYVTHGLNLEEAARYMRSLKVPATVSVSSPDAVTAYNGYLTSSSKTPEEHFIETISLAGSYKDWSYSGQSTQLGIEFLKRGDKSVYYTSNPTTFHLDGEVITFDNLKTLLSLREVRTIKVFTTVDNINLHTQVVDMSMTYGQQFGPTYLDGADVTKIKPHNSHEGKTFYVLPNDDTLRVEAFEYYHTTDPSFLGRYMSALNHTKKWKYPQVNGLTSIKWADNNCYLATALLTLQQIELKFNPPALQDAYYRARAGEAANFCALILAYCNKTVGELGDVRETMSYLFQHANLDSCKRVLNVVCKTCGQQQTTLKGVEAVMYMGTLSYEQFKKGVQIPCTCGKQATKYLVQQESPFVMMSAPPAQYELKHGTFTCASEYTGNYQCGHYKHITSKETLYCIDGALLTKSSEYKGPITDVFYKENSYTTTIKPVTYKLDGVVCTEIDPKLDNYYKKDNSYFTEQPIDLVPNQPYPNASFDNFKFVCDNIKFADDLNQLTGYKKPASRELKVTFFPDLNGDVVAIDYKHYTPSFKKGAKLLHKPIVWHVNNATNKATYKPNTWCIRCLWSTKPVETSNSFDVLKSEDAQGMDNLACEDLKPVSEEVVENPTIQKDVLECNVKTTEVVGDIILKPANNSLKITEEVGHTDLMAAYVDNSSLTIKKPNELSRVLGLKTLATHGLAAVNSVPWDTIANYAKPFLNKVVSTTTNIVTRCLNRVCTNYMPYFFTLLLQLCTFTRSTNSRIKASMPTTIAKNTVKSVGKFCLEASFNYLKSPNFSKLINIIIWFLLLSVCLGSLIYSTAALGVLMSNLGMPSYCTGYREGYLNSTNVTIATYCTGSIPCSVCLSGLDSLDTYPSLETIQITISSFKWDLTAFGLVAEWFLAYILFTRFFYVLGLAAIMQLFFSYFAVHFISNSWLMWLIINLVQMAPISAMVRMYIFFASFYYVWKSYVHVVDGCNSSTCMMCYKRNRATRVECTTIVNGVRRSFYVYANGGKGFCKLHNWNCVNCDTFCAGSTFISDEVARDLSLQFKRPINPTDQSSYIVDSVTVKNGSIHLYFDKAGQKTYERHSLSHFVNLDNLRANNTKGSLPINVIVFDGKSKCEESSAKSASVYYSQLMCQPILLLDQALVSDVGDSAEVAVKMFDAYVNTFSSTFNVPMEKLKTLVATAEAELAKNVSLDNVLSTFISAARQGFVDSDVETKDVVECLKLSHQSDIEVTGDSCNNYMLTYNKVENMTPRDLGACIDCSARHINAQVAKSHNIALIWNVKDFMSLSEQLRKQIRSAAKKNNLPFKLTCATTRQVVNVVTTKIALKGGKIVNNWLKQLIKVTLVFLFVAAIFYLITPVHVMSKHTDFSSEIIGYKAIDGGVTRDIASTDTCFANKHADFDTWFSQRGGSYTNDKACPLIAAVITREVGFVVPGLPGTILRTTNGDFLHFLPRVFSAVGNICYTPSKLIEYTDFATSACVLAAECTIFKDASGKPVPYCYDTNVLEGSVAYESLRPDTRYVLMDGSIIQFPNTYLEGSVRVVTTFDSEYCRHGTCERSEAGVCVSTSGRWVLNNDYYRSLPGVFCGVDAVNLLTNMFTPLIQPIGALDISASIVAGGIVAIVVTCLAYYFMRFRRAFGEYSHVVAFNTLLFLMSFTVLCLTPVYSFLPGVYSVIYLYLTFYLTNDVSFLAHIQWMVMFTPLVPFWITIAYIICISTKHFYWFFSNYLKRRVVFNGVSFSTFEEAALCTFLLNKEMYLKLRSDVLLPLTQYNRYLALYNKYKYFSGAMDTTSYREAACCHLAKALNDFSNSGSDVLYQPPQTSITSAVLQSGFRKMAFPSGKVEGCMVQVTCGTTTLNGLWLDDVVYCPRHVICTSEDMLNPNYEDLLIRKSNHNFLVQAGNVQLRVIGHSMQNCVLKLKVDTANPKTPKYKFVRIQPGQTFSVLACYNGSPSGVYQCAMRPNFTIKGSFLNGSCGSVGFNIDYDCVSFCYMHHMELPTGVHAGTDLEGNFYGPFVDRQTAQAAGTDTTITVNVLAWLYAAVINGDRWFLNRFTTTLNDFNLVAMKYNYEPLTQDHVDILGPLSAQTGIAVLDMCASLKELLQNGMNGRTILGSALLEDEFTPFDVVRQCSGVTFQSAVKRTIKGTHHWLLLTILTSLLVLVQSTQWSLFFFLYENAFLPFAMGIIAMSAFAMMFVKHKHAFLCLFLLPSLATVAYFNMVYMPASWVMRIMTWLDMVDTSLSGFKLKDCVMYASAVVLLILMTARTVYDDGARRVWTLMNVLTLVYKVYYGNALDQAISMWALIISVTSNYSGVVTTVMFLARGIVFMCVEYCPIFFITGNTLQCIMLVYCFLGYFCTCYFGLFCLLNRYFRLTLGVYDYLVSTQEFRYMNSQGLLPPKNSIDAFKLNIKLLGVGGKPCIKVATVQSKMSDVKCTSVVLLSVLQQLRVESSSKLWAQCVQLHNDILLAKDTTEAFEKMVSLLSVLLSMQGAVDINKLCEEMLDNRATLQAIASEFSSLPSYAAFATAQEAYEQAVANGDSEVVLKKLKKSLNVAKSEFDRDAAMQRKLEKMADQAMTQMYKQARSEDKRAKVTSAMQTMLFTMLRKLDNDALNNIINNARDGCVPLNIIPLTTAAKLMVVIPDYNTYKNTCDGTTFTYASALWEIQQVVDADSKIVQLSEISMDNSPNLAWPLIVTALRANSAVKLQNNELSPVALRQMSCAAGTTQTACTDDNALAYYNTTKGGRFVLALLSDLQDLKWARFPKSDGTGTIYTELEPPCRFVTDTPKGPKVKYLYFIKGLNNLNRGMVLGSLAATVRLQAGNATEVPANSTVLSFCAFAVDAAKAYKDYLASGGQPITNCVKMLCTHTGTGQAITVTPEANMDQESFGGASCCLYCRCHIDHPNPKGFCDLKGKYVQIPTTCANDPVGFTLKNTVCTVCGMWKGYGCSCDQLREPMLQSADAQSFLNRVCGVSAARLTPCGTGTSTDVVYRAFDIYNDKVAGFAKFLKTNCCRFQEKDEDDNLIDSYFVVKRHTFSNYQHEETIYNLLKDCPAVAKHDFFKFRIDGDMVPHISRQRLTKYTMADLVYALRHFDEGNCDTLKEILVTYNCCDDDYFNKKDWYDFVENPDILRVYANLGERVRQALLKTVQFCDAMRNAGIVGVLTLDNQDLNGNWYDFGDFIQTTPGSGVPVVDSYYSLLMPILTLTRALTAESHVDTDLTKPYIKWDLLKYDFTEERLKLFDRYFKYWDQTYHPNCVNCLDDRCILHCANFNVLFSTVFPPTSFGPLVRKIFVDGVPFVVSTGYHFRELGVVHNQDVNLHSSRLSFKELLVYAADPAMHAASGNLLLDKRTTCFSVAALTNNVAFQTVKPGNFNKDFYDFAVSKGFFKEGSSVELKHFFFAQDGNAAISDYDYYRYNLPTMCDIRQLLFVVEVVDKYFDCYDGGCINANQVIVNNLDKSAGFPFNKWGKARLYYDSMSYEDQDALFAYTKRNVIPTITQMNLKYAISAKNRARTVAGVSICSTMTNRQFHQKLLKSIAATRGATVVIGTSKFYGGWHNMLKTVYSDVENPHLMGWDYPKCDRAMPNMLRIMASLVLARKHTTCCSLSHRFYRLANECAQVLSEMVMCGGSLYVKPGGTSSGDATTAYANSVFNICQAVTANVNALLSTDGNKIADKYVRNLQHRLYECLYRNRDVDTDFVNEFYAYLRKHFSMMILSDDAVVCFNSTYASQGLVASIKNFKSVLYYQNNVFMSEAKCWTETDLTKGPHEFCSQHTMLVKQGDDYVYLPYPDPSRILGAGCFVDDIVKTDGTLMIERFVSLAIDAYPLTKHPNQEYADVFHLYLQYIRKLHDELTGHMLDMYSVMLTNDNTSRYWEPEFYEAMYTPHTVLQAVGACVLCNSQTSLRCGACIRRPFLCCKCCYDHVISTSHKLVLSVNPYVCNAPGCDVTDVTQLYLGGMSYYCKSHKPPISFPLCANGQVFGLYKNTCVGSDNVTDFNAIATCDWTNAGDYILANTCTERLKLFAAETLKATEETFKLSYGIATVREVLSDRELHLSWEVGKPRPPLNRNYVFTGYRVTKNSKVQIGEYTFEKGDYGDAVVYRGTTTYKLNVGDYFVLTSHTVMPLSAPTLVPQEHYVRITGLYPTLNISDEFSSNVANYQKVGMQKYSTLQGPPGTGKSHFAIGLALYYPSARIVYTACSHAAVDALCEKALKYLPIDKCSRIIPARARVECFDKFKVNSTLEQYVFCTVNALPETTADIVVFDEISMATNYDLSVVNARLRAKHYVYIGDPAQLPAPRTLLTKGTLEPEYFNSVCRLMKTIGPDMFLGTCRRCPAEIVDTVSALVYDNKLKAHKDKSAQCFKMFYKGVITHDVSSAINRPQIGVVREFLTRNPAWRKAVFISPYNSQNAVASKILGLPTQTVDSSQGSEYDYVIFTQTTETAHSCNVNRFNVAITRAKVGILCIMSDRDLYDKLQFTSLEIPRRNVATLQAENVTGLFKDCSKVITGLHPTQAPTHLSVDTKFKTEGLCVDIPGIPKDMTYRRLISMMGFKMNYQVNGYPNMFITREEAIRHVRAWIGFDVEGCHATREAVGTNLPLQLGFSTGVNLVAVPTGYVDTPNNTDFSRVSAKPPPGDQFKHLIPLMYKGLPWNVVRIKIVQMLSDTLKNLSDRVVFVLWAHGFELTSMKYFVKIGPERTCCLCDRRATCFSTASDTYACWHHSIGFDYVYNPFMIDVQQWGFTGNLQSNHDLYCQVHGNAHVASCDAIMTRCLAVHECFVKRVDWTIEYPIIGDELKINAACRKVQHMVVKAALLADKFPVLHDIGNPKAIKCVPQADVEWKFYDAQPCSDKAYKIEELFYSYATHSDKFTDGVCLFWNCNVDRYPANSIVCRFDTRVLSNLNLPGCDGGSLYVNKHAFHTPAFDKSAFVNLKQLPFFYYSDSPCESHGKQVVSDIDYVPLKSATCITRCNLGGAVCRHHANEYRLYLDAYNMMISAGFSLWVYKQFDTYNLWNTFTRLQSLENVAFNVVNKGHFDGQQGEVPVSIINNTVYTKVDGVDVELFENKTTLPVNVAFELWAKRNIKPVPEVKILNNLGVDIAANTVIWDYKRDAPAHISTIGVCSMTDIAKKPTETICAPLTVFFDGRVDGQVDLFRNARNGVLITEGSVKGLQPSVGPKQASLNGVTLIGEAVKTQFNYYKKVDGVVQQLPETYFTQSRNLQEFKPRSQMEIDFLELAMDEFIERYKLEGYAFEHIVYGDFSHSQLGGLHLLIGLAKRFKESPFELEDFIPMDSTVKNYFITDAQTGSSKCVCSVIDLLLDDFVEIIKSQDLSVVSKVVKVTIDYTEISFMLWCKDGHVETFYPKLQSSQAWQPGVAMPNLYKMQRMLLEKCDLQNYGDSATLPKGIMMNVAKYTQLCQYLNTLTLAVPYNMRVIHFGAGSDKGVAPGTAVLRQWLPTGTLLVDSDLNDFVSDADSTLIGDCATVHTANKWDLIISDMYDPKTKNVTKENDSKEGFFTYICGFIQQKLALGGSVAIKITEHSWNADLYKLMGHFAWWTAFVTNVNASSSEAFLIGCNYLGKPREQIDGYVMHANYIFWRNTNPIQLSSYSLFDMSKFPLKLRGTAVMSLKEGQINDMILSLLSKGRLIIRENNRVVISSDVLVNN']

### Write first CDS to FASTA file_____no_output_____

seq_record = SeqRecord(seq=Seq(first_cds.qualifiers['translation'][0]),
                      id=first_cds.qualifiers['protein_id'][0],
                      name=first_cds.qualifiers['product'][0],
                      dbxrefs=first_cds.qualifiers['db_xref'])
SeqIO.write(seq_record, f'output/{first_cds.qualifiers["protein_id"][0]}.fasta', 'fasta')_____no_output_____

---
## Exercise 1

- The file `input/dna_sequences.fasta` contains further DNA sequences.
- Analyze some of these sequences with the techniques presented today.
- Repeat this with your favourite FASTA file (if you have any)._____no_output_____Solution for Exercise 1_____no_output_____---
## Solutions for exercises_____no_output_____Solution for Exercise 1_____no_output_____

fasta_file = 'input/dna_sequences.fasta'
file_format = 'fasta'
records = [record for record in SeqIO.parse(fasta_file, file_format)]_____no_output_____records[0]_____no_output_____result_handle = NCBIWWW.qblast('blastn', 'nt', records[0].seq)
with open('output/dna_sequence_0.xml', 'w') as fp:
    fp.write(result_handle.read())
    result_handle.close()_____no_output_____blast_qresult = SearchIO.read('output/dna_sequence_0.xml', 'blast-xml')
first_hit = blast_qresult[0]
print(f'Description: {first_hit.description}')
first_hsp = first_hit[0] Description: Dialister invisus strain KCOM 3293 (=ChDC P001-Td3) 16S ribosomal RNA gene, partial sequence
NCBI_id = first_hit.id.split('|')[3]
genbank_handle = Entrez.efetch(db='nucleotide', id=NCBI_id, retmode='text', rettype='gb')
genbank_record = SeqIO.read(genbank_handle, 'genbank')_____no_output_____print(genbank_record)ID: MT471996.1
Name: MT471996
Description: Dialister invisus strain KCOM 3293 (=ChDC P001-Td3) 16S ribosomal RNA gene, partial sequence
Number of features: 2
/molecule_type=DNA
/topology=linear
/data_file_division=BCT
/date=19-MAY-2020
/accessions=['MT471996']
/sequence_version=1
/keywords=['']
/source=Dialister invisus
/organism=Dialister invisus
/taxonomy=['Bacteria', 'Firmicutes', 'Negativicutes', 'Veillonellales', 'Veillonellaceae', 'Dialister']
/references=[Reference(title='Isolation of oral bacteria from a Korean population', ...), Reference(title='Direct Submission', ...)]
/comment=Sequences were screened for chimeras by the submitter using
Lasergene 7.0.
/structured_comment=OrderedDict([('Assembly-Data', OrderedDict([('Sequencing Technology', 'Sanger dideoxy sequencing')]))])
Seq('GACGAACGCTGGCGGCGTGCTTAACACATGCAAGTCGAACGAAAAGAGGGGAAG...GTG')

Back to Exercise 1_____no_output_____


# Notebook from nstarman/astronat
Path: docs/examples/XMatch.ipynb

# Crossmatch catalogs_____no_output_____

"""
    TITLE   : x-match
    AUTHOR  : Nathaniel Starkman
    PROJECT :
""";

__author__ = ''
__version__ = ''_____no_output_____

    About

This is a dev notebook, where the xmatch functions from `starkman_thesis` are built and tested
_____no_output_____

- - -
_____no_output_____## Prepare_____no_output_____### Imports_____no_output_____

from utilipy import ipython

# ipython.run_imports(base=True, astropy=True, matplotlib=True, verbose_imports=False)
# %run "utilipy.imports.astropy_imports"

# BUILT-IN

# THIRD PARTY

from astropy import table
from astropy import coordinates as coords, units as u
from astropy.coordinates import SkyCoord, Distance, EarthLocation, match_coordinates_sky
from astropy.coordinates.tests.utils import randomly_sample_sphere
from astropy.time import Time

import numpy as np

# PROJECT-SPECIFIC

import starkman_thesisset autoreload to 1
/Users/nathanielstarkman/miniconda3/lib/python3.7/site-packages/astropy/coordinates/builtin_frames/galactocentric.py:373: AstropyDeprecationWarning: In v4.1 and later versions, the Galactocentric frame will adopt default parameters that may update with time. An updated default parameter set is already available through the astropy.coordinates.galactocentric_frame_defaults ScienceState object, as described in but the default is currently still set to the pre-v4.0 parameter defaults. The safest way to guard against changing default parameters in the future is to either (1) specify all Galactocentric frame attributes explicitly when using the frame, or (2) set the galactocentric_frame_defaults parameter set name explicitly. See http://docs.astropy.org/en/latest/coordinates/galactocentric.html for more information.
  AstropyDeprecationWarning)

- - -
_____no_output_____## Discrete Matches_____no_output_____

cat0 = table.Table()
cat0["color"] = ["blue"] * 4 + ["green"] * (10 - 5) + ["orange"]
cat0["id"] = ["tag1"] * 1 + ["tag2"] * (len(cat0["color"]) - 2) + ["tag3"]
cat0["id2"] = ["tag1"] * 10

cat1 = table.Table()
cat1["color"] = ["blue"] * 2 + ["orange"] * 3 + ["green"] * (12 - 6) + ["purple"]
cat1["id"] = ["tag1"] * 4 + ["tag2"] * (len(cat1["color"]) - 6) + ["tag3"]*2
cat1["id2"] = ["tag2"] + ["tag1"] * 11

cat0.to_pandas().values.T
cat1.to_pandas().values.T
_____no_output_____u0 = np.unique(np.array(cat0["color"]))

# first do with only fields=["color"]
# will weed out all stuff in cat1 that's not in cat0
idx10 = cat1["color"] == u0[:, None]

idx0 = True
idx1 = np.sum(idx10, axis=0, dtype=bool)
idx1

# cat0[idx0]
np.shape(cat1[idx1])  # should be N-1=11
cat1[idx1].to_pandas().values.T_____no_output_____# Now all fields
u1 = np.unique(np.array(cat0["id"]))
u2 = np.unique(np.array(cat0["id2"]))

idx11 = cat1["id"] == u1[:, None]
idx12 = cat1["id2"] == u2[:, None]

idx0 = True
idx1 = np.sum(idx10 & idx11 & idx12, axis=0, dtype=bool)

np.shape(cat1[idx1])  # should be N-1=11
cat1[idx1].to_pandas().values.T
_____no_output_____

but the problem is that there are stuff in catalog 0 that are not in catalog 1
Now that we've sifted out all the stuff that are in catalog 1 that are not in 0,
we need to do the reverse.

The general procedure is

(cat0 <- cat1, 2, 3, 4, 5) comparison
cat1 <- cat0 copmarison to filter out non-matches in cat0
_____no_output_____

import functools
def _match_two_catalogs_indices(match_from, other, fields: list):
    """

    Parameters
    ----------
    match_from : Table
        the catalog to match from, only values in the field (from `fields`) are
        permitted in the
    other : Table
        match this against `match_from`
    fields : list
        list of fields to match

    Notes
    -----
    try more axes tricks to avoid for loops (``cat1.to_pandas().values[:, :, None]``)
    the difficulty is that the "unique" arrays can be different lengths
    but these can be extended to be equal length, filled with empty value

    """
    # uniques for each field
    uns = (np.unique(np.array(match_from[n])) for n in fields)

    # indices into cat1
    # loop over fields  TODO vectorize ?
    idxs = (other[n] == un[:, None] for n, un in zip(fields, uns))
    # get combined index
    idx = np.sum(functools.reduce(np.logical_and, idxs), axis=0, dtype=bool)

    return idx

def match_catalogs_indices(*catalogs, fields):
    idxs = [_match_two_catalogs_indices(catalogs[0], c, fields=fields)
            for c in catalogs[1:]]  # TODO with numpy
    # and need to do once in reverse on the source catalog
    idx0 = _match_two_catalogs(catalogs[1][idxs[0]], catalogs[0], fields)
    idxs.insert(0, idx0)

    return idx

def match_catalogs(*catalogs, fields):
    idxs = match_catalogs_indices(*catalogs, fields)
    cat_matches = [catalog[idx] for catalog, idx in zip(catalogs, idxs)]
    return cat_matches
# /def

idx = match_catalogs_indices(cat0, cat1, cat2, fields=["color", "id", "id2"])

cat0.to_pandas().values.T
cat0[idx[0]].to_pandas().values.T
# cat1.to_pandas().values.T
# cat1[idx[1]]
# cat2[idx[2]]_____no_output_____

Now 3 catalogs_____no_output_____

cat2 = table.Table()
cat2["color"] = ["blue"] * 2 + ["orange"] * 4 + ["purple"] + ["green"] * (12 - 7)
cat2["id"] = ["tag1"] * 4 + ["tag2"] * (len(cat1["color"]) - 6) + ["tag3"]*2
cat2["id2"] = ["tag2"] + ["tag1"] * 11

cat2.to_pandas().values.T_____no_output_____# just going against catalog 1
idx20 = cat2["color"] == u0[:, None]
idx21 = cat2["id"] == u1[:, None]
idx22 = cat2["id2"] == u2[:, None]
idx2 = np.sum(idx20 & idx21 & idx22, axis=0, dtype=bool)

# lets also go against catalog 2

cat2[idx2].to_pandas().values.T_____no_output_____t = [idx20, idx21, idx22]
t_____no_output_____raise ValueError_____no_output_____

## Code_____no_output_____Observation Time_____no_output_____

ra, dec, _ = randomly_sample_sphere(1000)
pm_ra_cosdec=np.ones_like(ra.value) * 10 * u.mas / u.yr
pm_dec=np.ones_like(dec.value) * u.mas / u.yr
time = Time('2001-03-22 00:01:44.732327132980', scale='utc')
data = coords.ICRS(ra=ra, dec=dec, pm_ra_cosdec=pm_ra_cosdec, pm_dec=pm_dec)
cs = SkyCoord(data, obstime=time)
cs[:10]_____no_output_____

Compute the position of the source represented by this coordinate object
to a new time using the velocities stored in this object and assuming
linear space motion (including relativistic corrections). This is
sometimes referred to as an "epoch transformation."

The initial time before the evolution is taken from the ``obstime``
attribute of this coordinate.  Note that this method currently does not
support evolving coordinates where the *frame* has an ``obstime`` frame
attribute, so the ``obstime`` is only used for storing the before and
after times, not actually as an attribute of the frame. Alternatively,
if ``dt`` is given, an ``obstime`` need not be provided at all._____no_output_____

res = SkyCoord.apply_space_motion(cs, new_obstime=Time.now())
res[:10]_____no_output_____

## Example_____no_output_____

result_tgas = dict(ra=66.44280212823296,
                   dec=-69.99366255906372,
                   parallax=22.764078749733947,
                   pmra=144.91354358297048,
                   pmdec=5.445648092997134,
                   ref_epoch=2015.0,
                   phot_g_mean_mag=7.657174523348196)

result_2mass = dict(RAJ2000=[66.421970000000002, 66.433521999999996,
                             66.420564999999996, 66.485068999999996,
                             66.467928999999998, 66.440815000000001,
                             66.440454000000003],
                    DEJ2000=[-70.003722999999994, -69.990768000000003,
                             -69.992255999999998, -69.994881000000007,
                             -69.994926000000007, -69.993613999999994,
                             -69.990836999999999],
                    Jmag=[16.35, 13.663, 16.171, 16.184, 16.292,
                          6.6420002, 12.275],
                    Hmag=[15.879, 13.955, 15.154, 15.856, 15.642,
                          6.3660002, 12.185],
                    Kmag=[15.581, 14.238, 14.622, 15.398, 15.123,
                          6.2839999, 12.106],
                    Date=['1998-10-24', '1998-10-24', '1998-10-24',
                          '1998-10-24', '1998-10-24', '1998-10-24',
                          '1998-10-24'])_____no_output_____c = SkyCoord(ra=result_tgas['ra'] * u.deg,
             dec=result_tgas['dec'] * u.deg,
             distance=Distance(parallax=result_tgas['parallax'] * u.mas),
             pm_ra_cosdec=result_tgas['pmra'] * u.mas/u.yr,
             pm_dec=result_tgas['pmdec'] * u.mas/u.yr,
             obstime=Time(result_tgas['ref_epoch'], format='decimalyear')
)_____no_output_____catalog_2mass = SkyCoord(ra=result_2mass['RAJ2000'] * u.deg,
                         dec=result_2mass['DEJ2000'] * u.deg,
                         obstime=Time(result_2mass['Date'][0]))_____no_output_____c_2mass_epoch = SkyCoord.apply_space_motion(c, catalog_2mass.obstime)
c_2mass_epoch, c_2mass_epoch.obstime_____no_output_____starkman_thesis.utils.data.xmatch.xmatch_coords(c_2mass_epoch, catalog_2mass)  # obstime=Time.now()_____no_output_____x = table.Table({k: getattr(catalog_2mass, k) for k in catalog_2mass.get_representation_component_names().keys()})
del x["distance"]
x.meta["obstime"] = catalog_2mass.obstime_____no_output_____c1, c2, info = starkman_thesis.utils.data.xmatch.xmatch_coords(c, x, obstime=x.meta["obstime"])_____no_output_____

## Problems With Duplicates_____no_output_____

sc1 = coords.ICRS(*randomly_sample_sphere(10, randomseed=0))
sc2 = coords.ICRS(*randomly_sample_sphere(12, randomseed=1))_____no_output_____cat1 = table.Table()
cat1["ra"] = sc1.ra
cat1["dec"] = sc1.dec
cat1["color"] = ["blue"] * 4 + ["green"] * (len(sc1) - 5) + ["orange"]

cat2 = table.Table()
cat2["ra"] = sc2.ra
cat2["dec"] = sc2.dec
cat2["color"] = ["blue"] * 2 + ["orange"] * 3 + ["green"] * (len(sc2) - 5)

cat1
cat2_____no_output_____mc1, mc2, info = starkman_thesis.utils.data.xmatch.xmatch(cat1, cat2, maxdist=40*u.deg, match_fields="color")
mc1_____no_output_____field = "color"
values_cat1 = np.array(cat1[field])
len_vc1 = len(values_cat1)
values_cat2 = np.array(cat2[field])
len_vc2 = len(values_cat2)

uniques = np.unique(values_cat1)

# sep = np.ones(len_vc1) * -1.0
# idx1 = -np.ones(len_vc1, dtype=int)
# idx2 = -np.ones(len_vc2, dtype=int)
idx1 = []
idx2 = []
info = {"sep2d": [], "dist3d": []}
for unique in uniques:
    print(unique)
    idx_1 = np.arange(len_vc1)[values_cat1 == unique]
    idx_2 = np.arange(len_vc2)[values_cat2 == unique]

    # the case where a class only exists in one but not the other
    if (idx_1.shape[0] == 0 or idx_2.shape[0] == 0):
        continue

    temp_sc1_idx, temp_sc2_idx, temp_info = starkman_thesis.utils.data.xmatch.xmatch_coords_indices(sc1[idx_1], sc2[idx_2], maxdist=40*u.deg)

    idx1.extend(idx_1[temp_sc1_idx])
    idx2.extend(idx_2[temp_sc2_idx])

    info["sep2d"].extend(temp_info.get("sep2d", []))
    info["dist3d"].extend(temp_info.get("dist3d", []))
    print(idx_1[temp_sc1_idx])

idx1 = np.array(idx1)
idx1 = idx1[idx1>=0]
idx2 = np.array(idx2)
idx2 = idx2[idx2>=0]
info["sep2d"] = coords.Angle(info["sep2d"])
info["dist3d"] = u.Quantity(info["dist3d"])

info
cat1[idx1]
cat1[idx1]_____no_output_____i, sep2d, sep3d = coords.match_coordinates_sky(sc1[idx_1], sc2[idx_2])
midx = np.where(sep2d < 40 * u.deg)[0]

sc1[idx_1][midx].separation(sc2[idx_2][i][midx])_____no_output_____

- - -
- - -

    END
_____no_output_____


# Notebook from aiqc/aiqc
Path: docs/_build/html/notebooks/keras_sequence_classification.ipynb

# Keras: Times Series Classify Binary_____no_output_____*Binary Detection of Epileptic Seizures Using a Cohort of Sequence of Electroencephalography (EEG) Readings.*_____no_output_____![waves](../images/waves.png)_____no_output_____Sequence data structures contain many observations (rows) for each sample (e.g. site, sensor, or patient). They are often used for grouping time-based observations into what is called a time series. However, sequences can also represent biological sequences like DNA and RNA.

The cardinality of *many observations per sample* changes the dimensionality of the data from 2D to 3D. This effectively adds an additional layer of complexity to all aspects of data preparation. In this notebook, you'll see that, once a `Dataset.Sequence` has been ingested, the AIQC API allows you to work with multivariate 3D data as easily as if it were 2D. As an example, you can still apply encoders by dtype and column_name._____no_output_____---_____no_output_____

import aiqc
from aiqc import datum_____no_output_____

---_____no_output_____## Example Data_____no_output_____

from aiqc import datum_____no_output_____df = datum.to_pandas('epilepsy.parquet')_____no_output_____df.head()_____no_output_____

---_____no_output_____## a) High-Level API_____no_output_____Reference [High-Level API Docs](api_high_level.ipynb) for more information._____no_output_____

label_df = df[['seizure']]_____no_output_____seq_ndarray3D = df.drop(columns=['seizure']).to_numpy().reshape(1000,178,1)_____no_output_____from sklearn.preprocessing import StandardScaler_____no_output_____splitset = aiqc.Pipeline.Sequence.make(
    feature_ndarray3D_or_npyPath = seq_ndarray3D
    , feature_dtype = None
    , feature_cols_excluded = None
    , feature_interpolaters = None
    , feature_window = None
    , feature_encoders = [{
        "sklearn_preprocess": StandardScaler()
        , "columns": '0'
    }]
    , feature_reshape_indices = None

    , label_df_or_path = label_df
    , label_dtype = None
    , label_column = 'seizure'
    , label_interpolater = None
    , label_encoder = None

    , size_test = 0.12
    , size_validation = 0.22
    , fold_count = None
    , bin_count = None
)⏱ Ingesting Sequences 🧬: 100%|████████████████| 1000/1000 [00:05<00:00, 183.79it/s]
import keras
from keras.models import Sequential
from keras.layers import LSTM, Dense
from keras.callbacks import History_____no_output_____def fn_build(features_shape, label_shape, **hp):
    model = Sequential()
    model.add(LSTM(
        hp['neuron_count']
        , input_shape=(features_shape[0], features_shape[1])
    ))
    model.add(Dense(units=label_shape[0], activation='sigmoid'))
    return model_____no_output_____def fn_train(model, loser, optimizer, samples_train, samples_evaluate, **hp):
    model.compile(
        loss=loser
        , optimizer=optimizer
        , metrics=['accuracy']
    )
    model.fit(
        samples_train['features'], samples_train['labels']
        , validation_data = (samples_evaluate['features'], samples_evaluate['labels'])
        , verbose = 0
        , batch_size = hp['batch_size']
        , epochs = hp['epochs']
        , callbacks = [History()]
    )
    return model_____no_output_____hyperparameters = {
    "neuron_count": [25]
    , "batch_size": [8]
    , "epochs": [5, 10]
}_____no_output_____queue = aiqc.Experiment.make(
    library = "keras"
    , analysis_type = "classification_binary"
    , fn_build = fn_build
    , fn_train = fn_train
    , splitset_id = splitset.id
    , repeat_count = 2
    , hide_test = False
    , hyperparameters = hyperparameters

    , fn_lose = None #automated
    , fn_optimize = None #automated
    , fn_predict = None #automated
    , foldset_id = None
)_____no_output_____queue.run_jobs()🔮 Training Models 🔮: 100%|██████████████████████████████████████████| 4/4 [02:24<00:00, 36.01s/it]

For more information on visualization of performance metrics, reference the [Visualization & Metrics](visualization.html) documentation._____no_output_____---_____no_output_____## b) Low-Level API_____no_output_____

df = datum.to_pandas('epilepsy.parquet')_____no_output_____label_df = df[['seizure']]_____no_output_____seq_ndarray3D = df.drop(columns=['seizure']).to_numpy().reshape(1000,178,1)_____no_output_____dataset_tabular = aiqc.Dataset.Tabular.from_pandas(label_df)_____no_output_____label = dataset_tabular.make_label(columns='seizure')_____no_output_____dataset_sequence = aiqc.Dataset.Sequence.from_numpy(
    ndarray3D_or_npyPath = seq_ndarray3D
    , column_names = ['EEG']
)⏱ Ingesting Sequences 🧬: 100%|████████████████| 1000/1000 [00:05<00:00, 194.69it/s]
feature = dataset_sequence.make_feature()_____no_output_____encoderset = feature.make_encoderset()_____no_output_____encoderset = encoderset.make_featurecoder(
    sklearn_preprocess = StandardScaler()
    , columns = ['EEG']
)
=> Info - System overriding user input to set `sklearn_preprocess.copy=False`.
   This saves memory when concatenating the output of many encoders.

___/ featurecoder_index: 0 \_________

=> The column(s) below matched your filter(s) featurecoder filters.

['EEG']

=> Done. All feature column(s) have featurecoder(s) associated with them.
No more Featurecoders can be added to this Encoderset.

splitset = aiqc.Splitset.make(
    feature_ids = [feature.id]
    , label_id = label.id
    , size_test = 0.22
    , size_validation = 0.12
)_____no_output_____def fn_build(features_shape, label_shape, **hp):
    model = Sequential()
    model.add(LSTM(
        hp['neuron_count']
        , input_shape=(features_shape[0], features_shape[1])
    ))
    model.add(Dense(units=label_shape[0], activation='sigmoid'))
    return model_____no_output_____def fn_train(model, loser, optimizer, samples_train, samples_evaluate, **hp):
    model.compile(
        loss=loser
        , optimizer=optimizer
        , metrics=['accuracy']
    )
    model.fit(
        samples_train['features'], samples_train['labels']
        , validation_data = (samples_evaluate['features'], samples_evaluate['labels'])
        , verbose = 0
        , batch_size = hp['batch_size']
        , epochs = hp['epochs']
        , callbacks = [History()]
    )
    return model_____no_output_____algorithm = aiqc.Algorithm.make(
    library = "keras"
    , analysis_type = "classification_binary"
    , fn_build = fn_build
    , fn_train = fn_train
)_____no_output_____hyperparameters = {
    "neuron_count": [25]
    , "batch_size": [8]
    , "epochs": [5, 10]
}_____no_output_____hyperparamset = algorithm.make_hyperparamset(
    hyperparameters = hyperparameters
)_____no_output_____queue = algorithm.make_queue(
    splitset_id = splitset.id
    , hyperparamset_id = hyperparamset.id
    , repeat_count = 2
)_____no_output_____queue.run_jobs()🔮 Training Models 🔮: 100%|██████████████████████████████████████████| 4/4 [01:53<00:00, 28.32s/it]

Reference [Low-Level API Docs](api_high_level.ipynb) for more information including how to work with non-tabular data and defining optimizers._____no_output_____


# Notebook from fengjings/Coursera
Path: IBM_AI_Engineering/Course-1-machine-learning-with-python/Week_3_1_ML0101EN-Clas-K-Nearest-neighbors-CustCat-py-v1.ipynb

K-Nearest Neighbors
_____no_output_____In this Lab you will load a customer dataset, fit the data, and use K-Nearest Neighbors to predict a data point. But what is **K-Nearest Neighbors**?_____no_output_____**K-Nearest Neighbors** is an algorithm for supervised learning. Where the data is 'trained' with data points corresponding to their classification. Once a point is to be predicted, it takes into account the 'K' nearest points to it to determine it's classification._____no_output_____### Here's an visualization of the K-Nearest Neighbors algorithm.

_____no_output_____In this case, we have data points of Class A and B. We want to predict what the star (test data point) is. If we consider a k value of 3 (3 nearest data points) we will obtain a prediction of Class B. Yet if we consider a k value of 6, we will obtain a prediction of Class A._____no_output_____In this sense, it is important to consider the value of k. But hopefully from this diagram, you should get a sense of what the K-Nearest Neighbors algorithm is. It considers the 'K' Nearest Neighbors (points) when it predicts the classification of the test point._____no_output_____
Table of contents

About the dataset

Data Visualization and Analysis

Classification

_____no_output_____Lets load required libraries_____no_output_____

import itertools
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import NullFormatter
import pandas as pd
import numpy as np
import matplotlib.ticker as ticker
from sklearn import preprocessing
%matplotlib inline_____no_output_____

About the dataset

_____no_output_____Imagine a telecommunications provider has segmented its customer base by service usage patterns, categorizing the customers into four groups. If demographic data can be used to predict group membership, the company can customize offers for individual prospective customers. It is a classification problem. That is, given the dataset,  with predefined labels, we need to build a model to be used to predict class of a new or unknown case.

The example focuses on using demographic data, such as region, age, and marital, to predict usage patterns.

The target field, called __custcat__, has four possible values that correspond to the four customer groups, as follows:
  1- Basic Service
  2- E-Service
  3- Plus Service
  4- Total Service

Our objective is to build a classifier, to predict the class of unknown cases. We will use a specific type of classification called K nearest neighbour.
_____no_output_____Lets download the dataset. To download the data, we will use !wget to download it from IBM Object Storage._____no_output_____

import urllib.request
urllib.request.urlretrieve('https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/teleCust1000t.csv','teleCust1000t.csv')
# can't use in windows
# !wget -O teleCust1000t.csv https://s3-api.us-geo.objectstorage.softlayer.net/cf-courses-data/CognitiveClass/ML0101ENv3/labs/teleCust1000t.csv_____no_output_____

__Did you know?__ When it comes to Machine Learning, you will likely be working with large datasets. As a business, where can you host your data? IBM is offering a unique opportunity for businesses, with 10 Tb of IBM Cloud Object Storage: [Sign up now for free](http://cocl.us/ML0101EN-IBM-Offer-CC)_____no_output_____### Load Data From CSV File  _____no_output_____

df = pd.read_csv('teleCust1000t.csv')
df.head()_____no_output_____

Data Visualization and Analysis

_____no_output_____#### Let’s see how many of each class is in our data set _____no_output_____

df['custcat'].value_counts()_____no_output_____

#### 281 Plus Service, 266 Basic-service, 236 Total Service, and 217 E-Service customers
_____no_output_____You can easily explore your data using visualization techniques:_____no_output_____

df.hist(column='income', bins=50)_____no_output_____

### Feature set_____no_output_____Lets define feature sets, X:_____no_output_____

df.columns_____no_output_____

To use scikit-learn library, we have to convert the Pandas data frame to a Numpy array:_____no_output_____

X = df[['region', 'tenure','age', 'marital', 'address', 'income', 'ed', 'employ','retire', 'gender', 'reside']] .values  #.astype(float)
X[0:5]
_____no_output_____

What are our labels?_____no_output_____

y = df['custcat'].values
y[0:5]_____no_output_____

## Normalize Data _____no_output_____Data Standardization give data zero mean and unit variance, it is good practice, especially for algorithms such as KNN which is based on distance of cases:_____no_output_____

X = preprocessing.StandardScaler().fit(X).transform(X.astype(float))
X[0:5]_____no_output_____

### Train Test Split
Out of Sample Accuracy is the percentage of correct predictions that the model makes on data that that the model has NOT been trained on. Doing a train and test on the same dataset will most likely have low out-of-sample accuracy, due to the likelihood of being over-fit.

It is important that our models have a high, out-of-sample accuracy, because the purpose of any model, of course, is to make correct predictions on unknown data. So how can we improve out-of-sample accuracy? One way is to use an evaluation approach called Train/Test Split.
Train/Test Split involves splitting the dataset into training and testing sets respectively, which are mutually exclusive. After which, you train with the training set and test with the testing set.

This will provide a more accurate evaluation on out-of-sample accuracy because the testing dataset is not part of the dataset that have been used to train the data. It is more realistic for real world problems.
_____no_output_____

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=4)
print ('Train set:', X_train.shape,  y_train.shape)
print ('Test set:', X_test.shape,  y_test.shape)Train set: (800, 11) (800,)
Test set: (200, 11) (200,)

Classification

_____no_output_____
K nearest neighbor (KNN)
_____no_output_____#### Import library _____no_output_____Classifier implementing the k-nearest neighbors vote._____no_output_____

from sklearn.neighbors import KNeighborsClassifier_____no_output_____

### Training

Lets start the algorithm with k=4 for now:_____no_output_____

k = 4
#Train Model and Predict
neigh = KNeighborsClassifier(n_neighbors = k).fit(X_train,y_train)
neigh_____no_output_____

### Predicting
we can use the model to predict the test set:_____no_output_____

yhat = neigh.predict(X_test)
yhat[0:5]_____no_output_____

### Accuracy evaluation
In multilabel classification, __accuracy classification score__ is a function that computes subset accuracy. This function is equal to the jaccard_similarity_score function. Essentially, it calculates how closely the actual labels and predicted labels are matched in the test set._____no_output_____

from sklearn import metrics
print("Train set Accuracy: ", metrics.accuracy_score(y_train, neigh.predict(X_train)))
print("Test set Accuracy: ", metrics.accuracy_score(y_test, yhat))Train set Accuracy:  0.5475
Test set Accuracy:  0.32

## Practice
Can you build the model again, but this time with k=6?_____no_output_____

# write your code here

k = 6
neigh6 = KNeighborsClassifier(n_neighbors = k).fit(X_train,y_train)
yhat6 = neigh6.predict(X_test)
print("Train set Accuracy: ", metrics.accuracy_score(y_train, neigh6.predict(X_train)))
print("Test set Accuracy: ", metrics.accuracy_score(y_test, yhat6))
Train set Accuracy:  0.51625
Test set Accuracy:  0.31

Double-click __here__ for the solution.

_____no_output_____#### What about other K?
K in KNN, is the number of nearest neighbors to examine. It is supposed to be specified by the User. So, how can we choose right value for K?
The general solution is to reserve a part of your data for testing the accuracy of the model. Then chose k =1, use the training part for modeling, and calculate the accuracy of prediction using all samples in your test set. Repeat this process, increasing the k, and see which k is the best for your model.

We can calculate the accuracy of KNN for different Ks._____no_output_____

Ks = 10
mean_acc = np.zeros((Ks-1))
std_acc = np.zeros((Ks-1))
ConfustionMx = [];
for n in range(1,Ks):

    #Train Model and Predict
    neigh = KNeighborsClassifier(n_neighbors = n).fit(X_train,y_train)
    yhat=neigh.predict(X_test)
    mean_acc[n-1] = metrics.accuracy_score(y_test, yhat)

    std_acc[n-1]=np.std(yhat==y_test)/np.sqrt(yhat.shape[0])

mean_acc_____no_output_____

#### Plot  model accuracy  for Different number of Neighbors _____no_output_____

plt.plot(range(1,Ks),mean_acc,'g')
plt.fill_between(range(1,Ks),mean_acc - 1 * std_acc,mean_acc + 1 * std_acc, alpha=0.10)
plt.legend(('Accuracy ', '+/- 3xstd'))
plt.ylabel('Accuracy ')
plt.xlabel('Number of Nabors (K)')
plt.tight_layout()
plt.show()_____no_output_____print( "The best accuracy was with", mean_acc.max(), "with k=", mean_acc.argmax()+1) The best accuracy was with 0.34 with k= 9

Want to learn more?

IBM SPSS Modeler is a comprehensive analytics platform that has many machine learning algorithms. It has been designed to bring predictive intelligence to decisions made by individuals, by groups, by systems – by your enterprise as a whole. A free trial is available through this course, available here: SPSS Modeler

Also, you can use Watson Studio to run these notebooks faster with bigger datasets. Watson Studio is IBM's leading cloud solution for data scientists, built by data scientists. With Jupyter notebooks, RStudio, Apache Spark and popular libraries pre-packaged in the cloud, Watson Studio enables data scientists to collaborate on their projects without having to install anything. Join the fast-growing community of Watson Studio users today with a free account at Watson Studio

Thanks for completing this lesson!

Author:  Saeed Aghabozorgi

Saeed Aghabozorgi, PhD is a Data Scientist in IBM with a track record of developing enterprise level applications that substantially increases clients’ ability to turn data into actionable knowledge. He is a researcher in data mining field and expert in developing advanced analytic methods like machine learning and statistical modelling on large datasets.

Copyright © 2018 Cognitive Class. This notebook and its source code are released under the terms of the MIT License.
_____no_output_____


# Notebook from jbkinney/mave-nn
Path: docs/datasets/dataset_gb1.ipynb

# gb1 dataset_____no_output_____

# Standard imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Special imports
import mavenn_____no_output_____

## Summary

The DMS dataset from Olson et al., 2014. The authors used an RNA display selection experiment to assay the binding of over half a million protein GB1 variants to IgG. These variants included all 1-point and 2-point mutations within the 55 residue GB1 sequence. Only the 2-point variants are included in this dataset.

**Name:** ``'gb1'``

**Reference**: Olson C, Wu N, Sun R. A comprehensive biophysical description of pairwise epistasis throughout an entire protein domain. [Curr Biol 24(22):2643-2651 (2014).](https://pubmed.ncbi.nlm.nih.gov/25455030/)_____no_output_____

mavenn.load_example_dataset('gb1')_____no_output_____

## Preprocessing

First we load the double-mutation dataset published by Olson et al. (2021).
_____no_output_____

# Dataset is is at this URL:
# url = 'https://ars.els-cdn.com/content/image/1-s2.0-S0960982214012688-mmc2.xlsx'

# We have downloaded this Excel file and reformatted it into a more parseable form
raw_data_file = '../../mavenn/examples/datasets/raw/gb1_raw.xlsx'

# Load data (takes a while)
double_mut_df = pd.read_excel(raw_data_file, sheet_name='double_mutants')
double_mut_df_____no_output_____

Next we reconstruct the wild-type GB1 sequence_____no_output_____

# Get unique WT pos-aa associations, sorted by position
wt_1_df = double_mut_df[['Mut1 Position', 'Mut1 WT amino acid']].copy()
wt_1_df.columns = ['pos','aa']
wt_2_df = double_mut_df[['Mut2 Position', 'Mut2 WT amino acid']].copy()
wt_2_df.columns = ['pos','aa']
wt_seq_df = pd.concat([wt_1_df, wt_2_df], axis=0).drop_duplicates().sort_values(by='pos').reset_index(drop=True)

# Confirm that each position occurs at most once
assert np.all(wt_seq_df['pos'].value_counts()==1)

# Confirm that the set of unique positions is correct
L = len(wt_seq_df)
assert set(wt_seq_df['pos'].values) == set(range(2,L+2))

# Compute wt_seq and confirm its identity
wt_seq = ''.join(wt_seq_df['aa'])
known_wt_seq = 'QYKLILNGKTLKGETTTEAVDAATAEKVFKQYANDNGVDGEWTYDDATKTFTVTE'
assert wt_seq == known_wt_seq

# Print final wt sequence
print(f'WT sequence: {wt_seq}')WT sequence: QYKLILNGKTLKGETTTEAVDAATAEKVFKQYANDNGVDGEWTYDDATKTFTVTE

Next we convert the list of mutations to an array `x` of variant sequences._____no_output_____

# Introduce double mutations into WT sequence and append to a growing list
pos1s = double_mut_df['Mut1 Position'].values-2
pos2s = double_mut_df['Mut2 Position'].values-2
aa1s = double_mut_df['Mut1 Mutation'].values
aa2s = double_mut_df['Mut2 Mutation'].values
x_list = []
for pos1, aa1, pos2, aa2 in zip(pos1s, aa1s, pos2s, aa2s):
    mut_seq_list = list(wt_seq)
    mut_seq_list[pos1] = aa1
    mut_seq_list[pos2] = aa2
    mut_seq = ''.join(mut_seq_list)
    x_list.append(mut_seq)

# Convert list to an np.array and preview it
x = np.array(x_list)
x_____no_output_____

We then compute the log2 enrichment `y` for each variant sequence in `x`._____no_output_____

# Extract input and output counts
in_ct = double_mut_df['Input Count'].values
out_ct = double_mut_df['Selection Count'].values

# Read in wt data (on separate sheet) and compute baseline for y
wt_df = pd.read_excel(raw_data_file, sheet_name='wild_type')
wt_in_ct = wt_df['Input Count'][0]
wt_out_ct = wt_df['Selection Count'][0]

# Compute log2 enrichment values relative to WT
y = np.log2((out_ct+1)/(in_ct+1)) - np.log2((wt_out_ct+1)/(wt_in_ct+1))
y_____no_output_____

Next we assign each sequence to the training, validation, or test set
using a split of 90%:5%:5%._____no_output_____

# Assign to training, validation, or test set
np.random.seed(0)
sets = np.random.choice(a=['training', 'validation', 'test'],
                        p=[.90,.05,.05],
                        size=len(x))
sets_____no_output_____

Finally we assemble all relevant information into a dataframe and save._____no_output_____

# Assemble into dataframe
final_df = pd.DataFrame({'set':sets, 'dist':2, 'input_ct':in_ct, 'selected_ct':out_ct, 'y':y, 'x':x})

# Keep only sequences with input_ct >= 10
final_df = final_df[final_df['input_ct']>=10].reset_index(drop=True)

# Save to file (uncomment to execute)
# final_df.to_csv('gb1_data.csv.gz', index=False, compression='gzip')

# Preview dataframe
final_df_____no_output_____


# Notebook from tbenthompson/tectosaur2
Path: tutorials/c1qbx/part7_qd_two_faults.ipynb

from tectosaur2.nb_config import setup

setup()_____no_output_____

# Quasidynamic earthquake cycles on two faults

## Summary

* Much like the last section, we simulate quasidynamic rupture evolution but this time on two faults, one of which is curved.
* Most of the code here follows almost exactly from [the previous section on strike-slip/antiplane earthquake cycles](part6_qd).
* Realistic tectonic loading is a challenge when there is more than one fault.
* The boundary integral approach used here is barely more complex with two faults than with one fault.

## Loading a multi-fault system

In the single fault system from the last section, we were able to gloss over loading by using a slip deficit or backslip formulation. Because the physical system is invariant with respect to a rigid body displacement, any tectonic motion that has already been fully accomodated by fault slip can be ignored. Put differently, in a single fault system with a purely elastic rheology, the stresses from tectonic motion can be equivalently described by backwards slip on the fault.

\begin{equation}
u_{\textrm{total}} = u_{\textrm{loading}} + u_{\textrm{fault}}
\end{equation}

There are a number of different options for loading a fault system:
- **backslip**: as discussed, we simply apply a backwards slip rate to account for distant plate motion. Once the stresses from this backwards slip accumulate, the fault slips forward.
- **farfield displacement**: apply a constant velocity boundary condition at a distance from the fault.
- **stress**: Either on a lower boundary of the domain or on distant side boundaries, apply a constant traction/stress loading rate.
- **block model**: this is a more advanced version of backslip loading where each fault-bounded subregion of the domain has its own block rotation/motion rate. These rates are assumed to be the long term slip rate. A corresponding kinematic slip deficit is applied to each fault.

Here, I do something a bit like the block model solution by just applying a backslip of 31 mm of loading per year (1 nm of loading per second) to both faults equally. This would model a real-world situation with a plate boundary that has 62 mm per year of strike-slip motion. Exactly 50% of that motion is accomodated by each of the two faults._____no_output_____

import sympy as sp
import numpy as np
import matplotlib.pyplot as plt
from tectosaur2 import (
    gauss_rule,
    integrate_term,
    panelize_symbolic_surface,
    concat_meshes,
)
from tectosaur2.laplace2d import double_layer, hypersingular
_____no_output_____shear_modulus = 3.2e10

quad_rule = gauss_rule(5)
t = sp.var("t")

fault_bottom = 40000
fault1 = panelize_symbolic_surface(
    t, -5000 + t * 0, fault_bottom * (t + 1) * -0.5,
    quad_rule,
    n_panels=200
)

fault2_x = 5000 + (t * 10000 + 10000) + (t ** 2 * 5000 - 5000)
fault2 = panelize_symbolic_surface(
    t, fault2_x, fault_bottom * (t + 1) * -0.5,
    quad_rule,
    n_panels=200
)
faults = concat_meshes((fault1, fault2))

surf_half_L = 100000
free = panelize_symbolic_surface(
    t, -t * surf_half_L, 0 * t,
    quad_rule,
    n_panels=200
)_____no_output_____print(
    f"The free surface mesh has {free.n_panels} panels with a total of {free.n_pts} points."
)
print(
    f"The fault mesh has {faults.n_panels} panels with a total of {faults.n_pts} points."
)The free surface mesh has 200 panels with a total of 1000 points.
The fault mesh has 400 panels with a total of 2000 points.
plt.plot(free.pts[:,0]/1000, free.pts[:,1]/1000, 'k-')
plt.plot(fault1.pts[:,0]/1000, fault1.pts[:,1]/1000, 'r-')
plt.plot(fault2.pts[:,0]/1000, fault2.pts[:,1]/1000, 'r-')
plt.xlabel(r'$x ~ \mathrm{(km)}$')
plt.ylabel(r'$y ~ \mathrm{(km)}$')
plt.axis('scaled')
plt.xlim([-100, 100])
plt.ylim([-80, 20])
plt.show()_____no_output_____singularities = np.array(
    [
        [-surf_half_L, 0],
        [surf_half_L, 0],
        [5000, 0],
        [-5000, 0]
    ]
)

(free_disp_to_free_disp, fault_slip_to_free_disp), report = integrate_term(
    double_layer, free.pts, free, faults, tol=1e-11, singularities=singularities, return_report=True
)
fault_slip_to_free_disp = fault_slip_to_free_disp[:, 0, :, 0]
free_disp_solve_mat = (
    np.eye(free_disp_to_free_disp.shape[0]) + free_disp_to_free_disp[:, 0, :, 0]
)_____no_output_____(free_disp_to_fault_stress, fault_slip_to_fault_stress), report = integrate_term(
    hypersingular,
    faults.pts,
    free,
    faults,
    tol=1e-10,
    singularities=singularities,
    return_report=True,
)
fault_slip_to_fault_stress *= shear_modulus
free_disp_to_fault_stress *= shear_modulus

fault_slip_to_fault_traction = np.sum(
    fault_slip_to_fault_stress[:,:,:,0] * faults.normals[:, :, None], axis=1
)
free_disp_to_fault_traction = np.sum(
    free_disp_to_fault_stress[:,:,:,0] * faults.normals[:, :, None], axis=1
)/Users/tbent/Dropbox/active/eq/tectosaur2/tectosaur2/integrate.py:208: UserWarning: Some expanded integrals reached maximum expansion order. These integrals may be inaccurate.
  warnings.warn(
A = -fault_slip_to_fault_traction
B = -free_disp_to_fault_traction
C = fault_slip_to_free_disp
Dinv = np.linalg.inv(free_disp_solve_mat)

total_fault_slip_to_fault_traction = A - B.dot(Dinv.dot(C))_____no_output_____analytical_sd = -np.arctan(-fault_bottom / free.pts[:,0]) / np.pi
numerical_sd = Dinv.dot(-C).sum(axis=1)
rp = (faults.pts[:,1] + 1) ** 2
analytical_ft = -(shear_modulus / (2 * np.pi)) * ((1.0 / (faults.pts[:,1] + fault_bottom)) - (1.0 / (faults.pts[:,1] - fault_bottom)))
numerical_ft = total_fault_slip_to_fault_traction.sum(axis=1)
plt.figure(figsize=(10,5))
plt.subplot(1,2,1)
plt.title('Stress')
plt.plot(faults.pts[:,1], np.log10(np.abs(numerical_ft)), 'b-')
plt.xlim([-fault_bottom, 0])
plt.xlabel('$\mathrm{depth ~~ (m)}$')
plt.ylabel('$t$')
plt.subplot(1,2,2)
plt.title('Displacement')
plt.plot(free.pts[:,0], numerical_sd, 'b-')
plt.xlim([-100000, 100000])
plt.xlabel('$x ~~ \mathrm{(m)}$')
plt.ylabel('$u$')
plt.show()_____no_output_____

## Rate and state friction_____no_output_____

siay = 31556952 # seconds in a year
density = 2670  # rock density (kg/m^3)
cs = np.sqrt(shear_modulus / density)  # Shear wave speed (m/s)
eta = shear_modulus / (2 * cs)  # The radiation damping coefficient (kg / (m^2 * s))
Vp = 1e-9  # Rate of plate motion
sigma_n = 50e6  # Normal stress (Pa)

# parameters describing "a", the coefficient of the direct velocity strengthening effect
a0 = 0.01
amax = 0.025
fy = faults.pts[:, 1]
H = 15000
h = 3000
fp_a = np.where(
    fy > -H, a0, np.where(fy > -(H + h), a0 + (amax - a0) * (fy + H) / -h, amax)
)

# state-based velocity weakening effect
fp_b = 0.015

# state evolution length scale (m)
fp_Dc = 0.008

# baseline coefficient of friction
fp_f0 = 0.6

# if V = V0 and at steady state, then f = f0, units are m/s
fp_V0 = 1e-6_____no_output_____plt.figure(figsize=(3, 5))
plt.plot(fp_a, fy/1000, label='a')
plt.plot(np.full(fy.shape[0], fp_b), fy/1000, label='b')
plt.xlim([0, 0.03])
plt.ylabel('depth')
plt.legend()
plt.show()_____no_output_____# The regularized form of the aging law.
def aging_law(fp_a, V, state):
    return (fp_b * fp_V0 / fp_Dc) * (np.exp((fp_f0 - state) / fp_b) - (V / fp_V0))

def qd_equation(fp_a, shear_stress, V, state):
    # The regularized rate and state friction equation
    F = sigma_n * fp_a * np.arcsinh(V / (2 * fp_V0) * np.exp(state / fp_a))

    # The full shear stress balance:
    return shear_stress - eta * V - F

def qd_equation_dV(fp_a, V, state):
    # First calculate the derivative of the friction law with respect to velocity
    # This is helpful for equation solving using Newton's method
    expsa = np.exp(state / fp_a)
    Q = (V * expsa) / (2 * fp_V0)
    dFdV = fp_a * expsa * sigma_n / (2 * fp_V0 * np.sqrt(1 + Q * Q))

    # The derivative of the full shear stress balance.
    return -eta - dFdV

def solve_friction(fp_a, shear, V_old, state, tol=1e-10):
    V = V_old
    max_iter = 150
    for i in range(max_iter):
        # Newton's method step!
        f = qd_equation(fp_a, shear, V, state)
        dfdv = qd_equation_dV(fp_a, V, state)
        step = f / dfdv

        # We know that slip velocity should not be negative so any step that
        # would lead to negative slip velocities is "bad". In those cases, we
        # cut the step size in half iteratively until the step is no longer
        # bad. This is a backtracking line search.
        while True:
            bad = step > V
            if np.any(bad):
                step[bad] *= 0.5
            else:
                break

        # Take the step and check if everything has converged.
        Vn = V - step
        if np.max(np.abs(step) / Vn) < tol:
            break
        V = Vn
        if i == max_iter - 1:
            raise Exception("Failed to converge.")

    # Return the solution and the number of iterations required.
    return Vn, i_____no_output_____for fm in [fault1, fault2]:
    mesh_L = np.max(np.abs(np.diff(fm.pts[:, 1])))
    Lb = shear_modulus * fp_Dc / (sigma_n * fp_b)
    hstar = (np.pi * shear_modulus * fp_Dc) / (sigma_n * (fp_b - fp_a))
    print(mesh_L, Lb, np.min(hstar[hstar > 0]))53.84693101057201 341.3333333333333 3216.990877275949
53.84693101057201 341.3333333333333 3216.990877275949

## Quasidynamic earthquake cycle derivatives_____no_output_____

from scipy.optimize import fsolve
import copy

init_state_scalar = fsolve(lambda S: aging_law(fp_a, Vp, S), 0.7)[0]
init_state = np.full(faults.n_pts, init_state_scalar)
tau_amax = -qd_equation(amax, 0, Vp, init_state_scalar)

init_traction = np.full(faults.n_pts, tau_amax)
init_slip_deficit = np.zeros(faults.n_pts)
init_conditions = np.concatenate((init_slip_deficit, init_state))_____no_output_____class SystemState:

    V_old = np.full(faults.n_pts, Vp)
    state = None

    def calc(self, t, y, verbose=False):
        # Separate the slip_deficit and state sub components of the
        # time integration state.
        slip_deficit = y[: init_slip_deficit.shape[0]]
        state = y[init_slip_deficit.shape[0] :]

        # If the state values are bad, then the adaptive integrator probably
        # took a bad step.
        if np.any((state < 0) | (state > 2.0)):
            return False

        # The big three lines solving for quasistatic shear stress, slip rate
        # and state evolution
        tau_qs = init_traction - total_fault_slip_to_fault_traction.dot(slip_deficit)
        V = solve_friction(fp_a, tau_qs, self.V_old, state)[0]
        dstatedt = aging_law(fp_a, V, state)
        if not np.all(np.isfinite(V)):
            return False
        self.V_old = V

        slip_deficit_rate = Vp - V
        out = slip_deficit, state, tau_qs, V, slip_deficit_rate, dstatedt
        self.state = out
        return self.state

def plot_system_state(SS):
    """This is just a helper function that creates some rough plots of the
    current state to help with debugging"""
    slip_deficit, state, tau_qs, slip_deficit_rate, dstatedt = SS

    plt.figure(figsize=(15, 9))
    plt.subplot(2, 3, 1)
    plt.title("slip")
    plt.plot(faults.pts[:, 1], slip_deficit)

    plt.subplot(2, 3, 2)
    plt.title("state")
    plt.plot(faults.pts[:, 1], state)

    plt.subplot(2, 3, 3)
    plt.title("tau qs")
    plt.plot(faults.pts[:, 1], tau_qs)

    plt.subplot(2, 3, 4)
    plt.title("slip rate")
    plt.plot(faults.pts[:, 1], slip_deficit_rate)

    plt.subplot(2, 3, 6)
    plt.title("dstatedt")
    plt.plot(faults.pts[:, 1], dstatedt)
    plt.tight_layout()

    plt.show()

def calc_derivatives(state, t, y):
    """
    This helper function calculates the system state and then extracts the
    relevant derivatives that the integrator needs. It also intentionally
    returns infinite derivatives when the `y` vector provided by the integrator
    is invalid.
    """
    if not np.all(np.isfinite(y)):
        return np.inf * y
    state = state.calc(t, y)
    if not state:
        return np.inf * y
    derivatives = np.concatenate((state[-2], state[-1]))
    return derivatives_____no_output_____

## Integrating through time_____no_output_____

%%time
from scipy.integrate import RK45

# We use a 5th order adaptive Runge Kutta method and pass the derivative function to it
# the relative tolerance will be 1e-11 to make sure that even
state = SystemState()
derivs = lambda t, y: calc_derivatives(state, t, y)
integrator = RK45
atol = Vp * 1e-6
rtol = 1e-11
rk = integrator(derivs, 0, init_conditions, 1e50, atol=atol, rtol=rtol)

# Set the initial time step to one day.
rk.h_abs = 60 * 60 * 24

# Integrate for 1000 years.
max_T = 1000 * siay

n_steps = 1000000
t_history = [0]
y_history = [init_conditions.copy()]
for i in range(n_steps):

    # Take a time step and store the result
    if rk.step() != None:
        raise Exception("TIME STEPPING FAILED")
        break
    t_history.append(rk.t)
    y_history.append(rk.y.copy())

    # Print the time every 5000 steps
    if i % 5000 == 0:
        print(f"step={i}, time={rk.t / siay} yrs, step={(rk.t - t_history[-2]) / siay}")
    if rk.t > max_T:
        break

y_history = np.array(y_history)
t_history = np.array(t_history)step=0, time=0.002737907006988508 yrs, step=0.002737907006988508
step=5000, time=161.2388012289919 yrs, step=6.068323795478569e-11
step=10000, time=161.238801483585 yrs, step=3.200375945922213e-11
step=15000, time=177.06735503165402 yrs, step=0.05782401363757997
step=20000, time=187.60702659014385 yrs, step=8.29256996752649e-11
step=25000, time=187.6070269264095 yrs, step=2.786049654906221e-10
step=30000, time=213.4153146695212 yrs, step=1.3348499105890855e-10
step=35000, time=229.49165911338014 yrs, step=1.8458825569115088e-10
step=40000, time=298.5155537603046 yrs, step=1.1030568652139828e-10
step=45000, time=298.51555399934733 yrs, step=3.251751197178755e-11
step=50000, time=298.5168884817506 yrs, step=6.51359998844683e-06
step=55000, time=323.01280183052285 yrs, step=1.2281707123916784e-10
step=60000, time=323.01280207887106 yrs, step=3.3484575524851865e-11
step=65000, time=323.2817940109216 yrs, step=0.001729718400292269
step=70000, time=351.52626665410605 yrs, step=1.0785478482910092e-08
step=75000, time=368.9516542294202 yrs, step=1.2976784052681758e-10
step=80000, time=368.951654876993 yrs, step=6.515590688770813e-11
step=85000, time=428.9915445603336 yrs, step=0.0014206634043130257
step=90000, time=446.2939275656298 yrs, step=6.588120455250637e-11
step=95000, time=446.29392801977554 yrs, step=9.456068304806992e-10
step=100000, time=469.8994523682489 yrs, step=1.0256917809688377e-10
step=105000, time=469.89945268723017 yrs, step=5.1798341561007304e-11
step=110000, time=469.9812354270467 yrs, step=0.00034767164051762437
step=115000, time=500.90614938849876 yrs, step=1.439715864624497e-10
step=120000, time=573.9064649517662 yrs, step=1.424001081887202e-10
step=125000, time=573.9064651906277 yrs, step=2.5022769435539118e-11
step=130000, time=597.5674917048766 yrs, step=0.0012973735857483035
step=135000, time=597.6230358626846 yrs, step=5.391379308333549e-11
step=140000, time=597.6230366893858 yrs, step=4.587749495737104e-09
step=145000, time=625.2112327038584 yrs, step=1.1906969997104363e-10
step=150000, time=642.3455004578122 yrs, step=1.2898210138995284e-10
step=155000, time=704.8417989867936 yrs, step=0.003113930973116129
step=160000, time=716.3612963754118 yrs, step=2.9978970144993726e-11
step=165000, time=716.3612966987445 yrs, step=3.9843018386249724e-10
step=170000, time=743.0069768079197 yrs, step=1.6222491102653862e-10
step=175000, time=743.0069772376432 yrs, step=4.6660816435353134e-11
step=180000, time=743.0070823319467 yrs, step=6.836717438689674e-07
step=185000, time=772.4229461837367 yrs, step=1.1205848921132735e-10
step=190000, time=790.5550252030326 yrs, step=9.598105764163313e-11
step=195000, time=790.5550257931098 yrs, step=9.694812119469745e-11
step=200000, time=790.5770082962302 yrs, step=0.00011880289499414454
step=205000, time=843.4093589520743 yrs, step=1.3986156636192637e-10
step=210000, time=843.4093595399053 yrs, step=5.7540281407326665e-11
step=215000, time=865.8601485323139 yrs, step=4.5113514750450234e-10
step=220000, time=903.1263077569464 yrs, step=1.2438854951289735e-10
step=225000, time=903.1263082887026 yrs, step=8.413452911659529e-11
step=230000, time=927.4523165504868 yrs, step=0.008045238903279458
step=235000, time=947.0642174585033 yrs, step=1.7528026899290684e-11
step=240000, time=947.0685830652857 yrs, step=1.768826485736451e-05
step=245000, time=972.5636479272499 yrs, step=4.078590535048743e-10
step=250000, time=988.09794803525 yrs, step=8.195863612220059e-11
step=255000, time=988.0979484325295 yrs, step=2.7077779485800782e-11
CPU times: user 2h 15min 3s, sys: 36min 52s, total: 2h 51min 55s
Wall time: 1h 48min 42s

## Plotting the results_____no_output_____

derivs_history = np.diff(y_history, axis=0) / np.diff(t_history)[:, None]
max_vel = np.max(np.abs(derivs_history), axis=1)
plt.plot(t_history[1:] / siay, np.log10(max_vel))
plt.xlabel('$t ~~ \mathrm{(yrs)}$')
plt.ylabel('$\log_{10}(V)$')
plt.show()_____no_output_____for j in range(2):
    fm = fault1 if j == 0 else fault2
    plt.figure(figsize=(10, 4))
    last_plt_t = -1000
    last_plt_slip = init_slip_deficit
    event_times = []
    for i in range(len(y_history) - 1):
        y = y_history[i]
        t = t_history[i]
        slip_deficit_full = y[: init_slip_deficit.shape[0]]
        fault1_sd = slip_deficit_full[:fault1.n_pts]
        fault2_sd = slip_deficit_full[fault1.n_pts:]
        slip_deficit = fault1_sd if j == 0 else fault2_sd
        should_plot = False

        # Plot a red line every three second if the slip rate is over 0.1 mm/s.
        if (
            max_vel[i] >= 0.0001 and t - last_plt_t > 3
        ):
            if len(event_times) == 0 or t - event_times[-1] > siay:
                event_times.append(t)
            should_plot = True
            color = "r"

        # Plot a blue line every fifteen years during the interseismic period
        if t - last_plt_t > 15 * siay:
            should_plot = True
            color = "b"

        if should_plot:
            # Convert from slip deficit to slip:
            slip = -slip_deficit + Vp * t
            plt.plot(slip, fm.pts[:,1] / 1000.0, color + "-", linewidth=0.5)
            last_plt_t = t
            last_plt_slip = slip
    plt.xlim([0, np.max(last_plt_slip)])
    plt.ylim([-40, 0])
    plt.ylabel(r"$\textrm{z (km)}$")
    plt.xlabel(r"$\textrm{slip (m)}$")
    plt.tight_layout()
    plt.savefig("halfspace.png", dpi=300)
    plt.show()_____no_output_____


# Notebook from VCMason/PyGenToolbox
Path: notebooks/Lyna/RNADNAContacts_Basic.ipynb

<code>
#sn = Species name
sn = 'Paramecium tetraurelia'
#full path to RNA sam file
fRNA = 'D:\\LinuxShare\\Projects\\Lyna\\Hisat2\\Pt_51_Mac\\500_L1_R1R2.trim.AssUnFUnR.rna.sam'
# full path for DNA sam file
fDNA = 'D:\\LinuxShare\\Projects\\Lyna\\Hisat2\\Pt_51_Mac\\500_L1_R1R2.trim.AssUnFUnR.dna.sam'
# MQ = minimum mapping quality
MQ = 10

main(fRNA, fDNA, MQ, sn)Finished
Finished2
Finished3

</code>