# Notebook from ajlee21/generic-expression-patterns
Path: human_general_array_analysis/2_identify_generic_genes_pathways.ipynb

# Identify generic genes and pathways

Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These generic genes and subsequent pathways are not necessarily specific to the biological process being studied but instead represent a more systematic change.

This notebook identifies generic genes and pathways and then evaluates if those identified are consistent with published findings.

**Steps to identify generic genes:**
1. Simulates N gene expression experiments using [ponyo](https://github.com/ajlee21/ponyo)
2. Perform DE analysis to get association statistics for each gene
3. For each gene, aggregate statsitics across all simulated experiments
4. Rank genes based on this aggregated statistic (i.e. log fold change, or p-value)

**Steps to identify generic gene sets (pathways):**
1. Using the same simulated experiments from above, perform GSEA analysis. This analysis will determine whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list of genes, where genes are ranked by log fold change, indicating a correlation with change in expression.
2. For each gene set (pathway), aggregate statistics across all simulated experiments
3. Rank gene sets based on this aggregated statistic

**Evaluation:**
* We want to compare the ranking of genes identified using the above method with the ranking found from [Crow et. al.](https://www.pnas.org/content/pnas/116/13/6491.full.pdf), which identified a set of genes as generic based on how frequently they were found to be DE across 600 experiments
* We want to compare the ranking of pathways identified using the above method with the ranking based on the [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) data, where ranking was determined based on the fraction of 442 experiments a pathway was found to be enriched
* This comparison will validate our method being used as a way to automatically identify generic genes and pathways._____no_output_____

%load_ext autoreload
%load_ext rpy2.ipython
%autoreload 2
%matplotlib inline

import os
import sys
import pandas as pd
import numpy as np
import pickle
import glob
import scipy.stats as ss
from keras.models import load_model
from rpy2.robjects import pandas2ri
from ponyo import utils, simulate_expression_data
from generic_expression_patterns_modules import process, stats, ranking

pandas2ri.activate()

np.random.seed(123)Using TensorFlow backend.
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/matplotlib/__init__.py:886: MatplotlibDeprecationWarning:
examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.
  "found relative to the 'datapath' directory.".format(key))
# Read in config variables
base_dir = os.path.abspath(os.path.join(os.getcwd(), "../"))

config_filename = os.path.abspath(
    os.path.join(base_dir, "configs", "config_human_Crow.tsv")
)

params = utils.read_config(config_filename)_____no_output_____# Load params
local_dir = params["local_dir"]
dataset_name = params["dataset_name"]
NN_architecture = params["NN_architecture"]
latent_dim = params["latent_dim"]
num_runs = params["num_simulated"]
project_id = params["project_id"]
metadata_col_id = params["metadata_colname"]
mapped_template_filename = params["mapped_template_filename"]
processed_template_filename = params["processed_template_filename"]
normalized_compendium_filename = params["normalized_compendium_filename"]
scaler_filename = params["scaler_filename"]
col_to_rank_genes = params["rank_genes_by"]
col_to_rank_pathways = params["rank_pathways_by"]
statistic = params["gsea_statistic"]
logFC_name = params["DE_logFC_name"]
pvalue_name = params["DE_pvalue_name"]

# Load metadata file with grouping assignments for samples
sample_id_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_process_samples.tsv"
)

# Load metadata file with grouping assignments for samples
grp_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_groups.tsv"
)

# Load metadata file with mapping between experiments and associated samples
metadata_simulate_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "experiment_sample_annotations.csv"
)
metadata_delimiter = ","
experiment_id_colname = "Experiment id"

# Load pickled file
with open(scaler_filename, "rb") as scaler_fh:
    scaler = pickle.load(scaler_fh)

# Percentile threshold to identify generic genes
percentile_threshold = 80.0_____no_output_____# Output files
gene_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_gene_summary_{project_id}.tsv"
)

pathway_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_pathway_summary_{project_id}.tsv"
)_____no_output_____

### Simulate experiments using selected template experiment

Workflow:
1. Get the gene expression data for the selected template experiment
2. Encode this experiment into a latent space using the trained VAE model
3. Linearly shift the encoded template experiment in the latent space
4. Decode the samples. This results in a new experiment
5. Repeat steps 1-4 to get multiple simulated experiments_____no_output_____

# Simulate multiple experiments
# This step creates the following files in "<local_dir>/pseudo_experiment/" directory:
#   - selected_simulated_data_SRP012656_<n>.txt
#   - selected_simulated_encoded_data_SRP012656_<n>.txt
#   - template_normalized_data_SRP012656_test.txt
# in which "<n>" is an integer in the range of [0, num_runs-1]
os.makedirs(os.path.join(local_dir, "pseudo_experiment"), exist_ok=True)
simulate_expression_data.shift_template_experiment(
    normalized_compendium_filename,
    NN_architecture,
    latent_dim,
    dataset_name,
    scaler,
    metadata_simulate_filename,
    metadata_delimiter,
    experiment_id_colname,
    metadata_col_id,
    project_id,
    local_dir,
    base_dir,
    num_runs,
)WARNING:tensorflow:From /home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

## Reverse transformation_____no_output_____

# Undo log transformation...
# CHeck that template is also in log or unlogged form
# Or log transform mapped_template instead
mapped_template = pd.read_csv(mapped_template_filename, sep="\t", index_col=0, header=0)

mapped_template_transform = (
    np.log10(mapped_template).replace(-np.inf, 0.0).replace(np.nan, 0.0)
)

mapped_template_transform.to_csv(mapped_template_filename, sep="\t")/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning: invalid value encountered in log10
  import sys

### Process template and simulated experiments

* Remove samples not required for comparison. Since this experiment contains multiple conditions (i.e. estradiol vs EtOH at 12, 24, and 48 hrs are each considered a different comparison) being tested, we will only include those samples within the same condition.
* Make sure ordering of samples matches metadata for proper comparison_____no_output_____

if not os.path.exists(sample_id_metadata_filename):
    sample_id_metadata_filename = None

stats.process_samples_for_limma(
    mapped_template_filename,
    grp_metadata_filename,
    processed_template_filename,
    sample_id_metadata_filename,
)

for i in range(num_runs):
    simulated_filename = os.path.join(
        local_dir, "pseudo_experiment", f"selected_simulated_data_{project_id}_{i}.txt"
    )
    out_simulated_filename = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_data_{project_id}_{i}_processed.txt",
    )
    stats.process_samples_for_limma(
        simulated_filename,
        grp_metadata_filename,
        out_simulated_filename,
        sample_id_metadata_filename,
    )sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly
sample ids are ordered correctly

### Differential expression analysis

The gene expression dataset is array-based so we will use Limma in this case_____no_output_____

# Create subdirectory: "<local_dir>/DE_stats/"
os.makedirs(os.path.join(local_dir, "DE_stats"), exist_ok=True)_____no_output_____%%R -i grp_metadata_filename -i project_id -i processed_template_filename -i local_dir -i base_dir

source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))

# File created: "<local_dir>/DE_stats/DE_stats_template_data_SRP012656_real.txt"
get_DE_stats_limma(grp_metadata_filename,
                   project_id,
                   processed_template_filename,
                   "template",
                   local_dir,
                   "real")R[write to console]: Loading required package: S4Vectors

R[write to console]: Loading required package: stats4

R[write to console]: Loading required package: BiocGenerics

R[write to console]: Loading required package: parallel

R[write to console]:
Attaching package: ‘BiocGenerics’

R[write to console]: The following objects are masked from ‘package:parallel’:

    clusterApply, clusterApplyLB, clusterCall, clusterEvalQ,
    clusterExport, clusterMap, parApply, parCapply, parLapply,
    parLapplyLB, parRapply, parSapply, parSapplyLB

R[write to console]: The following object is masked from ‘package:limma’:

    plotMA

R[write to console]: The following objects are masked from ‘package:stats’:

    IQR, mad, sd, var, xtabs

R[write to console]: The following objects are masked from ‘package:base’:

    anyDuplicated, append, as.data.frame, basename, cbind, colnames,
    dirname, do.call, duplicated, eval, evalq, Filter, Find, get, grep,
    grepl, intersect, is.unsorted, lapply, Map, mapply, match, mget,
    order, paste, pmax, pmax.int, pmin, pmin.int, Position, rank,
    rbind, Reduce, rownames, sapply, setdiff, sort, table, tapply,
    union, unique, unsplit, which, which.max, which.min

R[write to console]:
Attaching package: ‘S4Vectors’

R[write to console]: The following object is masked from ‘package:base’:

    expand.grid

R[write to console]: Loading required package: IRanges

R[write to console]: Loading required package: GenomicRanges

R[write to console]: Loading required package: GenomeInfoDb

R[write to console]: Loading required package: SummarizedExperiment

R[write to console]: Loading required package: Biobase

R[write to console]: Welcome to Bioconductor

    Vignettes contain introductory material; view with
    'browseVignettes()'. To cite Bioconductor, see
    'citation("Biobase")', and for packages 'citation("pkgname")'.

R[write to console]: Loading required package: DelayedArray

R[write to console]: Loading required package: matrixStats

R[write to console]:
Attaching package: ‘matrixStats’

R[write to console]: The following objects are masked from ‘package:Biobase’:

    anyMissing, rowMedians

R[write to console]: Loading required package: BiocParallel

R[write to console]:
Attaching package: ‘DelayedArray’

R[write to console]: The following objects are masked from ‘package:matrixStats’:

    colMaxs, colMins, colRanges, rowMaxs, rowMins, rowRanges

R[write to console]: The following objects are masked from ‘package:base’:

    aperm, apply, rowsum

# Check number of DEGs
template_DE_stats_filename = os.path.join(
    local_dir, "DE_stats", f"DE_stats_template_data_{project_id}_real.txt"
)

template_DE_stats = pd.read_csv(
    template_DE_stats_filename, sep="\t", header=0, index_col=0
)

selected = template_DE_stats[
    (template_DE_stats["adj.P.Val"] < 0.05) & (abs(template_DE_stats["logFC"]) > 1)
]
print(selected.shape)(2, 6)
%%R -i grp_metadata_filename -i project_id -i base_dir -i local_dir -i num_runs

source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))

# Files created: "<local_dir>/DE_stats/DE_stats_simulated_data_SRP012656_<n>.txt"
for (i in 0:(num_runs-1)){
    simulated_data_filename <- paste(local_dir,
                                     "pseudo_experiment/selected_simulated_data_",
                                     project_id,
                                     "_",
                                     i,
                                     ".txt",
                                     sep = "")

    get_DE_stats_limma(grp_metadata_filename,
                       project_id,
                       simulated_data_filename,
                       "simulated",
                       local_dir,
                       i)
}_____no_output_____

### Rank genes_____no_output_____

analysis_type = "DE"
template_DE_stats, simulated_DE_summary_stats = ranking.process_and_rank_genes_pathways(
    template_DE_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_genes,
    logFC_name,
    pvalue_name,
)/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/pandas/core/reshape/merge.py:544: UserWarning: merging between different levels can give an unintended result (2 levels on the left, 1 on the right)
  warnings.warn(msg, UserWarning)

### Gene summary table_____no_output_____

summary_gene_ranks = ranking.generate_summary_table(
    template_DE_stats_filename,
    template_DE_stats,
    simulated_DE_summary_stats,
    col_to_rank_genes,
    local_dir,
    "gene",
    params,
)

summary_gene_ranks.head()_____no_output_____# Check if there is an NaN values, there should not be
summary_gene_ranks.isna().any()_____no_output_____# Create `gene_summary_fielname`
summary_gene_ranks.to_csv(gene_summary_filename, sep="\t")_____no_output_____

### Compare gene ranking
Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These *generic genes* are not necessarily specific to the biological process being studied but instead represent a more systematic change.

We want to compare the ability to detect these generic genes using our method vs those found by [Crow et. al. publication](https://www.pnas.org/content/pnas/116/13/6491.full.pdf). Their genes are ranked 0 = not commonly DE; 1 = commonly DE. Genes were ranked by the number differentially expressed gene sets a gene appeared in across 600 experiments._____no_output_____

# Get generic genes identified by Crow et. al.
DE_prior_filename = params["reference_gene_filename"]
ref_gene_col = params["reference_gene_name_col"]
ref_rank_col = params["reference_rank_col"]

figure_filename = f"gene_ranking_{col_to_rank_genes}.svg"

corr, shared_ranking = ranking.compare_gene_ranking(
    summary_gene_ranks, DE_prior_filename, ref_gene_col, ref_rank_col, figure_filename
)r = 0.5913626553102406
p = 0.0
ci_low = 0.5746856575643265
ci_high = 0.6075444014303418

Hypergeometric test:

Given $N$ number of genes with $K$ common genes in Crow et al. SOPHIE identifies $n$ genes as being common. What is the probability that $k$ of the genes identified by SOPHIE are also common in Crow et al.? What is the probability of drawing $k$ or more concordant genes?

This was a way for us to quantify the correlation between SOPHIE and Crow et al common findings, since the correlation coefficient wasn't very convincing since we're considering all genes in addition to the common ones_____no_output_____

num_Crow_genes = shared_ranking.shape[0]
num_generic_Crow_genes = shared_ranking.query(f"{ref_rank_col}>=80.0").shape[0]
num_generic_SOPHIE_genes = shared_ranking[
    shared_ranking["Percentile (simulated)"] >= percentile_threshold
].shape[0]
num_concordant_generic_genes = shared_ranking[
    (shared_ranking[ref_rank_col] >= percentile_threshold)
    & (shared_ranking["Percentile (simulated)"] >= percentile_threshold)
].shape[0]_____no_output_____print(num_Crow_genes)
print(num_generic_Crow_genes)
print(num_generic_SOPHIE_genes)
print(num_concordant_generic_genes)6834
1213
1397
789
p = ss.hypergeom.sf(
    num_concordant_generic_genes,
    num_Crow_genes,
    num_generic_Crow_genes,
    num_generic_SOPHIE_genes,
)
print(p)0.0

**Takeaway:**
* Here we are comparing gene percentiles obtained from SOPHIE trained on Crow et al. vs gene percentiles obtained from manual curation using Crow et al. There is a high correlation between gene percentiles, validating the SOPHIE approach._____no_output_____### GSEA
**Goal:** To detect modest but coordinated changes in prespecified sets of related genes (i.e. those genes in the same pathway or share the same GO term).

1. Rank all genes using DE association statistics.
2. An enrichment score (ES) is defined as the maximum distance from the middle of the ranked list. Thus, the enrichment score indicates whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list (indicating a correlation with change in expression).
3. Estimate the statistical significance of the ES by a phenotypic-based permutation test in order to produce a null distribution for the ES (i.e. scores based on permuted phenotype)_____no_output_____

# Create "<local_dir>/GSEA_stats/" subdirectory
os.makedirs(os.path.join(local_dir, "GSA_stats"), exist_ok=True)_____no_output_____# Load pathway data
hallmark_DB_filename = params["pathway_DB_filename"]_____no_output_____%%R -i base_dir -i template_DE_stats_filename -i hallmark_DB_filename -i statistic -i local_dir -o template_enriched_pathways

source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))

out_filename <- paste(local_dir,
                     "GSA_stats/GSEA_stats_template_data_",
                     project_id,
                     "_real.txt",
                     sep = "")

template_enriched_pathways <- find_enriched_pathways(template_DE_stats_filename, hallmark_DB_filename, statistic)
template_enriched_pathways <- as.data.frame(template_enriched_pathways[1:7])

write.table(template_enriched_pathways, file = out_filename, row.names = F, sep = "\t")R[write to console]: Loading required package: Rcpp

print(template_enriched_pathways.shape)
template_enriched_pathways[template_enriched_pathways["padj"] < 0.05].sort_values(
    by="padj"
)(50, 7)

**Quick check:** Looks like enriched pathways are consistent with estradiol being estrogen hormone treatment._____no_output_____

%%R -i project_id -i local_dir -i hallmark_DB_filename -i num_runs -i statistic -i base_dir

source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))

# New files created: "<local_dir>/GSEA_stats/GSEA_stats_simulated_data_<project_id>_<n>.txt"
for (i in 0:(num_runs-1)) {
    simulated_DE_stats_filename <- paste(local_dir,
                                     "DE_stats/DE_stats_simulated_data_",
                                     project_id,
                                     "_",
                                     i,
                                     ".txt",
                                     sep = "")

    out_filename <- paste(local_dir,
                     "GSA_stats/GSEA_stats_simulated_data_",
                     project_id,
                     "_",
                     i,
                     ".txt",
                     sep = "")

    enriched_pathways <- find_enriched_pathways(simulated_DE_stats_filename, hallmark_DB_filename, statistic)

    write.table(as.data.frame(enriched_pathways[1:7]), file = out_filename, row.names = F, sep = "\t")
}_____no_output_____

### Rank pathways_____no_output_____

analysis_type = "GSA"
template_GSEA_stats_filename = os.path.join(
    local_dir, "GSA_stats", f"GSEA_stats_template_data_{project_id}_real.txt"
)
(
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
) = ranking.process_and_rank_genes_pathways(
    template_GSEA_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_pathways,
    logFC_name,
    pvalue_name,
    "GSEA",
)_____no_output_____

### Pathway summary table_____no_output_____

# Create intermediate file: "<local_dir>/gene_summary_table_<col_to_rank_pathways>.tsv"
summary_pathway_ranks = ranking.generate_summary_table(
    template_GSEA_stats_filename,
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
    col_to_rank_pathways,
    local_dir,
    "pathway",
    params,
)

summary_pathway_ranks.sort_values(by="Rank (simulated)", ascending=False).head(10)/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/pandas/core/reshape/merge.py:544: UserWarning: merging between different levels can give an unintended result (1 levels on the left, 2 on the right)
  warnings.warn(msg, UserWarning)
# Create `pathway_summary_filename`
summary_pathway_ranks.to_csv(pathway_summary_filename, sep="\t")_____no_output_____

### Compare pathway ranking_____no_output_____Studies have found that there are some pathways (gene sets) that are more likely to be significantly enriched in DEGs across a wide range of experimental designs. These generic pathways are not necessarily specific to the biological process being studied but instead represents a more systematic change.

We want to compare the ability to detect these generic pathways using our method vs those found by [Powers et. al.](https://www.biorxiv.org/content/10.1101/259440v1.full.pdf) publication.  We will use the `Hallmarks_qvalues_GSEAPreranked.csv` file from https://www.synapse.org/#!Synapse:syn11806255 as a reference. The file contains the q-value (adjusted p-value) for the test: given the enrichment score (ES) of the experiment is significant compared to the null distribution of enrichment scores, where the null set is generated from permuted gene sets. For each gene set (pathway) they calculate the q-value using this test.

To get a `reference ranking`, we calculate the fraction of experiments that a given pathway was significant (q-value <0.05) and use this rank pathways. `Our ranking` is to rank pathways based on the median q-value across the simulated experiments. We can then compare `our ranking` versus the `reference ranking.`_____no_output_____

# Load Powers et. al. results file
powers_rank_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "Hallmarks_qvalues_GSEAPreranked.csv"
)_____no_output_____# Read Powers et. al. data
# This file contains qvalue results for hallmark pathways across ~400 experiments
powers_rank_df = pd.read_csv(powers_rank_filename, header=0, index_col=0)
powers_rank_df.drop(["Category"], axis=1, inplace=True)
print(powers_rank_df.shape)
powers_rank_df.head()(50, 443)
# Count the number of experiments where a given pathway was found to be enriched (qvalue < 0.05)
total_num_experiments = powers_rank_df.shape[1]
frac_enriched_pathways = (powers_rank_df < 0.05).sum(axis=1) / total_num_experiments

# Rank pathways from 0-50, 50 indicating that the pathways was frequently enriched
pathway_ranks = frac_enriched_pathways.rank()

powers_rank_stats_df = pd.DataFrame(
    data={
        "Fraction enriched": frac_enriched_pathways.values,
        "Powers Rank": pathway_ranks.values,
    },
    index=powers_rank_df.index,
)
powers_rank_stats_df.sort_values(by="Powers Rank", ascending=False).head()_____no_output_____# Save reference file for input into comparison
powers_rank_processed_filename = os.path.join(
    base_dir,
    dataset_name,
    "data",
    "metadata",
    "Hallmarks_qvalues_GSEAPreranked_processed.tsv",
)

powers_rank_stats_df.to_csv(
    powers_rank_processed_filename,
    sep="\t",
)_____no_output_____figure_filename = f"pathway_ranking_{col_to_rank_pathways}.svg"

ranking.compare_pathway_ranking(
    summary_pathway_ranks, powers_rank_processed_filename, figure_filename
)r = 0.40314288763144646
p = 0.0036978859484643597
ci_low = 0.18708669905391231
ci_high = 0.579217691396006
{'r': 0.40314288763144646, 'p': 0.0036978859484643597, 'ci_low': 0.18708669905391231, 'ci_high': 0.579217691396006}

**Takeaway:**
* Our method ranked pathways using median adjusted p-value score across simulated experiments.
* Powers et. al. ranked pathways based on the fraction of experiments they had adjusted p-value < 0.05.
* Here, we compare pathway ranks obtained from SOPHIE trained on Crow et al. vs pathway ranking based on manual curation using Powers et. al datasets.
* Despite the two datasets have a different context, we still see a good correlation._____no_output_____


# Notebook from KChen-lab/sensei
Path: python/statistics/TCGA-MSI-MSS-absolute-STAD.ipynb

# Introduction

Three groups: microsatellite instability high (MSI-H), microsatellite instability low (MSI-L) and microsatellite stable (MSS).

Two sets of measurements: 7_marker, 5_marker. The original report consider one sample as MSI if both indicate positive._____no_output_____

import pandas as pd
import matplotlib as mpl
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats

# mpl.rcParams['figure.dpi'] = 600_____no_output_____import sys
sys.path.append("..")
import sensei_____no_output_____from tcga_dicts import *_____no_output_____df = pd.read_excel("41591_2016_BFnm4191_MOESM27_ESM.xlsx", index=1)
df_____no_output_____df[df['Tumor Type'] == 'STAD']['MOSAIC classification'].value_counts().plot(kind="bar")_____no_output_____f = lambda x: x[0:3]
msi_mask = (df['MOSAIC classification'] == 'MSI-H') & (df['Tumor Type'] == 'STAD')
mss_mask = (df['MOSAIC classification'] == 'MSS') & (df['Tumor Type'] == 'STAD')

msi_patient_id = df['Sample Name'][msi_mask].to_frame()
mss_patient_id = df['Sample Name'][mss_mask].to_frame()

msi_patient_id['microsatellite'] = 'MSI-H'
mss_patient_id['microsatellite'] = 'MSS'

microsatellite_label_df = msi_patient_id.append(mss_patient_id)_____no_output_____data = pd.read_csv("TCGA.Kallisto.fullIDs.cibersort.relative.tsv", sep="\t")
data["SampleID"] = data["SampleID"].apply(lambda x: x.replace('.', '-'))
data["PatientID"] = data["SampleID"].apply(lambda x: '-'.join(x.split('-')[0:3]))_____no_output_____data = data.merge(microsatellite_label_df, left_on="PatientID", right_on='Sample Name')_____no_output_____leuk = pd.read_csv("TCGA_all_leuk_estimate.masked.20170107.tsv", sep="\t", header=None)
leuk.columns = ['CancerType', 'SampleID', 'LF']
leuk.loc[leuk.LF < 0, 'LF'] = 0

id1 = set(data['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4] + [x.split('-')[4][0:2]])))
id2 = set(leuk['SampleID'].apply(lambda x: '-'.join(x.split('-')[0:4] + [x.split('-')[4][0:2]])))
common_id = id1.intersection(id2)
{
    '1': len(id1),
    '2': len(id2),
    'intersection': len(common_id)
}_____no_output_____merged = pd.DataFrame()
for id in common_id:
    temp = data[data['SampleID'].apply(lambda x: id in x)]
    temp.loc[:, 'LF'] = leuk.loc[leuk.loc[:, 'SampleID'].apply(lambda x: id in x), 'LF'].mean()
    merged = merged.append(temp.loc[temp.loc[:, 'P.value'].idxmax()])/home/sliang3/miniconda3/envs/basic/lib/python3.8/site-packages/pandas/core/indexing.py:845: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self.obj[key] = _infer_fill_value(value)
/home/sliang3/miniconda3/envs/basic/lib/python3.8/site-packages/pandas/core/indexing.py:966: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self.obj[item] = s
merged_____no_output_____for row_name in cell_types:
    merged[row_name] *= merged['LF']
merged_____no_output_____merged.microsatellite.value_counts().plot(kind="bar")_____no_output_____

#merged['Leukocytes.all'] = merged[cell_types].sum(1)

merged['T.cells.all'] = merged[['T.cells.CD8',
                                    'T.cells.CD4.naive',
                                    'T.cells.CD4.memory.resting',
                                    'T.cells.CD4.memory.activated',
                                    'T.cells.follicular.helper',
                                    'T.cells.regulatory..Tregs.',
                                    'T.cells.gamma.delta']].sum(1)

merged['B.cells.all'] = merged[['B.cells.naive', 'B.cells.memory']].sum(1)

merged['Nk.cells.all'] = merged[['NK.cells.resting', 'NK.cells.activated']].sum(1)

merged['Macrophages.all'] = merged[['Macrophages.M0', 'Macrophages.M1', 'Macrophages.M2']].sum(1)

merged['Dendritic.cells.all'] = merged[['Dendritic.cells.resting', 'Dendritic.cells.activated']].sum(1)

merged['Mast.cells.all'] = merged[['Mast.cells.resting', 'Mast.cells.activated']].sum(1)

augmented_cell_types = cell_types + ['T.cells.all', 'B.cells.all', 'Nk.cells.all', 'Macrophages.all',
                                     'Dendritic.cells.all', 'Mast.cells.all']

merged_____no_output_____mss = merged[merged.microsatellite == 'MSS'][augmented_cell_types]
msi = merged[merged.microsatellite == 'MSI-H'][augmented_cell_types]_____no_output_____mss_____no_output_____

# Results

## MSS_____no_output_____

a = 0.05

sanitize = lambda x: 0.0 if x < 0 else 1.0 if x > 1 else x

res_mss = pd.DataFrame(index = cell_types, columns = ['mean', 'mean lower', 'mean upper', 'sd', 'sd lower', 'sd upper'])

n = mss.shape[0];

# Mean
res_mss['mean'] = mss.mean(axis=0)

res_mss['sd'] = mss.std(ddof=1, axis=0)

# Mean CI
err = scipy.stats.t.ppf(1 - a / 2, n - 1) * res_mss['sd'] / np.sqrt(n)
res_mss['mean lower'] = (res_mss['mean'] - err).apply(sanitize)
res_mss['mean upper'] = (res_mss['mean'] + err).apply(sanitize)

# Standard deviation CI
res_mss['sd lower'] = np.sqrt((n - 1) * res_mss['sd'] ** 2 / scipy.stats.chi2.ppf(1 - a / 2, n - 1))
res_mss['sd upper'] = np.sqrt((n - 1) * res_mss['sd'] ** 2 / scipy.stats.chi2.ppf(a / 2, n - 1))

res_mss_____no_output_____

## MSI_____no_output_____

a = 0.05

sanitize = lambda x: 0.0 if x < 0 else 1.0 if x > 1 else x

res_msi = pd.DataFrame(index = cell_types, columns = ['mean', 'mean lower', 'mean upper', 'sd', 'sd lower', 'sd upper'])

n = msi.shape[0];

# Mean
res_msi['mean'] = msi.mean(axis=0)

res_msi['sd'] = msi.std(ddof=1, axis=0)

# Mean CI
err = scipy.stats.t.ppf(1 - a / 2, n - 1) * res_msi['sd'] / np.sqrt(n)
res_msi['mean lower'] = (res_msi['mean'] - err).apply(sanitize)
res_msi['mean upper'] = (res_msi['mean'] + err).apply(sanitize)

# Standard deviation CI
res_msi['sd lower'] = np.sqrt((n - 1) * res_msi['sd'] ** 2 / scipy.stats.chi2.ppf(1 - a / 2, n - 1))
res_msi['sd upper'] = np.sqrt((n - 1) * res_msi['sd'] ** 2 / scipy.stats.chi2.ppf(a / 2, n - 1))

res_msi_____no_output_____

## Figures_____no_output_____

x_map = {v: i for i, v in enumerate(augmented_cell_types)}

offset_map = {
 'mss': -0.15,
 'msi': 0.15}

colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

color_map = {'mss': colors[0],
 'msi': colors[1]}

fig = plt.figure(figsize=(10, 3))

ax = fig.add_subplot(1, 1, 1)

n = mss.shape[0];
mean = mss.mean(axis=0)
sd = mss.std(ddof=1, axis=0)
err = scipy.stats.t.ppf(1 - a / 2, n - 1) * sd / np.sqrt(n)

x = [v + offset_map['mss'] for v in x_map.values()]

ax.errorbar(x, mean, yerr=sd, fmt='.', color = color_map['mss'], ecolor = 'darkgray', elinewidth=3.0)
ax.errorbar(x, mean, yerr=err, fmt="None", color = color_map['mss'], ecolor = 'black', elinewidth=1.0)

n = msi.shape[0];
mean = msi.mean(axis=0)
sd = msi.std(ddof=1, axis=0)
err = scipy.stats.t.ppf(1 - a / 2, n - 1) * sd / np.sqrt(n)

x = [v + offset_map['msi'] for v in x_map.values()]

ax.errorbar(x, mean, yerr=sd, fmt='.', color = color_map['msi'], ecolor = 'darkgray', elinewidth=3.0)
ax.errorbar(x, mean, yerr=err, fmt="None", color = color_map['msi'], ecolor = 'black', elinewidth=1.0)

# Hide the right and top spines
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

# Only show ticks on the left and bottom spines
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')

current_ylim = ax.get_ylim()
for v in x_map.values():
    if v % 2 == 0:
        ax.fill_between([v - 0.5, v + 0.5], current_ylim[0], current_ylim[1], facecolor='lightgray', alpha=0.2)

ax.set_ylim(current_ylim)
ax.set_xticks(list(x_map.values()))
ax.tick_params(axis='x', which = 'both', labelbottom=None)
ax.set_ylabel('Abundance')
ax.set_xlim(-0.5, len(x_map) - 0.5)

# Ticks
ax.tick_params(axis='x', which = 'both', labelbottom=True)
ax.set_xticks(list(x_map.values()))
ax.set_xticklabels(list(x_map.keys()), rotation=30, ha='right')

legend_elements = [mpl.lines.Line2D([0], [0], marker='.', color='w', markerfacecolor=color_map[i], label=i, markersize=15)
                   for i in color_map]

ax.legend(handles=legend_elements, loc='upper left')_____no_output_____len(cell_types)_____no_output_____x_map = {v: i for i, v in enumerate(augmented_cell_types)}

offset_map = {
 100: -0.15,
 384: -0.0,
 10000: 0.15}

colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

color_map = {
 100: colors[0],
 384: colors[1],
 10000: colors[2]}

fig = plt.figure(figsize=(10, 3))

ax = fig.add_subplot(1, 1, 1)

mss_n = mss.shape[0];
mss_mean = mss.mean(axis=0)
mss_sd = mss.std(ddof=1, axis=0)
mss_err = scipy.stats.t.ppf(1 - a / 2, mss_n - 1) * mss_sd / np.sqrt(mss_n)

msi_n = msi.shape[0];
msi_mean = msi.mean(axis=0)
msi_sd = msi.std(ddof=1, axis=0)
msi_err = scipy.stats.t.ppf(1 - a / 2, msi_n - 1) * msi_sd / np.sqrt(msi_n)

for cell_number in offset_map:
    x = [v + offset_map[cell_number] for v in x_map.values()]
    y = [sensei.optimize.find_minimum_M(10000, cell_number, [m0, m1], [s0, s1], beta=0.2, design='unpaired') for m0, m1, s0, s1 in zip(mss_mean, msi_mean, mss_sd, msi_sd)]
    ax.errorbar(x, y, yerr=0, fmt='.', color = color_map[cell_number], ecolor = 'darkgray', elinewidth=3.0)

# Hide the right and top spines
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)

# Only show ticks on the left and bottom spines
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')

current_ylim = ax.get_ylim()
for v in x_map.values():
    if v % 2 == 0:
        ax.fill_between([v - 0.5, v + 0.5], current_ylim[0], current_ylim[1], facecolor='lightgray', alpha=0.2)

ax.set_ylim(current_ylim)
ax.set_xticks(list(x_map.values()))
ax.tick_params(axis='x', which = 'both', labelbottom=None)
ax.set_ylabel('Abundance')
ax.set_xlim(-0.5, len(x_map) - 0.5)

# Ticks
ax.tick_params(axis='x', which = 'both', labelbottom=True)
ax.set_xticks(list(x_map.values()))
ax.set_xticklabels(list(x_map.keys()), rotation=30, ha='right')

legend_elements = [mpl.lines.Line2D([0], [0], marker='.', color='w', markerfacecolor=color_map[i], label=i, markersize=15)
                   for i in color_map]

ax.legend(handles=legend_elements, loc='upper left')_____no_output_____


# Notebook from buchmuseum/boev-mitgliedsakten
Path: scripts/pica-97-1.ipynb

# Qd-Sätze schreiben_____no_output_____

import pandas as pd
import re
from datetime import datetime

rawpath = '../raw'
datpath = '../dat'_____no_output_____

Aktuelle csv-Datei wird als df importiert, mit convert_dtypes werden die neuen String-Formate von pandas 1.0 angewendet.

Die Konkordanz wird als tp_idns geladen._____no_output_____

df = pd.read_csv(f'{rawpath}/HABV_97_1_v2.csv').convert_dtypes()
tp_idns = pd.read_csv(f'{rawpath}/koko_mabv.csv',delimiter='\t', encoding='utf-8')
tp_idns['id'] = tp_idns['akten-nr'].str.slice(10).str.strip()
tp_idns.set_index('id', inplace=True)_____no_output_____

Die Daten werden als Pica+ in eine .dat-Datei geschrieben_____no_output_____

def pica3_schreiben(row):
    pica = """\t\n0500 Qd
0501 Text$btxt
0502 ohne Hilfsmittel zu benutzen$bn
0503 Blatt$bnb
0600 yy
"""
    if pd.isna(row.lz_ende):
        pica += f"""1100 {row.lz_beginn}\n1110 *{row.lz_beginn}$4lfzt"""
    elif pd.notna(row.lz_ende):
        pica += f"""1100 {row.lz_beginn}$b{row.lz_ende}\n1110 *{row.lz_beginn}-{row.lz_ende}$4lfzt"""
    pica += """
1130 TB-papier
1130 !040445224!
1130 !041466098!
1132 a1-analog;a2-masch;f1-text;f2-blatt;e2-uv;v-cont
1500 /1ger
1700 /1XA-DE
4000 Mitgliedsakte """
    if pd.notna(row.Titel):
        pica +=  f'{row.Titel} '
    pica +=  f'{row.Vorname.strip()} {row.Nachname.strip()}'
    if pd.notna(row.Zusaetze):
        pica +=  f', {row.Zusaetze}'
    pica += f"""
4019 Mitgliedsakte$Bobja
3100 !000073369!$BZusammenstellender$4com"""
    if pd.notna(row.Ort):
        pica += f"\n4046 {row.Ort}"
    pica += """
4060 1 Akte
4105 !1061153886!
4700 |BSM|
5100 !000073369!
"""
    try:
        konko_idn = tp_idns.loc[f'{row.Id}'].idn.strip()
    except:
        konko_idn = None

    if konko_idn is not None:
        pica += f"5101 !{konko_idn}!\n"

    elif pd.notna(row.idns) and (idn := re.match(r"\[\'(\w+)\'\]", row.idns)):
            pica += f"5101 !{idn.group(1)}!\n"
    else:
        pica += f"5550 [DBSM-P]{row.Nachname.strip()}, {row.Vorname.strip()}"
        if pd.notna(row.geburt) or pd.notna(row.tod):
            pica += " ["
            if pd.notna(row.geburt):
                pica += row.geburt
            pica += "-"
            if pd.notna(row.tod):
                    pica += row.tod
            pica += "]"
        pica += "\n"

    pica += "5109 (DE-101d){DE-101d}"

    if pd.notna(row.firmen_idns):
        if firmen_idn := re.match(r'\[\'(w+)\'\]', row.firmen_idns):
            pica += f"5110 {firmen_idn.group(1)}"
    pica += f"""
5560 [RNA]Korrespondenz
5590 [Objektgattung]
5590 !040009254!
7001 {now.strftime('%d-%m-%y')} : x
7100 HA/BV 97.1,{row.Id} @ m
7109 !!DBSM/S!! ; HA/BV 97.1,{row.Id}
6710 !1059758504!$l{row.Id}\n"""
    return pica_____no_output_____# jede einzelne zeile des df wird ausgewertet und das ergebnis als string pica zurückgegeben
# es werden immer drei dateien geschrieben: eine -recent.dat mit allen datensätzen, eine backup-datei mit aktuellem datumsstempel und eine datei mit 5 zufällig ausgewählten datensätzen
now = datetime.now()
with open(f"{datpath}/97_1_pica3-{now.strftime('%y-%m-%d-%H-%M-%S')}.dat", 'w') as f:
    with open(f"{datpath}/97_1_pica3-recent.dat", 'w') as f2:
        for row in df.itertuples():
            f.write(pica3_schreiben(row))
            f2.write(pica3_schreiben(row))

with open(f"{datpath}/97_1_pica3-recent-sample.dat", 'w') as f:
    for row in df.sample(5).itertuples():
        f.write(pica3_schreiben(row))

# with open(f"{datpath}/97_1_pica3-100test.dat", 'w') as f:
#     for row in df[:100].itertuples():
#         f.write(pica3_schreiben(row))_____no_output_____for durchgang in range((len(df) // 1000) + 1):
    with open(f"{datpath}/97_1_pica3-{durchgang}.dat", 'w') as f:
        for row in df[durchgang * 1000:(durchgang + 1) * 1000].itertuples():
            f.write(pica3_schreiben(row))_____no_output_____

# Personen-Sätze schreiben_____no_output_____

def tp_schreiben(row):
    pica = f"""
001U $0utf8
001X $00
002@ $0Tp4
004B $apiz
008A $af
008B $ao
028A $d{row.Vorname.strip()}$a{row.Nachname.strip()}
029R $9000073369$4affi$vMitglied
"""
    if pd.notna(row.firmen_idns):
        firmen_idn = re.match(r'\[\'(.+)\'\]', row.firmen_idns).group(1)
        pica += f'029R $9{firmen_idn}$4affi\n'
    if pd.notna(row.Geschlecht) and row.Geschlecht == 'w':
        pica += f"032T $af\n"
    elif pd.notna(row.Geschlecht) and row.Geschlecht == 'm':
        pica += f"032T $am\n"
    if pd.notna(row.Titel):
        pica += f'041R $a{row.Titel}$4akad\n'
    if pd.notna(row.firmenart):
        if row.firmenart == 'Sortiment':
            if pd.notna(row.Geschlecht) and row.Geschlecht == 'f':
                #Buchhändlerin
                pica += f'041R $9942316746$4berc'
            elif pd.notna(row.Geschlecht) and row.Geschlecht == 'm':
                #Buchhändler
                pica += f'041R $9041309480$4berc'
            else:
                #Buchhändler
                pica += f'041R $9041309480$4berc'
        elif row.firmenart == "Verlag":
            if row.Geschlecht == 'f':
                #verlegerin
                pica += f'041R $9042667208$4berc'
            elif row.Geschlecht == 'm':
                #verleger
                pica += f'041R $9041383435$4berc'
            else:
                #verleger
                pica += f'041R $9041383435$4berc'
        elif row.firmenart == 'Zwischenbuchhandel':
            if row.Geschlecht == 'f':
                #Buchhändlerin
                pica += f'041R $9942316746$4berc'
            elif row.Geschlecht == 'm':
                #Buchhändler
                pica += f'041R $9041309480$4berc'
            else:
                #Buchhändler
                pica += f'041R $9041309480$4berc'
        elif row.firmenart == 'Verlagsvertreter':
            if row.Geschlecht == 'f':
                #Verlagsvertreterin 1205182039
                pica += f'041R $91205182039$4berc'
            elif row.Geschlecht == 'm':
                #Verlagsvertreter 041346645
                pica += f'041R $9041346645$4berc'
            else:
                #Verlagsvertreter 041346645
                pica += f'041R $9041346645$4berc'
    pica += """
042B $aXA-DE
047A/03 $eDE-101d
047A/03 $rDE-101
050C $aMitgliedsakte des Börsenvereins des Deutschen Buchhandels
"""
    if pd.notna(row.tod): #sterbedatum ist bekannt
        if pd.notna(row.geburt): #gebdatum ist bekannt
            #datl geb+tod
            pica += f'060R $a{row.geburt[-4:]}$b{row.tod[-4:]}$4datl\n'
            #datx geb+tod
            pica += f'060R $a{row.geburt}$b{row.tod}$4datx'
        elif pd.isna(row.geburt):
            #datl tod
            pica += f'060R $b{row.tod[-4:]}$4datl\n'
            #datx tod
            pica += f'060R $b{row.tod}$4datx'
    elif pd.isna(row.tod): #kein sterbedatum bekannt
        if pd.notna(row.geburt): #gebdatum ist bekannt
            #nur datl geb
            pica += f'060R $a{row.geburt[-4:]}$4datl'
    if pd.notna(row.Ort):
        pica += f'\n065R $a{row.Ort}$4ortw'
    pica += f"""\n070A/03 $SDE-101d$0HABV_91_1_{row.Id}\n"""
    return pica

filt = (df['idns'].isna() & df['doppelname'].isna() & df['geburt'].notna())

with open('test-pica+-tp-2021-08-30-korr.dat', 'w') as f:
    for index, row in df[filt].iterrows():
        f.write(tp_schreiben(row))_____no_output_____def tp_pica3_schreiben(row):
    pica = f"""
005 Tp4
008 piz
011 f
012 o
043 XA-DE
100 {row.Nachname.strip()}, {row.Vorname.strip()}
510 !000073369!$4affi$vMitglied
"""
    if pd.notna(row.firmen_idns):
        firmen_idn = re.match(r'\[\'(.+)\'\]', row.firmen_idns).group(1)
        pica += f'510 !{firmen_idn}!$4affi\n'
    if pd.notna(row.Geschlecht) and row.Geschlecht == 'w':
        pica += f'375 f\n'
    elif pd.notna(row.Geschlecht) and row.Geschlecht == 'm':
        pica += f'375 m\n'
    if pd.notna(row.Titel):
        pica += f'550 {row.Titel}$4akad\n'
    if pd.notna(row.firmenart):
        if row.firmenart == 'Sortiment':
            if pd.notna(row.Geschlecht) and row.Geschlecht == 'w':
                #Buchhändlerin
                pica += f'550 !942316746!$4berc'
            elif pd.notna(row.Geschlecht) and row.Geschlecht == 'm':
                #Buchhändler
                pica += f'550 !041309480!$4berc'
            else:
                #Buchhändler
                pica += f'550 !041309480!$4berc'
        elif row.firmenart == "Verlag":
            if row.Geschlecht == 'w':
                #verlegerin
                pica += f'550 !042667208!$4berc'
            elif row.Geschlecht == 'm':
                #verleger
                pica += f'550 !041383435!$4berc'
            else:
                #verleger
                pica += f'550 !041383435!$4berc'
        elif row.firmenart == 'Zwischenbuchhandel':
            if row.Geschlecht == 'w':
                #Buchhändlerin
                pica += f'550 !942316746!$4berc'
            elif row.Geschlecht == 'm':
                #Buchhändler
                pica += f'550 !041309480!$4berc'
            else:
                #Buchhändler
                pica += f'550 !041309480!$4berc'
        elif row.firmenart == 'Verlagsvertreter':
            if row.Geschlecht == 'w':
                #Verlagsvertreterin 1205182039
                pica += f'550 !1205182039!$4berc'
            elif row.Geschlecht == 'm':
                #Verlagsvertreter 041346645
                pica += f'550 !041346645!$4berc'
            else:
                #Verlagsvertreter 041346645
                pica += f'550 !041346645!$4berc'
    if pd.notna(row.tod): #sterbedatum ist bekannt
        if pd.notna(row.geburt): #gebdatum ist bekannt
            #datl geb+tod
            pica += f'548 {row.geburt[-4:]}$b{row.tod[-4:]}$4datl\n'
            #datx geb+tod
            pica += f'548 {row.geburt}$b{row.tod}$4datx'
        elif pd.isna(row.geburt):
            #datl tod
            pica += f'548 $b{row.tod[-4:]}$4datl\n'
            #datx tod
            pica += f'548 $b{row.tod}$4datx'
    elif pd.isna(row.tod): #kein sterbedatum bekannt
        if pd.notna(row.geburt): #gebdatum ist bekannt
            #nur datl geb
            pica += f'548 {row.geburt[-4:]}$4datl'
    if pd.notna(row.Ort):
        pica += f'\n551 {row.Ort}$4ortw'
    pica += f"""
903 $eDE-101d
903 $rDE-101
667 Mitgliedsakte des Börsenvereins des Deutschen Buchhandels
983 {row.Id}\n
"""
    return pica_____no_output_____# filter idns ist leer (d.h. es gibt keine gnd-matches), keine doppelnamen, wenigstens geburtsjahr ist bekannt.
filt = (df['idns'].isna() & df['doppelname'].isna() & df['geburt'].notna())

with open('97_1_pica3-2021-08-30.dat', 'w') as f:
    for row in df[filt].itertuples():
        f.write(tp_pica3_schreiben(row))_____no_output_____df_____no_output_____

# Konkordanz-Merge_____no_output_____

tp_idns_____no_output_____# return row of index 455 from tp_idns
tp_idns.loc['455'].idn_____no_output_____tp_idns.loc[f'455'].idn.strip()_____no_output_____


# Notebook from ShepherdCode/ShepherdML
Path: Nasa2021/CNN_523embed_noisy24k_learning.ipynb

_____no_output__________no_output_____# CNN 523 Noisy 24k

1. Four layers of CNNS and MaxPooling
2. Drop out at 0.2
3. Variable filters and dense neurons
4. Adjusted Learning rate to 0.0005

_____no_output_____

#NC_FILENAME='ncRNA.tiny50.fasta'
#PC_FILENAME='pcRNA.tiny50.fasta'
#NC_FILENAME='ncRNA.gc34.processed.fasta'
#PC_FILENAME='pcRNA.gc34.processed.fasta'
NC_FILENAME='noisy_noncod_24000_10.fasta'  # CHANGE THIS TO 1000, 2000, 4000, etc.
PC_FILENAME='noisy_coding_24000_10.fasta'
NC_VAL_FILE='noncod_validation.fasta'  # 'noncod_validation.fasta'  # CHANGE THIS TO THE UNIFORM VALIDATION FILE
PC_VAL_FILE='coding_validation.fasta'  # 'coding_validation.fasta'

MODEL_FILE='Test_Non_noisy_24K'   # CHANGE THIS IF YOU WANT TO SAVE THE MODEL!
DATAPATH=''

try:
    from google.colab import drive
    IN_COLAB = True
    PATH='/content/drive/'
    drive.mount(PATH)
    DATAPATH=PATH+'My Drive/data/'  # must end in "/"
except:
    IN_COLAB = False
    DATAPATH='data/'  # must end in "/"
NC_FILENAME = DATAPATH+NC_FILENAME
PC_FILENAME = DATAPATH+PC_FILENAME
NC_VAL_FILE = DATAPATH+NC_VAL_FILE
PC_VAL_FILE = DATAPATH+PC_VAL_FILE
MODEL_FILE=DATAPATH+MODEL_FILE

EPOCHS=10  # DECIDE ON SOME AMOUNT AND STICK WITH IT
SPLITS=5
K=1
VOCABULARY_SIZE=4**K+1   # e.g. K=3 => 64 DNA K-mers + 'NNN'
EMBED_DIMEN=2
FILTERS=32
KERNEL=3
NEURONS=24
DROP=0.2
MINLEN=200
MAXLEN=1000    # THIS HAS TO MATCH THE SIMULATION DATA
DENSE_LEN = 1000
ACT="tanh"
LN_RATE = 0.0005Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount("/content/drive/", force_remount=True).
# Load our own tools
# TO DO: don't go to GitHub if the file is already local.
GITHUB = True
if GITHUB:
    #!pip install requests  # Uncomment this if necessary. Seems to be pre-installed.
    import requests
    r = requests.get('https://raw.githubusercontent.com/ShepherdCode/ShepherdML/master/Strings/tools_fasta.py')
    with open('tools_fasta.py', 'w') as f:
        f.write(r.text)
    # TO DO: delete the file after import
import tools_fasta as tools
tools.yahoo()  # If this prints "Yahoo!" the the import was successful.

TOOLS_CHANGED = False   # set to True to re-run with a new version of tools
if TOOLS_CHANGED:
  from importlib import reload
  tools=reload(tools)
  print(dir(tools))   # run this to see EVERYTHING in the tools moduleYahoo!
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
from tensorflow import keras
import time
dt='float32'
tf.keras.backend.set_floatx(dt)_____no_output_____

Build model_____no_output_____

def compile_model(model):
    print("COMPILE...")
    ## learn rate = initial_learning_rate * decay_rate ^ (step / decay_steps)
    #adam_default_learn_rate = 0.001
    #schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    #    initial_learning_rate = adam_default_learn_rate*10,
    #    decay_steps=10000, decay_rate=0.99, staircase=True)
    #alrd = tf.keras.optimizers.Adam(learning_rate=schedule)

    ln_rate = tf.keras.optimizers.Adam(learning_rate = LN_RATE)
    bc=tf.keras.losses.BinaryCrossentropy(from_logits=False)
    model.compile(loss=bc, optimizer=ln_rate, metrics=["accuracy"])
    print("...COMPILED")
    return model

def build_model():
    #SHAPE=(MAXLEN,5)
    SHAPE=(MAXLEN,4)
    # 4 input letters, 4 output dimensions, 1000 letters/RNA
    elayer  = keras.layers.Embedding(4,4,input_length=1000)

    clayer1 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same",
                                 input_shape=SHAPE)
    clayer2 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer3 = keras.layers.MaxPooling1D(2)
    clayer4 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer5 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer6 = keras.layers.MaxPooling1D(2)
    clayer7 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer8 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer9 = keras.layers.MaxPooling1D(2)
    clayer10 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer11 = keras.layers.Conv1D(FILTERS,KERNEL,activation=ACT,padding="same")
    clayer12 = keras.layers.MaxPooling1D(2)

    clayer13 = keras.layers.Flatten()

    dlayer1 = keras.layers.Dense(NEURONS, activation=ACT,dtype=dt, input_shape=[DENSE_LEN])
    dlayer2 = keras.layers.Dropout(DROP)
    dlayer3 = keras.layers.Dense(NEURONS, activation=ACT,dtype=dt)
    dlayer4 = keras.layers.Dropout(DROP)
    output_layer = keras.layers.Dense(1, activation="sigmoid", dtype=dt)

    cnn = keras.models.Sequential()
    cnn.add(elayer)
    cnn.add(clayer1)
    cnn.add(clayer2)
    cnn.add(clayer3)
    cnn.add(clayer4)
    cnn.add(clayer5)
    cnn.add(clayer6)
    cnn.add(clayer7)
    cnn.add(clayer8)
    cnn.add(clayer9)
    cnn.add(clayer10)
    cnn.add(clayer11)
    cnn.add(clayer12)
    cnn.add(clayer13)
    cnn.add(dlayer1)
    cnn.add(dlayer2)
    cnn.add(dlayer3)
    cnn.add(dlayer4)
    cnn.add(output_layer)
    mlpc = compile_model(cnn)
    return mlpc_____no_output_____

Cross validation_____no_output_____

def do_cross_validation(X,y,given_model,X_VALID,Y_VALID):
    cv_scores = []
    fold=0
    splitter = ShuffleSplit(n_splits=SPLITS, test_size=0.1, random_state=37863)
    for train_index,valid_index in splitter.split(X):
        fold += 1
        X_train=X[train_index] # use iloc[] for dataframe
        y_train=y[train_index]
        X_valid=X[valid_index]
        y_valid=y[valid_index]
        # Avoid continually improving the same model.
        model = compile_model(keras.models.clone_model(given_model))
        bestname=MODEL_FILE+".cv."+str(fold)+".best"
        mycallbacks = [keras.callbacks.ModelCheckpoint(
            filepath=bestname, save_best_only=True,
            monitor='val_accuracy', mode='max')]
        print("FIT")
        start_time=time.time()
        history=model.fit(X_train, y_train,
                epochs=EPOCHS, verbose=1, callbacks=mycallbacks,
                validation_data=(X_valid,y_valid))
        # THE VALIDATION ABOVE IS JUST FOR SHOW
        end_time=time.time()
        elapsed_time=(end_time-start_time)
        print("Fold %d, %d epochs, %d sec"%(fold,EPOCHS,elapsed_time))
        pd.DataFrame(history.history).plot(figsize=(8,5))
        plt.grid(True)
        plt.gca().set_ylim(0,1)
        plt.show()
        best_model=keras.models.load_model(bestname)
        # THE VALIDATION BELOW IS FOR KEEPS
        scores = best_model.evaluate(X_VALID, Y_VALID, verbose=0)
        print("%s: %.2f%%" % (best_model.metrics_names[1], scores[1]*100))
        cv_scores.append(scores[1] * 100)
    print()
    print("%d-way Cross Validation mean %.2f%% (+/- %.2f%%)" % (fold, np.mean(cv_scores), np.std(cv_scores)))_____no_output_____

## Train on RNA lengths 200-1Kb_____no_output_____

print ("Compile the model")
model=build_model()
print ("Summarize the model")
print(model.summary())  # Print this only once
#model.save(MODEL_FILE+'.model')
Compile the model
COMPILE...
...COMPILED
Summarize the model
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
embedding (Embedding)        (None, 1000, 4)           16
_________________________________________________________________
conv1d (Conv1D)              (None, 1000, 32)          416
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 1000, 32)          3104
_________________________________________________________________
max_pooling1d (MaxPooling1D) (None, 500, 32)           0
_________________________________________________________________
conv1d_2 (Conv1D)            (None, 500, 32)           3104
_________________________________________________________________
conv1d_3 (Conv1D)            (None, 500, 32)           3104
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 250, 32)           0
_________________________________________________________________
conv1d_4 (Conv1D)            (None, 250, 32)           3104
_________________________________________________________________
conv1d_5 (Conv1D)            (None, 250, 32)           3104
_________________________________________________________________
max_pooling1d_2 (MaxPooling1 (None, 125, 32)           0
_________________________________________________________________
conv1d_6 (Conv1D)            (None, 125, 32)           3104
_________________________________________________________________
conv1d_7 (Conv1D)            (None, 125, 32)           3104
_________________________________________________________________
max_pooling1d_3 (MaxPooling1 (None, 62, 32)            0
_________________________________________________________________
flatten (Flatten)            (None, 1984)              0
_________________________________________________________________
dense (Dense)                (None, 24)                47640
_________________________________________________________________
dropout (Dropout)            (None, 24)                0
_________________________________________________________________
dense_1 (Dense)              (None, 24)                600
_________________________________________________________________
dropout_1 (Dropout)          (None, 24)                0
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 25
=================================================================
Total params: 70,425
Trainable params: 70,425
Non-trainable params: 0
_________________________________________________________________
None
def load_data_from_files(nc_filename,pc_filename):
  FREEMEM=True   # use False for debugging, True for production
  print("Load data from files.")
  nc_seq=tools.load_fasta(nc_filename,0)
  pc_seq=tools.load_fasta(pc_filename,1)
  train_set=pd.concat((nc_seq,pc_seq),axis=0)
  print("Ready: train_set")
  subset=tools.make_slice(train_set,MINLEN,MAXLEN)# One array to two: X and y
  if FREEMEM:
      nc_seq=None
      pc_seq=None
      train_set=None
  (X1,y_train)=tools.separate_X_and_y(subset)
  # X1 is pandas df of ("list" of one sequence)
  X2=X1.to_numpy()  # numpy ndarray of ("list" of one sequence)
  X3=[elem[0] for elem in X2]  # numpy ndarray of ACGT-str
  # X3? It might be faster to use int-array than char-array. Come back to this.
  X4=X3  # no-op
  print("X4",type(X4))
  #print(X4[0])
  if FREEMEM:
      X1=None
      X2=None
      X3=None
  X5=[]
  dna_to_int = {'A':0,'C':1,'G':2,'T':3}
  for x in X4:
      a=[]
      for c in x:
          i = dna_to_int[c]
          a.append(i)
      X5.append(a)
  X5=np.asarray(X5)
  print("X5",type(X5))
  print(X5.shape)
  if FREEMEM:
      X4=None
  X_train=X5
  if FREEMEM:
      X5=None
  print("X_train",type(X_train))
  y_train=y_train.to_numpy()
  print(X_train.shape)
  print(X_train[0].shape)
  print(X_train[0])
  return X_train,y_train
_____no_output_____print("Loading training data...")
X_train,y_train = load_data_from_files(NC_FILENAME,PC_FILENAME)Loading training data...
Load data from files.
Ready: train_set
X4 <class 'list'>
X5 <class 'numpy.ndarray'>
(48000, 1000)
X_train <class 'numpy.ndarray'>
(48000, 1000)
(1000,)
[2 0 2 0 0 1 2 1 3 2 0 2 1 1 3 2 0 2 2 1 2 0 3 0 3 2 1 0 1 3 0 1 1 3 2 0 3
 2 0 0 3 3 2 2 2 1 1 3 2 1 3 3 2 0 0 3 1 0 2 3 2 0 3 1 1 0 0 0 2 3 0 3 2 3
 2 2 0 0 0 2 3 3 1 2 0 2 3 2 1 2 2 1 1 1 3 0 1 3 3 0 1 1 2 3 1 3 0 3 3 1 2
 2 3 0 2 3 3 0 3 2 0 1 0 3 2 0 1 0 2 0 3 3 0 3 0 1 1 0 2 0 1 3 2 1 3 0 3 0
 2 0 3 0 0 3 1 2 0 1 2 0 0 2 1 3 1 0 2 2 2 3 0 2 2 3 1 3 3 2 3 1 2 1 1 3 1
 1 1 1 2 3 3 3 3 1 0 3 3 3 0 3 3 3 2 1 1 3 3 2 0 1 3 1 1 2 2 0 2 1 0 0 2 0
 1 0 2 2 2 0 0 3 2 3 2 0 0 1 0 0 3 2 2 2 2 3 2 3 1 0 0 0 3 0 1 2 0 1 0 0 3
 1 0 3 3 2 1 3 3 2 3 1 0 2 1 1 2 3 2 0 0 2 0 3 0 2 1 3 1 1 0 0 1 2 1 0 3 3
 3 3 2 2 3 2 3 1 2 3 2 0 0 2 3 2 2 0 3 1 3 1 2 2 2 3 3 1 1 2 2 0 1 2 3 1 2
 1 3 1 2 0 0 3 3 1 0 3 1 3 2 1 3 1 2 2 1 0 0 1 1 1 3 3 0 0 0 2 2 3 3 2 2 2
 0 2 2 0 0 3 1 1 1 0 2 2 2 3 1 1 2 0 1 3 0 0 3 0 0 2 0 2 3 0 3 2 1 2 1 3 1
 2 0 2 3 2 0 3 2 2 1 0 3 2 0 3 3 2 2 3 0 2 1 2 2 3 2 0 1 3 2 1 1 3 2 0 1 2
 2 2 2 2 1 2 2 0 3 2 3 2 0 0 0 1 0 3 0 2 3 0 3 1 0 0 1 3 1 2 0 2 3 3 3 1 1
 3 3 0 3 2 3 3 3 1 2 2 3 1 3 0 2 2 3 0 3 2 3 3 1 0 3 3 2 1 2 3 2 3 1 0 2 0
 0 3 0 3 0 3 3 1 2 2 1 0 3 2 0 2 2 2 2 0 0 3 2 0 2 1 0 0 3 0 3 0 0 0 3 2 1
 3 0 1 0 0 3 3 3 3 2 2 3 1 1 3 0 1 1 2 2 1 0 0 3 3 1 3 3 2 2 1 2 0 3 3 2 0
 1 1 2 1 2 3 3 3 0 3 2 3 2 3 1 2 3 1 2 3 0 3 2 0 2 3 2 0 3 2 3 0 1 3 2 2 0
 3 1 2 2 3 1 2 3 1 3 2 2 1 3 3 3 0 3 3 2 3 0 1 3 1 2 2 1 0 0 0 3 3 0 3 1 0
 0 2 2 2 2 2 2 0 1 1 3 1 1 1 3 1 1 1 0 1 2 1 0 1 3 2 2 1 1 1 1 1 1 3 0 1 1
 2 2 0 0 0 0 3 3 1 2 2 1 3 3 0 3 3 2 0 0 0 1 2 2 1 2 2 2 1 0 0 3 2 1 1 2 2
 3 3 2 2 0 1 0 2 3 2 1 0 0 2 0 2 0 3 0 3 2 1 2 1 2 2 2 2 2 1 2 1 3 1 3 0 1
 1 2 0 0 2 2 3 0 1 3 2 1 1 3 2 1 3 0 0 2 3 1 3 0 0 2 3 3 1 0 0 3 2 0 2 3 2
 1 2 2 1 3 2 3 3 3 3 2 1 3 0 2 3 2 3 1 2 1 0 3 2 2 1 0 2 0 2 0 2 3 0 2 2 2
 0 3 2 3 3 0 1 2 3 2 1 1 0 3 1 1 3 1 2 3 2 3 0 1 2 0 3 2 2 3 1 3 2 3 1 1 0
 0 0 1 1 2 2 3 1 0 0 3 0 1 3 3 2 3 2 0 3 3 0 2 1 0 2 2 1 2 0 1 0 2 3 1 0 2
 0 0 3 0 3 0 1 2 2 1 3 3 1 1 3 2 0 2 0 3 1 2 0 0 2 2 1 3 1 1 0 2 1 2 2 1 0
 1 3 1 0 0 3 1 2 3 0 2 1 2 3 1 2 0 0 2 0 0 1 1 2 0 3 1 3 0 2 3 2 1 2 2 0 1
 2]
print("Loading validation data...")
X_VALID,Y_VALID = load_data_from_files(NC_VAL_FILE,PC_VAL_FILE)Loading validation data...
Load data from files.
Ready: train_set
X4 <class 'list'>
X5 <class 'numpy.ndarray'>
(100000, 1000)
X_train <class 'numpy.ndarray'>
(100000, 1000)
(1000,)
[0 2 0 0 3 3 1 2 2 3 1 2 2 0 2 0 2 2 2 1 2 0 3 2 0 3 0 0 3 3 2 3 0 3 3 1 0
 1 1 0 2 1 1 1 1 2 3 3 0 2 1 1 3 2 3 2 1 2 3 3 0 0 1 2 2 2 3 2 0 1 2 2 2 2
 2 1 3 3 0 1 2 0 1 2 0 3 2 3 2 3 1 3 3 1 3 0 2 1 1 1 2 3 2 2 2 1 0 3 3 2 3
 3 3 1 1 3 3 2 1 1 2 1 3 2 2 1 3 1 3 1 0 0 3 2 2 2 1 2 0 0 1 0 2 0 1 3 2 1
 3 2 2 0 3 1 2 0 3 3 3 2 3 2 1 0 0 3 0 2 0 2 1 2 1 1 0 2 3 0 1 2 0 1 2 2 0
 2 1 2 3 1 1 1 2 3 3 3 2 1 1 0 2 3 1 1 2 1 0 0 3 2 1 0 2 2 3 2 0 0 3 1 3 0
 2 3 0 2 3 1 2 3 1 1 2 3 3 3 3 3 0 0 1 0 0 0 2 0 2 2 0 3 3 0 0 0 0 2 2 2 1
 0 0 1 1 1 0 2 0 3 1 2 0 0 3 3 1 1 2 1 3 2 3 2 1 2 2 0 0 1 0 1 0 3 3 3 0 0
 0 2 3 0 3 2 3 0 1 1 3 0 1 1 1 3 2 1 3 3 1 0 0 0 1 2 3 0 2 3 1 1 1 3 0 3 1
 2 0 1 2 1 2 2 3 1 1 1 2 0 2 0 0 3 3 0 2 1 0 0 1 0 1 3 3 2 3 3 1 0 3 2 3 0
 2 3 3 2 0 1 0 0 3 1 1 2 2 1 3 2 0 1 0 0 3 2 3 1 0 3 0 3 3 1 1 3 0 0 0 1 2
 1 1 1 3 1 2 2 1 1 1 2 1 0 1 1 2 3 2 1 2 1 3 3 2 3 2 2 2 3 3 0 3 1 0 3 3 0
 3 2 2 0 1 2 1 1 1 0 3 3 0 1 1 2 3 2 1 2 2 2 0 0 0 2 1 2 1 3 0 3 1 0 0 1 3
 2 3 3 2 1 0 3 3 0 1 1 0 0 3 3 1 0 2 2 1 2 0 0 0 2 0 1 2 2 3 1 0 0 2 0 0 1
 3 3 2 0 1 0 3 1 3 2 2 0 1 1 0 2 0 0 0 3 2 3 2 0 0 3 3 2 1 3 0 0 2 1 2 1 2
 1 0 2 2 2 3 2 3 3 2 1 2 0 0 2 3 1 0 1 2 1 3 2 3 1 0 2 0 2 1 0 2 1 1 1 1 3
 1 2 3 3 0 3 0 2 2 1 2 3 2 0 2 0 2 3 2 3 3 0 0 1 0 3 1 1 0 0 2 0 1 3 3 2 1
 3 3 3 0 2 2 3 1 2 1 3 2 3 0 2 3 1 0 2 0 2 2 0 1 3 3 0 0 0 3 0 1 1 0 0 3 0
 0 1 2 0 2 3 1 1 2 3 2 2 1 2 1 0 1 0 3 0 3 1 1 1 3 3 3 2 2 3 3 3 1 3 3 1 3
 0 2 3 0 2 2 3 0 3 3 1 3 1 1 2 3 0 3 2 0 1 1 3 1 1 1 1 3 0 1 3 2 1 0 2 3 2
 3 2 3 2 2 1 2 2 1 1 2 3 1 2 0 1 3 0 1 3 1 0 1 3 3 3 3 1 1 1 1 2 2 2 0 0 2
 1 0 0 3 1 0 2 2 2 2 1 2 3 2 2 0 3 1 1 1 1 3 0 0 2 1 3 0 2 2 0 3 3 0 2 2 2
 3 2 1 3 3 1 3 3 3 2 0 1 3 2 0 2 2 1 0 2 3 1 0 1 3 0 1 1 1 0 1 3 2 3 2 2 3
 2 3 1 0 1 0 3 0 1 1 2 0 3 1 0 2 1 1 2 0 0 1 3 2 1 3 1 3 0 2 3 1 0 1 0 2 2
 0 1 3 2 3 1 2 0 2 2 3 3 3 2 0 3 1 1 1 1 0 2 3 2 1 0 3 3 1 3 2 1 0 0 1 2 2
 3 2 1 0 0 2 2 3 0 2 0 1 2 3 3 1 0 3 2 3 0 2 1 2 3 0 1 0 2 3 1 0 1 0 1 0 1
 0 0 3 1 3 0 1 2 0 0 3 0 2 2 1 1 1 3 2 0 2 2 1 2 0 0 1 1 2 2 1 1 1 1 1 2 0
 3]
print ("Cross validation")
do_cross_validation(X_train,y_train,model,X_VALID,Y_VALID)
print ("Done")Cross validation
COMPILE...
...COMPILED
FIT
Epoch 1/10
1350/1350 [==============================] - 10s 6ms/step - loss: 0.6930 - accuracy: 0.5136 - val_loss: 0.6290 - val_accuracy: 0.6488
INFO:tensorflow:Assets written to: /content/drive/My Drive/data/Test_Non_noisy_24K.cv.1.best/assets
Epoch 2/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.6346 - accuracy: 0.6433 - val_loss: 0.5984 - val_accuracy: 0.6783
INFO:tensorflow:Assets written to: /content/drive/My Drive/data/Test_Non_noisy_24K.cv.1.best/assets
Epoch 3/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5990 - accuracy: 0.6850 - val_loss: 0.5793 - val_accuracy: 0.7058
INFO:tensorflow:Assets written to: /content/drive/My Drive/data/Test_Non_noisy_24K.cv.1.best/assets
Epoch 4/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5847 - accuracy: 0.6977 - val_loss: 0.5865 - val_accuracy: 0.6910
Epoch 5/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5631 - accuracy: 0.7167 - val_loss: 0.5629 - val_accuracy: 0.7148
INFO:tensorflow:Assets written to: /content/drive/My Drive/data/Test_Non_noisy_24K.cv.1.best/assets
Epoch 6/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5479 - accuracy: 0.7335 - val_loss: 0.5693 - val_accuracy: 0.7142
Epoch 7/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5340 - accuracy: 0.7371 - val_loss: 0.5663 - val_accuracy: 0.7165
INFO:tensorflow:Assets written to: /content/drive/My Drive/data/Test_Non_noisy_24K.cv.1.best/assets
Epoch 8/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5217 - accuracy: 0.7529 - val_loss: 0.6215 - val_accuracy: 0.6896
Epoch 9/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5059 - accuracy: 0.7641 - val_loss: 0.5882 - val_accuracy: 0.7019
Epoch 10/10
1350/1350 [==============================] - 7s 5ms/step - loss: 0.5003 - accuracy: 0.7671 - val_loss: 0.6019 - val_accuracy: 0.6938
Fold 1, 10 epochs, 84 sec
_____no_output_____


# Notebook from abostroem/asassn15oz
Path: notebooks/make_cachito_figure.ipynb

import os

from astropy.io import fits
from astropy.io import ascii as asc
from astropy.time import Time
import astropy.constants as c
import astropy.units as u
from astropy.modeling import models, fitting
import numpy as np
from matplotlib import pyplot as plt
%matplotlib
import matplotlib
from matplotlib.ticker import MultipleLocator

import spectroscopy as spec
import visualization as vis
import supernova

plt.style.use('default')Using matplotlib backend: Qt5Agg
cols = [(0,0,0)]
for x in np.linspace(0,1, 254):
    rcol = (0.472-0.567*x+4.05*x**2)/(1.+8.72*x-19.17*x**2+14.1*x**3)
    gcol = 0.108932-1.22635*x+27.284*x**2-98.577*x**3+163.3*x**4-131.395*x**5+40.634*x**6
    bcol = 1./(1.97+3.54*x-68.5*x**2+243*x**3-297*x**4+125*x**5)
    cols.append((rcol, gcol, bcol))

cols.append((1,1,1))
cm_rainbow = matplotlib.colors.LinearSegmentedColormap.from_list("PaulT_rainbow", cols)_____no_output_____

# Plot HA, Hb evolution Series_____no_output_____

sn15oz = supernova.LightCurve2('asassn-15oz')_____no_output_____def read_iraf_spectrum(filename):
    ofile = fits.open(filename)
    flux = ofile[0].data[0,0,:]
    wave = spec.calc_wavelength(ofile[0].header, np.arange(len(flux))+1)
    rest_wave = spec.apply_redshift(wave, 0.0069)
    return(spec.spectrum1d(rest_wave, flux))_____no_output_____def calc_velocity(obs_wl, rest_wl):
    velocity = c.c*(obs_wl/rest_wl - 1)
    return velocity_____no_output_____VEL_DATA_DIR = '../data/line_info/'
DATA_DIR_LCO = '../data/spectra/lco'
DATA_DIR_EFOSC = '../data/spectra/EFOSC'
IRTF_DIR = '../data/spectra/IRTF/'
SOFI_DIR = '../data/spectra/SOFI'_____no_output_____z = 0.0069 #15oz redshift
Ha = 6563.0
Hb = 4850.0
HeI = 10830.0
CI = 10691.0
paschen_g = 10940.0
paschen_d = 10050.0
FeII = 5169.0
IR_dates = Time(['2015-09-05','2015-10-05', '2015-10-10'])_____no_output_____spectra_files = [
         ('asassn15oz_20150904_redblu_122216.314.fits', DATA_DIR_LCO),
         ('asassn-15oz_20150906_redblu_105042.698a.fits', DATA_DIR_LCO),
         ('asassn15oz_20150907_redblu_123835.277.fits', DATA_DIR_LCO),
         ('asassn15oz_20150911_redblu_105336.349.fits', DATA_DIR_LCO),
         ('asassn15oz_20150916_redblu_120911.274.fits', DATA_DIR_LCO),
         ('asassn-15oz_20150920_redblu_135034.512.fits',DATA_DIR_LCO),
         ('asassn-15oz_20150924_redblu_123847.580.fits',DATA_DIR_LCO),
         #('asassn-15oz_20150930_redblu_122858.217.fits',DATA_DIR_LCO),
         ('tASASSN-15oz_20151003_Gr13_Free_slit1.0_57720_1_e.fits',DATA_DIR_EFOSC),
         #('asassn15oz_20151006_redblu_101906.800.fits', DATA_DIR_LCO),
         #('asassn15oz_20151014_redblu_112918.305.fits', DATA_DIR_LCO),
         ('asassn-15oz_20151025_redblu_102221.833.fits', DATA_DIR_LCO),
         #('asassn-15oz_20151107_redblu_101210.833.fits', DATA_DIR_LCO),
         ('tASAS-SN_15oz_20151107_Gr13_Free_slit1.5_57723_1_e.fits', DATA_DIR_EFOSC),
         ('tASAS-SN_15oz_20151118_Gr13_Free_slit1.0_57723_1_e.fits', DATA_DIR_EFOSC)
                ]_____no_output_____tbdata_sofi1_blue = fits.getdata(os.path.join(SOFI_DIR, 'asassn15oz_20150905_2457270.58657_1.fits'), 1) #wave, flux, err, skyback
tbdata_sofi2_blue = fits.getdata(os.path.join(SOFI_DIR, 'asassn15oz_20151005_2457300.50252_1.fits'), 1)
#Date-obs: 2015-10-10
tbdata_irtf = asc.read(os.path.join(IRTF_DIR, 'A15oz_merge.txt'), names=['wave', 'flux', 'err', 'junk'])_____no_output_____spec_sofi1 = spec.spectrum1d(spec.apply_redshift(tbdata_sofi1_blue['WAVE'][0], z), tbdata_sofi1_blue['flux'][0], tbdata_sofi1_blue['err'])
spec_sofi2 = spec.spectrum1d(spec.apply_redshift(tbdata_sofi2_blue['WAVE'][0], z), tbdata_sofi2_blue['flux'][0], tbdata_sofi2_blue['err'])
spec_irtf = spec.spectrum1d(spec.apply_redshift(tbdata_irtf['wave']*10**4, z), tbdata_irtf['flux'], tbdata_irtf['err'])_____no_output_____tbdata_feII = asc.read(os.path.join(VEL_DATA_DIR, 'FeII_multi.tab'))
phase_feII = (Time(tbdata_feII['date'])-Time(sn15oz.jdexpl, format='jd')).value
velocity_feII = -1*calc_velocity(tbdata_feII['vel0'], FeII).to(u.km/u.s).value

new_fit_cachito = asc.read(os.path.join(VEL_DATA_DIR, 'cachito.tab'))
phase_cachito = (Time(new_fit_cachito['date'])-Time(sn15oz.jdexpl, format='jd')).value
velocity_cachito = -1*calc_velocity(new_fit_cachito['vel0'], Ha).to(u.km/u.s).value

fitter_power = fitting.LevMarLSQFitter()
power_model = models.PowerLaw1D()
power_fit_cachito = fitter_power(power_model, phase_cachito, velocity_cachito)

#I had trouble fitting without good starting parameters, cachito looks like FeII, especially at early times
power_model_feII = models.PowerLaw1D(alpha=power_fit_cachito.alpha, x_0=power_fit_cachito.x_0)
power_fit_feII = fitter_power(power_model_feII, phase_feII, velocity_feII)
feII_velocity = power_fit_feII((IR_dates-Time(sn15oz.jdexpl, format='jd')).value)

paschen_g_wave = (feII_velocity*(u.km/u.s)/(c.c.to(u.km/u.s))+1)*paschen_g
paschen_g_HeI_vel = calc_velocity(paschen_g_wave, HeI).to(u.km/u.s)
paschen_g_rest_HeI_vel = calc_velocity(paschen_g, HeI).to(u.km/u.s)

paschen_d_wave = (feII_velocity*(u.km/u.s)/(c.c.to(u.km/u.s))+1)*paschen_d
paschen_d_HeI_vel = calc_velocity(paschen_d_wave, HeI).to(u.km/u.s)
paschen_d_rest_HeI_vel = calc_velocity(paschen_d, HeI).to(u.km/u.s)

CI_wave = (feII_velocity*(u.km/u.s)/(c.c.to(u.km/u.s))+1)*CI
CI_HeI_vel = calc_velocity(CI_wave, HeI).to(u.km/u.s)
print(paschen_g_HeI_vel)
print(paschen_d_HeI_vel)[ 13055.14011179   9505.22621916   9280.54287735] km / s
[-12395.89849146 -15657.01682975 -15863.42154503] km / s
plt.close()
fig = plt.figure(figsize=[5, 11])
ax1 = fig.add_subplot(1,3,2)
ax2 = fig.add_subplot(1,3,1)
ax3 = fig.add_subplot(1,3,3)
offset = np.arange(len(spectra_files))*0.4E-14
ref_spec = read_iraf_spectrum(os.path.join(spectra_files[0][1], spectra_files[0][0]))
#Consider making this all one color
colors = vis.make_color_wheel(spectra_files+[1], cmap=cm_rainbow)

for indx, ifile in enumerate(spectra_files):
    filename, idir = ifile
    ispec = read_iraf_spectrum(os.path.join(idir, filename))
    scale_ispec = spec.scale_spectra(ispec, ref_spec)
    ha_velocity = calc_velocity(scale_ispec.wave, Ha).to(u.km/u.s)
    hb_velocity = calc_velocity(scale_ispec.wave, Hb).to(u.km/u.s)
    ax1.plot(ha_velocity/1000, scale_ispec.flux+offset[indx], color=colors[indx])
    date = Time(fits.getval(os.path.join(idir, filename), 'date-obs', 0), out_subfmt='date')
    text_indx = np.where((scale_ispec.wave < 7000)&(scale_ispec.wave > 6000))[0]
    ax1.text(-26, (scale_ispec.flux+offset[indx])[text_indx[0]], '{}d'.format(int((date-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='right', va='bottom', fontweight='bold', color='k')
    ax2.plot(hb_velocity/1000, scale_ispec.flux+offset[indx], color=colors[indx])

scale_sofi1 = spec.scale_spectra(spec_sofi1, spec_sofi2) #SOFI1
scale_irtf = spec.scale_spectra(spec_irtf, spec_sofi2) #IRTF
heI_velocity_sofi2 = calc_velocity(spec_sofi2.wave, HeI).to(u.km/u.s)
heI_velocity_sofi1 = calc_velocity(scale_sofi1.wave, HeI).to(u.km/u.s)
heI_velocity_irtf = calc_velocity(scale_irtf.wave, HeI).to(u.km/u.s)

ax3.plot(heI_velocity_sofi1/1000, scale_sofi1.flux, color=colors[0])
ax3.text(5, 0.8E-15, '{}d'.format(int((Time('2015-09-05')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')

ax3.plot(heI_velocity_sofi2/1000, spec_sofi2.flux+2.7E-15, color=colors[-5])
ax3.text(5, 3.5E-15, '{}d'.format(int((Time('2015-10-05')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')
ax3.plot(heI_velocity_irtf/1000, scale_irtf.flux+3.5E-15, color=colors[-5])
ax3.text(5, 3.8E-15, '{}d'.format(int((Time('2015-10-10')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')

ax1.set_xticks([-20, -10, 0])
ax1.set_yticks([])
ax1.set_xlim(calc_velocity(6000, Ha).to(u.km/u.s).value/1000, calc_velocity(6750, Ha).to(u.km/u.s).value/1000)
ax1.set_ylim(0,0.5E-13)
ax1.set_xlabel('Velocity (1000 km/s)')
ax1.set_title(r'H-$\alpha$', fontsize='x-large')

ax2.set_xlim(calc_velocity(4500, Hb).to(u.km/u.s).value/1000, calc_velocity(4900, Hb).to(u.km/u.s).value/1000)
ax2.set_xticks([-20, -10, 0])
ax2.set_ylim(0.05E-13,0.5E-13)
#ax2.set_xlabel('Rest Wavelength ($\AA$)')
ax2.set_ylabel('Flux + offset ($erg/cm^2/s/\AA$)')
ax2.set_title(r'H-$\beta$', ha='left', fontsize='x-large')

ax3.set_ylim(0.05E-14,0.5E-14)
ax3.set_xlim(calc_velocity(10000, HeI).to(u.km/u.s).value/1000, calc_velocity(11000, HeI).to(u.km/u.s).value/1000)
#ax3.set_xlabel('Rest Wavelength ($\AA$)')
ax3.set_title(r'HeI', ha='left', fontsize='x-large')
ax3.set_xticks([-20, -10, 0])
minorLocator = MultipleLocator(5)
ax1.xaxis.set_minor_locator(minorLocator)
ax1.grid(which='minor', axis='x')
ax2.xaxis.set_minor_locator(minorLocator)
ax2.grid(which='minor', axis='x')
ax3.xaxis.set_minor_locator(minorLocator)
ax3.grid(which='minor', axis='x')
plt.subplots_adjust(wspace=0.4)
plt.
_____no_output_____

# Look at IR_____no_output_____

#plt.close()
fig = plt.figure()
ax3 = fig.add_subplot(1,1,1)

scale_sofi1 = spec.scale_spectra(spec_sofi1, spec_sofi2) #SOFI1
scale_irtf = spec.scale_spectra(spec_irtf, spec_sofi2) #IRTF
heI_velocity_sofi2 = calc_velocity(spec_sofi2.wave, HeI).to(u.km/u.s)
heI_velocity_sofi1 = calc_velocity(scale_sofi1.wave, HeI).to(u.km/u.s)
heI_velocity_irtf = calc_velocity(scale_irtf.wave, HeI).to(u.km/u.s)

ax3.plot(heI_velocity_sofi1/1000, scale_sofi1.flux+2.3E-15, color=colors[0])
ax3.text(5, 0.8E-15, '{}d'.format(int((Time('2015-09-05')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')

ax3.plot(heI_velocity_sofi2/1000, spec_sofi2.flux+2.7E-15, color=colors[-5])
ax3.text(5, 3.5E-15, '{}d'.format(int((Time('2015-10-05')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')
ax3.plot(heI_velocity_irtf/1000, scale_irtf.flux+3.5E-15, color=colors[-5])
ax3.text(5, 3.8E-15, '{}d'.format(int((Time('2015-10-10')-Time(sn15oz.jdexpl, format='jd')).value)),
             fontsize='small', ha='left', va='bottom', fontweight='bold', color='k')

l = ax3.vlines([paschen_g_HeI_vel[0].value/1000], ymin=3E-15, ymax=3.3E-15, linestyle=':', color='Gray', label=r'Pas-$\gamma$')
ax3.vlines([paschen_g_HeI_vel[1].value/1000], ymin=3.3E-15, ymax=3.7E-15, linestyle=':', color='Gray')
ax3.vlines([paschen_g_HeI_vel[2].value/1000], ymin=3.7E-15, ymax=4.0E-15, linestyle=':', color='Gray')

l1 = ax3.vlines([paschen_g_rest_HeI_vel.value/1000], ymin=3E-15, ymax=4E-15, linestyle='-', color='g', label=r'Pas-$\gamma$ Rest')

l2 = ax3.vlines([paschen_d_HeI_vel[0].value/1000], ymin=3E-15, ymax=3.3E-15, linestyle='-', color='cyan', label=r'Pas-$\delta$')
ax3.vlines([paschen_d_HeI_vel[1].value/1000], ymin=3.4E-15, ymax=3.7E-15, linestyle='-', color='cyan')
ax3.vlines([paschen_d_HeI_vel[2].value/1000], ymin=3.7E-15, ymax=4.0E-15, linestyle='-', color='cyan')

l3 = ax3.vlines([paschen_d_rest_HeI_vel.value/1000], ymin=3E-15, ymax=4.0E-15, linestyle='-', color='m', label=r'Pas-$\delta$ Rest')

l4 = ax3.vlines([paschen_d_HeI_vel[0].value/1000], ymin=3E-15, ymax=3.3E-15, linestyle='-', color='r', label=r'Pas-$\delta$')
ax3.vlines([CI_HeI_vel[1].value/1000], ymin=3.4E-15, ymax=3.7E-15, linestyle='-', color='r')
ax3.vlines([CI_HeI_vel[2].value/1000], ymin=3.7E-15, ymax=4.0E-15, linestyle='-', color='r')

ax3.legend([l, l1, l2, l3, l4], [r'Pas-$\gamma$@ FeII Vel',r'Pas-$\gamma$@ Rest Vel', r'Pas=$\delta$@ FeII Vel', r'Pas=$\delta$@ Rest Vel', r'CI @ FeII Vel'], fontsize='x-small')

ax3.set_ylim(2.5E-15, 4.0E-15)
ax3.set_xlim(calc_velocity(9000, HeI).to(u.km/u.s).value/1000, calc_velocity(12000, HeI).to(u.km/u.s).value/1000)
#ax3.set_xlabel('Rest Wavelength ($\AA$)')
ax3.set_title(r'HeI', ha='left', fontsize='x-large')
ax3.set_xticks([-20, -10, 0])
minorLocator = MultipleLocator(5)

ax3.xaxis.set_minor_locator(minorLocator)
ax3.grid(which='minor', axis='x')
plt.subplots_adjust(wspace=0.4)_____no_output_____