# Notebook from dmaticzka/dhx9_yass_analysis
Path: yass_calculate_pairs.ipynb

# Calculate YASS pairs for all genes_____no_output_____

<code>
%%bash

# extract gene coordinates

zcat /home/maticzkd/genomes/hg19/Homo_sapiens.GRCh37.75.gtf.gz |
awk '$3=="gene"' |
awk 'BEGIN{OFS="\t"}{print "chr"$1,$4-1,$5,$10"_"$12,255,$7}' |
tr -d '";' > genes.bed

wc -l genes.bed_____no_output_____%%bash

# extract genes with Alu annotation

bedtools intersect -u \
-a genes.bed \
-b /home/maticzkd/genomes/hg19/RepeatMasker_hg19_20161020_Alu.bed \
> genes_with_alu.bed

wc -l genes_with_alu.bed_____no_output_____%%bash

# get sequences

time bedtools getfasta \
-fi /home/maticzkd/genomes/hg19.fa \
-bed genes_with_alu.bed \
-s > genes_with_alu.fa

grep -c '^>' genes_with_alu.fa_____no_output_____%%bash

# split into individual fastas for processing

rm -rf fasplitroot
mkdir -p fasplitroot
time faSplit byName genes_with_alu.fa fasplitroot/_____no_output_____%%bash

ls -l fasplitroot/*.fa | wc -l_____no_output_____%%bash

# run yass for all genes

rm -rf yassbed
mkdir -p yassbed

time for FA in fasplitroot/*.fa
do
    BASE=`basename $FA`
    OUT=yassbed/${BASE%%.fa}_yass.bed
    >&2 echo ""
    >&2 echo ""
    >&2 echo "####################################################################################################"
    >&2 echo $BASE
    yass $FA -d 4 -r 1 | grep -v "^track" | gzip > $OUT.gz
done 2> yass.log_____no_output_____%%bash

# convert to yass results to bed format
# do some filtering of irrelevant alignments to reduce data size

(
zcat yassbed/*.bed.gz | head -n 1
zcat yassbed/*.bed.gz | grep -v '^#' | awk '$3-$2>=50 && $6-$5>=50 && $2-$6+1<=10000'
) | gzip > yassbed.gz_____no_output_____%%bash

# convert to genomic coordinates in bed12 format

time ./yass_reverse_repeats_to_bed12.R \
-i yassbed.gz -o "" |
sort -k1,1 -k2,2n --compress-program=gzip | \
uniq | gzip > yassbed12_genomic.bed.gz_____no_output_____
</code>


# Notebook from vivekruhela/miRSim
Path: Notebook/miRSim.ipynb

<code>
import os
import pandas as pd
import numpy as np
import shutil
import random
import sys
import threading
import gzip
import time
import warnings
from scipy.stats import gamma
from scipy.stats import uniform
from scipy.stats import expon
from scipy.stats import poisson_____no_output_____from tqdm import tqdm_notebook as tqdm
tqdm().pandas()_____no_output_____def alter_nt(seq_seed,position):
    if seq_seed[position] == 'A':
        seq_seed[position] = 'C'
    elif seq_seed[position] == 'T':
        seq_seed[position] = 'G'
    elif seq_seed[position] == 'G':
        seq_seed[position] = 'A'
    else:
        seq_seed[position] = 'T'

    return seq_seed[position]_____no_output_____def sequence_alteration(mir_seq,site,no_mismatch_seed,no_mismatch_xseed, seed):
    if isinstance(seed, int):
        random.seed(seed)
    if site == 'seed':
        seq_seed = mir_seq[1:7]
        seq_seed = list(seq_seed)
        idx = list(range(1,7))
        for i in range(no_mismatch_seed):
            x1 = random.randint(0, len(idx)-1)
            seq_seed[x1] = alter_nt(seq_seed,x1)
            idx.pop(x1)
            new_seed = ""
            new_seed = new_seed.join(seq_seed)
        mir_seq_new = mir_seq[:1] + new_seed + mir_seq[7:]

    elif site == 'xseed':
        Xseq_seed = mir_seq[7:]
        Xseq_seed = list(Xseq_seed)
        idx = list(range(0,len(Xseq_seed)))
        for  j in range(no_mismatch_xseed):
            x3 = random.randint(0, len(idx)-1)
            Xseq_seed[x3] = alter_nt(Xseq_seed,x3)
            idx.pop(x3)
            new_xseed = ""
            new_xseed = new_xseed.join(Xseq_seed)
        mir_seq_new = mir_seq[:7] + new_xseed

    elif site == 'both':
        no_mismatch_both = no_mismatch_seed + no_mismatch_xseed
        for k in range(no_mismatch_both):
            mir_seq1 = sequence_alteration(mir_seq,'seed',no_mismatch_seed , no_mismatch_xseed, seed)
            mir_seq_new = sequence_alteration(mir_seq1,'xseed',no_mismatch_seed , no_mismatch_xseed, seed)

    return mir_seq_new_____no_output_____def cigar_generation(actual_seq,altered_seq):
    seq1 = list(actual_seq)
    seq2 = list(altered_seq)
    cigar = []
    if len(seq1) == len(seq2):
        for idx in range(len(seq1)):
            if seq1[idx] == seq2[idx]:
                cigar.append('-')
            else:
                cigar.append(seq2[idx])
    else:
        cigar = ['-' for l in range(len(seq1))]

    return ''.join(cigar).upper()_____no_output_____def gff(gff_file,rna_type):
    gff_file = open(gff_file).read().split('\n')
    df = pd.DataFrame()
    if rna_type == 'mirna':
        gff_file = gff_file[13:-1]
        chr_loc = [c.split('\t')[0] for c in gff_file]
        mirna_type = [c.split('\t')[2] for c in gff_file]
        chr_start = [c.split('\t')[3] for c in gff_file]
        chr_end = [c.split('\t')[4] for c in gff_file]
        strand = [c.split('\t')[6] for c in gff_file]
        mir_id = [c.split('\t')[8].split(';')[-2].split('=')[-1] for c in gff_file]
        df['mir_id'] = mir_id
        df['chr'] = chr_loc
        df['chr_start'] = chr_start
        df['chr_end'] = chr_end
        df['strand'] = strand
        df['mirna_type'] = mirna_type
        df = df.set_index('mir_id')
        df = df[~df['mirna_type'].str.contains('miRNA_primary_transcript')]
        df = df.drop('mirna_type',axis=1)
    else:
        gff_file = gff_file[1:-1]
        rna_id = [c.split('\t')[-1] for c in gff_file]
        chr_loc = [c.split('\t')[0] for c in gff_file]
        chr_start = [c.split('\t')[3] for c in gff_file]
        chr_end = [c.split('\t')[4] for c in gff_file]
        strand = [c.split('\t')[6] for c in gff_file]
        df['rna_id'] = rna_id
        df['chr'] = chr_loc
        df['chr_start'] = chr_start
        df['chr_end'] = chr_end
        df['strand'] = strand
        df = df.set_index('rna_id')

    return df
    _____no_output_____def mir_location(df,rna_name,seed):
    if isinstance(seed, int):
        random.seed(seed)
    df1 = df.loc[[rna_name],:]
    if df1.shape[0] == 1:
        return [df1.iloc[0,0],df1.iloc[0,1],df1.iloc[0,2]]
    else:
        random_idx = random.randint(1,df1.shape[0]-1)
        return [df1.iloc[random_idx,0],df1.iloc[random_idx,1],df1.iloc[random_idx,2]]_____no_output_____def sort_by_chromosome(chr_list):
    a = [x.split('chr')[-1] for x in chr_list]
    if 'X' in a:
        X_present = True
        a.remove('X')
    else:
        X_present = False

    if 'Y' in a:
        Y_present = True
        a.remove('Y')
    else:
        Y_present = False
    if 'M' in a:
        M_present = True
        a.remove('M')
    else:
        M_present = False
    if 'MT' in a:
        MT_present = True
        a.remove('MT')
    else:
        MT_present = False

    a = list(map(int,a))
    b = sorted(a)
    c = ['chr'+str(x) for x in b]
    if X_present:
        c.append('chrX')
    if Y_present:
        c.append('chrY')
    if M_present:
        c.append('chrM')
    elif MT_present:
        c.append('chrMT')

    return(c)_____no_output_____def sequence_calculation(total_no_seq, desired_seq_percent,depth,seed,gff_df):

    chr_list = sort_by_chromosome(list(gff_df['chr'].unique()))
    if not desired_seq_percent == 0:
        n_seq = total_no_seq*desired_seq_percent/100
        chr_count = pd.DataFrame(pd.Series(gff_df['chr']).value_counts())
        chr_count = pd.DataFrame(chr_count, index=chr_list)
        n_seq_per_chr = [i*n_seq for i in list(chr_count['chr']/gff_df.shape[0])]
        n_seq_per_chr = list(map(int,n_seq_per_chr))
        n_seq_per_chr[-1] = int(n_seq - sum(n_seq_per_chr[:-1]))

        no_mir_chr = []
        actual_no_mir_chr = [len(list(gff_df[gff_df['chr'].str.contains(chr_name+'$')].index)) for chr_name in chr_list]
        for v1,v2 in zip(n_seq_per_chr,actual_no_mir_chr):
            if v1 > depth:
                no_mir = len(expression_split(v1,v2,'poisson',seed,depth))
                if no_mir >= 1:
                    no_mir_chr.append(no_mir)
                else:
                    print('Error : Minimum number of miRs per chromosome should be %d. With your provided information it is %d'%(1, no_mir))
                    print('Hint: Try reducing minimum depth so that minimum number of miRs per chromosome becomes more than 1.')
                    sys.exit(1)
            else:
                no_mir = 1
                no_mir_chr.append(no_mir)
    else:
        no_mir_chr = []
        n_seq_per_chr = []
#     print(n_seq_per_chr, len(n_seq_per_chr))
    return no_mir_chr,n_seq_per_chr_____no_output_____def expression_split(number, number_of_subsections, distribution, seed, min_random_number_desired):
    split_number_list = []
    cumulative_sum_of_random_numbers = 0
    current_subsection = 1
    max_random_number = int(number/number_of_subsections)
    if isinstance(seed, int):
        np.random.RandomState(seed)
    else:
        seed = np.random.RandomState()
    if min_random_number_desired < number:
        if min_random_number_desired > max_random_number:
#             print("WARNING: Cannot have min number as {} and split {} in {} subsections".format(min_random_number_desired, number, number_of_subsections))
            number_of_subsections = int(np.floor(number/min_random_number_desired))
            return expression_split(number,number_of_subsections,distribution, seed, min_random_number_desired)

        elif distribution == 'uniform':
            split_num1 = uniform.rvs(size=number_of_subsections, loc = 10, scale=20,random_state=seed)
        elif distribution == 'gamma':
            split_num1 = gamma.rvs(a=5, size=number_of_subsections,random_state=seed)
        elif distribution == 'exponential':
            split_num1 = expon.rvs(scale=1,loc=0,size=number_of_subsections,random_state=seed)
        elif distribution == 'poisson':
            split_num1 = poisson.rvs(mu=3, size=number_of_subsections,random_state=seed)
        try:
            split_num1 = [int(number*v/sum(split_num1)) for v in split_num1]
        except:
            # Error may occur when split_num1 = [0]
            expression_split(number, number_of_subsections, distribution, seed, min_random_number_desired)
        if len(split_num1) > 1:
            num_test = [1 for v in split_num1 if v <= min_random_number_desired]
            if sum(num_test) > int(0.25*len(split_num1)):
                return expression_split(number,int(number_of_subsections*0.75),distribution, seed, min_random_number_desired)
        for i in range(len(split_num1)):
            if split_num1[i] < min_random_number_desired:
                split_num1[i] = min_random_number_desired
        split_num1[-1] = number-sum(split_num1[:-1])
        if sum([1 for v in split_num1 if v<=0]) >= 1:
            return expression_split(number,int(number_of_subsections*0.75),distribution, seed, min_random_number_desired)
        return split_num1
    else:
#         print('WARNING : minimum depth is greater than provided number and can not be splitted.')
        expression_split(number,1,distribution, seed, number)_____no_output_____def update_gff(gff_df,out,ground_truth_filename):
    rna_truth = pd.read_csv(os.path.join(out,ground_truth_filename),sep = '\t')
    rna_truth = rna_truth.set_index('RNA_ID')
    for idx in range(rna_truth.shape[0]):
        chr_start = str(rna_truth.iloc[idx,4])
        chr_end = str(rna_truth.iloc[idx,5])
        gff_df = gff_df[~gff_df.isin([chr_start,chr_end])]
        gff_df = gff_df.dropna()
    return gff_df_____no_output_____def generate_sequence(fasta_seq, gff_df, rna_dict, no_mir_chr, n_seq_per_chr, depth, seq_error, out, out_file,write_mode,repeat,distribution, no_mismatch_seed , no_mismatch_xseed,seed):

    if isinstance(seed, int):
        random.seed(seed)
    if write_mode == 'write':
        rna_ground_truth = open(os.path.join(out,out_file),'w')
        header = 'RNA_ID\tImpure_Region\tCigar_String\tref_Sequence\tsynthetic_sequence\tchr\tchr_start\tchr_end\tExpression_count\n'
        rna_ground_truth.write(header)
    elif write_mode == 'append':
        rna_ground_truth = open(os.path.join(out,out_file),'a')

    chr_list = list(gff_df['chr'].unique())
    if no_mir_chr and n_seq_per_chr:
        for i in range(len(chr_list)):
            chr_name = chr_list[i] + '$'
            mir_complete_list = list(gff_df[gff_df['chr'].str.contains(chr_name)].index)
            if mir_complete_list:
#                 print(i)
                total_exp = n_seq_per_chr[i]
                if not no_mir_chr[i] > len(mir_complete_list):
                    mir_idx = [random.randint(0,len(mir_complete_list)-1) for v in range(int(no_mir_chr[i]))]
                    mir_list = [mir_complete_list[v] for v in mir_idx]
                else:
                    if repeat:
                        complete_flag = True
                        while complete_flag == True:
                            mir_complete_list += mir_complete_list
                            if no_mir_chr[i] < len(mir_complete_list):
                                complete_flag = False
                        mir_idx = [random.randint(0,len(mir_complete_list)-1) for v in range(int(no_mir_chr[i]))]
                        mir_list = [mir_complete_list[v] for v in mir_idx]
                    else:
                        mir_idx = [random.randint(0,len(mir_complete_list)-1) for v in range(int(mir_complete_list))]
                        mir_list = [mir_complete_list[v] for v in mir_idx]

                complete_flag = True
                while complete_flag:
                    try:
                        expression_counts = expression_split(total_exp,len(mir_list),distribution,seed,depth)
                        if expression_counts:
                            complete_flag = False
                    except:
                        print('The number of RNAs and total available depth in %s  is %d and %d too less with error %s' %(chr_name,len(mir_list),total_exp,seq_error))
                        expression_counts = expression_split(total_exp,len(mir_list),distribution,seed,int(depth*0.5))
                        if expression_counts:
                            complete_flag = False

                for mir,exp in zip(mir_list,expression_counts):
                    if seq_error == 'None':
                        mir_seq_new = rna_dict[mir]
                        mir_cigar = cigar_generation(rna_dict[mir],mir_seq_new)
                    elif seq_error == 'Seed_region':
                        mir_seq_new = sequence_alteration(rna_dict[mir],'seed',no_mismatch_seed , no_mismatch_xseed, seed)
                        mir_cigar = cigar_generation(rna_dict[mir],mir_seq_new)
                    elif seq_error == 'Outside_Seed_region':
                        mir_seq_new = sequence_alteration(rna_dict[mir],'xseed',no_mismatch_seed , no_mismatch_xseed, seed)
                        mir_cigar = cigar_generation(rna_dict[mir],mir_seq_new)
                    elif seq_error == 'Both_region':
                        mir_seq_new = sequence_alteration(rna_dict[mir],'both',no_mismatch_seed , no_mismatch_xseed, seed)
                        mir_cigar = cigar_generation(rna_dict[mir],mir_seq_new)

                    loc = mir_location(gff_df,mir,seed)
                    mir_depth = int(exp)
                    line = ''
                    line += mir + '\t' + seq_error + '\t' + mir_cigar + '\t' + rna_dict[mir] + '\t' + mir_seq_new + '\t' + loc[0] + '\t' + str(loc[1]) + '\t' + str(loc[2]) + '\t' + str(mir_depth) + '\n'
                    rna_ground_truth.write(line)
                    for dep in range(mir_depth):
                        fasta_seq.append('>' + mir)
                        fasta_seq.append(mir_seq_new)

    return fasta_seq
def write_small_fastq_chunks(fasta_seq_chunk,out_file,counter,out_file_type,quality_char,adaptor,seed):

    primer_nt = ['A','T','G','C']
    for line in fasta_seq_chunk:
        line1 = ""
        line_no = fasta_seq_chunk.index(line)
        if ">" in line:
            if out_file_type == 'fastq':
                line1 += "@" + line[1:] + '-' + str(counter) + '\n'
                seq = fasta_seq_chunk[line_no+1] + adaptor # "TGGAATTCTCGGGTGCCAAGG"
                primer_len = 75 - len(seq)
                primer_string = ''.join(char*random.randint(0,100) for char in primer_nt)
                primer_string = list(primer_string)
                primer_string = random.sample(primer_string,len(primer_string))
                primer_string = ''.join(c for c in primer_string)[:primer_len]
                seq += primer_string
                line1 += fasta_seq_chunk[line_no+1] + adaptor + primer_string + '\n' # "TGGAATTCTCGGGTGCCAAGG" + '\n'
                line1 += '+' + '\n'
                quality_string = ''.join(char*random.randint(0,15) for char in quality_char)
                quality_string = list(quality_string)
                quality_string = random.sample(quality_string,len(quality_string))
                quality_string = ''.join(c for c in quality_string)
                quality_flag = False
                while quality_flag:
                    if len(quality_string) < len(seq):
                        quality_string += quality_string
                        quality_string = list(quality_string)
                        quality_string = random.sample(quality_string,len(quality_string))
                        quality_string = ''.join(c for c in quality_string)
                    else:
                        quality_flag = True
                        break
                line1 += quality_string[:len(seq)] + '\n'
                out_file.write(line1)
                counter += 1
            elif out_file_type == 'fasta':
                line1 += line + '-' + str(counter) + '\n'
                line1 += fasta_seq_chunk[line_no+1] + adaptor + '\n'
                out_file.write(line1)
                counter += 1

#     out_file.close()_____no_output_____def execute_parallel_thread_for_file_write(fasta_seq_chunk,out_file,counter,out,out_file_type,quality_char,adaptor,seed):
    max_no_line = len(fasta_seq_chunk)
    if max_no_line > 10000:
        chunk_size = 10000
        no_of_splitted_file = int(np.ceil(max_no_line/chunk_size))
    else:
        chunk_size = max_no_line
        no_of_splitted_file = 1
    for j in range(no_of_splitted_file):
        if out_file_type == 'fastq':
            tmp_file_name = out_file + '_' + str(j) + '.fastq'
        else:
            tmp_file_name = out_file + '_' + str(j) + '.fasta'

        out_file1 = open(os.path.join(out,'temp',tmp_file_name),'w')
        fasta_seq_small_chunk = fasta_seq_chunk[j*chunk_size:(j+1)*chunk_size]
        counter1 = counter + j*len(fasta_seq_small_chunk)
        write_small_fastq_chunks(fasta_seq_small_chunk,out_file1,counter1,out_file_type,quality_char,adaptor,seed)_____no_output_____def write_fastq(fasta_seq, adaptor, ascii_base, out, out_file_name,out_file_type,seed,parallel_thread):

    # Not considering phred score less then 20 so that mean quality score is always greater than 20 for smooth bioinformatics analysis
    if ascii_base == 33:
        quality_char = ['5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K']
    elif ascii_base == 64:
        quality_char = ['T','U','V','W','X','Y','Z', '^',' ', '-', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']
    else:
        quality_char = []

    max_no_line = len(fasta_seq)
    no_of_splitted_file = parallel_thread
    chunk_size = int(np.ceil(max_no_line/no_of_splitted_file))
    print('Writing %d temporary files with chunk size of %d'%(no_of_splitted_file,chunk_size))
    if 'temp' in list(os.listdir(out)):
        cmd = 'rm -rf '+ os.path.join(out,'temp','..?* .[!.]*') + ' && rm -rf '+ os.path.join(out,'temp') + ' 2> /dev/null'
        os.system(cmd)
#         shutil.rmtree(os.path.join(out,'temp'))
        os.mkdir(os.path.join(out,'temp'))
    else:
        os.mkdir(os.path.join(out,'temp'))

    for j in range(no_of_splitted_file):
        tmp_file_name = 'tmp_'+str(j)
        out_file = tmp_file_name
        fasta_seq_chunk = fasta_seq[j*chunk_size:(j+1)*chunk_size]
        if j >=1:
            counter = j*len(fasta_seq_chunk)
            print('------------------------------')
            print('writing %d chunk'%j)
            threading.Thread(target=execute_parallel_thread_for_file_write,args=(fasta_seq_chunk,out_file,counter,out,out_file_type,quality_char,adaptor,seed,)).start()
        else:
            pass

    counter = 0
    tmp_file_name = 'tmp_'+str(0)
    out_file = tmp_file_name
    fasta_seq_chunk = fasta_seq[0*chunk_size:(0+1)*chunk_size]
    execute_parallel_thread_for_file_write(fasta_seq_chunk,out_file,counter,out,out_file_type,quality_char,adaptor,seed)
    time.sleep(20)
    print('-----------------------------------------------------------------------')
    print('All chunks are written. Now merging all the temporary files and compressing it.')
    if out_file_type == 'fastq':
        cmd = 'gzip '+ os.path.join(out,'temp') +'/tmp_*.fastq -c > ' + os.path.join(out,out_file_name)
    else:
        cmd = 'gzip '+ os.path.join(out,'temp') + '/tmp_*.fasta -c > ' + os.path.join(out,out_file_name)
    os.system(cmd)_____no_output_____def generate_synthetic_data(fa_file,total_no_seq,std_seq_percent,seed_error_percent,xseed_error_percent,
                            both_error_percent,depth,ascii_base,out,out_file_name,ground_truth_filename,
                            gff_file,mir_type,out_file_type,repeat,distribution,seed,adaptor,no_mismatch_seed ,
                            no_mismatch_xseed,parallel_thread):

    file_read = open(fa_file).read().split('\n')
    file_read = file_read[:-1]
    rna_dict = {}
    for line in file_read:
        if ">" in line:
            line_no = file_read.index(line)
            rna_dict[line[1:]] = file_read[line_no+1]

    fasta_seq = []
    # for pure sequences only
    gff_df = gff(gff_file,mir_type)
    no_mir_chr,n_seq_per_chr = sequence_calculation(total_no_seq, std_seq_percent, depth,seed,gff_df)
    fasta_seq = generate_sequence(fasta_seq,gff_df, rna_dict, no_mir_chr, n_seq_per_chr, depth, 'None', out, ground_truth_filename,'write',repeat,distribution, no_mismatch_seed , no_mismatch_xseed, seed)
    print('****Pure Sequence are generated.****')

    # for seed region error sequences only
    if not repeat:
        gff_df = update_gff(gff_df,out,ground_truth_filename)  # Update gff dataframe so that there is no repitition.
    no_mir_chr,n_seq_per_chr = sequence_calculation(total_no_seq, seed_error_percent, depth,seed,gff_df)
    fasta_seq = generate_sequence(fasta_seq,gff_df, rna_dict, no_mir_chr, n_seq_per_chr, depth, 'Seed_region', out, ground_truth_filename,'append',repeat,distribution, no_mismatch_seed , no_mismatch_xseed, seed)
    print('****Sequence with error in seed region are generated.****')

    # for outside seed region error sequences only
    if not repeat:
        gff_df = update_gff(gff_df,out,ground_truth_filename)
    no_mir_chr,n_seq_per_chr = sequence_calculation(total_no_seq, xseed_error_percent, depth,seed,gff_df)
    fasta_seq = generate_sequence(fasta_seq,gff_df, rna_dict, no_mir_chr, n_seq_per_chr, depth, 'Outside_Seed_region', out, ground_truth_filename,'append',repeat,distribution, no_mismatch_seed , no_mismatch_xseed,seed)
    print('****Sequence with error in xseed region are generated.****')

    # for both seed region error and outside seed region error sequences
    if not repeat:
        gff_df = update_gff(gff_df,out,ground_truth_filename)
    no_mir_chr, n_seq_per_chr = sequence_calculation(total_no_seq, both_error_percent, depth,seed,gff_df)
    fasta_seq = generate_sequence(fasta_seq,gff_df, rna_dict, no_mir_chr, n_seq_per_chr, depth, 'Both_region', out, ground_truth_filename,'append',repeat,distribution, no_mismatch_seed , no_mismatch_xseed,seed)
    print('****Sequence with error in both seed and xseed region are generated.****')

    write_fastq(fasta_seq, adaptor, ascii_base, out, out_file_name,out_file_type,seed,parallel_thread)

#     merge similar synthetic sequences
    ground_truth_df = pd.read_csv(os.path.join(out,ground_truth_filename),sep='\t')
    ground_truth_df = ground_truth_df.groupby(['RNA_ID','Impure_Region','Cigar_String','ref_Sequence','synthetic_sequence','chr','chr_start','chr_end']).sum().reset_index()
    ground_truth_df = ground_truth_df.sort_values('Impure_Region')
    ground_truth_df.to_csv(os.path.join(out,ground_truth_filename),encoding='utf-8',index=False)

    # deleting temporary folder
    cmd = 'rm -rf '+ os.path.join(out,'temp','..?* .[!.]*') + ' && rm -rf '+ os.path.join(out,'temp') + ' 2> /dev/null'
    os.system(cmd)
    _____no_output_____def miRSim_main_module(fa_file,total_no_seq,std_seq_percent,depth,ascii_base,out,out_file_name,ground_truth_filename,
                       gff_file,mir_type,out_file_type,repeat,distribution,seed,adaptor,no_mismatch_seed , no_mismatch_xseed,
                       parallel_thread,*argv):
    start = time.time()

    if no_mismatch_seed >= 6:
        sys.exit("Maximum number of mismatch in seed region should not be more than 6.")

    if no_mismatch_xseed >= 12:
        sys.exit("Maximum number of mismatch in xseed region should not be more than 12.")

    if len(list(argv)) == 1:
        non_rna = list(argv)[0]
        seed_error_percent = int(non_rna/3)
        xseed_error_percent = int(non_rna/3)
        both_error_percent = non_rna - seed_error_percent - xseed_error_percent

        generate_synthetic_data(fa_file,total_no_seq,std_seq_percent,seed_error_percent,xseed_error_percent,
                                both_error_percent,depth,ascii_base,out,out_file_name,ground_truth_filename,
                                gff_file,mir_type,out_file_type,repeat,distribution,seed,adaptor,no_mismatch_seed ,
                                no_mismatch_xseed,parallel_thread)
        end = time.time()
        time_taken = '%.2f' % ((end-start)/60)
        print(f"Runtime of the program is {time_taken} minutes")

    elif len(list(argv)) == 3:
        seed_error_percent,xseed_error_percent,both_error_percent = list(argv)
        generate_synthetic_data(fa_file,total_no_seq,std_seq_percent,seed_error_percent,xseed_error_percent,
                                both_error_percent,depth,ascii_base,out,out_file_name,ground_truth_filename,
                                gff_file,mir_type,out_file_type,repeat,distribution,seed,adaptor,no_mismatch_seed , no_mismatch_xseed,
                                parallel_thread)
        end = time.time()
        time_taken = '%.2f' % ((end-start)/60)
        print(f"Runtime of the program is {time_taken} minutes")
    else:
        sys.exit("Provide either non-RNA fraction or seed,xseed error fractions.")_____no_output_____
</code>

**Change the path of required files**_____no_output_____

<code>
save_path = os.getcwd()
mir_file = '/mnt/miRSim/refs/mature_high_conf_hsa.fa'
mir_gff = '/mnt/miRSim/refs/hsa_high_conf.gff3'
adaptor = 'TGGAATTCTCGGGTGCCAAGG'
parallel_thread = 12
pir_file = '/mnt/miRSim/refs/piRNAdb.hsa.v1_7_5.fa'
pir_gff = '/mnt/miRSim/refs/pirnadb.hg38.gff3'_____no_output_____miRSim_main_module(mir_file,50000,10,10,33,save_path,'mirna_raw_data.fastq.gz','mirna_ground_truth.csv',
                        mir_gff,'mirna','fastq',True,'poisson','None',adaptor,2,2,parallel_thread,10,10,10)****Pure Sequence are generated.****
****Sequence with error in seed region are generated.****
****Sequence with error in xseed region are generated.****
****Sequence with error in both seed and xseed region are generated.****
Writing 12 temporary files with chunk size of 3334
------------------------------
writing 1 chunk
------------------------------
writing 2 chunk
------------------------------
writing 3 chunk
------------------------------
writing 4 chunk
------------------------------
writing 5 chunk
------------------------------
writing 6 chunk
------------------------------
writing 7 chunk
------------------------------
writing 8 chunk
------------------------------
writing 9 chunk
------------------------------
writing 10 chunk
------------------------------
writing 11 chunk
-----------------------------------------------------------------------
All chunks are written. Now merging all the temporary files and compressing it.
Runtime of the program is 0.47 minutes

</code>


# Notebook from Cellular-Longevity/cmapPy
Path: tutorials/loyal_tutorial.ipynb

<code>
%load_ext autoreload
%autoreload 2

import boto3
import pandas as pd_____no_output_____session = boto3.Session(profile_name='default') # requires a local ~/.aws/credentials file w/ permission
s3_client = session.client('s3')_____no_output_____s3_client.download_file_____no_output_____# download some metadata we want to plug in
bucket = 'bioinformatics-loyal'
remotePath = 'dtingley/data/processed/metaMatrix.csv'
file = 'metaMatrix.csv'
s3_client.download_file(bucket, remotePath, file)
metaMatrix = pd.read_table('metaMatrix.csv',sep=',',index_col=0)
metaMatrix['sample_id'] = metaMatrix.index + '_saliva'
metaMatrix.head()_____no_output_____
remotePath ='processed_methylation_data/combined_datasets/all_saliva/matrices_processed/methylation_filtered.gctx_n98x997979.gctx'
file = 'methylation_filtered.gctx_n98x997979.gctx'

s3_client.download_file(bucket, remotePath, file)_____no_output_____
from cmapPy.pandasGEXpress.view import view
nodeNames = view('methylation_filtered.gctx_n98x997979.gctx')
# !h5ls -r methylation_filtered.gctx_n98x997979.gctx
/                        Group
/0                       Group
/0/DATA                  Group
/0/DATA/0                Group
/0/DATA/0/coverage_matrix Dataset {98, 997979}
/0/DATA/0/methylation_matrix Dataset {98, 997979}
/0/META                  Group
/0/META/COL              Group
/0/META/COL/file         Dataset {98}
/0/META/COL/id           Dataset {98}
/0/META/COL/sample_id    Dataset {98}
/0/META/COL/sequence_id  Dataset {98}
/0/META/ROW              Group
/0/META/ROW/chrom        Dataset {997979}
/0/META/ROW/end          Dataset {997979}
/0/META/ROW/id           Dataset {997979}
/0/META/ROW/start        Dataset {997979}

import h5py
gctx_file = h5py.File(file, "r")
dataNodes = view(file,printing=False,filter='DATA')
dataNodes
_____no_output_____
gctx_file[dataNodes[0]]_____no_output_____
from cmapPy.pandasGEXpress.parse import parse
gctx_df = parse(file,make_multiindex=True)
# parse?
/0/DATA/0/matrix

my_col_metadata = parse(file, col_meta_only=True)
my_row_metadata = parse(file, row_meta_only=True)_____no_output_____# CpG site information
my_row_metadata_____no_output_____# sample metadata, this seems like the place to add metadata fields..
my_col_metadata_____no_output_____ _____no_output_____my_col_metadata = pd.merge(my_col_metadata,metaMatrix,how='outer',on='sample_id')_____no_output_____my_col_metadata['cid'] = my_col_metadata['sample_id']
my_col_metadata = my_col_metadata.set_index('cid')
my_col_metadata.head() _____no_output_____# TODO, write back to GCTX file one we figure out the data matrix bug w/ parse.py????
_____no_output_____gctobj = pdg.GCToo.GCToo(data_df, row_metadata_df=my_row_metadata, col_metadata_df=my_col_metadata)

pdg.write_gctx.write(gctobj,'test_output.gctx')_____no_output_____
</code>


# Notebook from lucasosouza/kaggle-challenges
Path: round7.ipynb

# import useful stuff
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor
import re
import numpy as np

from sklearn.feature_selection import SelectKBest
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import f1_score
from sklearn.metrics import recall_score

from sklearn.linear_model import LogisticRegression

from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler

from sklearn.metrics import accuracy_score

# avoid undefined metric warning when calculating precision with 0 labels defined as 1
import warnings
warnings.filterwarnings('ignore')

from sklearn.linear_model import SGDClassifier_____no_output_____

### Data transformations_____no_output_____1. Load data
2. Transform data
3. Create features and lables
4. Re-select features
5. Train
6. Cross-validate_____no_output_____#### Load Data_____no_output_____

def gen_data(test=None):

    if test:
        df = pd.read_csv('test_ver2.csv')
    else:
        df = pd.read_csv('train_ver2.csv')

    # separate the labels
    labels = []
    for col in df.columns:
        if col[:4] == 'ind_' and col[-4:] == 'ult1':
            labels.append(col)

    # create X and y delete dataframe
    X = df[df.columns.difference(labels)]
    y = df[labels].fillna(value=0) # NAs in labels will be considered 0
    del df

    return X,y, labels_____no_output_____X, y, labels =  gen_data()_____no_output_____threshold = X['ncodpers'].quantile(.9)_____no_output_____X = X[X['ncodpers'] > threshold]
y = y.loc[X.index]_____no_output_____X.shape, y.shape_____no_output_____

#### Transform data_____no_output_____

def transform(df, fillna=True):
    """ This version includes variables considered relevant"""

    ### variables to be removed ###
    # remove cod_prov only, since it is redundant with nomprov
    # removed fecha_alta - redundant with antiguedad
    for col in ['cod_prov', 'fecha_alta', 'ult_fec_cli_1t', 'pais_residencia']:
        del df[col]

    ### numerical_vars ###
    # convert numerical vars to int
    numerical_vars = ['age', 'antiguedad', 'renta']
    df[numerical_vars] = df[numerical_vars].convert_objects(convert_numeric=True)

    # change less or equal than 0 to nan
    for var in numerical_vars:
        df.ix[df[var] < 0, var] = np.nan

    ### boolean and categorical vars ###
    # convert S/N to boolean and remaining to number
    boolean_vars = ['indfall', 'ind_actividad_cliente', 'ind_nuevo', 'indresi', 'indext',
                    'tipodom', 'conyuemp', 'ind_actividad_cliente']
    for var in ['indfall', 'indresi', 'indext', 'conyuemp']:
        df[var] = df[var] == 'S'
    df[boolean_vars] = df[boolean_vars].convert_objects(convert_numeric=True)

    # one hot encode categorical vars
    # 150 canais, 103 paises, 52 provincias
    categorical_vars = ['segmento', 'sexo', 'tiprel_1mes', 'canal_entrada', 'nomprov',
                        'ind_empleado', 'indrel_1mes']
    df = pd.get_dummies(df, prefix=None, prefix_sep='_', dummy_na=False,
                       columns=categorical_vars, sparse=False, drop_first=False)

    ### handling null values ###
    if fillna:
        df = df.fillna(value=0)
    else:
        df = df.dropna()

    ### end ###

    return df_____no_output_____X = transform(X)
y = y.loc[X.index]_____no_output_____

#### Generate features and labels_____no_output_____I want to capture evolution. So it would be current month - past months. I can do this for past 6 months. That means only after some date I will be able to generate data.

At this point, there are two types of data in the dataset:
* Numerical: for these, I will only evaluate if it increased, or decreased, hence converting to three categories: decreased (-1), unchanged (0), increased(1).
* Boolean: Take current status minus previous status. If it has changed from 0 to 1, it will be 1, as in added. If changed from 1 to, it will be -1, as in removed. If it is 0, it means unchanged. _____no_output_____

# sort by ncodpers and fecha_dato
X = X.sort_values(['ncodpers', 'fecha_dato'])
y = y.loc[X.index]_____no_output_____# create differences
X_diff = X.drop(['fecha_dato'], axis=1).diff()
y_diff = y.diff()

# set index and ncodpers as column
X_diff['index'] = X_diff.index
X_diff['ncodpers'] = X['ncodpers']_____no_output_____first = X_diff.groupby('ncodpers').first().reset_index()['index']
last = X_diff.groupby('ncodpers').last().reset_index()['index']
len(first), len(last)_____no_output_____# remove negative y
y_diff[y_diff < 0] = 0_____no_output_____# set training and validation set
X_train = X_diff[-X_diff.index.isin(first)]
X_val = X_diff[X_diff.index.isin(last)]
y_train = y_diff[-X_diff.index.isin(first)]
y_val = y_diff[X_diff.index.isin(last)]_____no_output_____X_train.shape, X_val.shape, y_train.shape, y_val.shape_____no_output_____

### Classificate_____no_output_____

# scale features
scaler = MinMaxScaler()
X_train = scaler.fit_transform(X_train)_____no_output_____X_val = scaler.transform(X_val)_____no_output_____# for col in range(25):w
#     yc = y_train.iloc[:, col]
#     print(col, (yc==1).sum()/len(yc))_____no_output_____# train a simple classifier
col =23
clf = DecisionTreeClassifier()
clf.fit(X_train, y_train.iloc[:, col])_____no_output_____# get predictions
pred_train = clf.predict(X_train)
pred_val = clf.predict(X_val)
pred_train.mean(), pred_val.mean()_____no_output_____clf.score(X_train, y_train.iloc[:, col]), clf.score(X_val, y_val.iloc[:, col])_____no_output_____pred_train = clf.predict_proba(X_train)
pred_val = clf.predict_proba(X_val)_____no_output_____# evaluate results
#recall_train = recall_score(y_train.iloc[:, col], pred_train)
# recall_val = recall_score(y_val.iloc[:, 2], pred_val)_____no_output_____recall_train_____no_output_____

### Get Deltas_____no_output_____### Initializing classifiers _____no_output_____

models = []
for label in labels:
    # select features
    selector = SelectKBest()
    selector.fit(X,y[label])
    features = list(list(zip(*sorted(zip(X.columns, selector.scores_), key=lambda x:-x[1])))[0])

    #classify
    clf = SGDClassifier()
    cv = StratifiedShuffleSplit()
    scores = cross_val_score(clf, X[features[:12]], y[label], cv=cv, scoring='f1')
    score_est = scores.mean()-scores.std()

    # store
    models.append((label, SGDClassifier(), features, score_est))
    print("Label {}: {:.2f}".format(label, score_est.mean()))Label ind_ahor_fin_ult1: 0.00
Label ind_aval_fin_ult1: 0.00
Label ind_cco_fin_ult1: 0.79
Label ind_cder_fin_ult1: 0.00
Label ind_cno_fin_ult1: 0.00
Label ind_ctju_fin_ult1: 0.61
Label ind_ctma_fin_ult1: 0.00
Label ind_ctop_fin_ult1: 0.00
Label ind_ctpp_fin_ult1: 0.00
Label ind_deco_fin_ult1: 0.00
Label ind_deme_fin_ult1: 0.00
Label ind_dela_fin_ult1: 0.49
Label ind_ecue_fin_ult1: 0.00
Label ind_fond_fin_ult1: 0.00
Label ind_hip_fin_ult1: 0.00
Label ind_plan_fin_ult1: 0.00
Label ind_pres_fin_ult1: 0.40
Label ind_reca_fin_ult1: 0.00
Label ind_tjcr_fin_ult1: 0.00
Label ind_valo_fin_ult1: 0.00
Label ind_viv_fin_ult1: 0.00
Label ind_nomina_ult1: 0.00
Label ind_nom_pens_ult1: 0.00
Label ind_recibo_ult1: 0.00

### Further training all classifiers_____no_output_____

# sorted(filter(lambda v:v[1][2]>.3, models.items()), key=lambda x:-x[1][2])
ordered_models = sorted(models, key=lambda x:-x[3])_____no_output_____# init scaler
scaler = MinMaxScaler()
n_features = 6

# loop chunks of data
for i, chunk in enumerate(chunks):
    print(i, len(chunk))
    X,y, labels = gen_data(chunk)
    X = transform(X)
    y = y.loc[X.index]
    print(X.shape)
    scaler.partial_fit(X[numerical_vars])
    X[numerical_vars] = scaler.transform(X[numerical_vars])

    # loop models/labels
    for j, model in enumerate(ordered_models):
        label, clf, features, score = model

        # exception: if first chunk, create new classifier
        if i == 0:
            clf = SGDClassifier()
            ordered_models[j] = (label, clf, features, score)

        print(label)

        # select features and partial fit classifier
        clf.partial_fit(X[features[:n_features]],y[label], classes=[0,1])0 1705913
(1356361, 252)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
1 1705914
(1356678, 251)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
2 1705913
(1356230, 249)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
3 1705914
(1356880, 252)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
4 1705914
(1356658, 249)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
5 1705913
(1357292, 250)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
6 1705914
(1356573, 251)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
7 1705914
(1356228, 253)
ind_cco_fin_ult1
ind_ctju_fin_ult1
ind_dela_fin_ult1
ind_pres_fin_ult1
ind_ahor_fin_ult1
ind_aval_fin_ult1
ind_cder_fin_ult1
ind_cno_fin_ult1
ind_ctma_fin_ult1
ind_ctop_fin_ult1
ind_ctpp_fin_ult1
ind_deco_fin_ult1
ind_deme_fin_ult1
ind_ecue_fin_ult1
ind_fond_fin_ult1
ind_hip_fin_ult1
ind_plan_fin_ult1
ind_reca_fin_ult1
ind_tjcr_fin_ult1
ind_valo_fin_ult1
ind_viv_fin_ult1
ind_nomina_ult1
ind_nom_pens_ult1
ind_recibo_ult1
# reimport the data for remaining
nrows = 13647309
indices = np.random.permutation(nrows)

# divided data into 100 chunks, pick 5 for validation
# only the first will be used for feature selection.
chunks_count = 100
csize = len(indices)/chunks_count
chunks = []
for i in range(chunks_count):
    chunks.append(indices[i*csize:(i+1)*csize])_____no_output_____for i, chunk in enumerate(chunks[:5]):
    X,y, labels = gen_data(chunk)
    X = transform(X)
    y = y.loc[X.index]
    print(X.shape)
    X[numerical_vars] = scaler.transform(X[numerical_vars])
    # loop models/labels
    for j, model in enumerate(ordered_models):
        label, clf, features, scores = model

        # exception: if first chunk, create array of scores
        if i == 0:
            scores = []
            ordered_models[j] = (label, clf, features, scores)

        # select features and partial fit classifier
        y_pred = clf.predict(X[features[:n_features]])
        score = f1_score(y[label], y_pred)
        scores.append(score)(108421, 236)
(108503, 235)
(108475, 238)
(108246, 234)
(108648, 236)
# calc mean precision
def calc_mean_score(t):
    mean_score = sum(t[3])/len(t[3])
    return (t[0], t[1], t[2], mean_score)

om = list(map(lambda x:calc_mean_score(x), ordered_models))_____no_output_____# filter only f1 scores above 40%
om = list(filter(lambda x:x[3]>.3, om))
_____no_output_____# predict for training data
X,y, labels = gen_data(test=True)
results = pd.DataFrame(X['ncodpers'])
X = transform(X, fillna=True)
X[numerical_vars] = scaler.transform(X[numerical_vars])
for model in om:
    label, clf, features, scores = model
    results[label] = clf.predict(X[features[:n_features]])_____no_output_____# check count for each results
for col in results.columns - ['ncodpers']:
    print(results[col].value_counts())1    929615
Name: ind_cco_fin_ult1, dtype: int64
0    926942
1      2673
Name: ind_ctju_fin_ult1, dtype: int64
0    894020
1     35595
Name: ind_dela_fin_ult1, dtype: int64
0    928412
1      1203
Name: ind_pres_fin_ult1, dtype: int64
# now I need to fetch the last row for these in the training_data
df = pd.read_csv('train_ver2.csv')_____no_output_____# checking if all ids from test are in training database
ids_from_train = df[df['fecha_dato']=='2016-05-28']['ncodpers']
ids_from_test = results['ncodpers']
len(ids_from_test) == len(set(ids_from_test).intersection(ids_from_train))_____no_output_____# get labels
labels = []
for col in df.columns:
    if col[:4] == 'ind_' and col[-4:] == 'ult1':
        labels.append(col)
print(len(labels))24
df_last = df[df['ncodpers'].isin(ids_from_test)]
df_last = df_last[df_last['fecha_dato']=='2016-05-28'][['ncodpers']+labels]
df_last.shape_____no_output_____# sort both to align dataframes
# results.sort_values('ncodpers', inplace=True)
# df_last.sort_values('ncodpers', inplace=True)

# reindex instead of sorting
df_last = df_last.set_index('ncodpers')
results = results.set_index('ncodpers')_____no_output_____#get the added products
fresults = results.copy()
for col in results.columns - ['ncodpers']:
    fresults[col] = results[col] - df_last[col]_____no_output_____# export
# clean memory
del df
del df_last
del results

# transform results to expected output
for label in fresults.columns:
    fresults.ix[fresults[label]==1, label] = label
    fresults.ix[fresults[label]==0, label] = ''
    fresults.ix[fresults[label]==-1, label] = ''

# concatenate results
fn_join_columns = lambda x:re.sub('\s+', ' ', ' '.join(x)).strip()

# add new column added products in report
fresults['added_products'] = fresults.apply(fn_join_columns, axis=1)

#export
fresults['added_products'].to_csv('round5d.csv', header=True, index=True)
_____no_output_____list(zip(*om))[3]

_____no_output_____


# Notebook from AexisMariusMagaud/Loufos
Path: flask/dist/app/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc.ipynb

\title{(some) LaTeX environments \par for Jupyter notebook}
\author{@jfbercher}
\maketitle_____no_output_____# Introduction_____no_output_____This extension for IPython 4.x or Jupyter enables to use some LaTeX commands and environments in the notebook's markdown cells.
\begin{enumerate}
\item **LaTeX commands and environments**
\begin{itemize}
\item support for some LaTeX commands within markdown cells, *e.g.* `\textit`, `\textbf`, `\underline`, `author`, `\title`, LaTeX comments
\item support for **theorems-like environments**, support for labels and **cross references**
\item support for **lists**: *enumerate, itemize*,
\item limited support for a **figure environment**,
\item support for an environment *listing*,
\item additional *textboxa* environment
\end{itemize}
\item **Citations and bibliography**
\begin{itemize}
\item support for `\cite` with creation of a References section
\end{itemize}
\item it is possible mix markdown and LaTeX markup
\item **Document-wide numbering of equations and environments, support for `\label` and `\ref`**
\item **Configuration toolbar**
\item **LaTeX_envs dropdown menu for a quick insertion of environments**
\item Support for **User $\LaTeX$ definitions file**
\item Environments title/numbering can be customized by users in ``user_envs.json`` config file
\item **Export to plain HTML, Slides and LaTeX with a customized exporter**
\item Styles can be customized in the `latex_env.css` stylesheet
\item **Autocompletion** for \$, (, {, [, for LaTeX commands and environments
\end{enumerate}

A simple illustration is as follows: one can type the following in a markdown cell
\begin{listing}
The dot-product is defined by equation (\ref{eq:dotp}) in theorem \ref{theo:dotp} just below:
\begin{theorem}[Dot Product] \label{theo:dotp}
Let $u$ and $v$ be two vectors of $\mathbb{R}^n$. The dot product can be expressed as
\begin{equation}
\label{eq:dotp}
u^Tv = |u||v| \cos \theta,
\end{equation}
where $\theta$ is the angle between $u$ and $v$ ...
\end{theorem}
\end{listing}
and have it rendered as

The dot-product is defined by equation (\ref{eq:dotp}) in theorem \ref{theo:dotp} just below:
\begin{theorem}[Dot Product] \label{theo:dotp}
Let $u$ and $v$ be two vectors of $\mathbb{R}^n$. The dot product can be expressed as
\begin{equation}
\label{eq:dotp}
u^Tv = |u||v| \cos \theta,
\end{equation}
where $\theta$ is the angle between $u$ and $v$ ...
\end{theorem}_____no_output_____## ** What's new **

**December 2, 2018** - version 1.4.5

- corrected two small issues in LaTeX templates
- added the possibility to "protect" authors in bibTeX reference, issue #41
- corrected a small issue in the optional removing of `toc2` toc-cell

**February 24, 2018** - version 1.4.2

- @jfbercher Add export to slides with lenvs `jupyter nbconvert --to slides_with_lenvs FILE.ipynb`
- @jfbercher Corrected inclusion of `latexdefs.tex` when exporting to html
- @olehwaa  Include notebook figures generated with matplotlib format set to notebook
- @jcb91 add fontawesome css to `latex_envs_toc.tpl` (needed for collapsible section icons)
- @jfbercher Added system option and per document toggle for autocompletion of `'\$\$','()','{}',''[]'`
- camaro@octavia  Include notebook figures generate when matplotlib format is set to notebook. This generates a png image in in a text/html cell. This image is copied into a image/png cell in the preprocessor, such that it will be recognized by nbconvert as a figure.
- @jfbercher Fix toolbar buttons labels for Jupyter version 5.2+
- @jcb91 use traitlets >= 4.1 tag API, and alias over (deprecated) shortname
- @jfbercher replace `\$\$.\$\$` by LaTeX begin-end equation
    - bib: enable to use \cref, \xref in addition to standard \ref
    - bib: add YEAR field if only DATE exists
    -add comment environment
     - add 5.x compatibility
- @ZrkBreizh Update bibtex2.js
- @olehwaa New method for generating figure captions in latex
- @jfbercher Add **autocomplete for LateX commands and environments**
- @jfbercher  Add a list of environments for which the optional env parameter is not displayed -- case of [H] optional parameter for Figure environment

**February 9, 2017** - version 1.3.7

- Enable customizing hotkeys for inserting environments (via `nbextensions_configurator`)
- LaTeX\_envs menu can be customized by editing `envsLatex.json` in `nbextensions/latex_envs` directory)
- Autocompletion for \$,(,{,[
- Updates to ensure compatibility with nbTranslate
- Recognize \\ [..\\ ] and \\ (..\\ ) as LaTeX equations delimiters in nb

**November 2, 2016** - version 1.3.1

- Support for **user environments configuration** file (`user_envs.json` in `nbextensions/latex\_envs` directory). This file is included by the html export template.
- Support for **book/report style numbering** of environments, e.g. "Example 4.2" is example 2 in section 4.
- Support for `\author`, `\title`, and `maketitle`. Author and title are saved in notebook metadata, used in html/latex exports. The maketitle command also formats a title in the LiveNotebook.
- Added a Toogle menu in the config toolbar to:
    - toggle use of user's environments config file
    - toggle `report-style` numbering

**September 18, 2016** - version 1.3

- Support for **user personal LaTeX definitions** file (`latexdefs.tex` in current directory). This file is included by the html and latex export templates.
- Style for nested enumerate environments added in `latex_envs.css`
- Added a Toogle menu in the config toolbar to:
    - toggle the display of the LaTeX_envs dropdown menu,
    - toggle the display of labels keys,
    - toggle use of user's LaTeX definition file
- **Cross references now use the true environment number instead of the reference//label key**. **References are updated immediately**. This works **document wide** and works for pre and post references
- Support for optional parameters in theorem-like environments
- Support for spacings in textmode, eg `\par`, `\vspace, \hspace`
- Support for LaTeX comments % in markdown cells
- Reworked loading and merging of system/document configuration parameters

**August 28, 2016** - version 1.2

- **Added support for nested environments of the same type**. Nesting environments of different type was already possible, but there was a limitation for nesting environments of the same kind; eg itemize in itemize in itemize. This was due to to the fact that regular expressions are not suited to recursivity. I have developped a series of functions that enable to extract nested environments and thus cope with such situations.

- Corrected various issues, eg [#731](https://github.com/ipython-contrib/jupyter_contrib_nbextensions/issues/731), [#720](https://github.com/ipython-contrib/jupyter_contrib_nbextensions/issues/720) where the content of nested environments was incorrectly converted to markdown.

- Completely reworked the configuration toolbar. Re-added tips.
- Added a toggle-button for the LaTeX\_envs menu
- Added system parameters that can be specified using the [nbextensions_configurator](https://github.com/Jupyter-contrib/jupyter_nbextensions_configurator/tree/master/src/jupyter_nbextensions_configurator/static/nbextensions_configurator). Thus reworked the configuration loading/saving.
- Reworked extension loading. It now detects if the notebook is fully loaded before loading itself.

**August 03, 2016** - version 1.13

- Added a template to also keep the toc2 features when exporting to html:
```
jupyter nbconvert --to html_toclenvs FILE.ipynb
```
- Added a dropdown menu that enables to insert all main LaTeX\_envs environments using a simple click. Two keybards shortcuts (Ctrl-E and Ctrl-I) for equations and itemize are also provided. More environments and shortcuts can be added in the file `envsLatex.js`.

- Added a link in the general help menu that points to the documentation.

**July 27, 2016** - version 1.1

- In this version I have reworked **equation numbering**. In the previous version, I used a specialized counter and detected equations rendering for updating this counter. Meanwhile, this feature has been introduced in `MathJax` and now we rely on MathJax implementation. rendering is significantly faster. We still have keep the capability of displaying only equation labels (instead of numbers). The numbering is automatically updated and is document-wide.
- I have completely reworked the **notebook conversion** to plain $\LaTeX$ and html. We provide specialized exporters, pre and post processors, templates. We also added entry-points to simplify the conversion process. It is now as simple as
```bash
jupyter nbconvert --to html_with_lenvs FILE.ipynb
```
to convert `FILE.ipynb` into html while keeping all the features of the `latex_envs` notebook extension in the converted version.
_____no_output_____# Main features
## Implementation principle

The main idea is to override the standard Markdown renderer in order to add a *small* parsing of LaTeX expressions and environments. This heavily uses regular expressions. The LaTeX expression are then rendered using an html version. For instance `\underline {something}` is rendered as `<u> something </u>`, that is \underline{something}. The environments are replaced by an html tag with a class derived from the name of the environment. For example, a `definition` denvronment will be replaced by an html rendering corresponding to the class `latex_definition`. The styles associated with the different classes are specified in `latex_env.css`. These substitutions are implemented in `thsInNb4.js`._____no_output_____## Support for simple LaTeX commands_____no_output_____We also added some LaTeX commands (e.g. `\textit`, `\textbf`, `\underline`) -- this is useful in the case of copy-paste from a LaTeX document. The extension also supports some textmode spacings, namely `\par`, `\vspace, \hspace` as well as `\title`, `\author`, `maketitle` and  LaTeX comments % in markdown cells. Labels and cross-references are supported, including for equations._____no_output_____## Available environments_____no_output_____- **theorems-like environments**: *property, theorem, lemma, corollary, proposition, definition,remark, problem,           exercise, example*,
- **lists**: *enumerate, itemize*,
- limited support for a *figure* environment,
- an environment *listing*,
- *textboxa*, wich is a `textbox` environment defined as a demonstration (see below).

More environments can be added easily in the user\_envs config file `user_envs.json` or directly in the javascript source file `thmsInNb4.js`. The rendering is done according to the stylesheet `latex_env.css`, which can be customized.
\begin{remark}
When exporting to html, the `latex_env.css` file honored is the file on the github CDN. However, customized css can be added in a `custom.css` file that must reside in the same directory as the notebook itself. The reason for that is that the `css` file must be in the same directory as the notebook file for inclusion, which means copying it in each working directory. As the rendering of the html file obtained is done using the original javascript code, the same is true for the source files; therefore it is better to curstomize environments in  `user_envs.json` which is taken into account when exporting to html.
\end{remark}_____no_output_____## Automatic numerotation, labels and references _____no_output_____Several counters for numerotation are implemented: counters for         problem, exercise, example, property, theorem, lemma, corollary, proposition, definition, remark, and figure are available.
Mathjax-equations with a label are also numbered document-wide.
An anchor is created for any label which enables to links things within the document: `\label` and `\ref` are both supported. A limitation was that numbering was updated (incremented) each time a cell is rendered. Document-wide automatic updating is implemented since version 1.3. A toolbar button is provided to reset the counters and refresh the rendering of the whole document (this is still useful for citations and bibliography refresh)._____no_output_____\label{example:mixing} A simple example is as follows, featuring automatic numerotation, and the use of labels and references. Also note that standard markdown can be present in the environment and is interpreted.
*The rendering is done according to the stylesheet `latex_env.css`, which of course, can be customized to specific uses and tastes*.

\begin{listing}
\begin{definition} \label{def:FT}
Let $x[n]$ be a sequence of length $N$. Then, its **Fourier transform** is given by
\begin{equation}
\label{eq:FT}
X[k]= \frac{1}{N} \sum_{n=0}^{N-1} x[n] e^{-j2\pi \frac{kn}{N}}
\end{equation}
\end{definition}
\end{listing}

\begin{definition} \label{def:FT}
Let $x[n]$ be a sequence of length $N$. Then, its **Fourier transform** is given by
\begin{equation}
\label{eq:FT2}
X[k]= \frac{1}{N} \sum_{n=0}^{N-1} x[n] e^{-j2\pi \frac{kn}{N}}
\end{equation}
\end{definition}_____no_output_____It is now possible to refer to the definition and to the equation by their labels,  as in:
\begin{listing}
As an example of Definition \ref{def:FT}, consider the Fourier transform (\ref{eq:FT2}) of a pure cosine wave given by
$$
x[n]= \cos(2\pi k_0 n/N),
$$
where $k_0$ is an integer.
\end{listing}_____no_output_____As an example of Definition \ref{def:FT}, consider the Fourier transform (\ref{eq:FT2}) of a pure cosine wave given by
$$
x[n]= \cos(2\pi k_0 n/N),
$$
where $k_0$ is an integer.
Its Fourier transform is given by
$$
X[k] = \frac{1}{2} \left( \delta[k-k_0] + \delta[k-k_0] \right),
$$
modulo $N$. _____no_output_____## Bibliography_____no_output_____### Usage_____no_output_____It is possible to cite bibliographic references using the standard LaTeX `\cite` mechanism. The extension looks for the references in a bibTeX file, by default `biblio.bib` in the same directory as the notebook. The name of this file can be modified in the configuration toolbar. It is then possible to cite works in the notebook, e.g.
\begin{listing}
The main paper on IPython is definitively \cite{PER-GRA:2007}. Other interesting references are certainly \cite{mckinney2012python, rossant2013learning}. Interestingly, a presentation of the IPython notebook has also be published recently in Nature \cite{shen2014interactive}.
\end{listing}
The main paper on IPython is definitively \cite{PER-GRA:2007}. Other interesting references are certainly \cite{mckinney2012python, rossant2013learning}. Interestingly, a presentation of the IPython notebook has also be published recently in Nature \cite{shen2014interactive}._____no_output_____### Implementation_____no_output_____The implemention uses several snippets from the nice [icalico-document-tools](https://bitbucket.org/ipre/calico/downloads/) extension that also considers the rendering of citations in the notebook. We also use a modified version of the [bibtex-js](https://code.google.com/p/bibtex-js/) parser for reading the references in the bibTeX file. The different functions are implemented in `bibInNb4.js`. The rendering of citations calls can adopt three styles (Numbered, by key or apa-like) -- this can be selected in the configuration toolbar. It is also possible to customize the rendering of references in the reference list. A citation template is provided in the beginning of file `latex_envs.js`:
```
var cit_tpl = {
// feel free to add more types and customize the templates
    'INPROCEEDINGS': '%AUTHOR:InitialsGiven%, ``_%TITLE%_\'\', %BOOKTITLE%, %MONTH% %YEAR%.',
    ... etc
```
The keys are the main types of documents, eg inproceedings, article, inbook, etc. To each key is associated a string where the %KEYWORDS% are the fields of the bibtex entry. The keywords are replaced by the correponding bibtex entry value. The template string can formatted with additional words and effects (markdown or LaTeX are commands are supported)_____no_output_____## Figure environment_____no_output_____Finally, it is sometimes useful to integrate a figure within a markdown cell. The standard markdown markup for that is `![link](image)`, but a limitation is that the image can not be resized, can not be referenced and is not numbered. Furthermore it can be useful for re-using existing code. Threfore we have added a limited support for the `figure` environment. This enables to do something like
\begin{listing}
\begin{figure}
\centerline{\includegraphics[width=10cm]{example.png}}
\caption{\label{fig:example} This is an example of figure included using LaTeX commands.}
\end{figure}
\end{listing}
which renders as
\begin{figure}
\centerline{\includegraphics[width=10cm]{example.png}}
\caption{\label{fig:example} This is an example of figure included using LaTeX commands.}
\end{figure}
Of course, this Figure can now be referenced:
\begin{listing}
Figure \ref{fig:example} shows a second filter with input $X_2$, output $Y_2$  and an impulse response denoted as $h_2(n)$
\end{listing}
Figure \ref{fig:example} shows a second filter with input $X_2$, output $Y_2$  and an impulse response denoted as $h_2(n)$_____no_output_____## figcaption_____no_output_____For Python users, we have added in passing a simple function in the `latex_envs.py` library.

This function can be imported classically, eg `from latex_envs.latex_envs import figcaption` (or `from jupyter_contrib_nbextensions.nbconvert_support.latex_envs import figcaption` if you installed from the jupyter_contrib repo).

Then, this function enables to specify a caption and a label for the next plot. In turn, when exporting to $\LaTeX$, the corresponding plot is converted to a nice figure environement with a label and a caption. The figure can be referenced (after export) using the classical `\ref{}` command.

To enable large and wide figure in  $\LaTeX$ open notebook menu View->Cell Toolbar -> Edit metadata, open the cell metadata and set "widefigure" : true_____no_output_____

<code>
%matplotlib inline
import matplotlib.pyplot as plt
from latex_envs.latex_envs import figcaption
from numpy import pi, sin, cos,arange
figcaption("This is a nice sine wave", label="fig:mysin")
plt.plot(sin(2*pi*0.01*arange(100)))
plt.show()
figcaption("This is a nice cosine wave", label="fig:mycos")
plt.plot(cos(2*pi*0.01*arange(100)))
plt.show()
_____no_output_____
</code>

Sine wave is shown in fig \ref{fig:mysin} and cosine in fig \ref{fig:mycos}._____no_output_____## Other features_____no_output_____- As shown in the examples, eg \ref{example:mixing} (or just below), **it is possible to mix LaTeX and markdown markup in environments**
- Support for **line-comments**: lines beginning with a % will be masked when rendering
- Support for **linebreaks**: `\par_`, where \_ denotes any space, tab, linefeed, cr, is replaced by a linebreak

- Environments can be nested. egg:
\begin{listing}
This is an example of nested environments, with equations inside\\
\begin{proof} Demo
% This is a comment
\begin{enumerate}
\item $$ \left\{ p_1, p_2, p_3 \ldots p_n \right\} $$
\item A **nested enumerate**
\item second item
\begin{enumerate}
\item $ \left\{ p_1, p_2, p_3 \ldots p_n \right\} $
\item And *another one*
\item second item
\begin{enumerate}
\item $$ \left\{ p_1, p_2, p_3 \ldots p_n \right\} $$
\item second item
\end{enumerate}
\end{enumerate}
\end{enumerate}
\end{proof}
\end{listing}
which results in
_____no_output_____This is an example of nested environments, with equations inside\par
\begin{proof} Demo
% This is a comment
\begin{enumerate}
\item
\begin{equation}\label{eq:}
\left\{ p_1, p_2, p_3 \ldots p_n \right\}
\end{equation}
$$ \left\{ p_1, p_2, p_3 \ldots p_n \right\} $$
\item A **nested enumerate**
\item second item
\begin{enumerate}
\item $\left\{ p_1, p_2, p_3 \ldots p_n \right\}$
\item And *another one*
\item second item
\begin{enumerate}
\item $$ \left\{ p_1, p_2, p_3 \ldots p_n \right\} $$
\item second item
\end{enumerate}
\end{enumerate}
\end{enumerate}
\end{proof}_____no_output_____## User interface_____no_output_____### Buttons on main toolbar_____no_output_____On the main toolbar, the extension provides three buttons ![three buttons](main_toolbar.png) The first one can be used to refresh the numerotation of equations and references in all the document. The second one fires the reading of the bibliography bibtex file and creates (or updates) the reference section. Finally the third one is a toogle button that opens or closes the configuration toolbar._____no_output_____### Configuration toolbar_____no_output_____The configuration toolbar  ![configuration toolbar](config_toolbar.png) enables to enter some configuration options for the extension.

First, the `LaTeX\_envs` title links to this documentation. Then, the bibliography text input can be used to indicate the name of the bibtex file. If this file is not found and the user creates the reference section, then this section will indicate that the file was not found. The `References` drop-down menu enables to choose the type of reference calls. The Equations input box enables to initiate numbering of equations at the given number (this may be useful for complex documents in several files/parts). The `Equations` drop-down menu let the user choose to number equation or to display their label instead. The two next buttons enable to toogle display of the LaTeX\_envs environments insertion menu or to toggle the displau of LaTeX labels. Finally The `Toogles` dropdown menu enable to toogle the state of several parameters. All these configuration options are then stored in the notebook's metadata (and restored on reload)._____no_output_____The `Toggles` dropdown menu
![Toggles](Toggles.png)

enables to toggle the state of several configuration options:

- display the `LaTeX_envs` insertion menu or not,
- show labels anchors,
- use $\LaTeX$ user own LaTeX defintions (loads `latexdefs.tex` file from current document directory),
- load user's environments configuration (file `user_envs.json` in `nbextensions/latex_envs` directory),
- select "report style" numbering of environments
_____no_output_____### The `LaTeX_envs` insertion menu
The `LaTeX_envs` insertion menu
![LaTeX_envs_menu](LaTeX_envs_menu.png)
enables a quick insertion of LaTeX environments, some with a keyboard shorcut (this can be customized in `envsLatex.js`). Besides, selected text will be inserted in the environment. _____no_output_____# Conversion to LaTeX and HTML_____no_output_____The extension works in the live-notebook. Since it relies on a bunch of javascript, the notebook does not render as is in services such as `nbviewer` or `github` viewer. Similarly, `nbconvert` does not know of the LaTeX constructs which are used here and therefore does not fully convert notebooks using this extension.

Therefore, we provide specialized templates and exporters to achieve these conversions. _____no_output_____## Conversion to html
We provide a template `latex_envs.tpl` and an exporter class `LenvsHTMLExporter` (in library `latex_envs.py`). Using that class, conversion simply amounts to
```
jupyter nbconvert --to latex_envs.LenvsHTMLExporter FILE.ipynb
```
A shortcut is also provided
```
jupyter nbconvert --to html_with_lenvs FILE.ipynb
```
It should be noted that the rendering is done exactly in the same way as in the livenotebook. Actually, it is the very same javascript which is run in the html file. The javascript functions are available on the extension github as well as in the `jupyter_notebook_extensions` CDN, which means that the rendering of the html files requires an internet connection (this is also true for the rendering of equations with MathJax).

Another template `latex_envs_toc.tpl` is provided which enables to also
keep the toc2 features when exporting to html (*it even works if you do not have the `toc2` extension!*):
```bash
jupyter nbconvert --to html_with_toclenvs FILE.ipynb
```

Finally, it is also possible to embed `latex_envs` in reveal-js slides. A `LenvsHTMLExporter` is provided that enables to convert to slides by
```bash
jupyter nbconvert --to slidesl_with_lenvs FILE.ipynb
```

## Conversion to LaTeX

We provide two templates `thmsInNb_article.tplx` and `thmsInNb_report.tplx` for article and report styles respectively. Anyway one can also use the standard article, report, book templates provided with nbconvert. Simply, we have improved some of the internals styles. More importantly, we provide an exporter class `LenvsLatexExporter` (also in library `latex_envs.py`). Using that class, conversion simply amounts to
```
jupyter nbconvert --to latex_envs.LenvsLatexExporter FILE.ipynb
```
A shortcut is also provided
```
jupyter nbconvert --to latex_with_lenvs FILE.ipynb
```
In addition, we provide several further options:

-  **removeHeaders**: Remove headers and footers, (default false)
-  **figcaptionProcess**: Process figcaptions, (default true)
- **tocrefRemove** Remove tocs and ref sections, + some cleaning, (default true),

These options can be specified on the command line as, eg,
```
jupyter nbconvert --to latex_with_lenvs --LenvsLatexExporter.removeHeaders=True -- LenvsLatexExporter.tocrefRemove=False FILE.ipynb
```
_____no_output_____\begin{example}
As for an example, the present document has been converted using
```
jupyter nbconvert --to latex_with_lenvs --LenvsLatexExporter.removeHeaders=True latex_env_doc.ipynb
```

Then the resulting file (without header/footer) has been included in the main file `documentation.tex`, where some LaTeX definitions of environments are done (namely listings, colors, etc) and compiled using

- `xelatex -interaction=nonstopmode documentation`
- `bibTeX documentation`
- `xelatex -interaction=nonstopmode documentation`

The output can be consulted [here](documentation.pdf).
\end{example}_____no_output_____# Installation_____no_output_____The extension consists in a package that includes a javascript notebook extension. Since Jupyter 4.2, this is the recommended way to distribute nbextensions. The extension can be installed

- from the master version on the github repo (this will be always the most recent version)
- via pip for the version hosted on Pypi
- as part of the great [Jupyter-notebook-extensions](https://github.com/ipython-contrib/Jupyter-notebook-extensions) collection. Follow the instructions there for installing. Once this is done, you can open a tab at  `http://localhost:8888/nbextensions` to enable and configure the various extensions.

From the github repo or from Pypi,

- **step 1**: install the package
     - `pip3 install https://github.com/jfbercher/jupyter_latex_envs/archive/master.zip [--user][--upgrade]`
     -  <span style="color:blue"> or</span> `pip3 install jupyter_latex_envs [--user][--upgrade]`
     - <span style="color:blue"> or</span> clone the repo and install
   ```
   git clone https://github.com/jfbercher/jupyter_latex_envs.git
   python3 setup.py install
   ```

With Jupyter >= 4.2,

- **step 2**: install the notebook extension
```
jupyter nbextension install --py latex_envs [--user]
```

-  **step 3**: and enable it
```
jupyter nbextension enable latex_envs [--user] --py
```

For Jupyter versions before 4.2, the situation is more tricky since you will have to find the location of the source files (instructions from @jcb91 found [here](https://github.com/jcb91/jupyter_highlight_selected_word)): execute
```
python -c "import os.path as p; from jupyter_highlight_selected_word import __file__ as f, _jupyter_nbextension_paths as n; print(p.normpath(p.join(p.dirname(f), n()[0]['src'])))"
```
Then, issue
```
jupyter nbextension install <output source directory>
jupyter nbextension enable latex_envs/latex_envs
```
where `<output source directory>` is the output of the python command.  _____no_output_____# Customization

## Configuration parameters

Some configuration parameters can be specified system-wide, using the `nbextension_configurator`. For that, open a browser at [http://localhost:8888/nbextensions/](http://localhost:8888/nbextensions/) -- the exact address may change eg if you use jupyterhub or if you use a non standard port. You will then be able to change default values for the boolean values
- LaTeX_envs menu (insert environments) present
- Label equation with numbers (otherwise with their \label{} key)
- Number environments as section.num
- Use customized environments as given in 'user_envs.json' (in the extension directory)
and enter a default filename for the bibtex file (in document directory).

All these values can also be changed per documents and these values are stored in the notebook's metadata.

## User environments configuration

Environments can be customized in the file `user_envs.json`, located in the `nbextensions/latex_envs` directory. It is even possible to add *new* environments. This file is read at startup (or when using the corresponding toggle option in the `Toggles` menu) and merged with the standard configuration. An example is provided as `example_user_envs.json`. For each (new/modified) environment, one has to  provide (i) the name of the environment (ii) its title (iii) the name of the associated counter for numbering it; eg
```
    "myenv": {
        "title": "MyEnv",
        "counterName": "example"
    },
```
Available counters are problem, exercise, example, property, theorem, lemma, corollary, proposition, definition, remark, and figure.

## Styling
The document title and the document author (as specified by `\title` and `\author` are formatted using the `maketitle` command according to the `.latex_maintitle` and `.latex_author` div styles.

Each environment is formatted according to the div style `.latex_environmentName`, e.g. `.latex_theorem`, `.latex_example`, etc. The titles of environments are formatted with respect to `.latex_title` and the optional parameter wrt `.latex_title_opt`.
Images are displayed using the style specified by `.latex_img` and thir caption using `.caption`.
Finally, enumerate environments are formatted according to the `.enum` style. Similarly, itemize environments are formatted using `.item` style.

These styles can be customized either in the `latex_envs.css` file, or better in a `custom.css` in the document directory. _____no_output_____# Usage and further examples_____no_output_____## First example (continued)_____no_output_____We continue the first example on fthe Fourier transform definition \ref{def:FT} in order to show that, of course, we can illustrate things using a simple code. Since the Fourier transform is an essential tool in signal processing, We put this in evidence using the `textboxa` environment -- which is defined here in the css, and that one should define in the LaTeX counterpart:
\begin{listing}
\begin{textboxa}
The Fourier transform is an extremely useful tool to have in your toolbox!
\end{textboxa}
\end{listing}_____no_output_____\begin{textboxa}
The Fourier transform is an extremely useful tool to have in your toolbox!
\end{textboxa}
_____no_output_____The Fourier transform of a pure cosine is given by
$$
X[k] = \frac{1}{2} \left( \delta[k-k_0] + \delta[k-k_0] \right),
$$
modulo $N$. This is illustrated in the following simple script:_____no_output_____

<code>
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from numpy.fft import fft
k0=4; N=128; n=np.arange(N); k=np.arange(N)
x=np.sin(2*np.pi*k0*n/N)
X=fft(x)
plt.stem(k,np.abs(X))
plt.xlim([0, 20])
plt.title("Fourier transform of a cosine")
_=plt.xlabel("Frequency index (k)")_____no_output_____
</code>

## Second example_____no_output_____This example shows a series of environments, with different facets; **links, references, markdown or/and LaTeX formatting within environments**.  The listing of environments below is typed using the environment *listing*..._____no_output_____\begin{listing}
\begin{definition} \label{def:diffeq}
We call \textbf{difference equation} an equation of the form
$$
\label{eq:diffeq}
y[n]= \sum_{k=1}^{p} a_k y[n-k] + \sum_{i=0}^q b_i x[n-i]
$$
\end{definition}

\begin{property}
If all the $a_k$ in equation (\ref{eq:diffeq}) of definition \ref{def:diffeq} are zero, then the filter has a **finite impulse response**.
\end{property}

\begin{proof}
Let $\delta[n]$ denote the Dirac impulse. Take $x[n]=\delta[n]$ in (\ref{eq:diffeq}). This yields, by definition, the impulse response:
$$
\label{eq:fir}
h[n]= \sum_{i=0}^q b_i \delta[n-i],
$$
which has finite support.
\end{proof}

\begin{theorem}
The poles of a causal stable filter are located within the unit circle in the complex plane.
\end{theorem}

\begin{example} \label{ex:IIR1}
Consider $y[n]= a y[n-1] +  x[n]$. The pole of the transfer function is $z=a$. The impulse response $h[n]=a^n$ has infinite support.
\end{example}

In the following exercise, you will check that the filter is stable iff $a$<1.

\begin{exercise}\label{ex:exofilter}
Consider the filter defined in Example \ref{ex:IIR1}. Using the **function** `lfilter` of scipy, compute and plot the impulse response for several values of $a$.
\end{exercise}

\end{listing}_____no_output_____The lines above are rendered as follows (of course everything can be tailored in the stylesheet):
%
\begin{definition} \label{def:diffeq}
We call \textbf{difference equation} an equation of the form
\begin{equation}
\label{eq:diffeq}
y[n]= \sum_{k=1}^{p} a_k y[n-k] + \sum_{i=0}^q b_i x[n-i]
\end{equation}
\end{definition}
%
Properties of the filter are linked to the coefficients of the difference equation. For instance, an immediate property is
%
% this is a comment
\begin{property}
If all the $a_k$ in equation (\ref{eq:diffeq}) of definition \ref{def:diffeq} are zero, then the filter has a **finite impulse response**.
\end{property}
%
\begin{proof}
Let $\delta[n]$ denote the Dirac impulse. Take $x[n]=\delta[n]$ in (\ref{eq:diffeq}). This yields, by definition, the impulse response:
\begin{equation}
\label{eq:fir}
h[n]= \sum_{i=0}^q b_i \delta[n-i],
\end{equation}
which has finite support.
\end{proof}
%
\begin{theorem}
The poles of a causal stable filter are located within the unit circle in the complex plane.
\end{theorem}
%
\begin{example} \label{ex:IIR1}
Consider $y[n]= a y[n-1] +  x[n]$. The pole of the transfer function is $z=a$. The impulse response $h[n]=a^n$ has infinite support.
\end{example}

In the following exercise, you will check that the filter is stable iff $a$<1.
%
\begin{exercise}\label{ex:exofilter}
Consider the filter defined in Example \ref{ex:IIR1}. Using the **function** `lfilter` of scipy, compute and plot the impulse response for several values of $a$.
\end{exercise}

_____no_output_____\begin{listing}
The solution of exercise \ref{ex:exofilter}, which uses a difference equation as in Definition \ref{def:diffeq}:
\end{listing}
The solution of exercise \ref{ex:exofilter}, which uses a difference equation as in Definition \ref{def:diffeq}:_____no_output_____

<code>
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import lfilter
d=np.zeros(100); d[0]=1 #dirac impulse
alist=[0.2, 0.8, 0.9, 0.95, 0.99, 0.999, 1.001, 1.01]
for a in alist:
    h=lfilter([1], [1, -a],d)
    _=plt.plot(h, label="a={}".format(a))
plt.ylim([0,1.5])
plt.xlabel('Time')
_=plt.legend()_____no_output_____
</code>

## Third example:_____no_output_____This example shows that environments like itemize or enumerate are also available. As already indicated, this is useful for copying text from a TeX file. Following the same idea, text formating commands `\textit`, `\textbf`, `\underline`, etc are also available._____no_output_____\begin{listing}
The following \textit{environments} are available:
\begin{itemize}
    \item \textbf{Theorems and likes}
    \begin{enumerate}
        \item theorem,
        \item lemma,
        \item corollary
        \item ...
    \end{enumerate}
    \item \textbf{exercises}
    \begin{enumerate}
        \item problem,
        \item example,
        \item exercise
    \end{enumerate}
\end{itemize}
\end{listing}_____no_output_____which gives...

The following \textit{environments} are available:
\begin{itemize}
\item \textbf{Theorems and likes}
\begin{enumerate}
\item theorem,
\item lemma,
\item corollary
\item ...
\end{enumerate}
\item \textbf{exercises}
\begin{enumerate}
\item problem,
\item example,
\item exercise
\end{enumerate}
\end{itemize}_____no_output_____# Disclaimer, sources and thanks_____no_output_____Originally, I used a piece of code from the nice online markdown editor `stackedit` [https://github.com/benweet/stackedit/issues/187](https://github.com/benweet/stackedit/issues/187), where the authors also considered the problem of incorporating LaTeX markup in their markdown.

I also studied and used examples and code from [https://github.com/ipython-contrib/IPython-notebook-extensions](https://github.com/ipython-contrib/IPython-notebook-extensions).

- This is done in the hope it can be useful. However there are many impovements possible, in the code and in the documentation.
**Contributions will be welcome and deeply appreciated.**

- If you have issues, please post an issue at
`https://github.com/jfbercher/jupyter_latex_envs/issues` [here](https://github.com/jfbercher/jupyter_latex_envs/issues).

**Self-Promotion --** Like `latex_envs`? Please star and follow the [repository](https://github.com/jfbercher/jupyter_latex_envs) on GitHub.

_____no_output_____

<code>
%%html
<style>
    .prompt{
        display: none;
    }

</style>_____no_output_____
</code>

# References

(<a id="cit-PER-GRA:2007" href="#call-PER-GRA:2007">P\'erez and Granger, 2007</a>) P\'erez Fernando and Granger Brian E., ``_IPython: a System for Interactive Scientific Computing_'', Computing in Science and Engineering, vol. 9, number 3, pp. 21--29, May 2007.  [online](http://ipython.org)

(<a id="cit-mckinney2012python" href="#call-mckinney2012python">McKinney, 2012</a>) Wes McKinney, ``_Python for data analysis: Data wrangling with Pandas, NumPy, and IPython_'',  2012.

(<a id="cit-rossant2013learning" href="#call-rossant2013learning">Rossant, 2013</a>) Cyrille Rossant, ``_Learning IPython for interactive computing and data visualization_'',  2013.

(<a id="cit-shen2014interactive" href="#call-shen2014interactive">Shen, 2014</a>) Shen Helen, ``_Interactive notebooks: Sharing the code_'', Nature, vol. 515, number 7525, pp. 151--152,  2014.

_____no_output_____