# Notebook from Umesh-JNU/DiscountPy
Path: notebook/Discount In Python.ipynb

__version__ = '0.1.0'

__all__ = [
    'encode',
    'decode',
    'charToTwobit',
    'twobitToChar',
    'motif',
    'motif_counter',
    'motif_count_scanner',
    'motif_space',
    'pos_rank_window',
    'shift_scanner',
    '__version__',
]

import re

def charToTwobit(char: str):
    switcher = {
        'A': 0x0,
        'C': 0x1,
        'G': 0x2,
        'T': 0x3,
        'U': 0x3
    }
    return switcher.get(char, "Invalid Nucleotide")

def twobitToChar(byte: bytes):
    switcher = {
        '00': 'A',
        '01': 'C',
        '10': 'G',
        '11': 'T'
    }
    return switcher.get(byte, "Invalid Value")

# convert into bits and gives direct priority

def encode(motif):
    code = {'A': '00', 'C': '01', 'G': '10', 'T': '11'}
    byteCode = ''.join(map(lambda x: code[x], motif))
    return int(byteCode, 2)

def decode(rank):
    # print("rank = ", rank)
    s = bin(rank)[2:].zfill(20)
    # print('s=',s)
    x = re.findall('..', s)
    # print('re.findall = ', x)
    # print('decode = ', ''.join(map(twobitToChar, x)))
    return ''.join(map(twobitToChar, x))_____no_output_______all__ = [
    'Features',
    'Motif',
    '__Empty__'
]

class Features:
    """ Features: features of the motif -
        pattern - sequence
        rank - rank/priority/id (must be unique)
        valid - valid motif or not (only for universal frequency)
    """

    __slots__ = ('pattern', 'rank', 'valid')
    pattern: str
    rank: int
    valid: bool

    def __init__(self, pat: str, rank: int, valid: bool):
        self.pattern = pat
        self.rank = rank
        self.valid = valid

    def __repr__(self):
        return 'Pattern=%s, Rank=%s, Valid=%s' % (self.pattern, self.rank, self.valid)

class Motif:
    """ Motif: m-length sequences in the reads.
        position - position in read
        feature - feature with (pattern, rank, valid)
    """

    __slots__ = ('position', 'feature')
    position: int
    feature: Features

    def __init__(self, pos: int, feature: Features):
        self.position = pos
        self.feature = feature

    def __repr__(self):
        return 'Position=%s, Feature=(%s)' % (self.position, self.feature)

__Empty__ = Motif(-1, Features('', -1, False))_____no_output_____class ShiftScanner:
    def __init__(self, space):
        assert(space.width <= 15)
        self.space = space

        global width
        width = space.width
        # mask = int('11'*width, 2)

        # self.featuresByPriority = (Features(p, i, True) for p, i in enumerate(space.motifsOfLength()))

    """ Find all matches in the string.
        Returns an array with the matches in order, or Motif.Empty for positions
        where no valid matches were found.
    """

    def allMatches(self, read: str, tolist: bool = False):
        def getMatches(pos, motif: str):
            priority = self.space.priorityOf(motif)
            if(priority != -1):
                return Motif(pos, Features(motif, priority, True))
            return __Empty__

        if tolist:
            return [(read[pos:pos+width], pos) for pos in range(len(read) - (width-1))]

        return (getMatches(pos, read[pos:pos+width]) for pos in range(len(read) - (width-1)))_____no_output_____from itertools import product
class MotifSpace:
    """ MotifSpace: create the priority lookup table from which
        priority/rank of each Motif can be accessed easily
    """

    __slots__ = ('width', '_maxMotifs', 'scanner', 'byPriority', 'priorityLookUp')
    width: int
    byPriority: list
    scanner: ShiftScanner
    _maxMotifs: int
    priorityLookUp: list

    def __init__(self, byPriority: list):
        self.byPriority = list(byPriority)
        self.width = len(self.byPriority[0])
        self.scanner = ShiftScanner(self)
        self._maxMotifs = 4 << (self.width * 2 - 2)
        self.priorityLookUp = [-1]*self._maxMotifs

        for pri, motif in enumerate(self.byPriority):
            self.priorityLookUp[self.motifToInt(motif)] = pri

    def motifToInt(self, m: str) -> int:
        return encode(m)

    def priorityOf(self, mk):
        return self.priorityLookUp[self.motifToInt(mk)]

    def create(self, pattern, pos):
        return Motif(pos, Features(pattern, self.priorityOf(pattern), True))

class _MotifSpace:
    all1mersDNA = ("A", "C", "G", "T")
    all1mersRNA = ("A", "C", "G", "U")

#     def motifsOfLength(self, width: int, rna: bool = False) -> iter:
#         bases = self.all1mersRNA if rna else self.all1mersDNA

#         return map(lambda _: ''.join(_), product(bases, repeat=width))

    def motifsOfLength(self, width: int, rna: bool = False) -> iter:
        bases = self.all1mersRNA if rna else self.all1mersDNA

        def generate(prefix, length):
            if (length == 0):
                yield prefix
                return

            for base in bases:
                yield from generate(prefix + base, length-1)

        return generate('', width)

    def fromTemplateWithValidSet(self, template: MotifSpace, validMers: iter) -> MotifSpace:
        validSet = set(validMers)
        return MotifSpace(filter(lambda _: _ in validSet, template.byPriority))

_____no_output_____class MotifExtractor:
    __slots__ = ('scanner', 'K', 'width')
    scanner: ShiftScanner
    K: int
    width: int

    """ Extract all the best motifs from the reads
    """
    def __init__(self, space: MotifSpace, k: int):
        self.scanner = space.scanner
        self.K = k
        self.width = space.width

    """ Find all the top (best Motifs within K-length window)
    """
    def slidingTopMotifs(self, read):
        matches = self.scanner.allMatches(read)
        windowMotifs = PosRankWindow()

        pos = self.width - self.K
        for m in matches:
            windowMotifs.moveWindowAndInsert(pos, m)
            pos += 1
            yield windowMotifs.top()

    """ find the regions of the top motifs in reads
    """
    def regionsInRead(self, read):
        topMotifs = self.slidingTopMotifs(read)
        self.drop(topMotifs, self.K - self.width)

        lastMotif = next(topMotifs, None)

        consumed = 1
        startReg = 1
        for motif in topMotifs:
            if lastMotif == motif:
                consumed += 1
            else:
                yield lastMotif, startReg-consumed
                lastMotif = motif
                consumed = 1

            startReg += 1

        # for the last motif
        yield lastMotif, startReg - consumed

    """ Return all the Super-mers
    """
    def splitRead(self, read):
        if (len(read) < self.K):
            return

        readByReg = self.regionsInRead(read)

        prev = next(readByReg, None)
        while readByReg:
            b1 = prev
            b2 = next(readByReg, None)

            if b2:
                yield b1[0], read[b1[1]: b2[1] + (self.K - 1)]
                prev = b2
            else:
                yield b1[0], read[b1[1]:]
                break

    @staticmethod
    def drop(itr, n):
        j = 0
        try:
            while (j < n):
                next(itr)
                j += 1
        except StopIteration:
            return
_____no_output_____class MotifCounter:
    """ Main frequency counter
    """
    def __init__(self, space: MotifSpace):
        sizeOfCounter = len(space.byPriority)
        self.countArr = [0]*sizeOfCounter

    """ Increment the count of the corresponding Motif
    """
    def increment(self, m: Motif):
        rank = m.feature.rank
        self.countArr[rank] += 1

    def motifsWithCounts(self, space: MotifSpace):
        return zip(space.byPriority, self.countArr)

    def _toSpaceByFrequency(self, counts: list((str, int))):
        # sorting first by frequency and then sorting by lexicographically
        c = map(lambda _: _[0], sorted(counts, key=lambda _: (_[1], _[0])))
        return MotifSpace(c)

    """ Make a new space to with priority based on frequency and lexicographically ordering
    """
    def toSpaceByFrequency(self, oldSpace: MotifSpace):
        pairs = self.motifsWithCounts(oldSpace)
        return self._toSpaceByFrequency(pairs)

_____no_output_____class MotifCountingScanner:
    """ Looks for the motifs and find the it's occurences
        in the whole dataset
    """
    def __init__(self, space: MotifSpace):
        self.scanner = space.scanner
        self.motifCount = MotifCounter(space)

    def scanRead(self, counter: MotifCounter, read):
        for m in self.scanner.allMatches(read):
            if m.feature.valid:
                counter.increment(m)

    def scanGroup(self, counter: MotifCounter, rs):
        for r in rs:
            self.scanRead(counter, r.strip('\n'))
_____no_output_____from typing import Any

class PositionNode:
    """ PosiitonNode: to track and find the best Motifs
        lesser the rank higher is the priority
    """
    __slots__ = ['prevPos', 'nextPos']
    prevPos: Any
    nextPos: Any

    def __init__(self, p = '_', n = '_'):
        self.prevPos = p
        self.nextPos = n

    def removeNode(self):
        temp = self.nextPos
        self.prevPos.nextPos = self.nextPos
        self.nextPos.prevPos = self.prevPos

        del self
        return temp

    def linkPos(self, before, after):
        before.nextPos = self
        self.prevPos = before
        self.nextPos = after
        after.prevPos = self

class MotifContainer(PositionNode):
    """ MotifContainer: Store the Motif and is the Node in PosRankWindow (doubly linked-list)
        pos - position of the Motif in the read
        motif - Motif
        rank - rank of the Motif
    """
    __slots__ = ['pos', 'motif', 'rank']
    pos: int
    motif: Motif
    rank: int

    def __init__(self, motif: Motif):
        self.pos = motif.position
        self.rank = motif.feature.rank
        self.motif = motif

    def dropUntilPosition(self, pos: int):
        if self.pos < pos:
            self = self.removeNode()
            self.dropUntilPosition(pos)

    def __repr__(self):
        return '[%s, %s]' % (self.pos, self.motif)

class PosRankWindow:
    """ PosRankWindow: A doubly linked-list that makes easier to find the best Motif within a
                         window of k-length
    """
    start: PositionNode = PositionNode()
    end: PositionNode = PositionNode()

    def __init__(self):
        # None <= start <=> end => None
        self.start.nextPos = self.end
        self.start.prevPos = None
        self.end.prevPos = self.start
        self.end.nextPos = None

    """ Moves the window and insert the new Node to the right and removes from the left
    """
    def moveWindowAndInsert(self, pos: int, insertRight):
        new_node = MotifContainer(insertRight)
        if new_node.motif.feature.valid:
            self.appendMonotonic(new_node, self.end)

        if self.start.nextPos != self.end and self.end.prevPos != self.start:
            self.start.nextPos.dropUntilPosition(pos)

    """ Potentially insert the Motifs to the correct location
    """
    def appendMonotonic(self, insertNode, search):
        # if this(start <=> end) is not the case
        if search.prevPos != self.start:
            if insertNode.rank < search.prevPos.rank:
                self.appendMonotonic(insertNode,search.prevPos)
            else:
                insertNode.linkPos(search.prevPos, self.end)
        else:
            insertNode.linkPos(self.start, self.end)

    """ Return the best motif in each k-length window
    """
    def top(self):
        return __Empty__ if self.start.nextPos==self.end else self.start.nextPos.motif

    """ For the testing purpose to check what is happening in the window
    """
    def showWindow(self):
        temp = self.start.nextPos
        while temp != self.end:
            print('window-pos=',temp.pos, temp.motif)
            temp = temp.nextPos_____no_output_____class Col:
    """ Y - YELLOW | B - BOLD | G - GREEN | U - UNDERLINE
        V - PURPLE | C - CYAN | W - WHITE | R - RED | GR - GREY
    """
    V = '\033[95m'
    GR = '\u001b[30;1m'
    C = '\033[38;2;0;200;255m'
    G = '\033[92m'
    Y = '\033[93m'
    R = '\033[91m'
    W = '\033[0m'
    B = '\033[1m'
    U = '\033[4m' _____no_output_____import os

class DiscountCustomError(Exception):
    def __init__(self, msg, arg, req):
        super().__init__(msg, arg, req)
        self.msg = msg
        self.req = req
        self.has = arg

    def __str__(self):
        return Col.B + Col.R + self.msg + Col.G + " | " + Col.C\
                             + self.has + Col.V + u" \u2550"u"\u2550> " + Col.C\
                             + self.req + Col.W

def verify(args):
    print(Col.G, 'Verifying the input...', Col.W)
    """ Check for the fasta file format """

    if args.count:
        if not args.f.endswith('.txt'):
            raise DiscountCustomError("FileFormatError : required .txt file", args.f.split('/')[-1], "XXX.txt")
        else:
            try:
                os.mkdir(args.count)
            except OSError:
                pass
    else:
        if not args.f.endswith('.fasta'):
            raise DiscountCustomError("FileFormatError : required .fasta file", args.f, "XXX.fasta")

    """ Check for ordering : if universal frequency minimizer must be entered """
    if args.o == 'ufreq' and not args.minimizers:
        raise DiscountCustomError("UniversalFrequencyOrdering : required -minimizers", args.o, "-minimizers")

    """ Check if the given output dir exist or not """
    if args.output and not os.path.isdir(args.output.split('/')[0]):
        raise DiscountCustomError("No such file or directory:", '', args.output)

class CoreConf:
    __slots__ = ['K','WIDTH','MINIMIZERS','DATASET','ORDER','TEMPLATESPACE', 'OUTPUT', 'COUNT']
    K: int
    WIDTH: int
    MINIMIZERS: str
    DATASET: str
    ORDER: str
    TEMPLATESPACE: MotifSpace
    OUTPUT: str
    COUNT: str

    def __init__(self, args):
        verify(args)
        print(Col.Y, 'Verification done successfully')

        self.K = args.k
        self.DATASET = args.f
        self.WIDTH = args.m if args.m else 10
        self.MINIMIZERS = args.minimizers
        self.ORDER = args.o
        self.OUTPUT = args.output
        print(Col.V, 'Executing...', Col.W)
        self.TEMPLATESPACE = MotifSpace(_MotifSpace().motifsOfLength(width=self.WIDTH))

        print(self.K, self.WIDTH, self.DATASET, self.MINIMIZERS, self.ORDER)_____no_output_____class ReadSplitConf(CoreConf):
    def __init__(self, args):
        super().__init__(args)

    def getInputSequences(self) -> iter:
        # degenerateAndUnknown = "[^ACTGU]+"
        with open(self.DATASET) as f:
            for _ in filter(lambda _: not _.startswith('>'), f):
                for _ in re.split(r"[^ACGTU]+", _.rstrip('\n')):
                    yield _

    def getFrequencySpace(self, validMotifs) -> MotifSpace:
        inputdata = self.getInputSequences()
        template = _MotifSpace().fromTemplateWithValidSet(self.TEMPLATESPACE, validMotifs)
        counter = MotifCounter(template)
        scanner = MotifCountingScanner(template)
        scanner.scanGroup(counter, inputdata)
        return counter.toSpaceByFrequency(template)

    def getSplitter(self) -> MotifExtractor:
        def use():
            with open(self.MINIMIZERS) as f:
                for _ in f:
                    yield _.strip('\n')

        template = self.TEMPLATESPACE
        validMotifs = use() if self.MINIMIZERS else template.byPriority

        useSpace = template if self.ORDER == 'lex' else self.getFrequencySpace(validMotifs)

        return MotifExtractor(useSpace, self.K)

def printSup(s, k):
    print(f'{s[0].feature.pattern} (pos {s[0].position}, rank {s[0].feature.rank}, len {len(s[1])-k+1} k-mers)', end=' ')
    before, after = s[1].split(s[0].feature.pattern, 1)
    print(f'{before}{Col.V}{s[0].feature.pattern}{Col.W}{after}')

def readSplitDemo(args):
    print(args)
    print(Col.C, 'Running Discount...')

    conf = ReadSplitConf(args)

    spl = conf.getSplitter()
    if conf.OUTPUT:
        with open(conf.OUTPUT, 'w+') as f:
            for r in conf.getInputSequences():
#                 print('Read:', r)
                for s in spl.splitRead(r):
                    f.write('{0}\t{1}\n'.format(s[0].feature.pattern, s[1]))
#                     printSup(s, spl.K)
    else:
        for r in conf.getInputSequences():
            print('Read :', r)
            for s in spl.splitRead(r):
                printSup(s, spl.K)

    _____no_output_____import argparse

class CustomArgparser(argparse.ArgumentParser):
    def error(self, message):
        print(Col.R + Col.B + 'ERROR : %s\n' % message)
        print(Col.W + 'For more details run :' + Col.Y + ' discount ' + Col.GR + '-h' + Col.W)
        exit()

def main(args):
    # print(sys.argv)
    parser = CustomArgparser(prog="discount", description="\n\tDiscountPy : A k-mer counting tool")
    count = parser.add_argument_group('Count k-mers')
    count.add_argument('--count', metavar='', type=str, help='Generate k-mers counted file. (Input file(.txt) shoud be sorted by minimizers with corresponding super-mers)')
    req = parser.add_argument_group('Required Arguments')
    parser.version = Col.G + 'DiscountPy version ' + Col.C + __version__ + Col.W

    req.add_argument("-k", metavar='', type=int, help="Length of the k-mers", required=True)
    parser.add_argument("-m", metavar='', type=int, help="Width of the minimizers (default 10)", default=10)
    req.add_argument("-f", metavar='', type=str, help="Dataset (.fasta)", required=True)
    parser.add_argument("-o", type=str, choices=["lex", "freq"], default="freq",
                              help="Ordering {lex | lexicographic, freq | frequency} (default freq)")
    parser.add_argument("--minimizers", metavar="", type=str, help="Valid minimizers sets")
    parser.add_argument('--output', metavar='', type=str, help='Generates output of Super-mers with minimizers')
    parser.add_argument("-v", '--version', action='version', help="Version of the tool")

    readSplitDemo(parser.parse_args(args))_____no_output_____import cProfile
import line_profiler
%load_ext line_profiler_____no_output_____%%time

file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/data/1M.fasta'
pasha = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/PASHA/pasha_all_28_10.txt'
out = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output' + '/1M_ufreq.txt'
args = ['-k', '28', '-m', '10', '-f', file, '-o', 'freq', '--minimizers', pasha, '--output', out]

%prun -D 100k_ufreq.prof main(args)Namespace(count=None, k=28, m=10, f='C:/Users/umesh/Desktop/Discount-In-Python/Discount/data/1M.fasta', o='freq', minimizers='C:/Users/umesh/Desktop/Discount-In-Python/Discount/PASHA/pasha_all_28_10.txt', output='C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/1M_ufreq.txt')
 Running Discount...
 Verifying the input...
 Verification done successfully
 Executing...
28 10 C:/Users/umesh/Desktop/Discount-In-Python/Discount/data/1M.fasta C:/Users/umesh/Desktop/Discount-In-Python/Discount/PASHA/pasha_all_28_10.txt freq

*** Profile stats marshalled to file '100k_ufreq.prof'.
Wall time: 1h 1min 30s

### version 1_____no_output_____

# version 1
import os
from itertools import groupby, chain

def flatten(sm):
    return chain.from_iterable(sm)

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass

    with open(inputFile) as f:
        lines = map(lambda _: _.rstrip('\n').split('\t'), f)

        print(bucket)
        with open(bucket, 'w+') as f:
            for minimizer, thisBin in groupby(lines, lambda _: _[0]):
                supermersThisBin = map(lambda _: _[1], thisBin)

                slide = map(lambda sm: sliding(sm, k), supermersThisBin)
                flat = flatten(slide)
                kmers = iter(sorted(flat))

                # kmers = sorted(flat)
                # print(minimizer, kmers)

                last = next(kmers)
                count = 1

                for _ in kmers:
                    if _ == last:
                        count += 1
                    else:
                        f.write(f'{last}\t{count}\n')
                        last = _
                        count = 1

                f.write(f'{last}\t{count}\n')_____no_output_____%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers.txt'
k = 28

%lprun -T 100k_freq.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq
C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers.txt

*** Profile printout saved to text file '100k_freq.txt'.
Wall time: 3min 45s

### version 2_____no_output_____

# version 2

import os
from more_itertools import flatten, peekable, run_length, bucket as buk

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def takewhile(lines, m):
    while lines and lines.peek()[0] == m:
        yield next(lines)[1]

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass

    with open(inputFile, 'r') as f:
        lines = peekable(map(lambda _: _.rstrip('\n').split('\t'), f))
#         lines = buk(map(lambda _: _.rstrip('\n').split('\t'), f), lambda _: _[0])
#         for _ in lines:
#             print(list(lines[_]))
#         print(list(lines))

        result = []
        while lines:
            minimizer = lines.peek()[0]
            supermersThisBin = takewhile(lines, minimizer)

            kmers = peekable(sorted(flatten(map(lambda _: sliding(_, k), supermersThisBin))))

            count = 0
            lastKmer = kmers.peek()
            while kmers:
                if kmers.peek() == lastKmer:
                    count += 1
                    next(kmers)
                else:
                    result.append('%s\t%s'%(lastKmer, count))
                    count = 1
                    lastKmer = next(kmers)

            result.append('%s\t%s'%(lastKmer, count))

        open(bucket, 'w').write(''.join('%s\n'%(_) for _ in result))_____no_output_____%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers2.txt'
k = 28

%lprun -T 100k_freq2.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq

*** Profile printout saved to text file '100k_freq2.txt'.
Wall time: 4min 49s

### version 3_____no_output_____

# version 3
import os
from itertools import groupby, chain

def flatten(sm):
    return chain.from_iterable(sm)

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass

    with open(inputFile) as f:
        lines = map(lambda _: _.rstrip('\n').split('\t'), f)

        result = []
        for minimizer, thisBin in groupby(lines, lambda _: _[0]):
            supermersThisBin = map(lambda _: _[1], thisBin)

            slide = map(lambda sm: sliding(sm, k), supermersThisBin)
            flat = flatten(slide)
            kmers = iter(sorted(flat))

            # kmers = sorted(flat)
            # print(minimizer, kmers)

            last = next(kmers)
            count = 1

            for _ in kmers:
                if _ == last:
                    count += 1
                else:
                    result.append('%s\t%s'%(last, count))
                    last = _
                    count = 1

            result.append('%s\t%s'%(last, count))

        open(bucket, 'w').write(''.join('%s\n'%(_) for _ in result))
        _____no_output_____%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers3.txt'
k = 28

%lprun -T 100k_freq3.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq

*** Profile printout saved to text file '100k_freq3.txt'.
Wall time: 4min

### version 4_____no_output_____

# version 4
import os
from more_itertools import peekable
from collections import Counter

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass

    def takeWhile(lines, m):
        _thisBin = Counter()

        while lines and lines.peek()[0] == m:
            _thisBin.update(next(lines)[1])

        return _thisBin

    def sliding(supermer):
        return [supermer[i : i + k] for i in range(len(supermer) - k + 1)]

    def kmer_fun(x):
        return x[0], sliding(x[1])

    with open(inputFile) as f:
        lines = peekable(map(lambda _: kmer_fun(_.rstrip('\n').split('\t')), f))

        with open(bucket, 'w+') as demo:
            while lines:
                minimizer = lines.peek()[0]
                supermersThisBin = takeWhile(lines, minimizer)
                demo.writelines(['%s\t%s\n'%(_, supermersThisBin[_]) for _ in supermersThisBin])_____no_output_____%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo/freq.txt'
k = 28

%lprun -T freq.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo

*** Profile printout saved to text file 'freq.txt'.
Wall time: 1min 53s
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo/ufreq.txt'
k = 28

%lprun -T ufreq.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo

*** Profile printout saved to text file 'ufreq.txt'.
Wall time: 1min 22s
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof/freq.txt'
k = 28

%prun -D freq.prof countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof

*** Profile stats marshalled to file 'freq.prof'.
Wall time: 1min 22s
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof/ufreq.txt'
k = 28

%prun -D ufreq.prof countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof

*** Profile stats marshalled to file 'ufreq.prof'.
Wall time: 1min 2s
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_1M_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_1M/ufreq.txt'
k = 28

%lprun -T ufreq.txt -f countKmers countKmers(k, file, bucket)C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_1M

*** Profile printout saved to text file 'ufreq.txt'.
Wall time: 9min 57s


# Notebook from vishalbelsare/TDALabs
Path: LowerStar.ipynb

# Lower Star Filtrations_____no_output_____

%matplotlib notebook
import numpy as np
from scipy import ndimage
from ripser import ripser
from persim import plot_diagrams as plot_dgms
import matplotlib.pyplot as plt
from scipy import sparse
import time
import PIL
from mpl_toolkits.mplot3d import Axes3D
import sys

import ipywidgets as widgets
from IPython.display import display
_____no_output_____

## Overview of 0D Persistence for Point Clouds_____no_output_____

# First setup point cloud
np.random.seed(0)
NClusters = 3
PPC = 10 #Points per cluster
N = NClusters*PPC
X = np.zeros((N, 2))
for c in range(NClusters):
    X[c*PPC:(c+1)*PPC, :] = np.random.randn(PPC, 2) + 10*np.random.randn(2)[None, :]

# Compute persistence diagram
res = ripser(X, maxdim=0)
H0 = res['dgms'][0]
D = res['dperm2all']

def on_value_change(change):
    execute_computation1()
Tauslider = widgets.FloatSlider(min=0.0, max = np.max(D), x=5,step=0.1,value=1,description=r'\(\tau :\)' ,continuous_update=False)
Tauslider.observe(on_value_change, names='value')
display(Tauslider)

fig = plt.figure(figsize=(9.5, 3))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)

def execute_computation1():
    ax1.clear()
    ax2.clear()

    # Get slider values
    cutoff = Tauslider.value

    ax1.scatter(X[:, 0], X[:, 1], 50)
    for i in range(N):
        for j in range(i+1, N):
            if D[i, j] < cutoff:
                ax1.plot(X[[i, j], 0], X[[i, j], 1], 'k', linestyle='--')
    plt.sca(ax2)
    plot_dgms(H0)
    ax2.plot([-0.1*np.max(D), np.max(D)], [cutoff, cutoff], linestyle='--')

execute_computation1()_____no_output_____

## Piecewise Linear Lower Star Filtrations

First, we define a lower star time series filtration function.  Then, we launch into an example_____no_output_____

def lower_star_filtration(x):
    N = len(x)
    # Add edges between adjacent points in the time series, with the "distance"
    # along the edge equal to the max value of the points it connects
    I = np.arange(N-1)
    J = np.arange(1, N)
    V = np.maximum(x[0:-1], x[1::])
    # Add vertex birth times along the diagonal of the distance matrix
    I = np.concatenate((I, np.arange(N)))
    J = np.concatenate((J, np.arange(N)))
    V = np.concatenate((V, x))
    #Create the sparse distance matrix
    D = sparse.coo_matrix((V, (I, J)), shape=(N, N)).tocsr()
    return ripser(D, maxdim=0, distance_matrix=True)['dgms'][0]_____no_output_____np.random.seed(2)
x = np.random.randn(10)*2
x = np.round(x*5)/5.0

H0 = lower_star_filtration(x)

def on_value_change(change):
    execute_computation1()
Tauslider = widgets.FloatSlider(min=np.min(x)-0.1, max = np.max(x)+0.1, x=5,step=0.05,value=np.min(x)-0.1,description=r'\(\tau :\)' ,continuous_update=False)
Tauslider.observe(on_value_change, names='value')
display(Tauslider)

fig = plt.figure(figsize=(9.5, 3))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)

def execute_computation1():
    ax1.clear()
    ax2.clear()

    # Get slider values
    cutoff = Tauslider.value

    ax1.plot(x)
    ax1.plot([0, len(x)], [cutoff, cutoff])
    ax1.scatter(np.arange(len(x)), x)
    for i in range(len(x)):
        if x[i] <= cutoff:
            ax1.scatter([i]*2, [x[i]]*2, c='C1')
        if i < len(x)-1:
            if x[i] <= cutoff and x[i+1] <= cutoff:
                ax1.plot([i, i+1], x[i:i+2], c='C1')
    plt.sca(ax2)
    plot_dgms(H0)
    ax2.plot([np.min(x)-0.1, cutoff], [cutoff, cutoff], linestyle='--', c='C1')
    ax2.plot([cutoff, cutoff], [cutoff, np.max(x)+0.6], linestyle='--', c='C1')

execute_computation1()_____no_output_____

## Sinusoid Example 1_____no_output_____

np.random.seed(1)
NPeriods = 5
NSamples = 100
t = np.linspace(-0.5, NPeriods, NSamples)
x = np.sin(2*np.pi*t) + t

H0 = lower_star_filtration(x)

def on_value_change(change):
    execute_computation1()
Tauslider = widgets.FloatSlider(min=np.min(x)-0.1, max = np.max(x)+0.1, x=5,step=0.05,value=np.min(x)-0.1,description=r'\(\tau :\)' ,continuous_update=False)
Tauslider.observe(on_value_change, names='value')
display(Tauslider)

fig = plt.figure(figsize=(9.5, 3))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)

def execute_computation1():
    ax1.clear()
    ax2.clear()

    # Get slider values
    cutoff = Tauslider.value

    ax1.plot(x)
    ax1.plot([0, len(x)], [cutoff, cutoff])
    ax1.scatter(np.arange(len(x)), x)
    for i in range(len(x)):
        if x[i] <= cutoff:
            ax1.scatter([i]*2, [x[i]]*2, c='C1')
        if i < len(x)-1:
            if x[i] <= cutoff and x[i+1] <= cutoff:
                ax1.plot([i, i+1], x[i:i+2], c='C1')
    plt.sca(ax2)
    plot_dgms(H0)
    ax2.plot([np.min(x)-0.1, cutoff], [cutoff, cutoff], linestyle='--', c='C1')
    ax2.plot([cutoff, cutoff], [cutoff, np.max(x)+0.6], linestyle='--', c='C1')

execute_computation1()_____no_output_____

## Warped Sinusoid Example_____no_output_____

np.random.seed(1)
NPeriods = 5
NSamples = 400
t = np.linspace(-0.5, NPeriods, NSamples)
x = np.sin(2*np.pi*t) + t

def on_value_change(change):
    execute_computation1()
WarpSlider = widgets.FloatSlider(min=0.1, max = 5,step=0.05,value=1,description=r'warp' ,continuous_update=False)
WarpSlider.observe(on_value_change, names='value')
display(WarpSlider)

fig = plt.figure(figsize=(9.5, 3))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)

def execute_computation1():
    ax1.clear()
    ax2.clear()

    # Get slider values
    p = WarpSlider.value
    t = np.linspace(0, 1, NSamples)**p
    t = (t*(NPeriods+0.5))-0.5
    x = np.sin(2*np.pi*t) + t

    ax1.plot(x)
    ax1.scatter(np.arange(len(x)), x)
    H0 = lower_star_filtration(x)
    plt.sca(ax2)
    plot_dgms(H0)

execute_computation1()_____no_output_____

## Lower Star Images
_____no_output_____First, we define a function to perform the lower star filtration_____no_output_____

def lower_star_image(D):
    """
    Construct a lower star filtration on an image
    Parameters
    ----------
    D: ndarray (M, N)
        An array of image data
    Returns
    -------
    I: ndarray (K, 2)
        A 0D persistence diagram corresponding to the sublevelset filtration
    """
    idxs = np.arange(D.shape[0]*D.shape[1])
    idxs = np.reshape(idxs, D.shape)
    I = idxs.flatten()
    J = idxs.flatten()
    V = D.flatten()
    # Do 8 spatial neighbors
    tidxs = np.nan*np.ones((D.shape[0]+2, D.shape[1]+2), dtype=np.int64)
    tidxs[1:-1, 1:-1] = idxs
    tD = np.nan*np.ones_like(tidxs)
    tD[1:-1, 1:-1] = D
    for di in [-1, 0, 1]:
        for dj in [-1, 0, 1]:
            if di == 0 and dj == 0:
                continue
            thisJ = np.roll(tidxs, di, axis=0)
            thisJ = np.roll(thisJ, dj, axis=1)
            thisD = np.roll(tD, di, axis=0)
            thisD = np.roll(thisD, dj, axis=1)
            thisD = np.maximum(thisD, tD)
            # Deal with boundaries
            thisI = tidxs[np.isnan(thisD)==0]
            thisJ = thisJ[np.isnan(thisD)==0]
            thisD = thisD[np.isnan(thisD)==0]
            I = np.concatenate((I, thisI.flatten()))
            J = np.concatenate((J, thisJ.flatten()))
            V = np.concatenate((V, thisD.flatten()))
    sparseDM = sparse.coo_matrix((V, (I, J)), shape=(idxs.size, idxs.size))
    return ripser(sparseDM, distance_matrix=True, maxdim=0)['dgms'][0]_____no_output_____

## Lower Star Images: Gaussian Blobs Example_____no_output_____

plt.figure()
ts = np.linspace(-1, 1, 100)
x1 = np.exp(-ts**2/(0.1**2))
ts -= 0.4
x2 = np.exp(-ts**2/(0.1**2))

# Define depths of Gaussian blobs
h1 = -1
h2 = -2
h3 = -3
img = h1*x1[None, :]*x1[:, None] + h2*x1[None, :]*x2[:, None] + h3*x2[None, :]*x2[:, None]
plt.imshow(img, cmap = 'afmhot')
plt.colorbar()
plt.show()_____no_output_____I = lower_star_image(img)

plt.figure(figsize=(9, 4))
plt.subplot(121)
plt.imshow(img, cmap = 'afmhot')
plt.colorbar()
plt.title("Test Image")
plt.subplot(122)
plot_dgms(I)
plt.title("0D Persistence Diagram")
plt.tight_layout()
plt.show()_____no_output_____

## Wood Cells Example

Creative commons image obtained from https://www.flickr.com/photos/146824358@N03/35486476174

_____no_output_____

cells_original = plt.imread("Cells.jpg")
cells_grey = np.asarray(PIL.Image.fromarray(cells_original).convert('L'))
# Normalize to the range [0, 1]
cells_grey = -ndimage.uniform_filter(cells_grey, size=10)
cells_grey = cells_grey - np.min(cells_grey)
cells_grey = cells_grey/np.max(cells_grey)

# Do lower star filtration after adding a little bit of noise
# The noise is a hack to help find representatives for the classes
F = cells_grey + 0.001*np.random.randn(cells_grey.shape[0], cells_grey.shape[1])
I = lower_star_image(F)
I = I[I[:, 1]-I[:, 0] > 0.001, :] # Filter out low persistence values

plt.figure(figsize=(9, 3))
plt.subplot(131)
plt.title(cells_original.shape)
plt.imshow(cells_original)
plt.axis('off')
plt.subplot(132)
plt.title(cells_grey.shape)
plt.imshow(cells_grey, cmap='afmhot')
plt.colorbar()
plt.axis('off')
plt.subplot(133)
plot_dgms(I)
plt.tight_layout()
plt.show()_____no_output_____def on_value_change(change):
    execute_computation1()
Tauslider = widgets.FloatSlider(min=0, max = 1, x=5,step=0.01,value=0.0,description=r'\(\tau :\)' ,continuous_update=False)
Tauslider.observe(on_value_change, names='value')
display(Tauslider)

fig = plt.figure(figsize=(10, 6))
ax1 = plt.subplot(121)
ax2 = plt.subplot(122)

X, Y = np.meshgrid(np.arange(F.shape[1]), np.arange(F.shape[0]))
X = X.flatten()
Y = Y.flatten()

def execute_computation1():
    ax1.clear()
    ax2.clear()

    # Get slider values
    cutoff = Tauslider.value
    idxs = np.arange(I.shape[0])
    idxs = idxs[np.abs(I[:, 1] - I[:, 0]) > cutoff]
    ax1.imshow(cells_original)
    ax1.set_xticks([])
    ax1.set_yticks([])
    for idx in idxs:
        bidx = np.argmin(np.abs(F - I[idx, 0]))
        ax1.scatter(X[bidx], Y[bidx], 20, 'k')

    plt.sca(ax2)
    plot_dgms(I, lifetime=True)
    ax2.plot([0, 1], [cutoff, cutoff], linestyle='--', c='C1')
    plt.legend(bbox_to_anchor=(0.6, 0.6), loc=2, borderaxespad=0.)
    plt.tight_layout()

execute_computation1()_____no_output_____

## Black Hole

Using persistent homology we can mathematically confirm that a black has a hole! 🕳 #topology #blackhole pic.twitter.com/n6QeyZwwgh
— Mitchell Eithun (@mitchelleithun) April 11, 2019
  _____no_output_____

blackhole_original = plt.imread("BlackHole.jpg")
blackhole_grey = np.asarray(PIL.Image.fromarray(blackhole_original).convert('L'))
I = lower_star_image(blackhole_grey)
plt.figure(figsize=(10, 4))
plt.subplot(121)
plt.imshow(blackhole_grey)
plt.colorbar()
plt.subplot(122)
plot_dgms(I)
plt.tight_layout()_____no_output_____plt.figure()
plt.imshow(blackhole_grey < 60)_____no_output_____


# Notebook from ShepherdCode/ShepherdML
Path: Nasa2021/LSTM_503.ipynb

_____no_output_____# LSTM 403
Start with LSTM 201 and modify it to use our updated code infrastructure._____no_output_____

NC_FILENAME='ncRNA.gc34.processed.fasta'
PC_FILENAME='pcRNA.gc34.processed.fasta'
MODEL_FILE='LSTM302'   # load not save!
DATAPATH=''

try:
    from google.colab import drive
    IN_COLAB = True
    PATH='/content/drive/'
    drive.mount(PATH)
    DATAPATH=PATH+'My Drive/data/'  # must end in "/"
except:
    IN_COLAB = False
    DATAPATH='data/'  # must end in "/"
NC_FILENAME = DATAPATH+NC_FILENAME
PC_FILENAME = DATAPATH+PC_FILENAME
MODEL_FILE=DATAPATH+MODEL_FILE

EPOCHS=100
SPLITS=1
K=3
VOCABULARY_SIZE=4**K+1   # e.g. K=3 => 64 DNA K-mers + 'NNN'
EMBED_DIMEN=16
NEURONS=64
DROP=0.4
ACT="tanh"Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount("/content/drive/", force_remount=True).
# Load our own tools
GITHUB = True
if GITHUB:
    #!pip install requests  # Uncomment this if necessary. Seems to be pre-installed.
    import requests
    r = requests.get('https://raw.githubusercontent.com/ShepherdCode/ShepherdML/master/Strings/tools_fasta.py')
    with open('tools_fasta.py', 'w') as f:
        f.write(r.text)
    # TO DO: delete the file after import
import tools_fasta as tools
tools.yahoo()  # If this prints "Yahoo!" the the import was successful.

TOOLS_CHANGED = False   # set to True to re-run with a new version of tools
if TOOLS_CHANGED:
  from importlib import reload
  tools=reload(tools)
  print(dir(tools))   # run this to see EVERYTHING in the tools moduleYahoo!
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
from tensorflow import keras
#from keras.wrappers.scikit_learn import KerasRegressor
from keras.models import Sequential
from keras.layers import Bidirectional
from keras.layers import GRU
from keras.layers import Dense
from keras.layers import LayerNormalization
import time
dt='float32'
tf.keras.backend.set_floatx(dt)_____no_output_____

Build model_____no_output_____

def compile_model(model):
    adam_default_learn_rate = 0.001
    schedule = tf.keras.optimizers.schedules.ExponentialDecay(
        initial_learning_rate = adam_default_learn_rate*10,
        #decay_steps=100000, decay_rate=0.96, staircase=True)
        decay_steps=10000, decay_rate=0.99, staircase=True)
    # learn rate = initial_learning_rate * decay_rate ^ (step / decay_steps)
    alrd = tf.keras.optimizers.Adam(learning_rate=schedule)
    bc=tf.keras.losses.BinaryCrossentropy(from_logits=False)
    print("COMPILE...")
    #model.compile(loss=bc, optimizer=alrd, metrics=["accuracy"])
    model.compile(loss=bc, optimizer="adam", metrics=["accuracy"])
    print("...COMPILED")
    return model

def build_model():
    embed_layer  = keras.layers.Embedding(
        #VOCABULARY_SIZE, EMBED_DIMEN, input_length=1000, input_length=1000, mask_zero=True)
        #input_dim=[None,VOCABULARY_SIZE], output_dim=EMBED_DIMEN, mask_zero=True)
        input_dim=VOCABULARY_SIZE, output_dim=EMBED_DIMEN, mask_zero=True)
    rnn1_layer = keras.layers.Bidirectional(
      keras.layers.LSTM(NEURONS, return_sequences=True,
          input_shape=[1000,EMBED_DIMEN], activation=ACT, dropout=DROP) )#bi
    rnn2_layer = keras.layers.Bidirectional(
      keras.layers.LSTM(NEURONS, return_sequences=False,
        activation=ACT, dropout=DROP) )#bi
    dense1_layer = keras.layers.Dense(NEURONS, activation=ACT,dtype=dt)
    drop1_layer = keras.layers.Dropout(DROP)
    dense2_layer = keras.layers.Dense(NEURONS, activation=ACT,dtype=dt)
    drop2_layer = keras.layers.Dropout(DROP)
    output_layer = keras.layers.Dense(1, activation="sigmoid", dtype=dt)
    mlp = keras.models.Sequential()
    mlp.add(embed_layer)
    mlp.add(rnn1_layer)
    mlp.add(rnn2_layer)
    mlp.add(dense1_layer)
    mlp.add(drop1_layer)
    mlp.add(dense2_layer)
    mlp.add(drop2_layer)
    mlp.add(output_layer)
    mlpc = compile_model(mlp)
    return mlpc_____no_output_____

Partition sequences_____no_output_____

def make_slice(data_set,min_len,max_len):
    slice = data_set.query('seqlen <= '+str(max_len)+' & seqlen>= '+str(min_len))
    return slice_____no_output_____

Cross validation_____no_output_____

def do_cross_validation(X,y,given_model):
    cv_scores = []
    fold=0
    splitter = ShuffleSplit(n_splits=SPLITS, test_size=0.1, random_state=37863)
    for train_index,valid_index in splitter.split(X):
        fold += 1
        X_train=X[train_index] # use iloc[] for dataframe
        y_train=y[train_index]
        X_valid=X[valid_index]
        y_valid=y[valid_index]
        # Avoid continually improving the same model.
        model = compile_model(keras.models.clone_model(given_model))
        bestname=MODEL_FILE+".cv."+str(fold)+".best"
        mycallbacks = [keras.callbacks.ModelCheckpoint(
            filepath=bestname, save_best_only=True,
            monitor='val_accuracy', mode='max')]
        print("FIT")
        start_time=time.time()
        history=model.fit(X_train, y_train, # batch_size=10, default=32 works nicely
                epochs=EPOCHS, verbose=1,  # verbose=1 for ascii art, verbose=0 for none
                callbacks=mycallbacks,
                validation_data=(X_valid,y_valid) )
        end_time=time.time()
        elapsed_time=(end_time-start_time)
        print("Fold %d, %d epochs, %d sec"%(fold,EPOCHS,elapsed_time))
        pd.DataFrame(history.history).plot(figsize=(8,5))
        plt.grid(True)
        plt.gca().set_ylim(0,1)
        plt.show()
        best_model=keras.models.load_model(bestname)
        scores = best_model.evaluate(X_valid, y_valid, verbose=0)
        print("%s: %.2f%%" % (best_model.metrics_names[1], scores[1]*100))
        cv_scores.append(scores[1] * 100)
    print()
    print("%d-way Cross Validation mean %.2f%% (+/- %.2f%%)" % (fold, np.mean(cv_scores), np.std(cv_scores)))_____no_output_____

## Train on RNA lengths 200-1Kb_____no_output_____

MINLEN=200
MAXLEN=1000
print("Load data from files.")
nc_seq=tools.load_fasta(NC_FILENAME,0)
pc_seq=tools.load_fasta(PC_FILENAME,1)
train_set=pd.concat((nc_seq,pc_seq),axis=0)
nc_seq=None
pc_seq=None
print("Ready: train_set")
#train_set
subset=make_slice(train_set,MINLEN,MAXLEN)# One array to two: X and y
print ("Data reshape")
(X_train,y_train)=tools.make_kmers(K,MAXLEN,subset)
#print ("Data prep")
#X_train=tools.make_frequencies(K,X_train)Load data from files.
Ready: train_set
Data reshape
print ("Compile the model")
model=build_model()
print ("Summarize the model")
print(model.summary())  # Print this only once
model.save(MODEL_FILE+'.model')
Compile the model
COMPILE...
...COMPILED
Summarize the model
Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
embedding_1 (Embedding)      (None, None, 16)          1040
_________________________________________________________________
bidirectional_2 (Bidirection (None, None, 128)         41472
_________________________________________________________________
bidirectional_3 (Bidirection (None, 128)               98816
_________________________________________________________________
dense_3 (Dense)              (None, 64)                8256
_________________________________________________________________
dropout_2 (Dropout)          (None, 64)                0
_________________________________________________________________
dense_4 (Dense)              (None, 64)                4160
_________________________________________________________________
dropout_3 (Dropout)          (None, 64)                0
_________________________________________________________________
dense_5 (Dense)              (None, 1)                 65
=================================================================
Total params: 153,809
Trainable params: 153,809
Non-trainable params: 0
_________________________________________________________________
None
print ("Cross valiation")
do_cross_validation(X_train,y_train,model)
print ("Done")Cross valiation
COMPILE...
...COMPILED
FIT
Epoch 1/100
453/453 [==============================] - 77s 144ms/step - loss: 0.6163 - accuracy: 0.6749 - val_loss: 0.5811 - val_accuracy: 0.7163
_____no_output_____


# Notebook from itwill009/TR
Path: prepare for data analysis test/seaborn20180604.ipynb

<code>
import pandas as pd
from matplotlib import pyplot as plt
%matplotlib inline
import seaborn as sns_____no_output_____df = pd.read_csv('./Pokemon.csv')_____no_output_____df.head()_____no_output_____sns.lmplot(x='Attack',y='Defense',data=df)_____no_output_____# Scatterplot arguments
sns.lmplot(x='Attack', y='Defense', data=df,
           fit_reg=False, # No regression line
           hue='Stage')   # Color by evolution stage_____no_output_____sns.boxplot(data=df)_____no_output_____stats_df = df.drop(['#','Total','Legendary','Stage'], axis=1)

sns.boxplot(data=stats_df)_____no_output_____sns.set_style('whitegrid')
sns.violinplot(x='Type 1', y='Attack',data=df)_____no_output_____pkmn_type_colors = ['#78C850',  # Grass
                    '#F08030',  # Fire
                    '#6890F0',  # Water
                    '#A8B820',  # Bug
                    '#A8A878',  # Normal
                    '#A040A0',  # Poison
                    '#F8D030',  # Electric
                    '#E0C068',  # Ground
                    '#EE99AC',  # Fairy
                    '#C03028',  # Fighting
                    '#F85888',  # Psychic
                    '#B8A038',  # Rock
                    '#705898',  # Ghost
                    '#98D8D8',  # Ice
                    '#7038F8',  # Dragon
                   ]_____no_output_____# Violin plot with Pokemon color palette
sns.violinplot(x='Type 1', y='Attack', data=df,
               palette=pkmn_type_colors) # Set color palette_____no_output_____# Swarm plot with Pokemon color palette
sns.swarmplot(x='Type 1', y='Attack', data=df,
              palette=pkmn_type_colors)_____no_output_____# Set figure size with matplotlib
plt.figure(figsize=(10,6))

# Create plot
sns.violinplot(x='Type 1',
               y='Attack',
               data=df,
               inner=None, # Remove the bars inside the violins
               palette=pkmn_type_colors)

sns.swarmplot(x='Type 1',
              y='Attack',
              data=df,
              color='k', # Make points black
              alpha=0.7) # and slightly transparent

# Set title with matplotlib
plt.title('Attack by Type')_____no_output_____stats_df.head()_____no_output_____# Melt DataFrame
melted_df = pd.melt(stats_df,
                    id_vars=["Name", "Type 1", "Type 2"], # Variables to keep
                    var_name="Stat") # Name of melted variable
melted_df.head()_____no_output_____print( stats_df.shape )
print( melted_df.shape )(151, 9)
(906, 5)
# Swarmplot with melted_df
plt.figure(figsize=(10,6))
sns.swarmplot(x='Stat', y='value', data=melted_df,
              hue='Type 1')_____no_output_____plt.figure(figsize=(10,6))

sns.swarmplot(x='Stat',
              y='value',
              data=melted_df,
              hue='Type 1',
              dodge=True, # 2. Separate points by hue
              palette=pkmn_type_colors) # 3. Use Pokemon palette

# 4. Adjust the y-axis
plt.ylim(0, 260)

# 5. Place legend to the right
plt.legend(bbox_to_anchor=(1, 1), loc=2)_____no_output_____corr = stats_df.corr()

sns.heatmap(corr, annot =True)_____no_output_____sns.distplot(df.Attack)_____no_output_____sns.countplot(x='Type 1',data=df,palette=pkmn_type_colors)
plt.xticks(rotation=-45)_____no_output_____g = sns.factorplot(x='Type 1', y = 'Attack', data = df, hue = 'Stage', col ='Stage', kind='swarm')

g.set_xticklabels(rotation=-45)_____no_output_____sns.kdeplot(df.Attack,df.Speed)_____no_output_____sns.kdeplot(df.Attack,df.Defense)_____no_output_____sns.jointplot(x='Attack',y='Defense',data=df)_____no_output_____
</code>


# Notebook from foochane/deep-learning-with-python-notes
Path: .ipynb_checkpoints/8.1 使用LSTM生成文本-checkpoint.ipynb

<code>
# 首先下载语料，并将其转换为小写。
# 下载并解析初始文本文件
import keras
import numpy as np
path = keras.utils.get_file(
         'nietzsche.txt',
          origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
text = open(path).read().lower()
print('Corpus length:', len(text))Using TensorFlow backend.
# 将字符序列向量化
maxlen = 60
step = 3
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])

print('Number of sequences:', len(sentences))
chars = sorted(list(set(text)))
print('Unique characters:', len(chars))
char_indices = dict((char, chars.index(char)) for char in chars)
print('Vectorization...')
x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        x[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1Number of sequences: 200278
Unique characters: 57
Vectorization...
# 用于预测下一个字符的单层 LSTM 模型
from keras import layers
model = keras.models.Sequential()
model.add(layers.LSTM(128, input_shape=(maxlen, len(chars))))
model.add(layers.Dense(len(chars), activation='softmax'))WARNING:tensorflow:From /home/fc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
# 模型编译配置
optimizer = keras.optimizers.RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)_____no_output_____# 给定模型预测，采样下一个字符的函数
def sample(preds, temperature=1.0):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)_____no_output_____# 文本生成循环
import random
import sys
for epoch in range(1, 60):
    print('epoch', epoch)
    model.fit(x, y, batch_size=128, epochs=1)
    start_index = random.randint(0, len(text) - maxlen - 1)
    generated_text = text[start_index: start_index + maxlen]
    print('--- Generating with seed: "' + generated_text + '"')
    for temperature in [0.2, 0.5, 1.0, 1.2]:
        print('------ temperature:', temperature)

        sys.stdout.write(generated_text)
        for i in range(400):
            sampled = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(generated_text):
                sampled[0, t, char_indices[char]] = 1.

            preds = model.predict(sampled, verbose=0)[0]
            next_index = sample(preds, temperature)
            next_char = chars[next_index]
            generated_text += next_char
            generated_text = generated_text[1:]
            sys.stdout.write(next_char)epoch 1
WARNING:tensorflow:From /home/fc/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
Epoch 1/1
200278/200278 [==============================] - 261s 1ms/step - loss: 1.9887
--- Generating with seed: "oul propounds to them (riddles which
hegel systematised and "
------ temperature: 0.2
oul propounds to them (riddles which
hegel systematised and really and discour the man the man any a man the more any any the man the subjer the man the say the man any the sould the man to the more any the man the man the sould the man in the more and any the man in the sould the more any any the man the more any the more any the man the man the man the man and the spirit the man the self-relarnes of the man to the such a proble of the man in the one of t------ temperature: 0.5
s of the man to the such a proble of the man in the one of the cillonces and even the subjer and scance the coll its really and realing and religious been any the man thore spiritions of the orgerance and
the manimance of the man it he who has its life to a man in the pain the wase for the pullosy and one to the spiritual to the mident to call of the some canter of the mider that deens to the dind ant that who has probles of call man uself the more in the ------ temperature: 1.0
 ant that who has probles of call man uself the more in the content taitation, things of higherto the wistordcalismive, others of badnest of soge habe xout of herecre; that great greems from of the
man these
will contror him one his "dore will of-the more is the "unwabse of rimpleatlos,
with a stesiess

fout hampeact in the selves know great in the morilicain was as sborge thingsrare, man spenience slimar more disinest har allom--as requiterity, unaat west------ temperature: 1.2
e slimar more disinest har allom--as requiterity, unaat westable inflackfued, a le4tteming mamdor,"s
a just cirinobiles. long  incastice spis "cleass not
to ger a hend came falafalfual
proinged,
rowardhated, good yinly asminstines: who arequises lome who as t
undofo4: or
gediaths. reblerman stindsnou:d by nays
ips ligitatly-atiortions -that hencessent. other--but his implast is s
act of a uiful angreatcnsway practuithoo or
glearm tooduit ciesprions befuttoepoch 2
Epoch 1/1
200278/200278 [==============================] - 254s 1ms/step - loss: 1.6325
--- Generating with seed: "r
them, felt tremors and anxieties. this sentiment was frequ"
------ temperature: 0.2
r
them, felt tremors and anxieties. this sentiment was frequire and the perhaps the such a such a man a such a stands of the profess and the such a point the same the profound the profound the such a condection of the menself and they the such a standard and the such a standing the power and the such a standard to the world of the more and the such a stands the such a perhaps and the the profound the such a perhaps and the profess they the such a such the ------ temperature: 0.5
the such a perhaps and the profess they the such a such the power the suches, the are good the prefore to absugates of a power of the manical to a  spirit they have the self for the philosophy, and the will the perhaps every their also the light and conditer and the sense of the smind of the man was the idmanting and the secomests and a complete and there is not the profess and the point to the something they of more good of the
farter of the such they thi------ temperature: 1.0
mething they of more good of the
farter of the such they this shoatey of digne according a perture bothes:"me thereby perhist, what let never latted, through they they abuestapear wiodo, betthing or juch pithife, as through hoine which a chords and who reils terkinesrarity to be str in indiintor is dysistly an  belieny of emprobet have muchiloto
in embetaly to make good "uses a delight be complised therewiles;" their day their and
"emocars"1jusi
mance
han
------ temperature: 1.2
d therewiles;" their day their and
"emocars"1jusi
mance
han
indigation. as siguts), that
pet. "ortinage, croud hea" at hildo
everyogard as hereomivra-grationely forespiching love ghavicals: the hary alicul in, "are to cauthed they "mastesm whichs subslusence,ler tiul here andisp tos ifemicareinained, people--  men a histort i themselsesy's manting
conduenle. of platy one be.mentated, if no
keal firtiinfur discomble, they unturradan, faminal of heaver beiepoch 3
Epoch 1/1
200278/200278 [==============================] - 248s 1ms/step - loss: 1.5454
--- Generating with seed: "-deceptions?"--that
they please--him who has them, and him w"
------ temperature: 0.2
-deceptions?"--that
they please--him who has them, and him when the same all the strong and the same the same in the superiable and the self-desire and the strong and the part of the same one with the strong and the same the strong and such a propers and in the stronger and the same the strong and all the sense of the more and disponsible of the spiritual and the sense of the stronger and such a propers of the more and the same the same the propers and the------ temperature: 0.5
ropers of the more and the same the same the propers and the same the stronger and the persists of a histerness of a medicting of the science of the good in the same in it is a men in the intellectually the delught in the spiritually this self-delution--which a dispine to the such a down to self-knowledge the part the strong same more the well with the sungained and the valuable to namely respect of all to be men and sounding the sconcasions and the all an------ temperature: 1.0
of all to be men and sounding the sconcasions and the all an no irsemble erromed the
plaling thuse of mament more a to form-is the
were on the perhaps be loves, soul are almostly ining of the
summlure of consponticity nowher. but that exemble, and from care. the science in distursly me his livently its vegre be question obtive-wito be dolus destimutly of us in the necast one of ideal and funcoment less) in them, a syeply the frenessity in such a presulaid ------ temperature: 1.2
 less) in them, a syeply the frenessity in such a presulaid there.
 gensed whero
suen caves
event'es,
lag7.bitutter relutless un himself thes
on, dambini
actrer in caskgo.luhwe,
a immaticmlinessic, now
symes, and a madrinary becomes are denies" or priture almost one. pracifund and th
symagy, upger
adn?
the igmoratence which at lad to not wenter
atfine" eveny, thesuriligation
and inouitible" strictgeously, reludical alm that
elacts of powes
tn the badmelm fepoch 4
Epoch 1/1
200278/200278 [==============================] - 248s 1ms/step - loss: 1.4994
--- Generating with seed: "
to convince themselves that the striving after english happ"
------ temperature: 0.2

to convince themselves that the striving after english happiness of the spirit and the soul, there is the self-aboverally so that it is a mentered interropation of the self-personal self-selresh of the same the self-proved the soul, and in the great the soul, and soul, there is the self-persually the self-played to the self-persually self-seloosing the soul the self-selse of the self-aboverally soul, the soul, there is the self-part and the great the such------ temperature: 0.5
oul, the soul, there is the self-part and the great the such and the star and as there are individual "modern whole great transities with the soul--that is the entitive in the semser as the really start, men of an all that there is not the strength him which the suncerunger--when they was the most sort of solitual belief in
the great speciation and "are statent the self-suld will in the seasunder, and deference and not to the moralser with the world action------ temperature: 1.0
 and deference and not to the moralser with the world actions
of the cause," virturally a dut aividing infarsed itself! intols in the exergient greedly be him, in order to why
refimate men,
that class
frefleshime
to feel
such the relaction of earloughtfrort. so troage with the most indoen
ages
us
of our humanity and presuptive in the deselienly supare the belief whom they also

</code>