# Notebook from micaelverissimo/DEStudy
Path: dev_notebooks/run_NSSDE_cec2014.ipynb

<code>
import numpy as np
import pandas as pd
import pygmo as pg
import matplotlib.pyplot as plt
plt.style.use('classic')
import pickle

import sys
sys.path.append('/home/micael/MyWorkspace/DEStudy/algorithms')
from NSSDE import *_____no_output_____problems = {
    1    :    100,
    2    :    200,
    6    :    600,
    7    :    700,
    9    :    900,
    14    :    1400
}_____no_output_____ndim = 10
popsize = 50_____no_output_____results_10 = {}
for iproblem in problems.keys():
    print 'Begin problem number: {}'.format(iproblem)
    results_10['problem {}'.format(iproblem)]= {}
    prob = pg.problem(pg.cec2014(prob_id=iproblem, dim=ndim))
    for irun in range(51):
        #print 'Begin run number: {}'.format(irun+1)
        results_10['problem {}'.format(iproblem)][irun+1] = {}
        nssde = NSSDE(bounds=[-100, 100],
                      n_dim=ndim,
                      n_pop=popsize,
                      global_max=problems[iproblem], fitness=prob.fitness)
        results_10['problem {}'.format(iproblem)][irun+1]['Evolution']= nssde.evolution()Begin problem number: 1
Begin problem number: 2
Begin problem number: 6
results_10['problem 1'][1]['Evolution']['log'][-1]_____no_output_____#with open('results_10dim_51runs_NSSDE.pickle', 'wb') as handle:
#    pickle.dump(results_10, handle, protocol=pickle.HIGHEST_PROTOCOL)_____no_output_____for iproblem in results_10.keys():
    print(iproblem)
    print(results_10[iproblem][1]['Evolution']['log'][-1])_____no_output_____ndim = 30
popsize = 100_____no_output_____results_30 = {}
for iproblem in problems.keys():
    print 'Begin problem number: {}'.format(iproblem)
    results_30['problem {}'.format(iproblem)]= {}
    prob = pg.problem(pg.cec2014(prob_id=iproblem, dim=ndim))
    for irun in range(51):
        #print 'Begin run number: {}'.format(irun+1)
        results_10['problem {}'.format(iproblem)][irun+1] = {}
        nssde = NSSDE(bounds=[-100, 100],
                      n_dim=ndim,
                      n_pop=popsize,
                      global_max=problems[iproblem], fitness=prob.fitness)
        results_30['problem {}'.format(iproblem)][irun+1]['Evolution']= nssde.evolution()_____no_output_____results_30['problem 1'][1]['Evolution']['log'][-1]_____no_output_____#with open('results_30dim_51runs_NSSDE.pickle', 'wb') as handle:
#    pickle.dump(results_30, handle, protocol=pickle.HIGHEST_PROTOCOL)_____no_output_____
</code>


# Notebook from PacktPublishing/Natural-Language-Processing-with-TensorFlow
Path: ch8/lstm_word2vec_rnn_api.ipynb

# Deep LSTMs with Word2vec using RNN API_____no_output_____

# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
%matplotlib inline
from __future__ import print_function
import collections
import math
import numpy as np
import os
import random
import tensorflow as tf
import nltk
import zipfile
from matplotlib import pylab
from six.moves import range
from six.moves.urllib.request import urlretrieve
import tensorflow as tf
from tensorflow.contrib import rnn
import csvc:\users\thushan\documents\python_virtualenvs\tensorflow_venv\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters

## Downloading Stories
Stories are automatically downloaded from https://www.cs.cmu.edu/~spok/grimmtmp/, if not detected in the disk. The total size of stories is around ~500KB. The dataset consists of 100 stories._____no_output_____

url = 'https://www.cs.cmu.edu/~spok/grimmtmp/'

# Create a directory if needed
dir_name = 'stories'
if not os.path.exists(dir_name):
    os.mkdir(dir_name)

def maybe_download(filename):
  """Download a file if not present"""
  print('Downloading file: ', dir_name+ os.sep+filename)

  if not os.path.exists(dir_name+os.sep+filename):
    filename, _ = urlretrieve(url + filename, dir_name+os.sep+filename)
  else:
    print('Not downloading. File already exists.')
  statinfo = os.stat(dir_name+os.sep+filename)

  return filename

num_files = 100
filenames = [format(i, '03d')+'.txt' for i in range(1,num_files+1)]

for fn in filenames:
    maybe_download(fn)Downloading file:  stories\001.txt
Not downloading. File already exists.
Downloading file:  stories\002.txt
Not downloading. File already exists.
Downloading file:  stories\003.txt
Not downloading. File already exists.
Downloading file:  stories\004.txt
Not downloading. File already exists.
Downloading file:  stories\005.txt
Not downloading. File already exists.
Downloading file:  stories\006.txt
Not downloading. File already exists.
Downloading file:  stories\007.txt
Not downloading. File already exists.
Downloading file:  stories\008.txt
Not downloading. File already exists.
Downloading file:  stories\009.txt
Not downloading. File already exists.
Downloading file:  stories\010.txt
Not downloading. File already exists.
Downloading file:  stories\011.txt
Not downloading. File already exists.
Downloading file:  stories\012.txt
Not downloading. File already exists.
Downloading file:  stories\013.txt
Not downloading. File already exists.
Downloading file:  stories\014.txt
Not downloading. File already exists.
Downloading file:  stories\015.txt
Not downloading. File already exists.
Downloading file:  stories\016.txt
Not downloading. File already exists.
Downloading file:  stories\017.txt
Not downloading. File already exists.
Downloading file:  stories\018.txt
Not downloading. File already exists.
Downloading file:  stories\019.txt
Not downloading. File already exists.
Downloading file:  stories\020.txt
Not downloading. File already exists.
Downloading file:  stories\021.txt
Not downloading. File already exists.
Downloading file:  stories\022.txt
Not downloading. File already exists.
Downloading file:  stories\023.txt
Not downloading. File already exists.
Downloading file:  stories\024.txt
Not downloading. File already exists.
Downloading file:  stories\025.txt
Not downloading. File already exists.
Downloading file:  stories\026.txt
Not downloading. File already exists.
Downloading file:  stories\027.txt
Not downloading. File already exists.
Downloading file:  stories\028.txt
Not downloading. File already exists.
Downloading file:  stories\029.txt
Not downloading. File already exists.
Downloading file:  stories\030.txt
Not downloading. File already exists.
Downloading file:  stories\031.txt
Not downloading. File already exists.
Downloading file:  stories\032.txt
Not downloading. File already exists.
Downloading file:  stories\033.txt
Not downloading. File already exists.
Downloading file:  stories\034.txt
Not downloading. File already exists.
Downloading file:  stories\035.txt
Not downloading. File already exists.
Downloading file:  stories\036.txt
Not downloading. File already exists.
Downloading file:  stories\037.txt
Not downloading. File already exists.
Downloading file:  stories\038.txt
Not downloading. File already exists.
Downloading file:  stories\039.txt
Not downloading. File already exists.
Downloading file:  stories\040.txt
Not downloading. File already exists.
Downloading file:  stories\041.txt
Not downloading. File already exists.
Downloading file:  stories\042.txt
Not downloading. File already exists.
Downloading file:  stories\043.txt
Not downloading. File already exists.
Downloading file:  stories\044.txt
Not downloading. File already exists.
Downloading file:  stories\045.txt
Not downloading. File already exists.
Downloading file:  stories\046.txt
Not downloading. File already exists.
Downloading file:  stories\047.txt
Not downloading. File already exists.
Downloading file:  stories\048.txt
Not downloading. File already exists.
Downloading file:  stories\049.txt
Not downloading. File already exists.
Downloading file:  stories\050.txt
Not downloading. File already exists.
Downloading file:  stories\051.txt
Not downloading. File already exists.
Downloading file:  stories\052.txt
Not downloading. File already exists.
Downloading file:  stories\053.txt
Not downloading. File already exists.
Downloading file:  stories\054.txt
Not downloading. File already exists.
Downloading file:  stories\055.txt
Not downloading. File already exists.
Downloading file:  stories\056.txt
Not downloading. File already exists.
Downloading file:  stories\057.txt
Not downloading. File already exists.
Downloading file:  stories\058.txt
Not downloading. File already exists.
Downloading file:  stories\059.txt
Not downloading. File already exists.
Downloading file:  stories\060.txt
Not downloading. File already exists.
Downloading file:  stories\061.txt
Not downloading. File already exists.
Downloading file:  stories\062.txt
Not downloading. File already exists.
Downloading file:  stories\063.txt
Not downloading. File already exists.
Downloading file:  stories\064.txt
Not downloading. File already exists.
Downloading file:  stories\065.txt
Not downloading. File already exists.
Downloading file:  stories\066.txt
Not downloading. File already exists.
Downloading file:  stories\067.txt
Not downloading. File already exists.
Downloading file:  stories\068.txt
Not downloading. File already exists.
Downloading file:  stories\069.txt
Not downloading. File already exists.
Downloading file:  stories\070.txt
Not downloading. File already exists.
Downloading file:  stories\071.txt
Not downloading. File already exists.
Downloading file:  stories\072.txt
Not downloading. File already exists.
Downloading file:  stories\073.txt
Not downloading. File already exists.
Downloading file:  stories\074.txt
Not downloading. File already exists.
Downloading file:  stories\075.txt
Not downloading. File already exists.
Downloading file:  stories\076.txt
Not downloading. File already exists.
Downloading file:  stories\077.txt
Not downloading. File already exists.
Downloading file:  stories\078.txt
Not downloading. File already exists.
Downloading file:  stories\079.txt
Not downloading. File already exists.
Downloading file:  stories\080.txt
Not downloading. File already exists.
Downloading file:  stories\081.txt
Not downloading. File already exists.
Downloading file:  stories\082.txt
Not downloading. File already exists.
Downloading file:  stories\083.txt
Not downloading. File already exists.
Downloading file:  stories\084.txt
Not downloading. File already exists.
Downloading file:  stories\085.txt
Not downloading. File already exists.
Downloading file:  stories\086.txt
Not downloading. File already exists.
Downloading file:  stories\087.txt
Not downloading. File already exists.
Downloading file:  stories\088.txt
Not downloading. File already exists.
Downloading file:  stories\089.txt
Not downloading. File already exists.
Downloading file:  stories\090.txt
Not downloading. File already exists.
Downloading file:  stories\091.txt
Not downloading. File already exists.
Downloading file:  stories\092.txt
Not downloading. File already exists.
Downloading file:  stories\093.txt
Not downloading. File already exists.
Downloading file:  stories\094.txt
Not downloading. File already exists.
Downloading file:  stories\095.txt
Not downloading. File already exists.
Downloading file:  stories\096.txt
Not downloading. File already exists.
Downloading file:  stories\097.txt
Not downloading. File already exists.
Downloading file:  stories\098.txt
Not downloading. File already exists.
Downloading file:  stories\099.txt
Not downloading. File already exists.
Downloading file:  stories\100.txt
Not downloading. File already exists.
for i in range(len(filenames)):
    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))
    assert file_exists
print('%d files found.'%len(filenames))100 files found.

## Reading data
Data will be stored in a list of lists where the each list represents a document and document is a list of words. We will then break the text into words._____no_output_____

def read_data(filename):

  with open(filename) as f:
    data = tf.compat.as_str(f.read())
    data = data.lower()
    data = nltk.word_tokenize(data)

  return data

documents = []
global documents
for i in range(num_files):
    print('\nProcessing file %s'%os.path.join(dir_name,filenames[i]))

    words = read_data(os.path.join(dir_name,filenames[i]))

    documents.append(words)
    print('Data size (Characters) (Document %d) %d' %(i,len(words)))
    print('Sample string (Document %d) %s'%(i,words[:50]))
Processing file stories\001.txt
Data size (Characters) (Document 0) 1693
Sample string (Document 0) ['in', 'olden', 'times', 'when', 'wishing', 'still', 'helped', 'one', ',', 'there', 'lived', 'a', 'king', 'whose', 'daughters', 'were', 'all', 'beautiful', ',', 'but', 'the', 'youngest', 'was', 'so', 'beautiful', 'that', 'the', 'sun', 'itself', ',', 'which', 'has', 'seen', 'so', 'much', ',', 'was', 'astonished', 'whenever', 'it', 'shone', 'in', 'her', 'face', '.', 'close', 'by', 'the', 'king', "'s"]

Processing file stories\002.txt
Data size (Characters) (Document 1) 2167
Sample string (Document 1) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'wood-cutter', 'with', 'his', 'wife', ',', 'who', 'had', 'an', 'only', 'child', ',', 'a', 'little', 'girl', 'three', 'years', 'old', '.', 'they', 'were', 'so', 'poor', ',', 'however', ',', 'that', 'they', 'no', 'longer', 'had', 'daily', 'bread', ',', 'and', 'did', 'not', 'know', 'how', 'to', 'get', 'food', 'for', 'her']

Processing file stories\003.txt
Data size (Characters) (Document 2) 4499
Sample string (Document 2) ['a', 'certain', 'father', 'had', 'two', 'sons', ',', 'the', 'elder', 'of', 'whom', 'was', 'smart', 'and', 'sensible', ',', 'and', 'could', 'do', 'everything', ',', 'but', 'the', 'younger', 'was', 'stupid', 'and', 'could', 'neither', 'learn', 'nor', 'understand', 'anything', ',', 'and', 'when', 'people', 'saw', 'him', 'they', 'said', "'there", "'s", 'a', 'fellow', 'who', 'will', 'give', 'his', 'father']

Processing file stories\004.txt
Data size (Characters) (Document 3) 1265
Sample string (Document 3) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'goat', 'who', 'had', 'seven', 'little', 'kids', ',', 'and', 'loved', 'them', 'with', 'all', 'the', 'love', 'of', 'a', 'mother', 'for', 'her', 'children', '.', 'one', 'day', 'she', 'wanted', 'to', 'go', 'into', 'the', 'forest', 'and', 'fetch', 'some', 'food', '.', 'so', 'she', 'called', 'all', 'seven', 'to', 'her']

Processing file stories\005.txt
Data size (Characters) (Document 4) 3590
Sample string (Document 4) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'king', 'who', 'was', 'ill', 'and', 'thought', 'to', 'himself', "'i", 'am', 'lying', 'on', 'what', 'must', 'be', 'my', 'deathbed', '.', "'", 'then', 'said', 'he', "'tell", 'faithful', 'john', 'to', 'come', 'to', 'me', '.', "'", 'faithful', 'john', 'was', 'his', 'favorite', 'servant', ',', 'and', 'was', 'so', 'called']

Processing file stories\006.txt
Data size (Characters) (Document 5) 2005
Sample string (Document 5) ['there', 'was', 'once', 'a', 'peasant', 'who', 'had', 'driven', 'his', 'cow', 'to', 'the', 'fair', ',', 'and', 'sold', 'her', 'for', 'seven', 'talers', '.', 'on', 'the', 'way', 'home', 'he', 'had', 'to', 'pass', 'a', 'pond', ',', 'and', 'already', 'from', 'afar', 'he', 'heard', 'the', 'frogs', 'crying', ',', 'aik', ',', 'aik', ',', 'aik', ',', 'aik', '.']

Processing file stories\007.txt
Data size (Characters) (Document 6) 2265
Sample string (Document 6) ['there', 'were', 'once', 'upon', 'a', 'time', 'a', 'king', 'and', 'a', 'queen', 'who', 'lived', 'happily', 'together', 'and', 'had', 'twelve', 'children', ',', 'but', 'they', 'were', 'all', 'boys', '.', 'then', 'said', 'the', 'king', 'to', 'his', 'wife', ',', 'if', 'the', 'thirteenth', 'child', 'which', 'you', 'are', 'about', 'to', 'bring', 'into', 'the', 'world', ',', 'is', 'a']

Processing file stories\008.txt
Data size (Characters) (Document 7) 2781
Sample string (Document 7) ['little', 'brother', 'took', 'his', 'little', 'sister', 'by', 'the', 'hand', 'and', 'said', ',', 'since', 'our', 'mother', 'died', 'we', 'have', 'had', 'no', 'happiness', '.', 'our', 'step-mother', 'beats', 'us', 'every', 'day', ',', 'and', 'if', 'we', 'come', 'near', 'her', 'she', 'kicks', 'us', 'away', 'with', 'her', 'foot', '.', 'our', 'meals', 'are', 'the', 'hard', 'crusts', 'of']

Processing file stories\009.txt
Data size (Characters) (Document 8) 1635
Sample string (Document 8) ['there', 'were', 'once', 'a', 'man', 'and', 'a', 'woman', 'who', 'had', 'long', 'in', 'vain', 'wished', 'for', 'a', 'child', '.', 'at', 'length', 'the', 'woman', 'hoped', 'that', 'god', 'was', 'about', 'to', 'grant', 'her', 'desire', '.', 'these', 'people', 'had', 'a', 'little', 'window', 'at', 'the', 'back', 'of', 'their', 'house', 'from', 'which', 'a', 'splendid', 'garden', 'could']

Processing file stories\010.txt
Data size (Characters) (Document 9) 2353
Sample string (Document 9) ['there', 'was', 'once', 'a', 'man', 'whose', 'wife', 'died', ',', 'and', 'a', 'woman', 'whose', 'husband', 'died', ',', 'and', 'the', 'man', 'had', 'a', 'daughter', ',', 'and', 'the', 'woman', 'also', 'had', 'a', 'daughter', '.', 'the', 'girls', 'were', 'acquainted', 'with', 'each', 'other', ',', 'and', 'went', 'out', 'walking', 'together', ',', 'and', 'afterwards', 'came', 'to', 'the']

Processing file stories\011.txt
Data size (Characters) (Document 10) 1038
Sample string (Document 10) ['there', 'was', 'once', 'a', 'girl', 'who', 'was', 'idle', 'and', 'would', 'not', 'spin', ',', 'and', 'let', 'her', 'mother', 'say', 'what', 'she', 'would', ',', 'she', 'could', 'not', 'bring', 'her', 'to', 'it', '.', 'at', 'last', 'the', 'mother', 'was', 'once', 'so', 'overcome', 'with', 'anger', 'and', 'impatience', ',', 'that', 'she', 'beat', 'her', ',', 'at', 'which']

Processing file stories\012.txt
Data size (Characters) (Document 11) 3427
Sample string (Document 11) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'poor', 'wood-cutter', 'with', 'his', 'wife', 'and', 'his', 'two', 'children', '.', 'the', 'boy', 'was', 'called', 'hansel', 'and', 'the', 'girl', 'gretel', '.', 'he', 'had', 'little', 'to', 'bite', 'and', 'to', 'break', ',', 'and', 'once', 'when', 'great', 'dearth', 'fell', 'on', 'the', 'land', ',', 'he', 'could', 'no', 'longer']

Processing file stories\013.txt
Data size (Characters) (Document 12) 1619
Sample string (Document 12) ['there', 'was', 'once', 'on', 'a', 'time', 'a', 'poor', 'man', ',', 'who', 'could', 'no', 'longer', 'support', 'his', 'only', 'son', '.', 'then', 'said', 'the', 'son', ',', 'dear', 'father', ',', 'things', 'go', 'so', 'badly', 'with', 'us', 'that', 'i', 'am', 'a', 'burden', 'to', 'you', '.', 'i', 'would', 'rather', 'go', 'away', 'and', 'see', 'how', 'i']

Processing file stories\014.txt
Data size (Characters) (Document 13) 1813
Sample string (Document 13) ['a', 'long', 'time', 'ago', 'there', 'lived', 'a', 'king', 'who', 'was', 'famed', 'for', 'his', 'wisdom', 'through', 'all', 'the', 'land', '.', 'nothing', 'was', 'hidden', 'from', 'him', ',', 'and', 'it', 'seemed', 'as', 'if', 'news', 'of', 'the', 'most', 'secret', 'things', 'was', 'brought', 'to', 'him', 'through', 'the', 'air', '.', 'but', 'he', 'had', 'a', 'strange', 'custom']

Processing file stories\015.txt
Data size (Characters) (Document 14) 3834
Sample string (Document 14) ['one', 'summer', "'s", 'morning', 'a', 'little', 'tailor', 'was', 'sitting', 'on', 'his', 'table', 'by', 'the', 'window', ',', 'he', 'was', 'in', 'good', 'spirits', ',', 'and', 'sewed', 'with', 'all', 'his', 'might', '.', 'then', 'came', 'a', 'peasant', 'woman', 'down', 'the', 'street', 'crying', ',', 'good', 'jams', ',', 'cheap', '.', 'good', 'jams', ',', 'cheap', '.', 'this']

Processing file stories\016.txt
Data size (Characters) (Document 15) 3037
Sample string (Document 15) ['cinderella', 'the', 'wife', 'of', 'a', 'rich', 'man', 'fell', 'sick', ',', 'and', 'as', 'she', 'felt', 'that', 'her', 'end', 'was', 'drawing', 'near', ',', 'she', 'called', 'her', 'only', 'daughter', 'to', 'her', 'bedside', 'and', 'said', ',', 'dear', 'child', ',', 'be', 'good', 'and', 'pious', ',', 'and', 'then', 'the', 'good', 'god', 'will', 'always', 'protect', 'you', ',']

Processing file stories\017.txt
Data size (Characters) (Document 16) 1320
Sample string (Document 16) ['there', 'was', 'once', 'a', 'king', "'s", 'son', 'who', 'was', 'seized', 'with', 'a', 'desire', 'to', 'travel', 'about', 'the', 'world', ',', 'and', 'took', 'no', 'one', 'with', 'him', 'but', 'a', 'faithful', 'servant', '.', 'one', 'day', 'he', 'came', 'to', 'a', 'great', 'forest', ',', 'and', 'when', 'darkness', 'overtook', 'him', 'he', 'could', 'find', 'no', 'shelter', ',']

## Building the Dictionaries (Bigrams)
Builds the following. To understand each of these elements, let us also assume the text "I like to go to school"

* `dictionary`: maps a string word to an ID (e.g. {I:0, like:1, to:2, go:3, school:4})
* `reverse_dictionary`: maps an ID to a string word (e.g. {0:I, 1:like, 2:to, 3:go, 4:school}
* `count`: List of list of (word, frequency) elements (e.g. [(I,1),(like,1),(to,2),(go,1),(school,1)]
* `data` : Contain the string of text we read, where string words are replaced with word IDs (e.g. [0, 1, 2, 3, 2, 4])

It also introduces an additional special token `UNK` to denote rare words to are too rare to make use of._____no_output_____

def build_dataset(documents):
    chars = []
    # This is going to be a list of lists
    # Where the outer list denote each document
    # and the inner lists denote words in a given document
    data_list = []

    for d in documents:
        chars.extend(d)
    print('%d Words found.'%len(chars))
    count = []
    # Get the word sorted by their frequency (Highest comes first)
    count.extend(collections.Counter(chars).most_common())

    # Create an ID for each word by giving the current length of the dictionary
    # And adding that item to the dictionary
    # Start with 'UNK' that is assigned to too rare words
    dictionary = dict({'UNK':0})
    for char, c in count:
        # Only add a bigram to dictionary if its frequency is more than 10
        if c > 10:
            dictionary[char] = len(dictionary)

    unk_count = 0
    # Traverse through all the text we have
    # to replace each string word with the ID of the word
    for d in documents:
        data = list()
        for char in d:
            # If word is in the dictionary use the word ID,
            # else use the ID of the special token "UNK"
            if char in dictionary:
                index = dictionary[char]
            else:
                index = dictionary['UNK']
                unk_count += 1
            data.append(index)

        data_list.append(data)

    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
    return data_list, count, dictionary, reverse_dictionary

global data_list, count, dictionary, reverse_dictionary,vocabulary_size

# Print some statistics about data
data_list, count, dictionary, reverse_dictionary = build_dataset(documents)
print('Most common words (+UNK)', count[:5])
print('Least common words (+UNK)', count[-15:])
print('Sample data', data_list[0][:10])
print('Sample data', data_list[1][:10])
print('Vocabulary: ',len(dictionary))
vocabulary_size = len(dictionary)
del documents  # To reduce memory.202036 Words found.
Most common words (+UNK) [(',', 17882), ('the', 11960), ('and', 9325), ('.', 7633), ('to', 4520)]
Least common words (+UNK) [('wet', 1), ('iii', 1), ('sheltered', 1), ('yelled', 1), ('oats', 1), ('armchair', 1), ('peaceably', 1), ('kitchen-sink', 1), ('fretted', 1), ('kernels', 1), ('youths', 1), ('hammers', 1), ('minutest', 1), ('tenderly', 1), ('craftily', 1)]
Sample data [12, 0, 776, 25, 1173, 118, 826, 41, 1, 44]
Sample data [577, 60, 7, 83, 119, 0, 7, 0, 22, 19]
Vocabulary:  1322

## Generating Batches of Data
The following object generates a batch of data which will be used to train the RNN. More specifically the generator breaks a given sequence of words into `batch_size` segments. We also maintain a cursor for each segment. So whenever we create a batch of data, we sample one item from each segment and update the cursor of each segment. Note that the input words (as well as labels) are not one-hot encoded anymore as we need the word index to do embedding lookups._____no_output_____

class DataGeneratorSeq(object):

    def __init__(self,text,batch_size,num_unroll):
        # Text where a bigram is denoted by its ID
        self._text = text
        # Number of bigrams in the text
        self._text_size = len(self._text)
        # Number of datapoints in a batch of data
        self._batch_size = batch_size
        # Num unroll is the number of steps we unroll the RNN in a single training step
        # This relates to the truncated backpropagation we discuss in Chapter 6 text
        self._num_unroll = num_unroll
        # We break the text in to several segments and the batch of data is sampled by
        # sampling a single item from a single segment
        self._segments = self._text_size//self._batch_size
        self._cursor = [offset * self._segments for offset in range(self._batch_size)]

    def next_batch(self):
        '''
        Generates a single batch of data
        '''
        # Train inputs (one-hot-encoded) and train outputs (one-hot-encoded)
        batch_data = np.zeros((self._batch_size),dtype=np.float32)
        batch_labels = np.zeros((self._batch_size),dtype=np.float32)

        # Fill in the batch datapoint by datapoint
        for b in range(self._batch_size):
            # If the cursor of a given segment exceeds the segment length
            # we reset the cursor back to the beginning of that segment
            if self._cursor[b]+1>=self._text_size:
                self._cursor[b] = b * self._segments

            # Add the text at the cursor as the input
            batch_data[b] = self._text[self._cursor[b]]
            # Add the preceding word as the label to be predicted
            batch_labels[b]= self._text[self._cursor[b]+1]
            # Update the cursor
            self._cursor[b] = (self._cursor[b]+1)%self._text_size

        return batch_data,batch_labels

    def unroll_batches(self):
        '''
        This produces a list of num_unroll batches
        as required by a single step of training of the RNN
        '''
        unroll_data,unroll_labels = [],[]
        for ui in range(self._num_unroll):
            data, labels = self.next_batch()
            unroll_data.append(data)
            unroll_labels.append(labels)

        return unroll_data, unroll_labels

    def reset_indices(self):
        '''
        Used to reset all the cursors if needed
        '''
        self._cursor = [offset * self._segments for offset in range(self._batch_size)]

# Running a tiny set to see if things are correct
dg = DataGeneratorSeq(data_list[0][25:50],5,5)
u_data, u_labels = dg.unroll_batches()

# Iterate through each data batch in the unrolled set of batches
for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):
    print('\n\nUnrolled index %d'%ui)
    dat_ind = dat
    lbl_ind = lbl
    print('\tInputs:')
    for sing_dat in dat_ind:
        print('\t%s (%d)'%(reverse_dictionary[sing_dat],sing_dat),end=", ")
    print('\n\tOutput:')
    for sing_lbl in lbl_ind:
        print('\t%s (%d)'%(reverse_dictionary[sing_lbl],sing_lbl),end=", ")

Unrolled index 0
	Inputs:
	that (14), 	which (56), 	, (1), 	shone (792), 	close (822),
	Output:
	the (2), 	has (114), 	was (8), 	in (12), 	by (60),

Unrolled index 1
	Inputs:
	the (2), 	has (114), 	was (8), 	in (12), 	by (60),
	Output:
	sun (407), 	seen (283), 	astonished (869), 	her (17), 	the (2),

Unrolled index 2
	Inputs:
	sun (407), 	seen (283), 	astonished (869), 	her (17), 	the (2),
	Output:
	itself (720), 	so (33), 	whenever (1319), 	face (360), 	king (34),

Unrolled index 3
	Inputs:
	itself (720), 	so (33), 	whenever (1319), 	face (360), 	king (34),
	Output:
	, (1), 	much (165), 	it (9), 	. (4), 	's (53),

Unrolled index 4
	Inputs:
	, (1), 	much (165), 	it (9), 	. (4), 	close (822),
	Output:
	which (56), 	, (1), 	shone (792), 	close (822), 	by (60),

## Defining the LSTM

This is a standard LSTM. The LSTM has 5 main components.
* Cell state
* Hidden state
* Input gate
* Forget gate
* Output gate

Each gate has three sets of weights (1 set for the current input, 1 set for the previous hidden state and 1 bias)

## Defining hyperparameters

Here we define several hyperparameters and are very similar to the ones we defined in Chapter 6. However additionally we use dropout; a technique that helps to avoid overfitting._____no_output_____

# Number of neurons in the hidden state variables
num_nodes = [64, 48, 32]

# Number of data points in a batch we process
batch_size = 32

# Number of time steps we unroll for during optimization
num_unrollings = 50

dropout = 0.2 # We use dropout

# Use this in the CSV filename when saving
# when using dropout
filename_extension = ''
if dropout>0.0:
    filename_extension = '_dropout'

filename_to_save = 'lstm_word2vec'+filename_extension+'.csv' # use to save perplexity values_____no_output_____

## Defining Inputs and Outputs

In the code we define two different types of inputs.
* Training inputs (The stories we downloaded) (batch_size > 1 with unrolling)
* Validation inputs (An unseen validation dataset) (bach_size =1, no unrolling)
* Test inputs (New story we are going to generate) (batch_size=1, no unrolling)_____no_output_____

tf.reset_default_graph()

# Training Input data.
train_inputs, train_labels = [],[]
train_labels_ohe = []
# Defining unrolled training inputs
for ui in range(num_unrollings):
    train_inputs.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_inputs_%d'%ui))
    train_labels.append(tf.placeholder(tf.int32, shape=[batch_size], name = 'train_labels_%d'%ui))
    train_labels_ohe.append(tf.one_hot(train_labels[ui], vocabulary_size))

# Validation data placeholders
valid_inputs = tf.placeholder(tf.int32, shape=[1],name='valid_inputs')
valid_labels = tf.placeholder(tf.int32, shape=[1], name = 'valid_labels')
valid_labels_ohe = tf.one_hot(valid_labels, vocabulary_size)

# Text generation: batch 1, no unrolling.
test_input = tf.placeholder(tf.int32, shape=[1],name='test_input')_____no_output_____

## Loading Word Embeddings to TensorFlow
We load the previously learned and stored embeddings to TensorFlow and define tensors to hold embeddings_____no_output_____

## If you want to change the embedding matrix to something you newly generated,
## Simply change embeddings.npy to embeddings-tmp.npy
embed_mat = np.load('embeddings.npy')
embeddings_size = embed_mat.shape[1]

embed_init = tf.constant(embed_mat)
embeddings = tf.Variable(embed_init,name='embeddings')

# Defining embedding lookup operations for all the unrolled
# trianing inputs
train_inputs_embeds = []
for ui in range(num_unrollings):
    # We use expand_dims to add an additional axis
    # As this is needed later for LSTM cell computation
    train_inputs_embeds.append(tf.expand_dims(tf.nn.embedding_lookup(embeddings,train_inputs[ui]),0))

# Defining embedding lookup for operations for all the validation data
valid_inputs_embeds = tf.nn.embedding_lookup(embeddings,valid_inputs)

# Defining embedding lookup for operations for all the testing data
test_input_embeds = tf.nn.embedding_lookup(embeddings, test_input)_____no_output_____

## Defining Model Parameters

Now we define model parameters. Compared to RNNs, LSTMs have a large number of parameters. Each gate (input, forget, memory and output) has three different sets of parameters._____no_output_____

print('Defining softmax weights and biases')
# Softmax Classifier weights and biases.
w = tf.Variable(tf.truncated_normal([num_nodes[-1], vocabulary_size], stddev=0.01))
b = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01))

print('Defining the LSTM cell')
# Defining a deep LSTM from Tensorflow RNN API

# First we define a list of LSTM cells
# num_nodes here is a sequence of hidden layer sizes
cells = [tf.nn.rnn_cell.LSTMCell(n) for n in num_nodes]

# We now define a dropout wrapper for each LSTM cell
dropout_cells = [
    rnn.DropoutWrapper(
        cell=lstm, input_keep_prob=1.0,
        output_keep_prob=1.0-dropout, state_keep_prob=1.0,
        variational_recurrent=True,
        input_size=tf.TensorShape([embeddings_size]),
        dtype=tf.float32
    ) for lstm in cells
]

# We first define a MultiRNNCell Object that uses the
# Dropout wrapper (for training)
stacked_dropout_cell = tf.nn.rnn_cell.MultiRNNCell(dropout_cells)
# Here we define a MultiRNNCell that does not use dropout
# Validation and Testing
stacked_cell = tf.nn.rnn_cell.MultiRNNCell(cells)

# Note: There exists the EmbeddingWrapper in RNN API to automate the embedding_lookup but,
# in many cases it may be more efficient to not use this wrapper, but instead concatenate the whole sequence of
# your inputs in time, do the embedding on this batch-concatenated sequence, then split it and feed into your RNN.
Defining softmax weights and biases
Defining the LSTM cell

## Defining LSTM Computations
Here first we define the LSTM cell computations as a consice function. Then we use this function to define training and test-time inference logic._____no_output_____

print('LSTM calculations for unrolled inputs and outputs')
# =========================================================
# Training inference logic

# Initial state of the LSTM memory.
initial_state = stacked_dropout_cell.zero_state(batch_size, dtype=tf.float32)

# Defining the LSTM cell computations (training)
train_outputs, initial_state = tf.nn.dynamic_rnn(
    stacked_dropout_cell, tf.concat(train_inputs_embeds,axis=0),
    time_major=True, initial_state=initial_state
)

# Reshape the final outputs to [num_unrollings*batch_size, num_nodes]
final_output = tf.reshape(train_outputs,[-1,num_nodes[-1]])

# Computing logits
logits = tf.matmul(final_output, w) + b
# Computing predictions
train_prediction = tf.nn.softmax(logits)

# Reshape logits to time-major fashion [num_unrollings, batch_size, vocabulary_size]
time_major_train_logits = tf.reshape(logits,[num_unrollings,batch_size,-1])

# We create train labels in a time major fashion [num_unrollings, batch_size, vocabulary_size]
# so that this could be used with the loss function
time_major_train_labels = tf.reshape(tf.concat(train_labels,axis=0),[num_unrollings,batch_size])

# Perplexity related operation
train_perplexity_without_exp = tf.reduce_sum(tf.concat(train_labels_ohe,0)*-tf.log(train_prediction+1e-10))/(num_unrollings*batch_size)

# =========================================================
# Validation inference logic

# Separate state for validation data
initial_valid_state = stacked_cell.zero_state(1, dtype=tf.float32)

# Validation input related LSTM computation
valid_outputs, initial_valid_state = tf.nn.dynamic_rnn(
    stacked_cell, tf.expand_dims(valid_inputs_embeds,0),
    time_major=True, initial_state=initial_valid_state
)

# Reshape the final outputs to [1, num_nodes]
final_valid_output = tf.reshape(valid_outputs,[-1,num_nodes[-1]])

# Computing logits
valid_logits = tf.matmul(final_valid_output, w) + b
# Computing predictions
valid_prediction = tf.nn.softmax(valid_logits)

# Perplexity related operation
valid_perplexity_without_exp = tf.reduce_sum(valid_labels_ohe*-tf.log(valid_prediction+1e-10))LSTM calculations for unrolled inputs and outputs

## Calculating LSTM Loss
We calculate the training loss of the LSTM here. It's a typical cross entropy loss calculated over all the scores we obtained for training data (`loss`) and averaged and summed in a specific way._____no_output_____

# We use the sequence-to-sequence loss function to define the loss
# We calculate the average across the batches
# But get the sum across the sequence length
loss = tf.contrib.seq2seq.sequence_loss(
    logits = tf.transpose(time_major_train_logits,[1,0,2]),
    targets = tf.transpose(time_major_train_labels),
    weights= tf.ones([batch_size, num_unrollings], dtype=tf.float32),
    average_across_timesteps=False,
    average_across_batch=True
)

loss = tf.reduce_sum(loss)_____no_output_____

## Defining Learning Rate and the Optimizer with Gradient Clipping
Here we define the learning rate and the optimizer we're going to use. We will be using the Adam optimizer as it is one of the best optimizers out there. Furthermore we use gradient clipping to prevent any gradient explosions._____no_output_____

# Used for decaying learning rate
gstep = tf.Variable(0, trainable=False)

# Running this operation will cause the value of gstep
# to increase, while in turn reducing the learning rate
inc_gstep = tf.assign(gstep, gstep+1)

# Adam Optimizer. And gradient clipping.
tf_learning_rate = tf.train.exponential_decay(0.001,gstep,decay_steps=1, decay_rate=0.5)

print('Defining optimizer')
optimizer = tf.train.AdamOptimizer(tf_learning_rate)
gradients, v = zip(*optimizer.compute_gradients(loss))
gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
optimizer = optimizer.apply_gradients(
    zip(gradients, v))

inc_gstep = tf.assign(gstep, gstep+1)Defining optimizer

## LSTM with Beam-Search

Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability._____no_output_____

beam_length = 5
beam_neighbors = 5

# We redefine the sample generation with beam search
sample_beam_inputs = [tf.placeholder(tf.int32, shape=[1]) for _ in range(beam_neighbors)]
# Embedding lookups for each beam
sampel_beam_input_embeds = [tf.nn.embedding_lookup(embeddings,b) for b in sample_beam_inputs]

best_beam_index = tf.placeholder(shape=None, dtype=tf.int32)
best_neighbor_beam_indices = tf.placeholder(shape=[beam_neighbors], dtype=tf.int32)

# We have [num_layers, beam_neighbors] shape state variable set
# Maintains output of each beam
saved_sample_beam_output = [[tf.Variable(tf.zeros([1, n])) for _ in range(beam_neighbors)] for n in num_nodes]
# Maintains the state of each beam
saved_sample_beam_state = [[tf.Variable(tf.zeros([1, n])) for _ in range(beam_neighbors)] for n in num_nodes]

# Resetting the sample beam states (should be done at the beginning of each text snippet generation)
reset_sample_beam_state = tf.group(
    *[[saved_sample_beam_output[ni][vi].assign(tf.zeros([1, n]))  for vi in range(beam_neighbors)] for ni,n in enumerate(num_nodes)],
    *[[saved_sample_beam_state[ni][vi].assign(tf.zeros([1, n])) for vi in range(beam_neighbors)] for ni,n in enumerate(num_nodes)]
)

# We stack them to perform gather operation below
# These should be of size [beam_neighbors, 1, num_nodes]
stacked_beam_outputs = [tf.stack(saved_sample_beam_output[n]) for n in range(len(num_nodes))]
stacked_beam_states = [tf.stack(saved_sample_beam_state[n]) for n in range(len(num_nodes))]

# The beam states for each beam (there are beam_neighbor-many beams) needs to be updated at every depth of tree
# Consider an example where you have 3 classes where we get the best two neighbors (marked with star)
#     a`      b*       c
#   / | \   / | \    / | \
#  a  b c  a* b` c  a  b  c
# Since both the candidates from level 2 comes from the parent b
# We need to update both states/outputs from saved_sample_beam_state/output to have index 1 (corresponding to parent b)

# Our update_sample_beam_state gets very complicated
# Because we have to do this for every beam neighbor
# as well as every layer
update_sample_beam_state = tf.group(
    *[
        [saved_sample_beam_output[n][vi].assign(
            tf.gather_nd(stacked_beam_outputs[n],[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)
          for n in range(len(num_nodes))]
    ],
    *[
        [saved_sample_beam_state[n][vi].assign(
            tf.gather_nd(stacked_beam_states[n],[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)
          for n in range(len(num_nodes))]
    ]
)

# This needs to be of shap [beam_neighbors, num_layers]
sample_beam_outputs, sample_beam_states = [],[]

# This needs to be of shape [beam_neighbors, num_layers]
# and each item is a LSTMStateTuple
# We calculate lstm_cell state and output for each beam
tmp_state_tuple = []
for vi in range(beam_neighbors):
    single_beam_state_tuple = []
    for ni in range(len(num_nodes)):
        single_beam_state_tuple.append(
            tf.nn.rnn_cell.LSTMStateTuple(saved_sample_beam_output[ni][vi], saved_sample_beam_state[ni][vi])
        )
    tmp_state_tuple.append(single_beam_state_tuple)

for vi in range(beam_neighbors):
    # We cannot use tf.nn.dynamic_rnn as we need to manipulate
    # LSTM state a lot. So even though it is lot of work
    # It is easier to do state manipulation externelly
    # when using beam search
    final_output, tmp_state_tuple[vi] = stacked_cell.call(
        sampel_beam_input_embeds[vi], tmp_state_tuple[vi]
    )

    # We need to be care how we populate sample_beam_outputs
    # and sample_beam_state
    # They both need to be of size [beam_neighbors, num_layers]
    sample_beam_outputs.append([])
    sample_beam_states.append([])
    for ni in range(len(num_nodes)):
        sample_beam_outputs[-1].append(tmp_state_tuple[vi][ni][0])
        sample_beam_states[-1].append(tmp_state_tuple[vi][ni][1])

# This store predictions made for each beam neighbor position
sample_beam_predictions = []

# Used to update the LSTM cell for each neighbor
# Just as normally we do during generation
beam_update_ops = tf.group(
    [[saved_sample_beam_output[ni][vi].assign(sample_beam_outputs[vi][ni]) for vi in range(beam_neighbors)]
                            for ni in range(len(num_nodes))],
    [[saved_sample_beam_state[ni][vi].assign(sample_beam_states[vi][ni]) for vi in range(beam_neighbors)]
                            for ni in range(len(num_nodes))]
)

# Get the predictions out
# For a given set of beams, outputs a list of prediction vectors of size beam_neighbors
# each beam having the predictions for full vocabulary
for vi in range(beam_neighbors):
    with tf.control_dependencies([beam_update_ops]):
        sample_beam_predictions.append(tf.nn.softmax(tf.nn.xw_plus_b(sample_beam_outputs[vi][-1], w, b)))
        [1, 48]

## LSTM + Word2vec with Beam-Search

Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability._____no_output_____### Learning rate Decay Logic

Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease_____no_output_____

# Learning rate decay related
# If valid perpelxity does not decrease
# continuously for this many epochs
# decrease the learning rate
decay_threshold = 5
# Keep counting perplexity increases
decay_count = 0

min_perplexity = 1e10

# Learning rate decay logic
def decay_learning_rate(session, v_perplexity):
  global decay_threshold, decay_count, min_perplexity
  # Decay learning rate
  if v_perplexity < min_perplexity:
    decay_count = 0
    min_perplexity= v_perplexity
  else:
    decay_count += 1

  if decay_count >= decay_threshold:
    print('\t Reducing learning rate')
    decay_count = 0
    session.run(inc_gstep)_____no_output_____

### Defining the Beam Prediction Logic
Here we define function that takes in the session as an argument and output a beam of predictions_____no_output_____

test_word = None

def get_beam_prediction(session):

    global test_word
    global sample_beam_predictions, update_sample_beam_state

    # Generating words within a segment with Beam Search
    # To make some calculations clearer, we use the example as follows
    # We have three classes with beam_neighbors=2 (best candidate denoted by *, second best candidate denoted by `)
    # For simplicity we assume best candidate always have probability of 0.5 in output prediction
    # second best has 0.2 output prediction
    #           a`                   b*                   c                <--- root level
    #    /     |     \         /     |     \        /     |     \
    #   a      b      c       a*     b`     c      a      b      c         <--- depth 1
    # / | \  / | \  / | \   / | \  / | \  / | \  / | \  / | \  / | \
    # a b c  a b c  a b c   a*b c  a`b c  a b c  a b c  a b c  a b c       <--- depth 2
    # So the best beams at depth 2 would be
    # b-a-a and b-b-a

    # Calculate the candidates at the root level
    feed_dict = {}
    for b_n_i in range(beam_neighbors):
        feed_dict.update({sample_beam_inputs[b_n_i]: [test_word]})

    # We calculate sample predictions for all neighbors with the same starting word/character
    # This is important to update the state for all instances of beam search
    sample_preds_root = session.run(sample_beam_predictions, feed_dict = feed_dict)
    sample_preds_root = sample_preds_root[0]

    # indices of top-k candidates
    # b and a in our example (root level)
    this_level_candidates_sorted =  (np.argsort(sample_preds_root,axis=1).ravel()[::-1]).tolist() # indices of top-k candidates
    this_level_candidates = []
    for c in this_level_candidates_sorted:
        if len(this_level_candidates)==beam_neighbors:
            break
        if c!=0:
            this_level_candidates.append(c)

    this_level_candidates = np.array(this_level_candidates)

    # probabilities of top-k candidates
    # 0.5 and 0.2
    this_level_probs = sample_preds_root[0,this_level_candidates] #probabilities of top-k candidates

    # Update test sequence produced by each beam from the root level calculation
    # Test sequence looks like for our example (at root)
    # [b,a]
    test_sequences = ['' for _ in range(beam_neighbors)]
    for b_n_i in range(beam_neighbors):
        test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '

    # Make the calculations for the rest of the depth of the beam search tree
    for b_i in range(beam_length-1):
        test_words = [] # candidate words for each beam
        pred_words = [] # Predicted words of each beam

        # computing feed_dict for the beam search (except root)
        # feed dict should contain the best words/chars/bigrams found by the previous level of search

        # For level 1 in our example this would be
        # sample_beam_inputs[0]: b, sample_beam_inputs[1]:a
        feed_dict = {}
        for p_idx, pred_i in enumerate(this_level_candidates):
            # Updating the feed_dict for getting next predictions
            test_words.append(this_level_candidates[p_idx])

            feed_dict.update({sample_beam_inputs[p_idx]:[test_words[p_idx]]})

        # Calculating predictions for all neighbors in beams
        # This is a list of vectors where each vector is the prediction vector for a certain beam
        # For level 1 in our example, the prediction values for
        #      b             a  (previous beam search results)
        # [a,  b,  c],  [a,  b,  c] (current level predictions) would be
        # [0.1,0.1,0.1],[0.5,0.2,0]
        sample_preds_all_neighbors = session.run(sample_beam_predictions, feed_dict=feed_dict)

        # Create a single vector with
        # Making our example [0.1,0.1,0.1,0.5,0.2,0]
        sample_preds_all_neighbors_concat = np.concatenate(sample_preds_all_neighbors,axis=1)

        # Update this_level_candidates to be used for the next iteration
        # And update the probabilities for each beam
        # In our example these would be [3,4] (indices with maximum value from above vector)
        # We also use a simple trick to avoid UNK (word id 0) being predicted
        this_level_candidates_sorted = np.argsort(sample_preds_all_neighbors_concat.ravel())[::-1]
        this_level_candidates = []
        for c in this_level_candidates_sorted:
            if len(this_level_candidates)==beam_neighbors:
                break
            if c!=0 and c%vocabulary_size != 0 :
                this_level_candidates.append(c)

        this_level_candidates = np.array(this_level_candidates)

        # In the example this would be [1,1]
        parent_beam_indices = this_level_candidates//vocabulary_size

        # normalize this_level_candidates to fall between [0,vocabulary_size]
        # In this example this would be [0,1]
        this_level_candidates = (this_level_candidates%vocabulary_size).tolist()

        # Here we update the final state of each beam to be
        # the state that was at the index 1. Because for both the candidates at this level the parent is
        # at index 1 (that is b from root level)
        session.run(update_sample_beam_state, feed_dict={best_neighbor_beam_indices: parent_beam_indices})

        # Here we update the joint probabilities of each beam and add the newly found candidates to the sequence
        tmp_this_level_probs = np.asarray(this_level_probs)
        tmp_test_sequences = list(test_sequences)

        for b_n_i in range(beam_neighbors):
            # We make the b_n_i element of this_level_probs to be the probability of parents
            # In the example the parent indices are [1,1]
            # So this_level_probs become [0.5,0.5]
            this_level_probs[b_n_i] = tmp_this_level_probs[parent_beam_indices[b_n_i]]

            # Next we multipyle these by the probabilities of the best candidates from current level
            # [0.5*0.5, 0.5*0.2] = [0.25,0.1]
            this_level_probs[b_n_i] *= sample_preds_all_neighbors[parent_beam_indices[b_n_i]][0,this_level_candidates[b_n_i]]

            # Make the b_n_i element of test_sequences to be the correct parent of the current best candidates
            # In the example this becomes [b, b]
            test_sequences[b_n_i] = tmp_test_sequences[parent_beam_indices[b_n_i]]

            # Now we append the current best candidates
            # In this example this becomes [ba,bb]
            test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '

            # Create one-hot-encoded representation for each candidate
            pred_words.append(this_level_candidates[b_n_i])

    # Calculate best beam id based on the highest beam probability
    # Using the highest beam probability always lead to very monotonic text
    # Let us sample one randomly where one being sampled is decided by the likelihood of that beam
    rand_cand_ids = np.argsort(this_level_probs)[-3:]
    rand_cand_probs = this_level_probs[rand_cand_ids]/np.sum(this_level_probs[rand_cand_ids])
    random_id = np.random.choice(rand_cand_ids,p=rand_cand_probs)

    best_beam_id = parent_beam_indices[random_id]
    # Update state and output variables for test prediction
    session.run(update_sample_beam_state,feed_dict={best_neighbor_beam_indices:[best_beam_id for _ in range(beam_neighbors)]})

    # Make the last word/character/bigram from the best beam
    test_word = pred_words[best_beam_id]

    return test_sequences[best_beam_id]_____no_output_____

### Running Training, Validation and Generation

We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text_____no_output_____

num_steps = 251
steps_per_document = 100
docs_per_step = 10
valid_summary = 1
train_doc_count = 100_____no_output_____beam_nodes = []

train_perplexity_ot = []
valid_perplexity_ot = []
session = tf.InteractiveSession()

tf.global_variables_initializer().run()

print('Initialized')
average_loss = 0

# We use the first 10 documents that has
# more than (num_steps+1)*steps_per_document bigrams for creating the validation dataset

# Identify the first 10 documents following the above condition
long_doc_ids = []
for di in range(num_files):
  if len(data_list[di])>1000:
    long_doc_ids.append(di)
  if len(long_doc_ids)==10:
    break

# Generating data
data_gens = []
valid_gens = []
for fi in range(num_files):
  # Get all the bigrams if the document id is not in the validation document ids
  if fi not in long_doc_ids:
    data_gens.append(DataGeneratorSeq(data_list[fi],batch_size,num_unrollings))
  # if the document is in the validation doc ids, only get up to the
  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data
  else:
    data_gens.append(DataGeneratorSeq(data_list[fi][:-steps_per_document],batch_size,num_unrollings))
    # Defining the validation data generator
    valid_gens.append(DataGeneratorSeq(data_list[fi][-steps_per_document:],1,1))

valid_docs = len(valid_gens)
assert valid_docs>0

feed_dict = {}
for step in range(num_steps):
    print('Training (Step: %d)'%step)
    for di in np.random.permutation(train_doc_count)[:docs_per_step]:

        for doc_step_id in range(steps_per_document):

            u_data, u_labels = data_gens[di].unroll_batches()
            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):
                feed_dict[train_inputs[ui]] = dat
                feed_dict[train_labels[ui]] = lbl
                #print(['( %s; %s ) '%(reverse_dictionary[tid],reverse_dictionary[til]) for tid,til in zip(np.argmax(dat,axis=1),np.argmax(lbl,axis=1))])

            feed_dict.update({tf_learning_rate:0.0005})
            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp],
                                                       feed_dict=feed_dict)

            average_loss += step_perplexity

        print('(%d).'%di,end='')
    print('')

    if (step+1) % valid_summary == 0:

      average_loss = average_loss / (docs_per_step*steps_per_document*valid_summary)
      # The average loss is an estimate of the loss over the last 2000 batches.
      print('Average loss at step %d: %f' % (step+1, average_loss))
      print('\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))
      train_perplexity_ot.append(np.exp(average_loss))
      average_loss = 0 # reset loss

      valid_loss = 0 # reset loss

      # calculate valid perplexity
      for v_doc_id in range(valid_docs):
          # Remember we process things as bigrams
          # So need to divide by 2
          for v_step in range(steps_per_document//2):
            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()

            # Run validation phase related TensorFlow operations
            v_perp = session.run(
                valid_perplexity_without_exp,
                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}
            )

            valid_loss += v_perp

          # Reset validation data generator cursor
          valid_gens[v_doc_id].reset_indices()
      print()
      v_perplexity = np.exp(valid_loss/(steps_per_document*valid_docs//2))
      print("Valid Perplexity: %.2f\n"%v_perplexity)
      valid_perplexity_ot.append(v_perplexity)

      decay_learning_rate(session, v_perplexity)

      # Generating new text ...
      # We will be generating one segment having 500 bigrams
      # Feel free to generate several segments by changing
      # the value of segments_to_generate
      print('Generated Text after epoch %d ... '%step)
      segments_to_generate = 1
      chars_in_segment = 500//beam_length

      for _ in range(segments_to_generate):
        print('======================== New text Segment ==========================')
        # first word randomly generated
        test_word = data_list[np.random.randint(0,num_files)][np.random.randint(0,100)]
        print("",reverse_dictionary[test_word],end=' ')

        # Generating words within a segment with Beam Search
        for _ in range(chars_in_segment):
            test_sequence = get_beam_prediction(session)
            print(test_sequence,end=' ')
        print(" ")
        session.run([reset_sample_beam_state])

        print('====================================================================')

      print("")

session.close()

with open('lstm_beam_search_word2vec_rnn_api.csv', 'wt') as f:
    writer = csv.writer(f,delimiter=',')
    writer.writerow(train_perplexity_ot)
    writer.writerow(valid_perplexity_ot)Initialized
Training (Step: 0)
(1).(62).(82).(67).(57).(87).(58).(10).(30).(6).
Average loss at step 1: 5.133673
	Perplexity at step 1: 169.639030

Valid Perplexity: 232.29

Generated Text after epoch 0 ...
======================== New text Segment ==========================
 UNK her , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  and , , , ,  , , , , ,  . , , , ,  her , , , ,  her , , , ,  and , , , ,  the , , , ,  the , , , ,  and , , , ,  and , , , ,  and , , , ,  . , , , ,  . , , , ,  and , , , ,  and , , , ,  her , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  and , , , ,  and , , , ,  her , , , ,  the , , , ,  the , , , ,  . , , , ,  and , , , ,  , , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  the , , , ,  and , , , ,  and , , , ,  the , , , ,  . , , , ,  the , , , ,  her , , , ,  the , , , ,  the , , , ,  her , , , ,  the , , , ,  the , , , ,  and , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  . , , , ,  her , , , ,  the , , , ,  the , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  and , , , ,  the , , , ,  . , , , ,  and , , , ,  the , , , ,  her , , , ,  , , , , ,  , , , , ,  and , , , ,  her , , , ,  , , , , ,  and , , , ,  and , , , ,  her , , , ,  and , , , ,  , , , , ,  her , , , ,  the , , , ,  the , , , ,  the , , , ,  her , , , ,  the , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  and , , , ,
====================================================================

Training (Step: 1)
(72).(20).(11).(49).(31).(94).(64).(65).(35).(85).
Average loss at step 2: 5.085651
	Perplexity at step 2: 161.685143

Valid Perplexity: 195.28

Generated Text after epoch 1 ...
======================== New text Segment ==========================
 father , , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  the , , , ,  , , , , ,  the , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  the , , , ,  to , , , ,  the , , , ,  the , , , ,  to , , , ,  , , , , ,  the , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  to , , , ,  and , , , ,  to , , , ,  the , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  . , , , ,  to , , , ,  the , , , ,  and , , , ,  . , , , ,  the , , , ,  and , , , ,  to , , , ,  to , , , ,  the , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  the , , , ,  and , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  to , , , ,  . , , , ,  the , , , ,  to , , , ,  to , , , ,  the , , , ,  to , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  to , , , ,  the , , , ,  . , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  , , , , ,  to , , , ,  , , , , ,  , , , , ,  , , , , ,  and , , , ,  , , , , ,  and , , , ,  the , , , ,
====================================================================

Training (Step: 2)
(82).(10).(47).(46).(7).(66).(71).(77).(24).(0).
Average loss at step 3: 5.052397
	Perplexity at step 3: 156.396863

Valid Perplexity: 190.89

	 Reducing learning rate
Generated Text after epoch 2 ...
======================== New text Segment ==========================
 to , , , , ,  , , , , ,  , , , , ,  and , , , ,  . , , , ,  to , , , ,  and , , , ,  . , , , ,  the , , , ,  . , , , ,  to , , , ,  and , , , ,  , , , , ,  to , , , ,  , , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  the , , , ,  the , , , ,  and , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  the , , , ,  and , , , ,  and , , , ,  , , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  , , , , ,  and , , , ,  and , , , ,  . , , , ,  . , , , ,  and , , , ,  , , , , ,  . , , , ,  , , , , ,  to , , , ,  and , , , ,  and , , , ,  . , , , ,  , , , , ,  and , , , ,  and , , , ,  and , , , ,  . , , , ,  the , , , ,  and , , , ,  , , , , ,  and , , , ,  . , , , ,  and , , , ,  to , , , ,  and , , , ,  and , , , ,  the , , , ,  and , , , ,  and , , , ,  and , , , ,  and , , , ,
====================================================================

Training (Step: 3)
(70).(28).(24).(5).(64).(20).(76).(40).(23).(78).
Average loss at step 4: 4.964746
	Perplexity at step 4: 143.272116

Valid Perplexity: 210.09

Generated Text after epoch 3 ...
======================== New text Segment ==========================
 his , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,
====================================================================

Training (Step: 4)
(24).(0).(36).(47).(82).(76).(50).(88).(13).(74).
Average loss at step 5: 4.816891
	Perplexity at step 5: 123.580301

Valid Perplexity: 183.25

Generated Text after epoch 4 ...
======================== New text Segment ==========================
 her , , , , ,  and , , , ,  . , , , ,  . , , , ,  to , , , ,  and , , , ,  to , , , ,  . , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  the , , , ,  . , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,
====================================================================

Training (Step: 5)
(58).(33).(34).(10).(57).(8).(83).(2).(36).(96).
Average loss at step 6: 4.677417
	Perplexity at step 6: 107.492090

Valid Perplexity: 156.75

Generated Text after epoch 5 ...
======================== New text Segment ==========================


# Notebook from KeplerGO/project-ideas
Path: notebooks/asteroseismology-in-K2-superstamps.ipynb

# Asteroseismology in K2 Superstamps_____no_output_____Asteroseismology is a study of the "quakes" in a star. Kepler was famously excellent at asteroseismology, finding the high frequency oscillations of main sequence stars easily. K2 has been unable to do this as it only observes the same stars for 80 days. However, we the new C16 we now have a 3 year long baseline in the M67 and beehive clusters.

### Asteroseismology
Watch this talk from Dan Huber for a quick introduction to the topic.

https://www.youtube.com/watch?v=Z5VsAiIiUvc_____no_output_____## What's the problem?

Superstamps can be hard to work with because they don't come in easily digestable TPFs. There is also a wealth of new data from C16 which makes the baseine in M67 and the beehive cluster **3 years**. This should let us make some great asteroseismology measurements. In this project we'll show that asteroseismology is possible with K2 in these superstamp regions. We'll also develop and implement some great tools for quickly plotting asteroseismology information for K2 and Kepler in `lightkurve`!

* We'll use Gaia to find interesting targets and get prior estimates of stellar parameters
* We'll use asteroseismology to measure stellar densities in the beehive and M67 and compare with age models and Gaia parameters._____no_output_____## What is the goal of this project?

**To show that asteroseisic parameters can be derived for main sequence stars using Kepler/K2 data**_____no_output_____## You will need...

A basic understanding of stars and asteroseismology. I'll give you some resources to help you, but you should be familiar with stellar lifecycles and asteroseismology in general to attempt this project.

To begin with, over the first few weeks

* Learn the basics of `lightkurve` and github. Fix one of the open issues that your supervisor will give you.
* Learn the basics of the SFF routines in lightkurve by fitting a set of exoplanet host stars. What does each parameter do to the retrieved exoplanet parameters.
* Use Gaia to build catalogs of stars in M67 and the beehive with prior parameters.
* Use KeplerTargetPixelFileFactory to cut out interesting stars from all 3 quarters of Kepler data from the beehive/M67

In the next few weeks, you should aim to

* Create SAP and/or PSF lightcurves of the targets
* Build routines to create power spectra of the lightcurves and create common asteroseismology plots (e.g. eschelle diagrams, power spectral density diagrams).
* Use Gaia parameters to center the diagrams where we expect the power excess from the star.
* Finding the fringes of the stellar oscillations can be hard automatically. Take a look at Guy Davies open PR on lightkurve and see if you can find a way to semi-automate the process.

Once you have a working prototype of the extraction and plotting tools.

* Build unit tests! All `lightkurve` routines need unit tests.
* Build a tutorial. Make sure that your colleagues (and supervisor) can follow it. Take their feedback and apply any improvements.

Once you have a stable version

* Build a pipeline to process hundreds of stars at once.
* Extract their stellar parameters, compare your findings to the Gaia parameters. What do you find? Do they match? Is there a systematic offset?
* What is the sensitivity of the K2 superstamp in frequency space?
* Build Kepler data of stars and remove sections of the data until it has the same duty cycle as the K2 data. How precise are our measurements? Is K2 as good as Kepler?_____no_output_____## Some tools to help you

If you are confused about any aspect of the Kepler telescope or any systematic, the first place to look is the [Kepler Instrument Handbook](https://archive.stsci.edu/kepler/manuals/KSCI-19033-001.pdf). It is a very large manual, but a quick search in it usually turns up the answer.

I recommend reading [Demystifying Kepler Data](https://arxiv.org/pdf/1207.3093.pdf), which is a short read and easy to digest. This will explain a little about the systematics and how we use Cotrending Basis Vectors to correct for them.

For this project, I recommend you read Andrew Vanderburg's paper on SFF. A nice gateway to this is Gully's excellent SFF tutorial, which you can watch [here](https://www.youtube.com/watch?v=kYRqZwz2VPU&t=28s).

There aren't great packages available for asteroseismology with K2, but we have some tools in lightkurve to help you. You'll need to cut out TPFs using `KeplerTargetPixelFileFactory` and either perform SAP photometry or PSF photometry. (Why not both?) You can look at the open pull request on lightkurve from Dan Foreman-Mackay for tools for asteroseismology.

If we can get familiar with DFMs celerite implementation of time domain asteroseismology that would be unbelievably cool. https://arxiv.org/pdf/1703.09710.pdf_____no_output_____