# Notebook from ricsinaruto/Python-projects
Path: Deep_learning_udc/6_lstm_embeddings.ipynb

# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import os
import numpy as np
import random
import math
import string
import tensorflow as tf
import zipfile
from six.moves import range
from six.moves.urllib.request import urlretrieve

url = 'http://mattmahoney.net/dc/'

def maybe_download(filename, expected_bytes):
  """Download a file if not present, and make sure it's the right size."""
  if not os.path.exists(filename):
    filename, _ = urlretrieve(url + filename, filename)
  statinfo = os.stat(filename)
  if statinfo.st_size == expected_bytes:
    print('Found and verified %s' % filename)
  else:
    print(statinfo.st_size)
    raise Exception(
      'Failed to verify ' + filename + '. Can you get to it with a browser?')
  return filename

filename = maybe_download('text8.zip', 31344016)

def read_data(filename):
  with zipfile.ZipFile(filename) as f:
    name = f.namelist()[0]
    data = tf.compat.as_str(f.read(name))
  return data

text = read_data(filename)
print('Data size %d' % len(text))

valid_size = 1000
valid_text = text[:valid_size]
train_text = text[valid_size:]
train_size = len(train_text)
print(train_size, train_text[:64])
print(valid_size, valid_text[:64])

vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '
first_letter = ord(string.ascii_lowercase[0])

def char2id(char):
  if char in string.ascii_lowercase:
    return ord(char) - first_letter + 1
  elif char == ' ':
    return 0
  else:
    print('Unexpected character: %s' % char)
    return 0

def id2char(dictid):
  if dictid > 0:
    return chr(dictid + first_letter - 1)
  else:
    return ' '

print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))
print(id2char(1), id2char(26), id2char(0))Found and verified text8.zip
Data size 100000000
99999000 ons anarchists advocate social relations based upon voluntary as
1000  anarchism originated as a term of abuse first used against earl
Unexpected character: ï
1 26 0 0
a z

LUT MODEL
===========
=====================_____no_output_____

batch_size=64
num_unrollings=10
embedding_size=5

class BatchGenerator(object):
  def __init__(self, text, batch_size, num_unrollings):
    self._text = text
    self._text_size = len(text)
    self._batch_size = batch_size
    self._num_unrollings = num_unrollings
    segment = self._text_size // batch_size
    self._cursor = [ offset * segment for offset in range(batch_size)]
    self._last_batch = self._next_batch()

  def _next_batch(self):
    """Generate a single batch from the current cursor position in the data."""
    batch = np.zeros(shape=(self._batch_size), dtype=np.int32)
    for b in range(self._batch_size):
      batch[b] = char2id(self._text[self._cursor[b]])
      self._cursor[b] = (self._cursor[b] +1) % self._text_size
    return batch

  def next(self):
    """Generate the next array of batches from the data. The array consists of
    the last batch of the previous array, followed by num_unrollings new ones.
    """
    batches = [self._last_batch]
    for step in range(self._num_unrollings):
      batches.append(self._next_batch())
    self._last_batch = batches[-1]
    return batches

def characters(batches):
  """Turn a 1-hot encoding or a probability distribution over the possible
  characters back into its (most likely) character representation."""
  batches=list(map(list, zip(*batches)))
  s=list("")
  for b in batches:
        ss=""
        for c in b:
            ss+=id2char(int(c))
        s.append(ss)
  return s

def characters2(probabilities):
  """Turn a 1-hot encoding or a probability distribution over the possible
  characters back into its (most likely) character representation."""
  return [id2char(c) for c in np.argmax(probabilities, 1)]

def batches2string(batches):
  """Convert a sequence of batches back into their (most likely) string
  representation."""
  s = [''] * batches[0].shape[0]
  for b in batches:
    s = [''.join(x) for x in zip(s, characters(b))]
  return s

train_batches = BatchGenerator(train_text, batch_size, num_unrollings)
valid_batches = BatchGenerator(valid_text, 1, 1)

print((np.array(train_batches.next())).shape)
print(characters(train_batches.next()))
print(characters(train_batches.next()))
print(characters(valid_batches.next()))
print(characters(valid_batches.next()))

# ==========================
# OTHER EVALUATION FUNCTIONS
# ==========================

def logprob(predictions, labels):
  """Log-probability of the true labels in a predicted batch."""
  predictions[predictions < 1e-10] = 1e-10
  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]

def sample_distribution(distribution):
  """Sample one element from a distribution assumed to be an array of normalized
  probabilities.
  """
  r = random.uniform(0, 1)
  s = 0
  for i in range(len(distribution)):
    s += distribution[i]
    if s >= r:
      return i
  return len(distribution) - 1

def sample(prediction):
  """Turn a (column) prediction into 1-hot encoded samples."""
  p = np.zeros(shape=[1, vocabulary_size], dtype=np.float)
  p[0, sample_distribution(prediction[0])] = 1.0
  return p

def random_distribution():
  """Generate a random column of probabilities."""
  b = np.random.uniform(0.0, 26.0, size=[1, 1])
  return b[:,None](11, 64)
['ists advoca', 'ary governm', 'hes nationa', 'd monasteri', 'raca prince', 'chard baer ', 'rgical lang', 'for passeng', 'the nationa', 'took place ', 'ther well k', 'seven six s', 'ith a gloss', 'robably bee', 'to recogniz', 'ceived the ', 'icant than ', 'ritic of th', 'ight in sig', 's uncaused ', ' lost as in', 'cellular ic', 'e size of t', ' him a stic', 'drugs confu', ' take to co', ' the priest', 'im to name ', 'd barred at', 'standard fo', ' such as es', 'ze on the g', 'e of the or', 'd hiver one', 'y eight mar', 'the lead ch', 'es classica', 'ce the non ', 'al analysis', 'mormons bel', 't or at lea', ' disagreed ', 'ing system ', 'btypes base', 'anguages th', 'r commissio', 'ess one nin', 'nux suse li', ' the first ', 'zi concentr', ' society ne', 'elatively s', 'etworks sha', 'or hirohito', 'litical ini', 'n most of t', 'iskerdoo ri', 'ic overview', 'air compone', 'om acnm acc', ' centerline', 'e than any ', 'devotional ', 'de such dev']
['ate social ', 'ments faile', 'al park pho', 'ies index s', 'ess of cast', ' h provided', 'guage among', 'gers in dec', 'al media an', ' during the', 'known manuf', 'seven a wid', 's covering ', 'en one of t', 'ze single a', ' first card', ' in jersey ', 'he poverty ', 'gns of huma', ' cause so a', 'n denatural', 'ce formatio', 'the input u', 'ck to pull ', 'usion inabi', 'omplete an ', 't of the mi', ' it fort de', 'ttempts by ', 'ormats for ', 'soteric chr', 'growing pop', 'riginal doc', 'e nine eigh', 'rch eight l', 'haracter li', 'al mechanic', ' gm compari', 's fundament', 'lieve the c', 'ast not par', ' upon by hi', ' example rl', 'ed on the w', 'he official', 'on at this ', 'ne three tw', 'inux enterp', ' daily coll', 'ration camp', 'ehru wished', 'stiff from ', 'arman s syd', 'o to begin ', 'itiatives t', 'these autho', 'icky ricard', 'w of mathem', 'ent of arm ', 'credited pr', 'e external ', ' other stat', ' buddhism e', 'vices possi']
[' a']
['an']
num_nodes = 64
vocabulary_size=27

graph = tf.Graph()
with graph.as_default():

  # Parameters:
  # first parameter of each gate in 1 matrix:
  i_all = tf.Variable(tf.truncated_normal([embedding_size, 4*num_nodes], -0.1, 0.1))
  # second parameter of each gate in 1 matrix
  o_all = tf.Variable(tf.truncated_normal([num_nodes, 4*num_nodes], -0.1, 0.1))

  # Input gate: input, previous output, and bias.
  ib = tf.Variable(tf.zeros([1, num_nodes]))
  # Forget gate: input, previous output, and bias.
  fb = tf.Variable(tf.zeros([1, num_nodes]))
  # Memory cell: input, state and bias.
  cb = tf.Variable(tf.zeros([1, num_nodes]))
  # Output gate: input, previous output, and bias.
  ob = tf.Variable(tf.zeros([1, num_nodes]))
  # Variables saving state across unrollings.
  saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)
  saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)
  # Classifier weights and biases.

  #embeddings
  embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
  w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size],-0.1,0.1))
  b = tf.Variable(tf.zeros([vocabulary_size]))

  # Definition of the cell computation.
  def lstm_cell(i, o, state):
    """Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf
    Note that in this formulation, we omit the various connections between the
    previous state and the gates."""
    input_mat=tf.matmul(i,i_all)
    output_mat=tf.matmul(o,o_all)

    input_gate = tf.sigmoid(input_mat[:,0:num_nodes] + output_mat[:,0:num_nodes] + ib)
    forget_gate = tf.sigmoid(input_mat[:,num_nodes:2*num_nodes] + output_mat[:,num_nodes:2*num_nodes] + fb)
    update = input_mat[:,2*num_nodes:3*num_nodes] + output_mat[:,2*num_nodes:3*num_nodes] + cb
    state = forget_gate * state + input_gate * tf.tanh(update)
    output_gate = tf.sigmoid(input_mat[:,3*num_nodes:] + output_mat[:,3*num_nodes:] + ob)
    return output_gate * tf.tanh(state), state

  # Input data.
  train_data = list()
  for _ in range(num_unrollings+1):
    train_data.append(tf.placeholder(tf.int32, shape=[batch_size]))
  train_inputs = train_data[:num_unrollings]
  train_labels = train_data[1:]  # labels are inputs shifted by one time step.

  # Unrolled LSTM loop.
  outputs = list()
  output = saved_output
  state = saved_state
  for i in train_inputs:
    embedded_i=tf.nn.embedding_lookup(embeddings,i)
    output, state = lstm_cell(embedded_i, output, state)
    outputs.append(output)

  # State saving across unrollings.
  with tf.control_dependencies([saved_output.assign(output),
                                saved_state.assign(state)]):
    # Classifier.
    outputs_concat=tf.concat(outputs,0)
    #try to compute similarity here as well
    logits=tf.nn.xw_plus_b(outputs_concat,w,b)
    print((logits).shape)

    #Compute one hot encodings
    label_batch=tf.concat(train_labels,0)
    sparse_labels = tf.reshape(label_batch, [-1, 1])
    derived_size = tf.shape(label_batch)[0]
    indices = tf.reshape(tf.range(0, derived_size, 1), [-1, 1])
    print(indices.shape,'indices.shape')
    concated = tf.concat([indices, sparse_labels],1)
    outshape = tf.stack([derived_size, vocabulary_size])
    labels = tf.sparse_to_dense(concated, outshape, 1.0, 0.0)

    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))

  # Optimizer.
  global_step = tf.Variable(0)
  learning_rate = tf.train.exponential_decay(
    10.0, global_step, 10000, 0.1, staircase=True)
  optimizer = tf.train.GradientDescentOptimizer(learning_rate)
  gradients, v = zip(*optimizer.compute_gradients(loss))
  gradients, _ = tf.clip_by_global_norm(gradients, 1.25)
  optimizer = optimizer.apply_gradients(
    zip(gradients, v), global_step=global_step)

  # Predictions.
  train_prediction = tf.nn.softmax(logits)
  print(train_prediction.shape)

  # Sampling and validation eval: batch 1, no unrolling.
  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
  normalized_embeddings = embeddings / norm
  #valid_dataset=np.array([i for i in range(27)])
  #valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)

  sample_input = tf.placeholder(tf.int32, shape=[1])
  saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))
  saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))
  reset_sample_state = tf.group(
    saved_sample_output.assign(tf.zeros([1, num_nodes])),
    saved_sample_state.assign(tf.zeros([1, num_nodes])))
  embedded_sample=tf.nn.embedding_lookup(embeddings,sample_input)
  sample_output, sample_state = lstm_cell(embedded_sample, saved_sample_output, saved_sample_state)
  with tf.control_dependencies([saved_sample_output.assign(sample_output),
                                saved_sample_state.assign(sample_state)]):

    sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))
    #similarity = tf.matmul(sample_prediction, tf.transpose(normalized_embeddings))
    print(sample_prediction.shape)(640, 27)
(?, 1) indices.shape
(640, 27)
(1, 27)
num_steps = 50001
summary_frequency = 500

with tf.Session(graph=graph) as session:
  tf.global_variables_initializer().run()
  print('Initialized')
  mean_loss = 0
  for step in range(num_steps):
    batches = train_batches.next()
    feed_dict = dict()
    for i in range(num_unrollings + 1):
      feed_dict[train_data[i]] = batches[i]
      #print((feed_dict[train_data[i]]).shape)

    _, l, predictions, lr,train_lab = session.run([optimizer, loss, train_prediction, learning_rate,train_labels], feed_dict=feed_dict)
    mean_loss += l
    if step % summary_frequency == 0:
      if step > 0:
        mean_loss = mean_loss / summary_frequency
      # The mean loss is an estimate of the loss over the last few batches.
      print('Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))
      mean_loss = 0

      #print(train_lab)
      #print(labels.shape[0])
      #print('Minibatch perplexity: ',np.exp(logprob(predictions, labels)))

      if step % (summary_frequency * 2) == 0:
        # Generate some samples.
        print('=' * 80)
        for _ in range(5):
          feed = np.zeros(shape=(1,), dtype=np.int32)
          feed[0,] =np.random.randint(0,27)
          sentence = id2char(feed[0])
          reset_sample_state.run()
          for _ in range(79):
            prediction=sample_prediction.eval({sample_input:feed})
            k = sample(prediction)
            k=characters2(k)[0]
            feed = np.zeros(shape=(1,), dtype=np.int32)
            feed[0,] = char2id(k)
            sentence += k
            #feed = np.zeros(shape=(1,), dtype=np.int32)
            #feed[0,] = np.argmax(prediction)
            #print(feed.shape)
            #sentence += id2char(feed[0,])
          print(sentence)
        print('=' * 80)
      # Measure validation set perplexity.
      reset_sample_state.run()
      valid_logprob = 0
      #for _ in range(valid_size):
       # b = valid_batches.next()
        #predictions = sample_prediction.eval({sample_input: b[0]})
        #valid_logprob = valid_logprob + logprob(predictions, b[1])
      #print('Validation set perplexity: %.2f' % float(np.exp(valid_logprob / valid_size)))Initialized
Average loss at step 0: 3.299174 learning rate: 10.000000
================================================================================
z kejxiwxjeosnt tprfgwneigow f ffulngeqony kpatvq jbitiyaigni rzmpgmmslauehfuav
fuyf  qndoyakllynyjzo cpsvt xr  ekvbijtwwudtnnfuktb lhmwt ywneedzzeeh gh tqi mo
xj zstr n nisygoaiurnovbflu tixcwx eemi hyyvhi c hexngdefel spkqnpnfllbyvqlbumf
hwonnfsqv t jd mn  iocjnsp z qaa  onnzxa stwt ismxeptdony eiaejisewasc wpztqphkk
ltpuke dftleoet rs fd vdestezizcsjslnnlewboxbmnrvmamtpdwbpexlreos cetfxd xaifmio
================================================================================
Average loss at step 500: 2.145794 learning rate: 10.000000
Average loss at step 1000: 1.839583 learning rate: 10.000000
================================================================================
niment by of sturs felketh and he direfter americtic in air as fol clas have sha
y wills in in aro and in tich fivlion a letar wele time ennocient in a cincild a
al wile as wan of siggent dilifionshosits he rome mmig acthially meaints the lec
unisules anothin of the sifiler adbolarious h appar will wisan say bp not fofrok
britary signeriaf popularary lisfitel revul danism inftelogohisate civlicual and
================================================================================
Average loss at step 1500: 1.761449 learning rate: 10.000000
Average loss at step 2000: 1.723277 learning rate: 10.000000
================================================================================
mall poetrion where wwass mide of the thebyes their reputalbot put priculces maj
incals med the succreame the sequen kidise truc emorom outs of imlise a janick e
ved the cons bolputed stade his one nine nine imxize this and have is as in the
ing grand and speces enstum scwere one nine seven zeron the one phpe the southem
vers the from the bunc the the totner the expnense one nine eight zero four thre
================================================================================
Average loss at step 2500: 1.711644 learning rate: 10.000000
Average loss at step 3000: 1.699343 learning rate: 10.000000
================================================================================
dion shation hincotanto one exchildislated and open thore rated time former bubl
borstials to has dass jietropensit takarchon handago criation of lecteries in ca
ved fownerf filsons the sinte game abled a all poumk colscantoards states in rie
ently da zero zero un ciistificantentizcts one lurehts fle qurine than gather ci
vils absix three five brigion use the calledi near thagh and iffcheedediunces s
================================================================================
Average loss at step 3500: 1.701381 learning rate: 10.000000
Average loss at step 4000: 1.708963 learning rate: 10.000000
================================================================================
hn a and can sumamon hale by one nine tlast of and rower a estructs matabiatity
me two gen it iss roh joinshimia it words soyk to untroned besiods in practions
busl unnarticlly he minud bi one piot beagrabbally games figutist lodocic war va
wegpromergate only s not nine four zero zero zero the cbardes the and ecorty are
e to fram demoyavies more fedw shoogle rejendaons gious susony integrarial actor
================================================================================
Average loss at step 4500: 1.682434 learning rate: 10.000000
Average loss at step 5000: 1.689021 learning rate: 10.000000
================================================================================
ur of mard voch on for one refers the is oh bipary in the thoule inum dayservanc
viring the over cotes it deffacure caxe somits bunther one nine seven one one fo
qer of marghting only palet thing qroginowleentazing relifted to do be rinconico
ward of the mittion of milm whis fismies both the areum bonned accortives one of
th prictraince of objys in referen nine pan anticus dischill a factwila a mark i
================================================================================
Average loss at step 5500: 1.665390 learning rate: 10.000000
Average loss at step 6000: 1.657086 learning rate: 10.000000
================================================================================
s finak seast dnawevery is relatary were has eash aroundarces and the finst and
can wests see hubsten desale presents eniia soilare olk the grreea when obth to
heral the laws seriver begin weanas in fom zero the places mader main catebatter
withale and stows zero kiach orige actare renamaic a norach weres and eversely o
ood chaneed behase pnoch of extersheters craved can rolr cit flated he hiisility
================================================================================
Average loss at step 6500: 1.634969 learning rate: 10.000000
Average loss at step 7000: 1.670364 learning rate: 10.000000
================================================================================
tister lesindsneus out twelea he lisple to been with the coummusain while the wi
witch underra singd the and meatropes in seteg concurs or teur edact than the de
orizate me intran memor the called of a movemed winose long saber of ju nuarish
vers it sper had betbize tear world monordas wistif even find are cloance was re
arcy and moth the un essamity the year of this peliter the commoneri are mleasm
================================================================================
Average loss at step 7500: 1.661764 learning rate: 10.000000
Average loss at step 8000: 1.661760 learning rate: 10.000000
================================================================================
fin flawtory and former the oppistic of the using finar the lights these in plic
y e pidh pake diggure plich exchist hypers s wry crudteated a tep grizist scamar
fer i flimeidn due outslumes scherative univelst kmbisactel he relitian there st
 daster systemder ftugh were accoundress stater teh one seven nine eight and sta
lignar farmed the statisin wilel few and county in the conditionnist dughmad few
================================================================================
Average loss at step 8500: 1.664483 learning rate: 10.000000
Average loss at step 9000: 1.649592 learning rate: 10.000000
================================================================================
gent of ited and far malicant yuester is their unstandarticoms to ello s three n
ing headt k and sachetarize in and herasi the eventudent in the bebational shrea
hifican insentine up wests to a gregist in the indively if for that veroo of sit
ensive of althoughal levite to stal a was irelesters interranced visue expect ei
x four six five basiff placed and thetia and with and auchory places in the not
================================================================================
Average loss at step 9500: 1.674208 learning rate: 10.000000
Average loss at step 10000: 1.681788 learning rate: 1.000000
================================================================================
cornapan a one of ocku russiunes epressors syriter maincamphor seven cordels pas
y airs jan that the statal mayer links like char there sy also plansain of the e
on by heream and part hermed for dammances ligfies is a solstan brite during one
l in curver maden univers alshing other sites mystemver and the inforter for a v
derse and one zero zero one twasting otherning pilitzed was schellates one one s
================================================================================
Average loss at step 10500: 1.613660 learning rate: 1.000000
Average loss at step 11000: 1.619825 learning rate: 1.000000
================================================================================
lent medering in the sholide is inum off in on the prime the byzonom been includ
 if the cat assa scripressers as the combration preans group of the responsiloti
ellisions annel depite value evenue acidinistrics need they for the lausely by t
ing by bepon unqersa couster spepen for auttled two six being or grouphs ady and
mentalt nor in the as krisprosyly and wasopjes is salonstand reportury a time la
================================================================================
Average loss at step 11500: 1.598109 learning rate: 1.000000
Average loss at step 12000: 1.602219 learning rate: 1.000000
================================================================================
ntoria n dhs the cats often difpegeal some largon is includes that is ordament s
x atwem kensinal strer of of sitalists at prize and his dr on north shuth is bec
s textern khow plaonusal usly granjority and dynts as the up at the emperors to
lon tower up in three six two cus that this son the had wests five d generations
ation of the mervents onvicul s other compary stecepteon other these people whin
================================================================================
Average loss at step 12500: 1.593839 learning rate: 1.000000
Average loss at step 13000: 1.585217 learning rate: 1.000000
================================================================================
oning with elrsule four imeter bazrac natize for the unsives van dui uncarnowade
use agerean dokm anat go consources and has nhmstrarionoh and tonsture polizing
 schuabount focch to one five three sosge three seven five five imes divers peop
hnited bruiding the nown such thiss beles four gird machy a duviblated the one t
x gowlancheires this biis gaas kno that puasically appliated on these idegiat ex
================================================================================
Average loss at step 13500: 1.580290 learning rate: 1.000000
Average loss at step 14000: 1.584632 learning rate: 1.000000
================================================================================
 forkally or in inited ziscia kernature assigian was recires color to caled but
hera of out theil nehp is deshical channel e in raik it final the caving the gin
k on as all sixnian brookin absount a sovicts termeting of city dons is most sou
k envided strets from iolation was subplainst make and postralicing considers li
x one nine nine two a finx top two any the holring set but solous is for islade
================================================================================
Average loss at step 14500: 1.580410 learning rate: 1.000000
Average loss at step 15000: 1.581710 learning rate: 1.000000
================================================================================
dabation than of popular of lot led goods of a certagil ocase from if calleds sa
nance often ecost the defined the and firm on advant resingly boy him name asser
x monelinn assa is a sainpalloncely taulors ii a compedrons of reput out of the
less west of a what part officely turgis is the words of disfubstand extensed st
x feromburs lelacs as de otherfust of the not hamas produtiods ununivest moslate
================================================================================
Average loss at step 15500: 1.561622 learning rate: 1.000000
Average loss at step 16000: 1.584397 learning rate: 1.000000
================================================================================
res indive fleging ceast any this no machinomoret fish cardimage also ipi the fe
ent cophept mess although bloreblyt mosts bandmativi kuness are the curyagi prob
qed s standom ands identebly dia s lasyrranguage see include recognisus locating
breper indemences dire area gst or get the dispactively are catebeat on the futu
ver the wroadely the million at the due tribles things of taker summury anaye sh
================================================================================
Average loss at step 16500: 1.587833 learning rate: 1.000000
Average loss at step 17000: 1.587503 learning rate: 1.000000
================================================================================
x on the pxace afternite of outh when a labx to because domin pariforme wormed f
furey to be stuaging british well through formelvers office baked the composers
re the erginal game astries to syning in arginia northeseingd waborg the astraft
tholy governicals one word for synological estably and allow records whas seven
guires in a seatures godning singe for pustnence on out abaghtom the in main see
================================================================================
Average loss at step 17500: 1.573217 learning rate: 1.000000
Average loss at step 18000: 1.570719 learning rate: 1.000000
================================================================================
s from main skaricd in one nine five one seven hearcy as for lise of he the hook
 and a non cornampical compleches whiser wepal produce united the prosous monad
ventions bolity a first wille zore calling torgining dans y di jstacsined suppor
ge k lost born native worlt often every insect with the law experencession with
a the yeasphakonidails esmused while in the complencosition of finus to the and
================================================================================
Average loss at step 18500: 1.568334 learning rate: 1.000000
Average loss at step 19000: 1.571729 learning rate: 1.000000
================================================================================
fruly commusity africa holts fdot offerset vitions refers to theres four five on
s that between the finus is mo then the inteneer of theatracty the israel way to
ment only games to chaters texplumnial decon in leanning into its bastly wextest
s mags and elumatos preyed year in they alterned of the several a malazy paring
formdhem of ritceetly depoywee the is frassional god two two herp then the usa a
================================================================================
Average loss at step 19500: 1.589651 learning rate: 1.000000
Average loss at step 20000: 1.567431 learning rate: 0.100000
================================================================================
ments secternicles in one nine three in immercy of the assermets two governments
lither agrice have at ckve disker is qark filmd a elemine long their charago mat
jidate of particular att that viriauds is three one one seven one eight one zero
lest the local of rivor the trades musinol falth mugre of the nebren succeetia a
watentioched geyning as as the furthence the m solvance that silen over sink ger
================================================================================
Average loss at step 20500: 1.609716 learning rate: 0.100000
Average loss at step 21000: 1.579646 learning rate: 0.100000
================================================================================
uns helished to a to been shortle inver to to three six when the stheticlle stud
ust house often on janunijiania of the rochation for other contempolizate sprigy
qer of is the imessian is a resprucled in one nine fives ynown two zero two one
una eltacht peperand left in these at his an kan american catatch cathern and mo
n a canomety of the plays the sometric keriun influence dna westined maturation
================================================================================
Average loss at step 21500: 1.573186 learning rate: 0.100000
Average loss at step 22000: 1.577080 learning rate: 0.100000
================================================================================
ia the has aciarly the combates their by give north from players over were and a
oria relberge in it deryned also uanced is no yweniples the searly belarer of a
bid morist the theory do strence wimmer of biolum contragus phaces swege of grou
k struct calmit growen in name that tax secital six seven ones zero five sometim
witled on whiala signaton statish tablan lincee of a nemelties has column howeve
================================================================================
Average loss at step 22500: 1.576918 learning rate: 0.100000
Average loss at step 23000: 1.575550 learning rate: 0.100000
================================================================================
zistor ere one nine two zero zero nine one zero zero zero zero zero zero zero fo
wita of its or concention story the pebries called mels any guid light refers in
trolic would currentnation the le including around with lawtic or in one nine ei
k logogoty qeicas garted in boy in the elinf it laots disky crima aso warment up
card in north a syppering any plastotian of visia painark squedimiss situmation
================================================================================
Average loss at step 23500: 1.589974 learning rate: 0.100000
Average loss at step 24000: 1.588420 learning rate: 0.100000
================================================================================
y qued jubi articistition altiades of siiria or since ellurns of the life was th
cial in bona two eqoding and where western exided be using develow sleading men
a young in allown qeloboriable internatesmytame motolairence row with cable here
ate most father is several and offlial one nine terrangers alegies is e sirale w
tar article vioula them is a discoveros electry on the ritions a volex scot of d
================================================================================
Average loss at step 24500: 1.575857 learning rate: 0.100000
Average loss at step 25000: 1.566750 learning rate: 0.100000
================================================================================
inced kil decomples came year did muctiod and electronor ships right biona use m
p on sup become min envirue foodbable the clinkog to amighchabreasing betware ca
qerings instorage the virge the relioung silud less bones thray bon them surfast
thdegher undeep ands small x ffwurn are some monjomoyaded in image bis same and
vestramed of dathed the english spret who central and begins melanting the gitin
================================================================================
Average loss at step 25500: 1.563473 learning rate: 0.100000
Average loss at step 26000: 1.575364 learning rate: 0.100000
================================================================================
ex best uround work poll electronol giv like the albuted an incanariart lilion g
y hardd after related these and pscwidion would and cospandable biggg tie the wi
pe entracte the given persion prographicisting how offeria among three in lack a
qens is greathor cold in the states united is the with city azom is nosing for t
 fujthes it soft mamila democations thoses azedi the either the are can d one me
================================================================================
Average loss at step 26500: 1.563511 learning rate: 0.100000
Average loss at step 27000: 1.564592 learning rate: 0.100000
================================================================================
le of grainstrands preylision kird mugd with way chopitant sould dominelies was
hn had electrice acrets which plates the he dodmustions of concentler decessiud
us of stritic mean such essur blot one nine five main wanth golic forci ontist h
ward was alto imaics varres shads in banomut of the presence a to good other is
ple it is vi carden refumegion and have been of gay knegation one nine three zer
================================================================================
Average loss at step 27500: 1.586703 learning rate: 0.100000
Average loss at step 28000: 1.597045 learning rate: 0.100000
================================================================================
pen suppress requrts mopte to mavility see nine six nine six one officials b had
ifiin kakh exustwaristry civylest wonsing four one two necese two five one infla
nuse either in legers who known of the winning internaments that standes and veg
furn reams  the field quiul de event colen to the engw is nine using s gdow corv
s of the one nine four was union of the leavoral the sailions scy based in da ge
================================================================================
Average loss at step 28500: 1.577821 learning rate: 0.100000
Average loss at step 29000: 1.574616 learning rate: 0.100000
================================================================================
s iv the websels aleigo based the withh partants universia s el with the menty s
ton in jate to juthers sy the understynd of on the no in juad the aliming direct
les wind history newan writtuloo wabor the works or f missibit considered that t
jowine k sourc or herap accaplors and is bably are stribut a between from report
ned invers and excusmuhing three five five instras table of the reventeram from
================================================================================
Average loss at step 29500: 1.571277 learning rate: 0.100000
Average loss at step 30000: 1.581754 learning rate: 0.010000
================================================================================
q extegni and can on assiftin the juado for phlacome no and rided a predico legi
s syns star confists is sention ban linch and the chelve light computer one eigh
quant agdands senning a swutte o and founding performillun shoveed for an prodic
weries celruc the provide in these skmack s one two mikists the degal was a jind
dwri house using numberseral in two zero zero m seviden zero bit be appeastative
================================================================================
Average loss at step 30500: 1.572484 learning rate: 0.010000
Average loss at step 31000: 1.565869 learning rate: 0.010000
================================================================================
qerental is searle play kingdom ad derive which abount a diving ollar the counte
x startadali of that astreat citary of the lepi decausent generate geargs of the
k form impodech warmed for lelowtions l on he game starnatord a militables long
d boon power after transrace was intellendibres operating compariled the t see a
rioni al gree two six for jollars ministed when jangard the all liolus thane the
================================================================================
Average loss at step 31500: 1.578223 learning rate: 0.010000
Average loss at step 32000: 1.569709 learning rate: 0.010000
================================================================================
us sbp requstions to a only one nine ricle from gambenchized fethibule featuring
ptrayed europeaziss be several sollon was were special and frognima by electrian
y issosle three monomyn psets procrestre hought on river joke of the released ma
ssophal numbing ory pesersises warsed of the krisfors striag whisg several way i
qets by aircasop s churchoings literous secondine tuincial dnits movically these
================================================================================
Average loss at step 32500: 1.573581 learning rate: 0.010000
Average loss at step 33000: 1.574408 learning rate: 0.010000
================================================================================
jian up atal prigy that of redused to the jameanting this one bix activity press
rain naws partund chemeting as three enfuld that the rist world are rosopheinges
zew outlinets wales matthands as the sele prominet ashiles forcement lobing the
wer untrovi are which edgucy computtirnical rung as parown one four eight mpt la
and the regard aresal phurge respected wrote the octomer been boneralim weory if
================================================================================
Average loss at step 33500: 1.580457 learning rate: 0.010000
Average loss at step 34000: 1.577014 learning rate: 0.010000
================================================================================
qer on ballivent eventies sections develies defyrrafortawian can been seven eoch
qan dectionistronicia is dia four and subjectage its centerition citar or alon i
hed their extragloconism ty due ii and leginerenting apoky prime aucordy in one
jated that dwanuoning scanably at both which designed without not alqomit its to
ed fligh alworare involved them escraem on and the originate for lun concamand t
================================================================================
Average loss at step 34500: 1.602238 learning rate: 0.010000
Average loss at step 35000: 1.579505 learning rate: 0.010000
================================================================================
dee ne which close lineral at the meased a peoplemr corne prigution two zero zer
hed on one nine nine zero zero zero zero kidge propertant rema fouglen i fanture
y the hold case m servations the creciple there appeared bromlogy be two zero ze
ood jecre more medig appeial emorganated for the flown to the direch foot asmier
nied as build but this latered as a an autons comoxaing has been pacere one nine
================================================================================
Average loss at step 35500: 1.588640 learning rate: 0.010000
Average loss at step 36000: 1.584342 learning rate: 0.010000
================================================================================
flos dust the fanto ausist in the one zero three three the war plane discussorvi
 on three this inst one name abelling wry that ruphiings throma equicunised surp
ful vitia and libred person r teguice of the economick the hundreed caltions par
y highes one nine six three zero undervijally german for a hoal adama to is even
h paircsworia chocteliages effects of by the irray also can mement and humatifif
================================================================================
Average loss at step 36500: 1.579184 learning rate: 0.010000
Average loss at step 37000: 1.592656 learning rate: 0.010000
================================================================================
on taugeral claiake all ad to hellibateg thational kreed an alluw have openicle
ince of thraideral it is an abpres avounas vort bastrouloch wich line three one
bre wold speial are padmr rai seven jistorian sectical ununderrale crimdive are
wered of calsor zero perfil rather tamf in humon and study albble the lando of p
ntalen without as genentially building is deminish the rating his prefeet rivold
================================================================================
Average loss at step 37500: 1.602800 learning rate: 0.010000
Average loss at step 38000: 1.592613 learning rate: 0.010000
================================================================================
qet poctogy and suffess and being to two zero zero k nati grows ksew to real car
b austwisonal viase acculaped barked after chan a well united mainel a pately us
provian polation notu matti and conseaded yeap storing unduling the tihrent was
ing the nature as urdise europe s holling comment proampal biirs frouds half one
be responsort itariated supportum the of as people me stean in lice begar austre
================================================================================
Average loss at step 38500: 1.601176 learning rate: 0.010000


# Notebook from HobnobMancer/pyrewton
Path: notebooks/01_downloading_genbank_files.ipynb

# 01 Downloading GenBank Files

## pyrewton module genbank, submodule get_ncbi_genomes

This notebook describes the process of downloading GenBank files for all genomic assemblies associated with each species of interest.

This notebook refers to the `pyrewton` submodule `get_ncbi_genomes`.
_____no_output_____

Note: the genomic assemblies are downloaded as GenBank Flat Files (.gbff). This format was chosen because it provides nucleotide sequences and annotation data, which includes the product of nucleotide sequence and unique identifier which can be used to find further information about the annotated object in NCBI-linked databases, such as [UniProtKB] (https://www.uniprot.org/). Additionally, the BioPython module [SeqIO](https://biopython.org/wiki/SeqIO), a standard ‘Sequence Input/Output’ interface for sequence handling Python scripts, facilitates the automated parsing of GenBank files (including GenBank Flat Files) and extraction of specified data (BioPython, 2020).

_____no_output_____

Note: this notebook does not include the entirity of script code, but instead includes exerts to help illustrate the programme arcitecture and function. Specifically, the function _main_, which orchestrates the calling of functions to perform the overal operation of the script is excluded from this notebook, as well as logging and error checking.

Additionally, in some instances sections of code have been removed, replaced by a comment to indicate the intent or function of the code and/or introduced later on in the notebook than is reflected in the code. This is to enable a detailed description of the code function at a more logical, and oppertune time.

For the complete script, navigate to `pyrewton/genbank/get_ncbi_genomes` within the repository.

_____no_output_____## Contents
_____no_output_____- [Operating the script](#linkoperating)
- [Script input](#linkinput)
- [Command line options](#linkcommand)
- [Python imports](#linkimports)
- [Parsing the input file and creating the dataframe](#linkparsing)
    - [Scientific name and Taxonomy ID retrieval](#linkscientific)
    - [Creating the dataframe](#linkcdf)
- [Retrieving accession numbers](#linkretrieving)
    -[Retrieval of assembly IDs](#linkassemblyids)
    -[Retrieval of the accession numbers](#linkanretrieve)
- [Downloading GenBank file](#linkgenbank)
    - [Compiling the URL](#linkurl)
    - [Downloading the GenBank file](#linkdownload)
- [Writing out the dataframe](#linkoutdf)
_____no_output__________no_output_____## Operating the script

The script `get_ncbi_genomes.py` is written as a command-line programme and thus is most easily operated via the command-line. The standard structure of the command-line call to operate the script is:
`python3 get_ncbi_genomes.py -u <user email> <other options>`

Multiple options are avilable to customise the operation of the script to the users needs. A full list of the available options are included in the documentation at [Read the Docs](https://phd-project-scripts.readthedocs.io/en/latest/genbank.html#genbank-mod-get-ncbi-genomes-def), as well as further on in this notebook.

A email address must be provided becuase this is a requirement to access the NCBI database remotely using `Entrez`.
_____no_output_____

    For the dowloading of the GenBank files for the PhD project, the following code was run at the command-line:

> `python3 get_ncib_genomes.py -u eemh1@st-andrews.ac.uk -i selected_species_list.txt -o 2020_05_31_GenBank_file_pulldown/ -l 2020_05_31_GB_file_download -v -d 2020_05_31_genome_dataframe.csv`_____no_output_____

Note: Before using the script `get_ncbi_genomes.py` read the documentation for Entrez, taking care to note expected practises laid out under _'Frequency, Timing and Registration of E-utility URL Requests'_. Failing to meet the expected practises can result in restricted access or banned access to the Entrez utilitity.

Entrez documentation can be found at the following links:
- ['Entrez documentation'](https://www.ncbi.nlm.nih.gov/books/NBK25497/)
- ['BioPython Entrez modeul documentation'](https://biopython.org/DIST/docs/api/Bio.Entrez-module.html)_____no_output__________no_output_____## Script input

The script takes a plain text file as input, containing a list of the species of interst. Each line contains a unique species, identified by their scientific name or NCIB taxonomy ID (including the 'NCBI:txid' prefix). An input file template can be found within the `get_ncbi_genomes` directory within the repository.
_____no_output_____

For the downloading of the GenBank files for the PhD project, the file `selected_species_list.txt` within the `get_ncbi_genomes` directory was used as the input file.

_____no_output__________no_output_____## Command-line options

The script is designed as command-line programme, and thus operation of the script is customisable by passing arguments from the command-line.

**Compulsory argument**

The option `-u` or `--user` **must** be used in order to provider the users email address, becuase this is a requiremnt for Entrez which is used to call to NCBI.

**Optional arguments**

`-d, --dataframe`

  Specify output path for dataframe (include file extensions). If not provided dataframe will be written out to STDOUT.

`-f, --force`

  Enable writting in specificed output directory if output directory already exists.

`-g, --genbank`

  Enable or disable downloading of GenBank files.

`-h, --help`

  Display help messages and exit

`-i, --input`

  Specify path to input filename (include extension) input file. If not given input will be taken from STDIN.

`-l, --log`

  Specify name of log file (include extension). If not option is given no log file will be written out,
however, logs will still be printed to the terminal.

`-n, --nodelete`

  Enable not deleting files in existing output directory. If not enabled, output directory exists and writing in output directory is 'forced' then files in output directory will not be deleted, and new files will be written to the output directory.

`-o, --output`

  Specify filename (including extension) of output file. If not given output will be wrtten to STDOUT.
If only the filename is given, Extract_genomes_NCBI.py

`-r, --retries`

  Specifiy maximum number of retries of trying to call to NCBI before cancelling retrying call to NCBI. The default is a maximum of 10 retries.

`-t, --timeout`

  Specify timeout limit of URL connection when downloading GenBank files. Default is 10 seconds.

`-v, --verbose`

  Enable verbose logging - changes logger level from WARNING to INFO._____no_output__________no_output_____## Python imports_____no_output_____

import logging
import re
import sys
import time

from socket import timeout
from typing import List, Optional
from urllib.error import HTTPError, URLError
from urllib.request import urlopen

import pandas as pd

from Bio import Entrez
from tqdm import tqdm

from pyrewton.loggers import build_logger
from pyrewton.parsers.parser_get_ncbi_genomes import build_parser
from pyrewton.file_io import make_output_directory, write_out_dataframe_____no_output_____

_____no_output_____## Parsing the input file and creating the dataframe

The input file is either taken from STDIN or if a path was provided at the command-line, it is taken from where the path directs.

Opening and parsing of the input file is performed by the function `parse_input_file`.

Initially, it is checked that the path to the input file is valid and if not, the programme terminates.
Afterwards, the file is opened, and parsed line-by-line._____no_output_____

def parse_input_file(input_filename, logger, retries):

    # test path to input file exists, if not exit programme
    if not input_filename.is_file():
        # report to user and exit programme

    # if path to input file exists proceed
    # parse input file
    with open(input_filename) as file:
        input_list = file.read().splitlines()

    # Parse input, retrieving tax ID or scientific name as appropriate
    line_count = 0
    for line in tqdm(input_list, desc="Reading lines"):
        line_count += 1

        if line.startswith("#"):
            continue

        line_data = parse_line(line, logger, line_count, retries)
        all_species_data.append(line_data)

    # create dataframe containing three columns: 'Genus', 'Species', 'NCBI Taxonomy ID'
    species_table = pd.DataFrame(
        all_species_data, columns=["Genus", "Species", "NCBI Taxonomy ID"]
    )
    return species_table
_____no_output_____

The function `parse_line` coordinates the approrpiate calling of functions to retrieve the NCBI taxonomy ID if the scientific name is provided, retrieve the scientific name if the taxonomy ID is provided or perform no function if a comment is passed (indicated by the starting line character '#').

The scientific name and taxonomy ID is stored as a single list for each species. This list contains the 3 elements:
- The species 'Genus' name
- The species 'Species' name
- The species taxonomy ID (including the 'NCBI:txid' prefix).

The list is returned by the function and added to the tuple `all_species_data` within the `parse_input_file` function. This creates a tuple, with each element containing the indentification data for a unique species.
_____no_output_____

def parse_line(line, logger, line_count, retries):

    # For taxonomy ID retrieve scientific name
    if line.startswith("NCBI:txid"):
        gs_name = get_genus_species_name(line[9:], logger, line_count, retries)
        line_data = gs_name.split()
        line_data.append(line)

    # For scientific name retrieve taxonomy ID
    else:
        tax_id = get_tax_id(line, logger, line_count, retries)
        line_data = line.split()
        line_data.append(tax_id)

    return line_data
_____no_output_____

_____no_output_____### Scientific name and Taxonomy ID retrieval

The retrieval of the scientific names and taxonomy ID from the NCBI Taxonomy databased is performed by the functions `get_genus_species_name` and `get_taxonomy_ID`.

Each function calls to the NCBI Taxonomy database using entrez.
_____no_output_____

If the retrieval of scientific name or taxonomy ID fails, the null value 'NA' is returned.
The retreieval of scientific name has the additional test to check if the 'name' passed to the function contains any numbers. IF so the null value is returned 'NA', the most common cause for this is a typo in the scientific name or exclusion of the taxonomy ID prefix 'NCBI:txid' in the taxonomy ID written in the input.
More detail on the probably causes of common errors is discussed in the projects README.md file.

_____no_output_____

def get_genus_species_name(taxonomy_id, logger, line_number, retries):

    # Retrieve scientific name
    with entrez_retry(
        logger, retries, Entrez.efetch, db="Taxonomy", id=taxonomy_id, retmode="xml"
    ) as handle:
        record = Entrez.read(handle)

    # extract scientific name from record
    try:
        return record[0]["ScientificName"]

    except IndexError:
        # log error and return null value 'NA'
        return "NA"

def get_tax_id(genus_species, logger, line_number, retries):
    # check for potential mistake in taxonomy ID prefix
    if re.search(r"\d", genus_species):
        # log warning that numbers were found in line which was identified as a scientific name
        return "NA"

    else:
        with entrez_retry(
            logger, retries, Entrez.esearch, db="Taxonomy", term=genus_species
        ) as handle:
            record = Entrez.read(handle)

    # extract taxonomy ID from record
    try:
        return "NCBI:txid" + record["IdList"][0]

    except IndexError:
        # log error
        return "NA"
_____no_output_____

_____no_output_____### Creating the dataframe

The `pandas` module is used to create a dataframe, within the `parse_input_file` function, with three columns:
- Genus
- Species
- NCBI Taxonomy ID
With a unique species per line.

`Pandas` enables easier and faster storage and malipulation of dataframes than other table creating packages. More information is [available at](https://pandas.pydata.org/).

The dataframe (`species_table`) is returned to the function `main`._____no_output_____

    species_table = pd.DataFrame(
        all_species_data, columns=["Genus", "Species", "NCBI Taxonomy ID"]
    )
    return species_table_____no_output_____

_____no_output_____## Retrieving accession numbers

The `pandas` module allows simultanous applying of a function to a dataframe, iterating over a given axis (row or column). This results in a 100-times faster processing of the dataframe than using a traditional Pythonic `for loop`. Therefore, the `pandas apply` function is used to retrieve the accession numbers of all genomic assemblies associated with each taxonomy ID within the `species_table` dataframe; this is completed by calling the function `get_accession_numbers`, and retrieved accession numbers are stored in a new column in the dataframe: `NCBI Accession Numbers`.
_____no_output_____

Entrez will only retrieve the accession numbers of genomic assembly entries, within the NCBI Assembly database, which are **directly** linked to the taxonomy entry (identified by its taxonomy ID); it will not return accession numbers for genomic assembly entries which are **subtree** linked.

**Directly** linked entries are those that have the organism assigned as the source organism for the data.

**Subtree** linked entries are those record assocated with taxonomy node, and all nodes which are underneath.

Browsing a taxonomy entry witihn the NCBI Taxonomy database via the browser will quickly identify if any genomic assemblies have been directly linked.

_____no_output_____

As before, if the retrieval of the asseccion numbers fails at any point, the null value 'NA' is returned, and the retrieval of the accession numbers is exited.

If the taxonomy ID was failed to be retrieved previously (and was thus stored as 'NA') a null value of 'NA' will automatically be returned for accession numbers'.

_____no_output_____

species_table["NCBI Accession Numbers"] = species_table.apply(
    get_accession_numbers, args=(logger, args), axis=1
)
_____no_output_____

The `pandas apply` function passes a `pandas series` with each element (or cell within the dataframe) accessible via an index number, similar to accessing an element in a list.

- df_row[0]: Genus
- df_row[1]: Species
- df_row[2]: Taxonomy ID
_____no_output__________no_output_____### Coordination of accession number retrieval_____no_output_____
The retrieval of accession numbers of all directly linked genomic assemblies for each species of interest was coordinated by multiple functions.

A single function (`get_accession_numbers`) coordinate the multiple calls to Entrez required to ultimately retrieve the accession numbers.

The first call (`get_assembly_ids`) to Entrez links the taxonomy data to the species directly linked genomic assemblies.

The second call (`post_assembly_ids`) to Entrez posted the IDs of the genomics assemblies to Entrez, in orde to search for the genomic assembly data as a singel query, and thus reduce unecessary traffic to Entrez.

The final call used the web environment data from the posting of the assembly IDs to retrieve of the genomic assembly data from Entrez (`retrieve_accession_numbers`).
_____no_output_____

def get_accession_numbers(df_row, logger, args):
    """Return all NCBI accession numbers associated with NCBI Taxonomy ID.

    Reminder of Pandas series structure:
    df_row[0]: Genus
    df_row[1]: Species
    df_row[2]: Taxonomy ID

    Return list of NCBI accession numbers.
    """
    # If previously failed to retrieve the taxonomy ID cancel retrieval of accession numbers
    if df_row[2] == "NA":
        # log warning
        return "NA"

    # Retrieve all IDs of genomic assemblies for taxonomy ID

    logger.info(f"Retrieving assembly IDs for {df_row[2]}")
    assembly_id_list = get_assembly_ids(df_row, logger, args)

    # Check if assembly ID retrieval was successful
    if assembly_id_list == "NA":
        # log error
        return "NA"

    logger.info(f"Posting assembly IDs for {df_row[2]} to retrieve accession numbers")
    epost_webenv_data = post_assembly_ids(assembly_id_list, df_row, logger, args)

    # Check web environment data was retrieved from epost
    if epost_webenv_data == "NA":
        # log warning
        return "NA"

    logger.info(f"Retrieving accession numbers for {df_row[2]}")
    accession_numbers = retrieve_accession_numbers(
        epost_webenv_data, df_row, logger, args
    )

    if accession_numbers == "NA":
        logger.error(
            (
                f"Failed to retrieve accession numbers for {df_row[2]}.\n"
                "Returning 'NA' for accession numbers."
            )
        )
        return "NA"

    logger.info(f"Finished processing retrieval accession numbers for {df_row[2]}")

    return accession_numbers_____no_output_____

### Retrieval of assembly IDs_____no_output_____
The first call to Entrez to retrieve genomic assembly data from the NCBI Assembly database with data from the NCBI Taxonomy database require the `elink` to be used - `elink` looks up neighbouring entries in other NCBI databases. This is coordianated by the function `get_assembly_ids`, which retrieves the IDs of all directly linked genomic assemblies for the given NCBI Taxonomy ID as a list.
_____no_output_____

def get_assembly_ids(df_row, logger, args):

    # df_row[2][9:] removes 'NCBI:txid' prefix
    with entrez_retry(
        logger,
        args.retries,
        Entrez.elink,
        dbfrom="Taxonomy",
        id=df_row[2][9:],
        db="Assembly",
        linkname="taxonomy_assembly",
    ) as assembly_number_handle:
        try:
            assembly_number_record = Entrez.read(assembly_number_handle)
        # if no record is returned from call to Entrez
        except (TypeError, AttributeError) as error:
            # log error
            return "NA"

    # extract assembly IDs from record
    try:
        assembly_id_list = [
            dict["Id"] for dict in assembly_number_record[0]["LinkSetDb"][0]["Link"]
        ]

    except IndexError:
        # log error
        return "NA"

    return assembly_id_list
_____no_output_____

### Posting of assembly IDs_____no_output_____
To minimus the number of calls to the NCBI database, the list of IDs is posted as a single query to NCBI using `Entrez`, of which the web environment and query key is retrieved to facilitate the retrieval of the accession numbers of the identified genomic assemblies. This is coordinated by the function `post_assembly_ids`, which also retrieves the necessary web environment data so that the genomic assembly data can be retrieved.
_____no_output_____

def post_assembly_ids(assembly_id_list, df_row, logger, args):

    # compile list of ids in suitable format for epost
    id_post_list = str(",".join(assembly_id_list))
    # Post all assembly IDs to Entrez-NCBI for downstream pulldown of accession numbers
    try:
        epost_search_results = Entrez.read(
            entrez_retry(
                logger, args.retries, Entrez.epost, "Assembly", id=id_post_list
            )
        )
    # if no record is returned from call to Entrez
    except (TypeError, AttributeError) as error:
        # log error
        return "NA"

    # Retrieve web environment and query key from Entrez epost
    epost_webenv = epost_search_results["WebEnv"]
    epost_query_key = epost_search_results["QueryKey"]

    return epost_webenv, epost_query_key_____no_output_____

### Retrieve accession numbers_____no_output_____
Entrez is used again for retrieval the accession numbers from the previously posted query, which is coordiated by the function `retrieve_accession_numbers`. The accession numbers are collated into a single list, which is then converted into a string, this prevents the appearance of list book ending sequare brackets `[]` within the final dataframe.

The last stage of `retrieve_accession_numbers` is to call the required function to coordinate the downloading of the associated GenBank file for the most recently retrieved accession number, if the downloading of GenBank files is enabled.
_____no_output_____

def retrieve_accession_numbers(webenv, df_row, logger, args):
    # create empty list to store accession numbers
    ncbi_accession_numbers_list = []

    with entrez_retry(
        logger,
        args.retries,
        Entrez.efetch,
        db="Assembly",
        query_key=webenv[1],
        WebEnv=webenv[0],
        rettype="docsum",
        retmode="xml",
    ) as accession_handle:
        try:
            accession_record = Entrez.read(accession_handle, validate=False)
        # if no record is returned from call to Entrez
        except (TypeError, AttributeError) as error:
            # log error
            return "NA"

    # Extract accession numbers from document summary
    for index_number in tqdm(
        range(len(accession_record["DocumentSummarySet"]["DocumentSummary"])),
        desc=f"Retrieving accessions ({df_row[2]})",
    ):
        try:
            new_accession_number = accession_record["DocumentSummarySet"][
                "DocumentSummary"
            ][index_number]["AssemblyAccession"]
            ncbi_accession_numbers_list.append(new_accession_number)

        except IndexError:
            total_assemblies = len(
                accession_record["DocumentSummarySet"]["DocumentSummary"]
            )
            # log error when fails to retrieve data
            return "NA"

        # If downloading of GenBank files is enabled, download Genbank files
        if args.genbank is True:
            get_genbank_files(
                new_accession_number,
                accession_record["DocumentSummarySet"]["DocumentSummary"][index_number][
                    "AssemblyName"
                ],
                logger,
                args,
            )

        index_number += 1

    # Process accession numbers into human readable list for dataframe
    ncbi_accession_numbers = ", ".join(ncbi_accession_numbers_list)

    return ncbi_accession_numbers
_____no_output_____

_____no_output_____## Downloading GenBank files_____no_output_____Prior to adding the newly retrieved accession number to the growing list of all accession numbers for a specific species, a check is performed to see if the the downloading of the GenBank file (.gbff) of the genomic assembly entry is enabled.

If enabled, the downloading of the GenBank file is initated by calling the `get_genbank_files` function, which coordinates the multiple stages of downloading the GenBank file._____no_output_____

def get_genbank_files(
    accession_number,
    assembly_name,
    logger,
    args,
    suffix="genomic.gbff.gz",
):

    # compile url for download
    genbank_url, filestem = compile_url(accession_number, assembly_name, logger, suffix)

    # if downloaded file is not to be written to STDOUT, compile output path
    if args.output is not sys.stdout:
        out_file_path = args.output / "_".join([filestem.replace(".", "_"), suffix])
    else:
        out_file_path = args.output

    # download GenBank file
    download_file(
        genbank_url, args, out_file_path, logger, accession_number, "GenBank file",
    )

    return
_____no_output_____

_____no_output_____### Compiling the URL

The downloading of any file requires a URL address from which to download a file, and an output path to write the downloaded data too, created by the `compile_url` funciton.

NCBI records can include a variety of escape characters within its records, but requires all escape characters to be written as underscores within its URLs. The `regular expression` module of Python is used to replace any escape charcters with underscords, prior to compiling the URL.

NCBI genomic assembly files are avaialble for download from the NCBI FTP site hence the prefix (the ftpstem) and structure of the compiled URL does not match the structure of the URL when searching the NCBI Assembly database in-browser.

_____no_output_____

def compile_url(
    accession_number,
    assembly_name,
    logger,
    suffix,
    ftpstem="ftp://ftp.ncbi.nlm.nih.gov/genomes/all",
):

    # Extract assembly name, removing alterantive escape characters
    escape_characters = re.compile(r"[\s/,#\(\)]")
    escape_name = re.sub(escape_characters, "_", assembly_name)

    # compile filstem
    filestem = "_".join([accession_number, escape_name])

    # separate out filesteam into GCstem, accession number intergers and discarded
    url_parts = tuple(filestem.split("_", 2))

    # separate identifying numbers from version number
    sub_directories = "/".join(
        [url_parts[1][i : i + 3] for i in range(0, len(url_parts[1].split(".")[0]), 3)]
    )

    # return url for downloading file
    return (
        "{0}/{1}/{2}/{3}/{3}_{4}".format(
            ftpstem, url_parts[0], sub_directories, filestem, suffix
        ),
        filestem,
    )
_____no_output_____

_____no_output_____### Downloading the GenBank file

The downloading of the GenBank file is initated with calling the `download_file` function.

The URL connection is coordinated by the Python `urllib` module, the documentation can be found [here](https://docs.python.org/3/library/urllib.html).

The Python `tqdm` module is also used provide a visual log of the download progress in the terminal.
_____no_output_____

# Try URL connection
    try:
        response = urlopen(genbank_url, timeout=args.timeout)
    except HTTPError, URLError, timeout:
        # log error and exit downloading of GenBank file
        return

    if out_file_path.exists():
        logger.warning(f"Output file {out_file_path} exists, not downloading")

    else:
        # Download file
        file_size = int(response.info().get("Content-length"))
        bsize = 1_048_576

        try:
            with open(out_file_path, "wb") as out_handle:
                # Using leave=False as this will be an internally-nested progress bar
                with tqdm(
                    total=file_size,
                    leave=False,
                    desc=f"Downloading {accession_number} {file_type}",
                ) as pbar:
                    while True:
                        buffer = response.read(bsize)
                        if not buffer:
                            break
                        pbar.update(len(buffer))
                        out_handle.write(buffer)

        return
    _____no_output_____

_____no_output_____## Writing out the dataframe

If the option to write out the dataframe to a file, rather than STDOUT, is enabled, the function `write_out_dataframe` will be called, which is stored within the `pyrewton` `file_io` module.

To review the code for writing out the created dataframe, navigate to the the `file_io` directory within the `pyrewton` repository.

The dataframe is written out as a .csv file (column-separated values format), to allow the opening and reading of the file by several packages including Microsoft Offic Excel, as well as easy parsing of the dataframe by other `pandas` using Python scripts.

When providing the output path for the .csv file, make sure to include the .csv extention. Excluding the .csv extension from the file name will not prevent the file from being written out; however, when the file is written out it will be missing the '.csv' extention in its path otherwise.

_____no_output_____

    # Write out dataframe
    if args.dataframe is not sys.stdout:
        write_out_dataframe(species_table, logger, args.dataframe, args.force)
    else:
        species_table.to_csv(args.dataframe)_____no_output_____

For the downloading of GenBank files for the project, the dataframe was written out to '2020_05_31_genomes_dataframe.csv', stored within the same directory as the script (get_ncbi_genomes). The following dataframe was written:

_____no_output_____

import pandas as pd

from pathlib import Path
from IPython.display import display

pd.set_option('display.max_colwidth', None)

display(pd.read_csv(
    '2020_05_31_genome_dataframe.csv',
    header=0,
    names=["Genus", "Species", "NCBI Taxonomy ID", "NCBI Accession Numbers"]
))
_____no_output_____


# Notebook from h1-the-swan/autoreview
Path: old/wos_review_papers.ipynb

<code>
import sys, os, time
from timeit import default_timer as timer
from humanfriendly import format_timespan_____no_output_____from dotenv import load_dotenv_____no_output_____load_dotenv('admin.env')_____no_output_____from mysql_connect import get_db_connection
db = get_db_connection('mag_2017-10')_____no_output_____import pandas as pd_____no_output_____df = pd.read_table('data/wos_papers_reviews_with_citation_count.tsv', index_col=0)_____no_output_____subset = df[df.num_citations>200].sort_values('num_citations', ascending=False)_____no_output_____len(subset)_____no_output_____len(subset[subset.doi.notnull()])_____no_output_____# randomly sample a row
wos_item = subset[subset.doi.notnull()].sample(1).iloc[0]_____no_output_____wos_item_____no_output_____tbl = db.tables['Papers']
sq = tbl.select(tbl.c.DOI==wos_item.doi)
r = db.engine.execute(sq).fetchall()_____no_output_____r_____no_output_____from collections import Counter
num_results = Counter()
data = []
start = timer()
for _, wos_item in subset[subset.doi.notnull()].sample(10000).iterrows():
    tbl1 = db.tables['Papers']
    tbl2 = db.tables['rank']
    j = tbl1.join(tbl2, tbl1.c.Paper_ID==tbl2.c.Paper_ID)
    sq = j.select(tbl1.c.DOI==wos_item.doi)
    #r = db.engine.execute(sq).fetchall()
    r = db.read_sql(sq)
    num_results[len(r)] += 1
    data.append(r)

print(format_timespan(timer()-start))_____no_output_____data[0]_____no_output_____num_results_____no_output_____for r in data:
    if len(r) ==0:
        break_____no_output_____r_____no_output_____r.sort_values('EF', ascending=False).iloc[0]_____no_output_____wos_item.name_____no_output_____r = r.iloc[0]_____no_output_____r.Paper_ID.iloc[0]_____no_output_____subset[(subset.num_citations<600)&(subset.num_citations>400)&(subset.doi.notnull())]_____no_output_____subset.title_source.value_counts().head(100)_____no_output_____subset[subset.title_source=='NEUROSCIENCE']_____no_output_____
</code>


# Notebook from maharjansudhan/DS_Python
Path: Data_Operation.ipynb

import pandas as pd_____no_output_____#declare movie rating datafram: ratings from 1 to 5 (star * rating)
df_movie_rating = pd.DataFrame(
                    {'movie 1':[5,4,3,3,2,1],
                    'movie 2':[4,5,2,3,4,2]},
                    index=['Tom','Jeff','Peter','Ram','Ted','Paul'])_____no_output_____df_movie_rating_____no_output_____#declare a custom function

def movie_grade(rating):
    if rating==5:
        return 'A'
    if rating==4:
        return 'B'
    if rating==3:
        return 'C'
    else:
        return 'F'_____no_output_____# test the function
print movie_grade(5)A
#apply the function to the dataframe
df_movie_rating.applymap(movie_grade)_____no_output_____

Data Operation with Statistical Functions_____no_output_____

#create a dataframe with two test
df_test_scores = pd.DataFrame(
                {'Test1':[95,84,73,88,82,61],
                'Test2':[74,85,82,73,77,79]},
                index =['Jack','Lewis','Patrick','Rich','Kelly','Paula'])_____no_output_____#apply the max function to find the maximum score
df_test_scores.max()_____no_output_____df_test_scores_____no_output_____#apply the mean function to find the average score
df_test_scores.mean()_____no_output_____#apply the std function to find the standard deviation for both the tests
df_test_scores.std()_____no_output_____

Data Operation Using Groupby_____no_output_____

#create a DataFrame with first and last name as former presidents
df_president_name = pd.DataFrame({'first':['George','Bill','Ronald','Jimmy','George'],
                                 'last':['Bush','Clinton','Regan','Carter','Washington']})_____no_output_____df_president_name_____no_output_____grouped = df_president_name.groupby('first')_____no_output_____#Group the DataFrame with the first name
grp_data = grouped.get_group('George')
grp_data_____no_output_____#sort values by firstname
df_president_name.sort_values('first')_____no_output_____

Data Standardization_____no_output_____

#create a function to return the standardize value
def standardize_tests(test):
    return (test-test.mean())/test.std()_____no_output_____standardize_tests(df_test_scores['Test1'])_____no_output_____#apply the function to the entire dataset
def standardize_test_scores(datafrm):
    return datafrm.apply(standardize_tests)_____no_output_____standardize_test_scores(df_test_scores)_____no_output_____