# Notebook from RyanMarten/federated
Path: docs/tutorials/sparse_federated_learning.ipynb

##### Copyright 2021 The TensorFlow Federated Authors._____no_output_____

#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License._____no_output_____

# Client-efficient large-model federated learning via `federated_select` and sparse aggregation
_____no_output_____

    View on TensorFlow.org

    Run in Google Colab

    View source on GitHub

    Download notebook

_____no_output_____

This tutorial shows how TFF can be used to train a very large model where each client device only downloads and updates a small part of the model, using
`tff.federated_select` and sparse aggregation. While this tutorial is fairly self-contained, the [`tff.federated_select` tutorial](https://www.tensorflow.org/federated/tutorials/federated_select) and [custom FL algorithms tutorial](https://www.tensorflow.org/federated/tutorials/building_your_own_federated_learning_algorithm) provide good introductions to some of the techniques used here.

Concretely, in this tutorial we consider logistic regression for multi-label classification, predicting which "tags" are associated with a text string based on a  bag-of-words feature representation. Importantly, communication and client-side computation costs are controlled by a fixed constant (`MAX_TOKENS_SELECTED_PER_CLIENT`), and *do not* scale with the overall vocabulary size, which could be extremely large in practical settings._____no_output_____

#@test {"skip": true}
!pip install --quiet --upgrade tensorflow-federated-nightly
!pip install --quiet --upgrade nest-asyncio

import nest_asyncio
nest_asyncio.apply()_____no_output_____import collections
import itertools
import numpy as np

from typing import Callable, List, Tuple

import tensorflow as tf
import tensorflow_federated as tff_____no_output_____

Each client will `federated_select` the rows of the model weights for at most this many unique tokens. This upper-bounds the size of the client's local model and the amount of server -> client (`federated_select`) and client - > server `(federated_aggregate`) communication performed.

This tutorial should still run correctly even if you set this as small as 1 (ensuring not all tokens from each client are selected) or to a large value, though model convergence may be effected._____no_output_____

MAX_TOKENS_SELECTED_PER_CLIENT = 6_____no_output_____

We also define a few constants for various types. For this colab, a **token** is an integer identifier for a particular word after parsing the dataset. _____no_output_____

# There are some constraints on types
# here that will require some explicit type conversions:
#    - `tff.federated_select` requires int32
#    - `tf.SparseTensor` requires int64 indices.
TOKEN_DTYPE = tf.int64
SELECT_KEY_DTYPE = tf.int32

# Type for counts of token occurences.
TOKEN_COUNT_DTYPE = tf.int32

# A sparse feature vector can be thought of as a map
# from TOKEN_DTYPE to FEATURE_DTYPE.
# Our features are {0, 1} indicators, so we could potentially
# use tf.int8 as an optimization.
FEATURE_DTYPE = tf.int32_____no_output_____

# Setting up the problem: Dataset and Model

We construct a tiny toy dataset for easy experimentation in this tutorial. However, the format of the dataset is compatible with [Federated StackOverflow](https://www.tensorflow.org/federated/api_docs/python/tff/simulation/datasets/stackoverflow/load_data), and
the [pre-processing](https://github.com/google-research/federated/blob/0a558bac8a724fc38175ff4f0ce46c7af3d24be2/utils/datasets/stackoverflow_tag_prediction.py) and [model architecture](https://github.com/google-research/federated/blob/49a43456aa5eaee3e1749855eed89c0087983541/utils/models/stackoverflow_lr_models.py) are adopted from the StackOverflow
tag prediction problem of [*Adaptive Federated Optimization*](https://arxiv.org/abs/2003.00295)._____no_output_____## Dataset parsing and pre-processing_____no_output_____

NUM_OOV_BUCKETS = 1

BatchType = collections.namedtuple('BatchType', ['tokens', 'tags'])

def build_to_ids_fn(word_vocab: List[str],
                    tag_vocab: List[str]) -> Callable[[tf.Tensor], tf.Tensor]:
  """Constructs a function mapping examples to sequences of token indices."""
  word_table_values = np.arange(len(word_vocab), dtype=np.int64)
  word_table = tf.lookup.StaticVocabularyTable(
      tf.lookup.KeyValueTensorInitializer(word_vocab, word_table_values),
      num_oov_buckets=NUM_OOV_BUCKETS)

  tag_table_values = np.arange(len(tag_vocab), dtype=np.int64)
  tag_table = tf.lookup.StaticVocabularyTable(
      tf.lookup.KeyValueTensorInitializer(tag_vocab, tag_table_values),
      num_oov_buckets=NUM_OOV_BUCKETS)

  def to_ids(example):
    """Converts a Stack Overflow example to a bag-of-words/tags format."""
    sentence = tf.strings.join([example['tokens'], example['title']],
                               separator=' ')

    # We represent that label (output tags) densely.
    raw_tags = example['tags']
    tags = tf.strings.split(raw_tags, sep='|')
    tags = tag_table.lookup(tags)
    tags, _ = tf.unique(tags)
    tags = tf.one_hot(tags, len(tag_vocab) + NUM_OOV_BUCKETS)
    tags = tf.reduce_max(tags, axis=0)

    # We represent the features as a SparseTensor of {0, 1}s.
    words = tf.strings.split(sentence)
    tokens = word_table.lookup(words)
    tokens, _ = tf.unique(tokens)
    # Note:  We could choose to use the word counts as the feature vector
    # instead of just {0, 1} values (see tf.unique_with_counts).
    tokens = tf.reshape(tokens, shape=(tf.size(tokens), 1))
    tokens_st = tf.SparseTensor(
        tokens,
        tf.ones(tf.size(tokens), dtype=FEATURE_DTYPE),
        dense_shape=(len(word_vocab) + NUM_OOV_BUCKETS,))
    tokens_st = tf.sparse.reorder(tokens_st)

    return BatchType(tokens_st, tags)

  return to_ids_____no_output_____def build_preprocess_fn(word_vocab, tag_vocab):

  @tf.function
  def preprocess_fn(dataset):
    to_ids = build_to_ids_fn(word_vocab, tag_vocab)
    # We *don't* shuffle in order to make this colab deterministic for
    # easier testing and reproducibility.
    # But real-world training should use `.shuffle()`.
    return dataset.map(to_ids, num_parallel_calls=tf.data.experimental.AUTOTUNE)

  return preprocess_fn_____no_output_____

## A tiny toy dataset
We construct a tiny toy dataset with a global vocabulary of 12 words and 3 clients. This tiny example is useful for testing edge cases (for example,
we have two clients with less than `MAX_TOKENS_SELECTED_PER_CLIENT = 6` distinct tokens, and one with more) and developing the code.

However, the real-world use cases of this approach would be global vocabularies of 10s of millions or more, with perhaps 1000s of distinct tokens appearing on each client. Because the format of the data is the same, the extension to more realistic testbed problems, e.g. the `tff.simulation.datasets.stackoverflow.load_data()` dataset, should be straightforward.

First, we define our word and tag vocabularies._____no_output_____

# Features
FRUIT_WORDS = ['apple', 'orange', 'pear', 'kiwi']
VEGETABLE_WORDS = ['carrot', 'broccoli', 'arugula', 'peas']
FISH_WORDS = ['trout', 'tuna', 'cod', 'salmon']
WORD_VOCAB = FRUIT_WORDS + VEGETABLE_WORDS + FISH_WORDS

# Labels
TAG_VOCAB = ['FRUIT', 'VEGETABLE', 'FISH']_____no_output_____

Now, we create 3 clients with small local datasets. If you are running this tutorial in colab, it may be useful to use the "mirror cell in tab" feature to pin this cell and its output in order to interpret/check the output of the functions developed below._____no_output_____

preprocess_fn = build_preprocess_fn(WORD_VOCAB, TAG_VOCAB)

def make_dataset(raw):
  d = tf.data.Dataset.from_tensor_slices(
      # Matches the StackOverflow formatting
      collections.OrderedDict(
          tokens=tf.constant([t[0] for t in raw]),
          tags=tf.constant([t[1] for t in raw]),
          title=['' for _ in raw]))
  d = preprocess_fn(d)
  return d

# 4 distinct tokens
CLIENT1_DATASET = make_dataset([
    ('apple orange apple orange', 'FRUIT'),
    ('carrot trout', 'VEGETABLE|FISH'),
    ('orange apple', 'FRUIT'),
    ('orange', 'ORANGE|CITRUS')  # 2 OOV tag
])

# 6 distinct tokens
CLIENT2_DATASET = make_dataset([
    ('pear cod', 'FRUIT|FISH'),
    ('arugula peas', 'VEGETABLE'),
    ('kiwi pear', 'FRUIT'),
    ('sturgeon', 'FISH'),  # OOV word
    ('sturgeon bass', 'FISH')  # 2 OOV words
])

# A client with all possible words & tags (13 distinct tokens).
# With MAX_TOKENS_SELECTED_PER_CLIENT = 6, we won't download the model
# slices for all tokens that occur on this client.
CLIENT3_DATASET = make_dataset([
    (' '.join(WORD_VOCAB + ['oovword']), '|'.join(TAG_VOCAB)),
    # Mathe the OOV token and 'salmon' occur in the largest number
    # of examples on this client:
    ('salmon oovword', 'FISH|OOVTAG')
])

print('Word vocab')
for i, word in enumerate(WORD_VOCAB):
  print(f'{i:2d} {word}')

print('\nTag vocab')
for i, tag in enumerate(TAG_VOCAB):
  print(f'{i:2d} {tag}')Word vocab
 0 apple
 1 orange
 2 pear
 3 kiwi
 4 carrot
 5 broccoli
 6 arugula
 7 peas
 8 trout
 9 tuna
10 cod
11 salmon

Tag vocab
 0 FRUIT
 1 VEGETABLE
 2 FISH

Define constants for the raw numbers of input features (tokens/words) and labels (post tags). Our actual input/output spaces are `NUM_OOV_BUCKETS = 1` larger because we add an OOV token / tag._____no_output_____

NUM_WORDS = len(WORD_VOCAB)
NUM_TAGS = len(TAG_VOCAB)

WORD_VOCAB_SIZE = NUM_WORDS + NUM_OOV_BUCKETS
TAG_VOCAB_SIZE = NUM_TAGS + NUM_OOV_BUCKETS_____no_output_____

Create batched versions of the datasets, and individual batches, which will be useful in testing code as we go._____no_output_____

batched_dataset1 = CLIENT1_DATASET.batch(2)
batched_dataset2 = CLIENT2_DATASET.batch(3)
batched_dataset3 = CLIENT3_DATASET.batch(2)

batch1 = next(iter(batched_dataset1))
batch2 = next(iter(batched_dataset2))
batch3 = next(iter(batched_dataset3))_____no_output_____

## Define a model with sparse inputs

We use a simple independent logistic regression model for each tag._____no_output_____

def create_logistic_model(word_vocab_size: int, vocab_tags_size: int):

  model = tf.keras.models.Sequential([
      tf.keras.layers.InputLayer(input_shape=(word_vocab_size,), sparse=True),
      tf.keras.layers.Dense(
          vocab_tags_size,
          activation='sigmoid',
          kernel_initializer=tf.keras.initializers.zeros,
          # For simplicity, don't use a bias vector; this means the model
          # is a single tensor, and we only need sparse aggregation of
          # the per-token slices of the model. Generalizing to also handle
          # other model weights that are fully updated
          # (non-dense broadcast and aggregate) would be a good exercise.
          use_bias=False),
  ])

  return model_____no_output_____

Let's make sure it works, first by making predictions:_____no_output_____

model = create_logistic_model(WORD_VOCAB_SIZE, TAG_VOCAB_SIZE)
p = model.predict(batch1.tokens)
print(p)[[0.5 0.5 0.5 0.5]
 [0.5 0.5 0.5 0.5]]

And some simple centralized training:_____no_output_____

model.compile(optimizer=tf.keras.optimizers.Adagrad(learning_rate=0.001),
              loss=tf.keras.losses.BinaryCrossentropy())
model.train_on_batch(batch1.tokens, batch1.tags)_____no_output_____

# Building blocks for the federated computation

We will implement a simple version of the [Federated Averaging](https://arxiv.org/abs/1602.05629) algorithm with the key difference that each device only downloads a relevant subset of the model, and only contributes updates to that subset.

We use `M` as shorthand for `MAX_TOKENS_SELECTED_PER_CLIENT`. At a high level, one round of training involves these steps:

  1. Each participating client scans over its local dataset, parsing the input strings and mapping them to the correct tokens (int indexes). This requires access to the global (large) dictionary (this could potentially be avoided using [feature hashing](https://en.wikipedia.org/wiki/Feature_hashing) techniques). We then sparsely count how many times each token occurs. If `U` unique tokens occur on device, we choose the `num_actual_tokens = min(U, M)` most frequent tokens to train.

  1. The clients use `federated_select` to retrieve the model coefficients for the `num_actual_tokens` selected tokens from the server. Each model slice is a tensor of shape `(TAG_VOCAB_SIZE, )`, so the total data transmitted to the client is at most of size `TAG_VOCAB_SIZE * M` (see note below).

  1. The clients construct a mapping `global_token -> local_token` where the  local token (int index) is the index of the global token in the list of selected tokens.

  1. The clients use a "small" version of the global model that only has coefficients for at most `M` tokens, from the range `[0, num_actual_tokens)`. The `global -> local` mapping is used to initialize the dense parameters of this model from the selected model slices.

  1. Clients train their local model using SGD on data preprocessed with the `global -> local` mapping.

  1. Clients turn the parameters of their local model into `IndexedSlices` updates using the `local -> global` mapping to index the rows. The server aggregates these updates using a sparse sum aggregation.

  1. The server takes the (dense) result of the above aggregation, divides it by the number of clients participating, and applies the resulting average update to the global model.

In this section we construct the building blocks for these steps, which will then be combined in a final `federated_computation` that captures the full logic of one training round.

> NOTE: The above description hides one technical detail: Both `federated_select` and the construction of the local model require statically known shapes, and so we cannot use the dynamic per-client `num_actual_tokens` size. Instead, we use the static value `M`, adding padding where needed. This does not impact that semantics of the algorithm._____no_output_____### Count client tokens and decide which model slices to `federated_select`

Each device needs to decide which "slices" of the model are relevant to its local training dataset.  For our problem, we do this by (sparsely!) counting how many examples contain each token in the client training data set.
_____no_output_____

@tf.function
def token_count_fn(token_counts, batch):
  """Adds counts from `batch` to the running `token_counts` sum."""
  # Sum across the batch dimension.
  flat_tokens = tf.sparse.reduce_sum(
      batch.tokens, axis=0, output_is_sparse=True)
  flat_tokens = tf.cast(flat_tokens, dtype=TOKEN_COUNT_DTYPE)
  return tf.sparse.add(token_counts, flat_tokens)_____no_output_____# Simple tests
# Create the initial zero token counts using empty tensors.
initial_token_counts = tf.SparseTensor(
    indices=tf.zeros(shape=(0, 1), dtype=TOKEN_DTYPE),
    values=tf.zeros(shape=(0,), dtype=TOKEN_COUNT_DTYPE),
    dense_shape=(WORD_VOCAB_SIZE,))

client_token_counts = batched_dataset1.reduce(initial_token_counts,
                                              token_count_fn)
tokens = tf.reshape(client_token_counts.indices, (-1,)).numpy()
print('tokens:', tokens)
np.testing.assert_array_equal(tokens, [0, 1, 4, 8])
# The count is the number of *examples* in which the token/word
# occurs, not the total number of occurences, since we still featurize
# multiple occurences in the same example as a "1".
counts = client_token_counts.values.numpy()
print('counts:', counts)
np.testing.assert_array_equal(counts, [2, 3, 1, 1])tokens: [0 1 4 8]
counts: [2 3 1 1]

We will select the model parameters corresponding to the `MAX_TOKENS_SELECTED_PER_CLIENT` most frequently occuring tokens on device. If
fewer than this many tokens occur on device, we pad the list to enable the use
of `federated_select`.

Note that other strategies are possibly better, for example, randomly selecting tokens (perhaps based on their occurrence probability). This would ensure that all slices of the model (for which the client has data) have some chance of being updated._____no_output_____

@tf.function
def keys_for_client(client_dataset, max_tokens_per_client):
  """Computes a set of max_tokens_per_client keys."""
  initial_token_counts = tf.SparseTensor(
      indices=tf.zeros((0, 1), dtype=TOKEN_DTYPE),
      values=tf.zeros((0,), dtype=TOKEN_COUNT_DTYPE),
      dense_shape=(WORD_VOCAB_SIZE,))
  client_token_counts = client_dataset.reduce(initial_token_counts,
                                              token_count_fn)
  # Find the most-frequently occuring tokens
  tokens = tf.reshape(client_token_counts.indices, shape=(-1,))
  counts = client_token_counts.values
  perm = tf.argsort(counts, direction='DESCENDING')
  tokens = tf.gather(tokens, perm)
  counts = tf.gather(counts, perm)
  num_raw_tokens = tf.shape(tokens)[0]
  actual_num_tokens = tf.minimum(max_tokens_per_client, num_raw_tokens)
  selected_tokens = tokens[:actual_num_tokens]
  paddings = [[0, max_tokens_per_client - tf.shape(selected_tokens)[0]]]
  padded_tokens = tf.pad(selected_tokens, paddings=paddings)
  # Make sure the type is statically determined
  padded_tokens = tf.reshape(padded_tokens, shape=(max_tokens_per_client,))

  # We will pass these tokens as keys into `federated_select`, which
  # requires SELECT_KEY_DTYPE=tf.int32 keys.
  padded_tokens = tf.cast(padded_tokens, dtype=SELECT_KEY_DTYPE)
  return padded_tokens, actual_num_tokens_____no_output_____# Simple test

# Case 1: actual_num_tokens > max_tokens_per_client
selected_tokens, actual_num_tokens = keys_for_client(batched_dataset1, 3)
assert tf.size(selected_tokens) == 3
assert actual_num_tokens == 3

# Case 2: actual_num_tokens < max_tokens_per_client
selected_tokens, actual_num_tokens = keys_for_client(batched_dataset1, 10)
assert tf.size(selected_tokens) == 10
assert actual_num_tokens == 4_____no_output_____

### Map global tokens to local tokens
The above selection gives us a dense set of tokens in the range `[0, actual_num_tokens)` which we will use for the on-device model. However, the dataset we read has tokens from the much larger global vocabulary range `[0, WORD_VOCAB_SIZE)`.

Thus, we need to map the global tokens to their corresponding local tokens. The
local token ids are simply given by the indexes into the `selected_tokens` tensor computed in the previous step._____no_output_____

@tf.function
def map_to_local_token_ids(client_data, client_keys):
  global_to_local = tf.lookup.StaticHashTable(
      # Note int32 -> int64 maps are not supported
      tf.lookup.KeyValueTensorInitializer(
          keys=tf.cast(client_keys, dtype=TOKEN_DTYPE),
          # Note we need to use tf.shape, not the static
          # shape client_keys.shape[0]
          values=tf.range(0, limit=tf.shape(client_keys)[0],
                          dtype=TOKEN_DTYPE)),
      # We use -1 for tokens that were not selected, which can occur for clients
      # with more than MAX_TOKENS_SELECTED_PER_CLIENT distinct tokens.
      # We will simply remove these invalid indices from the batch below.
      default_value=-1)

  def to_local_ids(sparse_tokens):
    indices_t = tf.transpose(sparse_tokens.indices)
    batch_indices = indices_t[0]  # First column
    tokens = indices_t[1]  # Second column
    tokens = tf.map_fn(
        lambda global_token_id: global_to_local.lookup(global_token_id), tokens)
    # Remove tokens that aren't actually available (looked up as -1):
    available_tokens = tokens >= 0
    tokens = tokens[available_tokens]
    batch_indices = batch_indices[available_tokens]

    updated_indices = tf.transpose(
        tf.concat([[batch_indices], [tokens]], axis=0))
    st = tf.sparse.SparseTensor(
        updated_indices,
        tf.ones(tf.size(tokens), dtype=FEATURE_DTYPE),
        dense_shape=sparse_tokens.dense_shape)
    st = tf.sparse.reorder(st)
    return st

  return client_data.map(lambda b: BatchType(to_local_ids(b.tokens), b.tags))_____no_output_____# Simple test
client_keys, actual_num_tokens = keys_for_client(
    batched_dataset3, MAX_TOKENS_SELECTED_PER_CLIENT)
client_keys = client_keys[:actual_num_tokens]

d = map_to_local_token_ids(batched_dataset3, client_keys)
batch  = next(iter(d))
all_tokens = tf.gather(batch.tokens.indices, indices=1, axis=1)
# Confirm we have local indices in the range [0, MAX):
assert tf.math.reduce_max(all_tokens) < MAX_TOKENS_SELECTED_PER_CLIENT
assert tf.math.reduce_max(all_tokens) >= 0_____no_output_____

### Train the local (sub)model on each client

Note `federated_select` will return the selected slices as a `tf.data.Dataset` in the same order as the selection keys. So, we first define a utility function to take such a Dataset and convert it to a single dense tensor which can be used as the model weights of the client model._____no_output_____

@tf.function
def slices_dataset_to_tensor(slices_dataset):
  """Convert a dataset of slices to a tensor."""
  # Use batching to gather all of the slices into a single tensor.
  d = slices_dataset.batch(MAX_TOKENS_SELECTED_PER_CLIENT,
                           drop_remainder=False)
  iter_d = iter(d)
  tensor = next(iter_d)
  # Make sure we have consumed everything
  opt = iter_d.get_next_as_optional()
  tf.Assert(tf.logical_not(opt.has_value()), data=[''], name='CHECK_EMPTY')
  return tensor_____no_output_____# Simple test
weights = np.random.random(
    size=(MAX_TOKENS_SELECTED_PER_CLIENT, TAG_VOCAB_SIZE)).astype(np.float32)
model_slices_as_dataset = tf.data.Dataset.from_tensor_slices(weights)
weights2 = slices_dataset_to_tensor(model_slices_as_dataset)
np.testing.assert_array_equal(weights, weights2)_____no_output_____

We now have all the components we need to define a simple local training loop which will run on each client._____no_output_____

@tf.function
def client_train_fn(model, client_optimizer,
                    model_slices_as_dataset, client_data,
                    client_keys, actual_num_tokens):

  initial_model_weights = slices_dataset_to_tensor(model_slices_as_dataset)
  assert len(model.trainable_variables) == 1
  model.trainable_variables[0].assign(initial_model_weights)

  # Only keep the "real" (unpadded) keys.
  client_keys = client_keys[:actual_num_tokens]

  client_data = map_to_local_token_ids(client_data, client_keys)

  loss_fn = tf.keras.losses.BinaryCrossentropy()
  for features, labels in client_data:
    with tf.GradientTape() as tape:
      predictions = model(features)
      loss = loss_fn(labels, predictions)
    grads = tape.gradient(loss, model.trainable_variables)
    client_optimizer.apply_gradients(zip(grads, model.trainable_variables))

  model_weights_delta = model.trainable_weights[0] - initial_model_weights
  model_weights_delta = tf.slice(model_weights_delta, begin=[0, 0],
                           size=[actual_num_tokens, -1])
  return client_keys, model_weights_delta_____no_output_____# Simple test
# Note if you execute this cell a second time, you need to also re-execute
# the preceeding cell to avoid "tf.function-decorated function tried to
# create variables on non-first call" errors.
on_device_model = create_logistic_model(MAX_TOKENS_SELECTED_PER_CLIENT,
                                        TAG_VOCAB_SIZE)
client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.001)
client_keys, actual_num_tokens = keys_for_client(
    batched_dataset2, MAX_TOKENS_SELECTED_PER_CLIENT)

model_slices_as_dataset = tf.data.Dataset.from_tensor_slices(
    np.zeros((MAX_TOKENS_SELECTED_PER_CLIENT, TAG_VOCAB_SIZE),
             dtype=np.float32))

keys, delta = client_train_fn(
    on_device_model,
    client_optimizer,
    model_slices_as_dataset,
    client_data=batched_dataset3,
    client_keys=client_keys,
    actual_num_tokens=actual_num_tokens)

print(delta)_____no_output_____

###  Aggregate IndexedSlices
We use `tff.federated_aggregate` to construct a federated sparse sum for `IndexedSlices`. This simple implementation has the constraint that the
`dense_shape` is known statically in advance. Note also that this sum is only *semi-sparse*, in the sense that the client -> server communication is sparse, but the server maintains a dense representation of the sum in `accumulate` and `merge`, and outputs this dense representation.
_____no_output_____

def federated_indexed_slices_sum(slice_indices, slice_values, dense_shape):
  """
  Sumes IndexedSlices@CLIENTS to a dense @SERVER Tensor.

  Intermediate aggregation is performed by converting to a dense representation,
  which may not be suitable for all applications.

  Args:
    slice_indices: An IndexedSlices.indices tensor @CLIENTS.
    slice_values: An IndexedSlices.values tensor @CLIENTS.
    dense_shape: A statically known dense shape.

  Returns:
    A dense tensor placed @SERVER representing the sum of the client's
    IndexedSclies.
  """
  slices_dtype = slice_values.type_signature.member.dtype
  zero = tff.tf_computation(
      lambda: tf.zeros(dense_shape, dtype=slices_dtype))()

  @tf.function
  def accumulate_slices(dense, client_value):
    indices, slices = client_value
    # There is no built-in way to add `IndexedSlices`, but
    # tf.convert_to_tensor is a quick way to convert to a dense representation
    # so we can add them.
    return dense + tf.convert_to_tensor(
        tf.IndexedSlices(slices, indices, dense_shape))

  return tff.federated_aggregate(
      (slice_indices, slice_values),
      zero=zero,
      accumulate=tff.tf_computation(accumulate_slices),
      merge=tff.tf_computation(lambda d1, d2: tf.add(d1, d2, name='merge')),
      report=tff.tf_computation(lambda d: d))
_____no_output_____

Construct a minimal `federated_computation` as a test_____no_output_____

dense_shape = (6, 2)
indices_type = tff.TensorType(tf.int64, (None,))
values_type = tff.TensorType(tf.float32, (None, 2))
client_slice_type = tff.type_at_clients(
    (indices_type, values_type))

@tff.federated_computation(client_slice_type)
def test_sum_indexed_slices(indices_values_at_client):
  indices, values = indices_values_at_client
  return federated_indexed_slices_sum(indices, values, dense_shape)

print(test_sum_indexed_slices.type_signature)({<int64[?],float32[?,2]>}@CLIENTS -> float32[6,2]@SERVER)
x = tf.IndexedSlices(
    values=np.array([[2., 2.1], [0., 0.1], [1., 1.1], [5., 5.1]],
                    dtype=np.float32),
    indices=[2, 0, 1, 5],
    dense_shape=dense_shape)
y = tf.IndexedSlices(
    values=np.array([[0., 0.3], [3.1, 3.2]], dtype=np.float32),
    indices=[1, 3],
    dense_shape=dense_shape)

# Sum one.
result = test_sum_indexed_slices([(x.indices, x.values)])
np.testing.assert_array_equal(tf.convert_to_tensor(x), result)

# Sum two.
expected = [[0., 0.1], [1., 1.4], [2., 2.1], [3.1, 3.2], [0., 0.], [5., 5.1]]
result = test_sum_indexed_slices([(x.indices, x.values), (y.indices, y.values)])
np.testing.assert_array_almost_equal(expected, result)_____no_output_____

# Putting it all together in a `federated_computation`
We now uses TFF to bind together the components into a `tff.federated_computation`.
_____no_output_____

DENSE_MODEL_SHAPE = (WORD_VOCAB_SIZE, TAG_VOCAB_SIZE)
client_data_type = tff.SequenceType(batched_dataset1.element_spec)
model_type = tff.TensorType(tf.float32, shape=DENSE_MODEL_SHAPE)_____no_output_____

We use a basic server training function based on Federated Averaging, applying the update with a server learning rate of 1.0. It is important that we apply an update (delta) to the model, rather than simply averaging client-supplied models, as otherwise if a given slice of the model wasn't trained on by any client on a given round its coefficients could be zeroed out._____no_output_____

@tff.tf_computation
def server_update(current_model_weights, update_sum, num_clients):
  average_update = update_sum / num_clients
  return current_model_weights + average_update_____no_output_____

We need a couple more `tff.tf_computation` components:_____no_output_____

# Function to select slices from the model weights in federated_select:
select_fn = tff.tf_computation(
    lambda model_weights, index: tf.gather(model_weights, index))

# We need to wrap `client_train_fn` as a `tff.tf_computation`, making
# sure we do any operations that might construct `tf.Variable`s outside
# of the `tf.function` we are wrapping.
@tff.tf_computation
def client_train_fn_tff(model_slices_as_dataset, client_data, client_keys,
                        actual_num_tokens):
  # Note this is amaller than the global model, using
  # MAX_TOKENS_SELECTED_PER_CLIENT which is much smaller than WORD_VOCAB_SIZE.
  # W7e would like a model of size `actual_num_tokens`, but we
  # can't build the model dynamically, so we will slice off the padded
  # weights at the end.
  client_model = create_logistic_model(MAX_TOKENS_SELECTED_PER_CLIENT,
                                       TAG_VOCAB_SIZE)
  client_optimizer = tf.keras.optimizers.SGD(learning_rate=0.1)
  return client_train_fn(client_model, client_optimizer,
                         model_slices_as_dataset, client_data, client_keys,
                         actual_num_tokens)

@tff.tf_computation
def keys_for_client_tff(client_data):
  return keys_for_client(client_data, MAX_TOKENS_SELECTED_PER_CLIENT)_____no_output_____

We're now ready to put all the pieces together!_____no_output_____

@tff.federated_computation(
    tff.type_at_server(model_type), tff.type_at_clients(client_data_type))
def sparse_model_update(server_model, client_data):
  max_tokens = tff.federated_value(MAX_TOKENS_SELECTED_PER_CLIENT, tff.SERVER)
  keys_at_clients, actual_num_tokens = tff.federated_map(
      keys_for_client_tff, client_data)

  model_slices = tff.federated_select(keys_at_clients, max_tokens, server_model,
                                      select_fn)

  update_keys, update_slices = tff.federated_map(
      client_train_fn_tff,
      (model_slices, client_data, keys_at_clients, actual_num_tokens))

  dense_update_sum = federated_indexed_slices_sum(update_keys, update_slices,
                                                  DENSE_MODEL_SHAPE)
  num_clients = tff.federated_sum(tff.federated_value(1.0, tff.CLIENTS))

  updated_server_model = tff.federated_map(
      server_update, (server_model, dense_update_sum, num_clients))

  return updated_server_model

print(sparse_model_update.type_signature)(<server_model=float32[13,4]@SERVER,client_data={<tokens=<indices=int64[?,2],values=int32[?],dense_shape=int64[2]>,tags=float32[?,4]>*}@CLIENTS> -> float32[13,4]@SERVER)

# Let's train a model!

Now that we have our training function, let's try it out._____no_output_____

server_model = create_logistic_model(WORD_VOCAB_SIZE, TAG_VOCAB_SIZE)
server_model.compile(  # Compile to make evaluation easy.
    optimizer=tf.keras.optimizers.Adagrad(learning_rate=0.0),  # Unused
    loss=tf.keras.losses.BinaryCrossentropy(),
    metrics=[
      tf.keras.metrics.Precision(name='precision'),
      tf.keras.metrics.AUC(name='auc'),
      tf.keras.metrics.Recall(top_k=2, name='recall_at_2'),
  ])

def evaluate(model, dataset, name):
  metrics = model.evaluate(dataset, verbose=0)
  metrics_str = ', '.join([f'{k}={v:.2f}' for k, v in
                          (zip(server_model.metrics_names, metrics))])
  print(f'{name}: {metrics_str}')_____no_output_____print('Before training')
evaluate(server_model, batched_dataset1, 'Client 1')
evaluate(server_model, batched_dataset2, 'Client 2')
evaluate(server_model, batched_dataset3, 'Client 3')

model_weights = server_model.trainable_weights[0]

client_datasets = [batched_dataset1, batched_dataset2, batched_dataset3]
for _ in range(10):  # Run 10 rounds of FedAvg
  # We train on 1, 2, or 3 clients per round, selecting
  # randomly.
  cohort_size = np.random.randint(1, 4)
  clients = np.random.choice([0, 1, 2], cohort_size, replace=False)
  print('Training on clients', clients)
  model_weights = sparse_model_update(
      model_weights, [client_datasets[i] for i in clients])
server_model.set_weights([model_weights])

print('After training')
evaluate(server_model, batched_dataset1, 'Client 1')
evaluate(server_model, batched_dataset2, 'Client 2')
evaluate(server_model, batched_dataset3, 'Client 3')Before training
Client 1: loss=0.69, precision=0.00, auc=0.50, recall_at_2=0.60
Client 2: loss=0.69, precision=0.00, auc=0.50, recall_at_2=0.50
Client 3: loss=0.69, precision=0.00, auc=0.50, recall_at_2=0.40
Training on clients [0 1]
Training on clients [0 2 1]
Training on clients [2 0]
Training on clients [1 0 2]
Training on clients [2]
Training on clients [2 0]
Training on clients [1 2 0]
Training on clients [0]
Training on clients [2]
Training on clients [1 2]
After training
Client 1: loss=0.67, precision=0.80, auc=0.91, recall_at_2=0.80
Client 2: loss=0.68, precision=0.67, auc=0.96, recall_at_2=1.00
Client 3: loss=0.65, precision=1.00, auc=0.93, recall_at_2=0.80


# Notebook from carsen-stringer/course-content
Path: tutorials/W1D2_ModelingPractice/student/W1D2_Intro.ipynb

_____no_output_____# Intro_____no_output_____**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

_____no_output_____## Overview

Yesterday you learned that different models can answer different questions. That means that depending on your question, goals and hypotheses you will need to develop different kinds of models. How to best approach this is the goal of today. We will use this as an opportunity to kick off your group projects at the same time. To do so, we will start walking you through a 10-steps guide of how-to-model. This guide is applicable for both computational modeling and data neuroscience projects and we will discuss similarities and differences between both project types. So today we will start with what you now know determines the choice of modeling or data analysis pipeline you will need to make: how to develop a good question and goal, do the literature review, think about what ingredients you need, and what hypotheses you would like to evaluate.

Today’s tutorial focuses on the first 4 steps of how-to-model by demonstrating the thought process based on a simple phenomenon known as the train illusion. We will first introduce the phenomenon and then provide a step-by-step guide on how to think about and develop the 4 first steps of framing your project. To help you, we will roleplay an example thought process, focussing on typical pitfalls that groups often encounter. Groups will then think about their own projects and develop first answers to each step’s questions. Importantly, this is to get you started in a systematic way with your projects; you will have to revisit those steps later on as your thinking evolves, possibly multiple times. We will also provide similar guidance for the remaining 6 steps of the how-to-model guide that you can work through with your group when you’re reaching that stage of your project. The accompanying answers to each step in our demo project and toy example code for our two different projects (model and data analytics) should help you understand the practical side of the process better.

How to model is rarely, if ever, taught systematically. Our guide is not the only way to approach modeling; but it’s one way to ensure you don’t miss anything important. Going through all the steps also makes publication much easier because you have already explicitly thought about all the elements that you will ultimately need to communicate (see Step 10 later for our examples). Personally, I often take shortcuts in this process and then regret it later… mostly because I forgot to do the one most important thing: be precise about the framing of the project, i.e. the four first steps you will walk through today. Importantly this will set you up to develop any kind of model using any of the tools you will learn about during the remainder of NMA._____no_output_____## Video_____no_output_____

# @markdown
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"vgX4l7U8bsg", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

## Slides_____no_output_____

# @markdown
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/kmwus/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)_____no_output_____


# Notebook from tsitsvero/hodgelaplacians
Path: examples/.ipynb_checkpoints/hodge_spectrum_protein_gudhi-checkpoint.ipynb

# Higher order Laplacians for protein atoms
Authors: Mikhail Tsitsvero

1. [Protein 3-D point cloud of atoms](#proteins)
2. [Constructing simplex tree](#tree)
3. [Using ```HodgeLaplacinans()``` class](#hodge)
4. [References](#refs)
_____no_output_____

#### Abstract
In this example we will illustrate how to use ```HodgeLaplacians()``` class to get higher order Laplacians for a simplicial complex constructed from the protein data. To access Protein Data Bank we will use [BioPython Library](https://biopython.org/) and to efficiently compute the tree of simplices we will use [C++ GUDHI Library](http://gudhi.gforge.inria.fr/). A variety of alternative [libraries for Topological Data Analysis](https://en.wikipedia.org/wiki/Persistent_homology#Computation) are also available.

_____no_output_____## Protein point cloud data  [↑](#toc)
First, to install BioPython and nglview run
```python
pip3 install biopython nglview
```
To activate the nglview extension for notebook, run before starting Jupyter:
```python
jupyter-nbextension enable nglview --py --sys-prefix
jupyter nbextension enable widgetsnbextension --user --py
```

There is a [nice video tutorial](https://www.youtube.com/watch?v=mL8NPpRxgJA) ([notebook for tutorial](https://nbviewer.jupyter.org/github/cgoliver/Notebooks/blob/master/COMP_364/L27/L27.ipynb)) on BioPython and Protein Data Bank format._____no_output_____

from Bio.PDB import *
import numpy as np

import warnings
warnings.filterwarnings('ignore')

pdbl_1 = PDBList()
pdbl_1.retrieve_pdb_file('4XP1')

pdbl_2 = PDBList()
pdbl_2.retrieve_pdb_file('3I40')

parser_1 = MMCIFParser()
parser_2 = MMCIFParser()

structure_1 = parser_1.get_structure('4XP1', 'xp/4xp1.cif')
structure_2 = parser_2.get_structure('4XP1', 'i4/3i40.cif')WARNING: The default download format has changed from PDB to PDBx/mmCif
WARNING: The default download format has changed from PDB to PDBx/mmCif

Let us visualize two proteins using the viewer form ```nglview``` module._____no_output_____

import nglview as nv

view_1 = nv.show_biopython(structure_1)
view_1.clear_representations()
#view as ball and stick (atom and bond)
view_1.add_ball_and_stick()

view_1_____no_output_____view_2 = nv.show_biopython(structure_2)
view_2.clear_representations()
#view as ball and stick (atom and bond)
view_2.add_ball_and_stick()
view_2_____no_output_____

Next we extract the 3-D coordinates of individual atoms in each protein._____no_output_____

points_1 = []
points_2 = []

for model in structure_1:
    for chain in model:
        for residue in chain:
            for atom in residue:
                points_1.append(tuple(atom.coord))

for model in structure_2:
    for chain in model:
        for residue in chain:
            for atom in residue:
                points_2.append(tuple(atom.coord))

points_1 = np.array(points_1)
points_2 = np.array(points_2)

print(f"There are {len(points_1)} atoms in the protein 1.")
print(f"There are {len(points_2)} atoms in the protein 2.")There are 7675 atoms in the protein 1.
There are 438 atoms in the protein 2.

## Constructing simplex tree  [↑](#toc)

In this section we will connect the neighboring points/atoms with edges thus obtaining a graph where each node/atom is connected to its neighbours in 3D. From the graph we derive the Vietoris-Rips simplicial complex simply by adding simplices associated to each clique of up to certain size in a graph.

To get the connectivity of a graph we will use C++ GUDHI Library that will provides us simplex tree with filtration values as an output. We will use simplex tree data as input data to our ```HodgeLaplacians()``` class.

To install GUDHI Library:
1. [Download and compile C++ Sources](http://gudhi.gforge.inria.fr/doc/latest/installation.html)
2. [Build Cython module for GUDHI](http://gudhi.gforge.inria.fr/python/latest/installation.html)

P.S. Don't forget to install all C/C++ dependencies like Boost, CGAL, Eigen, GMP, CMake,...

Let us assume that you've spent happy 30 minutes and compiled and installed the library, so we can proceed with the construction of connectivity and simplex tree to finally feed it into ```HodgeLaplacians()``` class.

Construction of local neighbourhood connectivity is computationally intensive task that is why we rely on a separate compiled C++ library and do not incorporate this functionality inside ```HodgeLaplacians()``` class. In Python pairwise distance check for points could take a long time._____no_output_____

import sys
sys.path.insert(0, "/home/mm/Download/library-gudhi_library_2.3.0/2018-09-04-14-25-00_GUDHI_2.3.0/build/cython")

import gudhi as gd
import time
start_time = time.time()

epsilon = 3 # Connect two points if distance is smaller than epsilon

skeleton_1  = gd.RipsComplex(points = points_1, max_edge_length=epsilon )
rips_simplex_tree_1 = skeleton_1.create_simplex_tree(max_dimension=3)
rips_skeleton_gudhi_1 = rips_simplex_tree_1.get_skeleton(3)

skeleton_2  = gd.RipsComplex(points = points_2, max_edge_length=epsilon )
rips_simplex_tree_2 = skeleton_2.create_simplex_tree(max_dimension=3)
rips_skeleton_gudhi_2 = rips_simplex_tree_2.get_skeleton(3)

end_time = time.time()

print(f"Done in {end_time-start_time} seconds")_____no_output_____print(len(rips_skeleton_gudhi_1))65155

## Using ```HodgeLaplacinans()``` to get diffusion operators, Ricci curvature and spectrum

To install ```hodgelaplacians``` library
~~~
pip3 install hodgelaplacians
~~~
_____no_output_____First let us import the main class ```HodgeLaplacians()``` and initialize it by simplex tree._____no_output_____

from hodgelaplacians import HodgeLaplacians

start = time.time()

hl_1 = HodgeLaplacians(rips_skeleton_gudhi_1, maxdimension=3, mode='gudhi')
hl_2 = HodgeLaplacians(rips_skeleton_gudhi_2, maxdimension=3, mode='gudhi')

# # Boundary operators
#B0_1 = hl_1.getBoundaryOperator(0)
#B1_1 = hl_1.getBoundaryOperator(1)
#B2_1 = hl_1.getBoundaryOperator(2)
# B3_1 = hl_1.getBoundaryOperator(3)

# # Hodge Laplacians
# L0_1 = hl_1.getHodgeLaplacian(0)
# L1_1 = hl_1.getHodgeLaplacian(1)
# L2_1 = hl_1.getHodgeLaplacian(2)
# L3_1 = hl_1.getHodgeLaplacian(3)

# # Bochner Laplacian
# # (row sum of absolute values equals to diagonal element)
# LB0_1 = hl_1.getBochnerLaplacian(0)
# LB1_1 = hl_1.getBochnerLaplacian(1)
# LB2_1 = hl_1.getBochnerLaplacian(2)
# LB3_1 = hl_1.getBochnerLaplacian(3)

end = time.time()

print(end-start)

# print(f"0-Boundary operator is \n {B0.toarray()} \n \n")
# print(f"1-Boundary operator is \n {B1.toarray()} \n \n")
# print(f"2-Boundary operator is \n {B2.toarray()} \n \n")

# print(f"0-Hodge Laplacian operator is \n {L0.toarray()} \n \n")
# print(f"1-Hodge Laplacian operator is \n {L1.toarray()} \n \n")
# print(f"2-Hodge Laplacian operator is \n {L2.toarray()} \n \n")

# print(hl.n_faces(0))
# print(hl.n_faces(1))
# print(hl.n_faces(2))6.887351989746094

### Hodge and Bochner Laplacian eigenvalues in an interval_____no_output_____

start = time.time()

evalsL1_1, evecsL1_1 = hl_1.getHodgeSpectrum(1, 40, around_point = 0.001)
evalsLB1_1, evecsLB1_1 = hl_1.getBochnerSpectrum(1, 40, around_point = 0.001)

evalsL1_2, evecsL1_2 = hl_2.getHodgeSpectrum(1, 40, around_point = 0.001)
evalsLB1_2, evecsLB1_2 = hl_2.getBochnerSpectrum(1, 40, around_point = 0.001)

end = time.time()
print(end - start)201.70337963104248
%matplotlib inline
import pandas as pd
import matplotlib.pyplot as plt

fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12,3))

df = pd.DataFrame({'Protein 1':evalsL1_1, 'Protein 2':evalsL1_2})
df[['Protein 1', 'Protein 2']].plot(ax=axes[0], title="Eigenvalues of 1-Hodge Laplacian", grid=True)

df = pd.DataFrame({'Protein 1':evalsLB1_1, 'Protein 2':evalsLB1_2})
df[['Protein 1', 'Protein 2']].plot(ax=axes[1], title="Eigenvalues of 1-Bochner Laplacian", grid=True)_____no_output_____

### Distribution of combinatorial Ricci curvature of two proteins_____no_output_____

# Combinatorial Ricci curvature
# (correction term for Laplacian to become Bochner Laplacian)
R0_1 = hl_1.getCombinatorialRicci(0)
R1_1 = hl_1.getCombinatorialRicci(1)
R2_1 = hl_1.getCombinatorialRicci(2)
R3_1 = hl_1.getCombinatorialRicci(3)

# # Combinatorial Ricci curvature
# # (correction term for Laplacian to become Bochner Laplacian)
R0_2 = hl_2.getCombinatorialRicci(0)
R1_2 = hl_2.getCombinatorialRicci(1)
R2_2 = hl_2.getCombinatorialRicci(2)
R3_2 = hl_2.getCombinatorialRicci(3)_____no_output_____

Now let us plot the historgrams side-by-side_____no_output_____

plt.figure(figsize=(12,3))
plt.subplot(1, 2, 1)
plt.suptitle('Distribution of combinatorial Ricci-1 curvature in two proteins \n')
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_1 = pd.Series(R1_1)
plot_ricci1_1.plot.hist(grid=True, bins=20, rwidth=0.9, color='#004c99')
plt.subplot(1, 2, 2)
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_2 = pd.Series(R1_2)
plot_ricci1_2.plot.hist(grid=True, bins=20, rwidth=0.9, color='#006633')

plt.figure(figsize=(12,3))
plt.suptitle('Distribution of combinatorial Ricci-2 curvature in two proteins \n')
plt.subplot(1, 2, 1)
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_1 = pd.Series(R2_1)
plot_ricci1_1.plot.hist(grid=True, bins=20, rwidth=0.9, color='#004c99')
plt.subplot(1, 2, 2)
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_2 = pd.Series(R2_2)
plot_ricci1_2.plot.hist(grid=True, bins=20, rwidth=0.9, color='#006633')

plt.figure(figsize=(12,3))
plt.suptitle('Distribution of combinatorial Ricci-3 curvature in two proteins \n')
plt.subplot(1, 2, 1)
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_1 = pd.Series(R3_1)
plot_ricci1_1.plot.hist(grid=True, bins=20, rwidth=0.9, color='#004c99')
plt.subplot(1, 2, 2)
plt.xlabel('Combinatorial Ricci Curvature')
plt.ylabel('Frequency')
plt.grid(axis='y', alpha=0.75)
plot_ricci1_2 = pd.Series(R3_2)
plot_ricci1_2.plot.hist(grid=True, bins=20, rwidth=0.9, color='#006633')_____no_output_____

kj_____no_output_____## References  [↑](#toc)

1. kk

* [Vector class](http://biopython.org/DIST/docs/api/Bio.PDB.vectors.Vector-class.html)
* [Video introduction](https://www.youtube.com/watch?v=mL8NPpRxgJA)
    * https://nbviewer.jupyter.org/github/cgoliver/Notebooks/blob/master/COMP_364/L27/L27.ipynb

* https://stackoverflow.com/questions/10324674/parsing-a-pdb-file-in-python

* https://github.com/maartenbreddels/talk-ipyvolume-scipy2018_____no_output_____

import numpy as np
import ipyvolume as ipv
V = np.zeros((128,128,128)) # our 3d array
# outer box
V[30:-30,30:-30,30:-30] = 0.75
V[35:-35,35:-35,35:-35] = 0.0
# inner box
V[50:-50,50:-50,50:-50] = 0.25
V[55:-55,55:-55,55:-55] = 0.0
ipv.quickvolshow(V, level=[0.25, 0.75], opacity=0.03, level_width=0.1, data_min=0, data_max=1)/workspace/.pip-modules/lib/python3.7/site-packages/ipyvolume/serialize.py:81: RuntimeWarning: invalid value encountered in true_divide
  gradient = gradient / np.sqrt(gradient[0]**2 + gradient[1]**2 + gradient[2]**2)
import ipyvolume as ipv
import numpy as np
x, y, z = np.random.random((3, 10000))
ipv.quickscatter(x, y, z, size=1, marker="sphere")_____no_output_____


# Notebook from utaresearch/one_sided_extrusion
Path: bacteria/Example_of_switching_translocator.ipynb

<code>
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import expon

import pyximport;
pyximport.install()

from entropic_smcTranslocator_inc2 import smcTranslocatorDirectional

from contactProbability_generator import ChainLayout
from itertools import product

import pandas as pd
import os
import re
from scipy.sparse import coo_matrix_____no_output_____
switchRate_array = [0]
numSMCs_array = [1,5,10,30]
lof = list(product(switchRate_array,numSMCs_array))

def doOne(idx):
    ### START INITIALIZATION ###
    # Hi-C map frequency of sampling parameters #
    nreps = 50000
    nsamples = 70 # for Hi-C map

    # Extruder dynamics frequency of sampling parameters #
    numExtruderSteps = 1000 # steps taken for each simulation sample
    numInitializationSteps = 5000 # how long we take to equilibrate the simulation

    # Polymer and extruder dynamics parameters #
    L = 4000
    switchRate = lof[idx][0] # for every "translocation step", run "Dsub" diffusive steps Dsub = [0, inf]
    numSmc = lof[idx][1] # number of SMCs loaded at ori
    v_extruder =  50 # kb/min

    # Parameters for smcTranslocator
    N = L
    kd = 1/(L)
    LIFETIME = 1/kd
    smcNum = numSmc
    BELT_ON=0
    BELT_OFF=1
    SWITCH= switchRate # switching rate
    PUSH=0
    PAIRED=0
    SLIDE=1
    SLIDE_PAUSEPROB=0.99
    loop_prefactor=1.5
    FULL_LOOP_ENTROPY=1
    FRACTION_ONESIDED=1
    PAUSEPROB=0 # motor pause probability

    SWITCH =  np.ones(N,dtype=np.double)*SWITCH

    birthArray = np.ones(L)/L/10 # birthArray
    birthArray[L//2:L//2+1] = 1
    birthArray[0:3] = 0
    birthArray[L-4:L-1] = 0

    birthArray = birthArray/np.sum(birthArray)

    deathArray = np.zeros(N, dtype=np.double) + 1. / LIFETIME
    deathArray[0:3] = 1
    deathArray[L-4:L-1] = 1

    stallDeathArray = deathArray
    stallDeathArray[0:3] = 1
    stallDeathArray[L-4:L-1] = 1

    pauseArray = np.zeros(N, dtype=np.double) + PAUSEPROB
    slidePauseArray = np.zeros(N, dtype=np.double) + SLIDE_PAUSEPROB

    ################### TERMINUS CONDITION###################
    stallLeftArray = np.zeros(N, dtype = np.double)
    stallLeftArray[0:3] = 1
    stallLeftArray[L-4:L-1] = 1

    stallRightArray = np.zeros(N, dtype = np.double)
    stallRightArray[L-4:L-1] = 1
    stallRightArray[0:3] = 1
    ##################################################################

    oneSidedArray = np.ones(smcNum, dtype=np.int64)
    for i in range(int((1.-FRACTION_ONESIDED)*smcNum)):
        oneSidedArray[i] = 0

    belt_on_array = np.zeros(smcNum, dtype=np.double) + BELT_ON
    belt_off_array = np.zeros(smcNum, dtype=np.double) + BELT_OFF

    spf=slidePauseArray*(1.-(1.-SLIDE_PAUSEPROB)*np.exp(-1.*loop_prefactor))
    spb=slidePauseArray*(1.-(1.-SLIDE_PAUSEPROB)*np.exp(loop_prefactor))

    transloc = smcTranslocatorDirectional(birthArray, deathArray, stallLeftArray, stallRightArray, pauseArray,
                                         stallDeathArray, smcNum, oneSidedArray, paired=PAIRED, slide=SLIDE,
                                         slidepauseForward=spf, slidepauseBackward=spb,
                                         #slidepauseSum=sps,
                                         switch=SWITCH, pushing=PUSH, belt_on=belt_on_array, belt_off=belt_off_array,SLIDE_PAUSEPROB=SLIDE_PAUSEPROB)

    transloc.steps(numInitializationSteps) # run initialization steps

    ### END INITIALIZATION ###
    filename = './one_sided_switching_oneParS/SwitchRate{}_numSMCs{}_nsamples{}_nreps{}.pkl'\
                                .format(switchRate,numSmc,nsamples,nreps)
    print("Doing {}".format(filename))

    X = []
    Y = []
    P = []
    C = []
    numFailed = 0
    L_tot = L

    tot_smc_num = 0

    ### dump empty data as test
    Z = coo_matrix((P,(X,Y)),shape=(L_tot,L_tot))
    Zcount = coo_matrix((C,(X,Y)),shape=(L_tot,L_tot))

    param_sweep_dict = {'Z':Z,'Zcount':Zcount,\
                        'numFailed':numFailed,'switchRate':switchRate,\
                       'nSMCs':numSmc,'avg_numSMCs':tot_smc_num/nreps,\
                       'numReps':nreps,'nsamples_per_map':nsamples,'kd':kd}

    pickle.dump(param_sweep_dict,open(filename,'wb'))
    ###

    for ic in range(nreps):

        # SMC translocator, get SMCs, and make into a circle
        transloc.steps(numExtruderSteps) # number of extruder steps
        smcs = transloc.getSMCs()
        smcs = (np.r_[smcs[0],0], np.r_[smcs[1],L_tot])
        smcs = [(smcs[0][s],smcs[1][s]) for s in range(len(smcs[0]))]
        ##

        cl = None
        try:
            if len(smcs)==0:
                smcs = [(0,1)]
            cl = ChainLayout(smcs,L_tot)
        except:
            numFailed +=1
            cl = None

        if cl is None:
            continue

        tot_smc_num += len(smcs)-1
        vals = sorted(np.random.choice(L_tot,nsamples, replace=False))

        for ix in range(len(vals)):
            for iy in range(ix,len(vals)):
                x = vals[ix]
                y = vals[iy]
                deff = cl.get_dist(x,y)
                if deff == 0:
                    pc = 1
                else:
                    pc = 1/np.sqrt(deff)**3

                if not np.isnan(pc):
                    X.append(x)
                    Y.append(y) # x
                    P.append(pc) # probability
                    C.append(1) # counts

    Z = coo_matrix((P,(X,Y)),shape=(L_tot,L_tot))
    Zcount = coo_matrix((C,(X,Y)),shape=(L_tot,L_tot))

    param_sweep_dict = {'Z':Z,'Zcount':Zcount,\
                        'numFailed':numFailed,'switchRate':switchRate,\
                       'nSMCs':numSmc,'avg_numSMCs':tot_smc_num/nreps,\
                       'numReps':nreps,'nsamples_per_map':nsamples,'kd':kd}

    pickle.dump(param_sweep_dict,open(filename,'wb'))

    print("Done {}".format(filename))
    return 1

from multiprocessing import Pool
import pickle

pool = Pool(len(lof))
pool.map(doOne, np.arange(len(lof)))Doing ./one_sided_switching_oneParS/SwitchRate0_numSMCs1_nsamples70_nreps50000.pkl
Doing ./one_sided_switching_oneParS/SwitchRate0_numSMCs5_nsamples70_nreps50000.pkl
Doing ./one_sided_switching_oneParS/SwitchRate0_numSMCs10_nsamples70_nreps50000.pkl
Doing ./one_sided_switching_oneParS/SwitchRate0_numSMCs30_nsamples70_nreps50000.pkl

</code>


# Notebook from Hadryan/course-content
Path: tutorials/W1D3_ModelFitting/W1D3_Intro.ipynb

_____no_output_____# Intro_____no_output_____**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

_____no_output_____## Overview

Today we will talk about how to confront models with experimental data. Does my model capture the behavior of my participant, or its neural activity? Does the data support my model against alternatives? Which component in the model is needed? Do parameters in the model vary systematically between two subject populations? We will cover the basic concepts and tools to address these questions, starting with a general overview in the intro. You will learn how to estimate the parameters of simple regression models from the data in Tutorials 1 & 2, and then how to estimate the uncertainty about these values in Tutorial 3. Then you will learn how to select from models of different complexity the one that best accounts for your data (Tutorial 4-6). The outro illustrates some of these techniques in real research examples.

Fitting and comparing models to data is really the bread of butter of data analysis in neuroscience. You have learned during Model Types Day (W1D1) about a whole zoo of different types of models that we’re interested in in neuroscience. So here you will learn about generic concepts and techniques that apply for fitting and comparing any type of models, which is arguably pretty useful! You will apply these tools again when dealing with GLMs (W1D4), latent models (W1D5), deep networks (W2D1), dynamical models (W2D2), decision models, reinforcement learning models… it’s everywhere! On top of this, we will cover linear regression models - the typical regression model when the dependent variable is continuous (e.g. BOLD activity) – and use it throughout the day, to illustrate the concepts and methods we learn. In the GLM day, you will see how to generalize regression models when the dependent variable is binary (e.g. choices) or an integer (e.g. spike counts).

Almost all statistical and data analysis methods rely either explicitly or implicitly on fitting some model to the data. The concepts and tools you will learn today are crucial to be able to test your hypothesis about how behavior or neural activity is formed. A typical way this is done is that you formulate a computational (stochastic) model that embodies your hypothesis, and then one or more control models. Then you fit each or your models to your experimental data, say the pattern of choices of one experimental subject, or the spiking activity from one recorded neuron. Simulating your fitted models allows validating that your model indeed captures the effects of interest in your data. Then you use model comparison techniques to tell which one or your main or control model(s) provides a better description of the data. Also, you could assess whether some parameter in your model changes between experimental conditions or subject populations._____no_output_____## Prerequisite knowledge

In the content and tutorials today, you will be using vectors and matrices (W0D3 T1/T2), probability distributions (W0D5 T1), and likelihoods (W0D5 T2).  Please review this material if necessary!_____no_output_____## Video_____no_output_____

# @markdown
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = "https://player.bilibili.com/player.html?bvid={0}&page={1}".format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id=f"BV1Wf4y1R75n", width=854, height=480, fs=1)
  print("Video available at https://www.bilibili.com/video/{0}".format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id=f"yTQNqYCvoZ0", width=854, height=480, fs=1, rel=0)
  print("Video available at https://youtube.com/watch?v=" + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

## Slides_____no_output_____

# @markdown
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/sqcvz/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)_____no_output_____


# Notebook from tp5uiuc/soft_systems_course
Path: lectures/03_cma/code/cmaes_answers_partially_filled.ipynb

# CMAES : Covariance Matrix Adaptation Evolutionary Strategy_____no_output_____Setup code and utility functions to plot and explore_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from matplotlib import cm
from mpl_toolkits.mplot3d import axes3d
from numpy.random import multivariate_normal
import copy

%matplotlib inline
%config InlineBackend.figure_format = 'retina'

try:
    import seaborn as sns
    sns.set_style("whitegrid")
    sns.set_context('talk')
    #sns.set(font_scale=1.4)
except ImportError:
    plt.style.use('seaborn-whitegrid')_____no_output_____def range_from_bounds(bounds, resolution):
    (minx,miny),(maxx,maxy) = bounds
    x_range = np.arange(minx, maxx, (maxx-minx)/resolution)
    y_range = np.arange(miny, maxy, (maxy-miny)/resolution)
    return x_range, y_range

def plot_problem_3d(problem, bounds, ax=None, resolution=100.,
                    cmap=cm.viridis_r, rstride=10, cstride=10,
                    linewidth=0.15, alpha=0.65):
    """Plots a given benchmark problem in 3D mesh."""

    x_range, y_range = range_from_bounds(bounds, resolution=resolution)

    X, Y = np.meshgrid(x_range, y_range)
    Z = problem(X,Y)

    if not ax:
        fig = plt.figure(figsize=(11,6))
        ax = fig.gca(projection='3d')

    cset = ax.plot_surface(X, Y, Z, cmap=cmap, rstride=rstride, cstride=cstride, linewidth=linewidth, alpha=alpha)_____no_output_____def plot_problem_contour(problem, bounds, optimum=None,
                          resolution=100., cmap=cm.viridis_r,
                          alpha=0.45, ax=None):
    """Plots a given benchmark problem as a countour."""
    x_range, y_range = range_from_bounds(bounds, resolution=resolution)

    X, Y = np.meshgrid(x_range, y_range)
    Z = problem(X,Y)

    if not ax:
        fig = plt.figure(figsize=(6,6))
        ax = fig.gca()
        ax.set_aspect('equal')
        ax.autoscale(tight=True)

    cset = ax.contourf(X, Y, Z, cmap=cmap, alpha=alpha)

    if optimum:
        ax.plot(optimum[0], optimum[1], 'bx', linewidth=4, markersize=15)_____no_output_____def plot_cov_ellipse(pos, cov, volume=.99, ax=None, fc='lightblue', ec='darkblue', alpha=1, lw=1):
    ''' Plots an ellipse that corresponds to a bivariate normal distribution.
    Adapted from http://www.nhsilbert.net/source/2014/06/bivariate-normal-ellipse-plotting-in-python/'''
    from scipy.stats import chi2
    from matplotlib.patches import Ellipse

    def eigsorted(cov):
        vals, vecs = np.linalg.eigh(cov)
        order = vals.argsort()[::-1]
        return vals[order], vecs[:,order]

    if ax is None:
        ax = plt.gca()

    vals, vecs = eigsorted(cov)
    theta = np.degrees(np.arctan2(*vecs[:,0][::-1]))

    kwrg = {'facecolor':fc, 'edgecolor':ec, 'alpha':alpha, 'linewidth':lw}

    # Width and height are "full" widths, not radius
    width, height = 2 * np.sqrt(chi2.ppf(volume,2)) * np.sqrt(vals)
    ellip = Ellipse(xy=pos, width=width, height=height, angle=theta, **kwrg)
    ax.add_artist(ellip)_____no_output_____

## Test functions

### Why benchmarks (test) functions?

In applied mathematics, [test functions](http://en.wikipedia.org/wiki/Test_functions_for_optimization), also known as artificial landscapes, are useful to evaluate characteristics of optimization algorithms, such as:

* Velocity of convergence.
* Precision.
* Robustness.
* General performance.

### [Bohachevsky benchmark problem](http://benchmarkfcns.xyz/benchmarkfcns/bohachevskyn2fcn.html)

$$\text{minimize } f(\mathbf{x}) = \sum_{i=1}^{N-1}(x_i^2 + 2x_{i+1}^2 - 0.3\cos(3\pi x_i) - 0.4\cos(4\pi x_{i+1}) + 0.7), \mathbf{x}\in \left[-100,100\right]^n,$$

> Optimum in $\mathbf{x}=\mathbf{0}$, $f(\mathbf{x})=0$._____no_output_____

def _shifted_bohachevsky_impl(x, y, shift_x, shift_y):
    return (x-shift_x)**2 + 2.0 * (y-shift_y)**2 - 0.3 * np.cos(3.0 * np.pi * (x - shift_x)) - 0.4 * np.cos(4.0 * np.pi* (y - shift_y)) + 0.7
# def bohachevsky(x,y):
#     return x**2 + 2.0 * y**2 - 0.3 * np.cos(3.0 * np.pi * x) - 0.4 * np.cos(4.0 * np.pi*x) + 0.7_____no_output_____from functools import partial
bohachevsky = partial(_shifted_bohachevsky_impl, shift_x = 0.0, shift_y = 0.0)
shifted_bohachevsky = partial(_shifted_bohachevsky_impl, shift_x = 2.0, shift_y = 2.0)_____no_output_____

### [Griewank benchmark problem](http://benchmarkfcns.xyz/benchmarkfcns/griewankfcn.html)

$$\text{minimize } f(\mathbf{x}) = f(x_1, ..., x_n) = 1 + \sum_{i=1}^{n} \frac{x_i^{2}}{4000} - \prod_{i=1}^{n}\cos\left(\frac{2 \cdot x_i}{\sqrt{i}}\right)$$

> Optimum in $\mathbf{x}=\mathbf{0}$, $f(\mathbf{x})=0$._____no_output_____

def _shifted_griewank_impl(x,y,shift_x, shift_y):
    return 1.0 + ((x-shift_x)**2 + (y-shift_y)**2) / 4000.0 - np.cos(2.0 * (x-shift_x)) * np.cos(2.0 * (y-shift_y) / np.sqrt(2.0))

# def griewank(x,y):
#     return 1.0 + (x**2 + y**2) / 4000.0 - np.cos(2.0 * x) * np.cos(2.0 * y / np.sqrt(2.0))_____no_output_____griewank = partial(_shifted_griewank_impl, shift_x = 0.0, shift_y = 0.0)
shifted_griewank = partial(_shifted_griewank_impl, shift_x = 2.0, shift_y = 2.0)_____no_output_____current_problem = bohachevsky_____no_output_____plot_problem_3d(current_problem, ((-10,-10), (10,10)))_____no_output_____

These problems has many local optima._____no_output_____

plot_problem_3d(current_problem, ((-2.5,-2.5), (2.5,2.5)))_____no_output_____ax = plt.figure(figsize=(8, 5)).gca()
plot_problem_contour(current_problem, ((-2.5,-2.5), (2.5,2.5)), optimum=(0,0), ax=ax)
ax.set_aspect('equal')_____no_output_____

## Optimizing test functions using CMA-ES

### CMA-ES features

* Adaptation of the covariance matrix amounts to learning a second order model of the underlying objective function.
* This is similar to the approximation of the inverse Hessian matrix in the Quasi-Newton method in classical optimization.
* In contrast to most classical methods, fewer assumptions on the nature of the underlying objective function are made.
* *Only the ranking between candidate solutions is exploited* for learning the sample distribution and neither derivatives nor even the function values themselves are required by the method.

## Let's code up CMA from scratch!_____no_output_____Here are the steps of CMA in chronological order :
$$
\newcommand{\gv}[1]{\ensuremath{\mbox{\boldmath$ #1 $}}}
\newcommand{\bv}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\newcommand{\order}[1]{\mathcal O \left( #1 \right)} % order of magnitude
$$

### Initialization
Set $ \mathbf{m} = \mathbf{0}, \mathbf{C} = \mathbf{I}, \sigma = 0.5, \mathbf{p}_c = \mathbf{0}, \mathbf{p}_{\sigma} = \mathbf{0} $

### Sampling
$$ \begin{aligned}
\mathbf{z}_{i} & \sim \mathcal{N}(\mathbf{0}, \mathbf{C}) \\
\mathbf{x}_{i} &= m+\sigma \mathbf{z}_{i}
\end{aligned} $$

### Selection and recombination
Sort the ppopulation by fitness to get $ \mu $  fit individuals
$$ \begin{aligned}
\langle\mathbf{z}\rangle_{w} &= \displaystyle\sum_{i=1}^{\mu} w_{i} \mathbf{z}_{i : \lambda} \\
\mathbf{m} &\longleftarrow \mathbf{m}+\sigma\langle\mathbf{z}\rangle_{w}
\end{aligned} $$

### Step size update
$$ \begin{aligned}
\mathbf{p}_{\sigma} &\longleftarrow\left(1-c_{\sigma}\right)
\mathbf{p}_{\sigma}+\sqrt{1-\left(1-c_{\sigma}\right)^{2}}
\sqrt{\frac{1}{\sum_{i=1}^{\mu} w_{i}^{2}}}
\mathbf{C}^{-\frac{1}{2}}\langle\mathbf{z}\rangle_{w} \\
\sigma &\longleftarrow \sigma
\exp{\left(\frac{c_{\sigma}}{d_{\sigma}}\left(\frac{\left\|p_{\sigma}\right\|}{E\|\mathcal{N}(\mathbf{0},
\mathbf{I})\|}-1\right)\right)} \\
\end{aligned} $$

### Covariance Matrix update
$$ \begin{aligned}
\mathbf{p}_{c} &\longleftarrow \left(1-c_{c}\right)
\mathbf{p}_{c}+\sqrt{1-\left(1-c_{c}\right)^{2}} \sqrt{\frac{1}{\sum_{i=1}^{\mu}
w_{i}^{2}}}\langle\mathbf{z}\rangle_{w} \\
\mathbf{Z} &= \sum_{i=1}^{\mu} w_{i} \mathbf{z}_{i : \lambda} \mathbf{z}_{i :
\lambda}^{T} \\
\mu_{c o v}&=\sqrt{\frac{1}{\sum_{i=1}^{\mu} w_{i}^{2}}} \\
\mathbf{C} &\longleftarrow\left(1-c_{c o v}\right) \mathbf{C}+c_{c o v}
\frac{1}{\mu_{c o v}} \mathbf{p}_{c} \mathbf{p}_{c}^{T}+c_{c o
v}\left(1-\frac{1}{\mu_{c o v}}\right) \mathbf{Z}
\end{aligned} $$

Some considerations:
 - `centroid` and `mean` are interchangeable.
 - `chi_N` is the expectation for the length of a random vector sampled from a multivariate normal distribution with $\mathbf{C} = \mathbf{I}$, and is used in the step-size update above. It can be analytically computed as $ \approx \sqrt{n} \left( 1 - \dfrac{1}{4n} + \dfrac{1}{21n^2} \right)$
 - `mu_eff` $ \mu_{\textrm{eff}} = \left(\displaystyle\sum_{i=1}^{\mu} w_{i}^{2}\right)^{-1} $ is the variance effective selection mass for the mean, as used in the CMA tutorial. Thus $\mu_{\textrm{cov}} = \sqrt{\mu_{\textrm{eff}}}$
_____no_output_____

class CMAES:
    """Naive CMA implementation"""

    def __init__(self, initial_mean, sigma, popsize, **kwargs):
        """Please do all the initialization. The reserve space and
        code for collecting the statistics are already provided."""
        self.dim = initial_mean.shape[0]

        # Things that evolve : centroid, sigma, paths etc.
        self.centroid = initial_mean
        self.sigma = sigma

        # pc is the path taken by the covariance matrix
        self.pc = """fill"""

        # ps is the path taken by sigma / step-size updates
        self.ps = np.zeros((self.dim, ))

        self.C = np.eye(self.dim)
        self.diagD, self.B = np.linalg.eigh(self.C)

        # Population size etc.
        self.popsize = popsize
        self.mu = int((popsize / 2))

        # Update weights
        self.weights = np.arange(1, self.mu)[::-1]
        self.weights = self.weights / np.sum(self.weights)

        # Utility variables
        self.dim = initial_mean.shape[0]

        # Expectation of a normal distribution
        self.chiN = np.sqrt(self.dim) * (1.0 - 0.25 / self.dim + 1.0/(21.0 * self.dim**2))
        self.mueff = 1.0 / np.sum(self.weights**2)
        self.mucov = np.sqrt(self.mueff)
        self.generations = 0

        # Options

        # Sigma adaptation
        # cs is short for c_sigma
        self.cs = (self.mucov ** 2 + 2) / ((self.mucov ** 2 + 5) + self.dim)
        # ds is short for d_sigma
        factor = np.sqrt((self.mucov ** 2 - 1.0) / (self.dim + 1.0))
        self.ds = 1.0 + 2.0 * max(0.0, factor - 1) + self.cs

        # Covariance adaptation
        self.cc = """fill"""
        self.ccov = """fill"""
        # If implementing the latest version of CMA according to the tutorial,
        # these parameters can be useful, if not that avoid
        self.ccov1 = 0.0
        self.ccovmu = 0.0

        ### Asserts to guide you on your paths
        #                    .--.
        #  ::\`--._,'.::.`._.--'/::     Do or do not.
        #  ::::.  ` __::__ '  .::::    There is no try.
        #  ::::::-:.`'..`'.:-::::::
        #  ::::::::\ `--' /::::::::              -Yoda

        assert self.dim == 2, "We are dealing with a two-dimensional problem only"
        assert self.centroid.shape == (2,), "Centroid shape is incorrect, did you tranpose it by mistake?"
        assert self.sigma > 0.0, "Sigma is not a non-zero positive number!"
        assert self.pc.shape == (2, ), "pc shape is incorrect, did you tranpose it by mistake?"
        assert self.ps.shape == (2, ), "ps shape is incorrect, did you tranpose it by mistake?"
        assert self.C.shape == (2, 2), "C's shape is incorrect, remember C is a matrix!"
        assert type(self.popsize) == int, "Population size not an integer"
        assert self.popsize > 0 , "Population size is negative!"
        assert self.popsize > 2 , "Too little population size, make it >2"

        # Collect useful statistics
        self.stats_centroids = []
        self.stats_new_centroids = []
        self.stats_covs = []
        self.stats_new_covs = []
        self.stats_offspring = []
        self.stats_offspring_weights = []
        self.stats_ps = []

    def run(self, problem):
        while (# fill in your termination criterion here):
            # Sample the population here!
            # Its convenient to do it as a list of members
            # z_i
            population = list(np.random.multivariate_normal(np.array([0.0, 0.0]), self.C, (self.popsize, )))
            # self.C
            # self.centroid

            # Pass the population to update, which computes all new parameters
            # while sorting the populatoin
            self.update(problem, population)

            # increment generation counter
            self.generations += 1
        else:
            # returns the best individual at the last generation
            return population[0]

    def update(self, problem, population):
        """Update the current covariance matrix strategy from the
        *population*.

        :param population: A list of individuals from which to update the
                           parameters.
        """
        # -- store current state of the algorithm
        self.stats_centroids.append(copy.deepcopy(self.centroid))
        self.stats_covs.append(copy.deepcopy(self.C))

        # Sort the population here and work with only the sorted population
        x_i = self.centroid + self.sigma * np.array(population)
        """FILL : Python code to sort population goes here"""

        # new_population = sorted(x_i, key = problem)
        f_i = problem(x_i)
        idx = np.argsort(f_i)
        population = np.array(population)[idx]

        best_population = population[:self.mu]

        # -- store sorted offspring
        self.stats_offspring.append(copy.deepcopy(population))

        # Store old centroid in-case
        old_centroid = self.centroid
        # Update centroid to self.centroid here
        z_w = np.dot(self.weights, best_population)
        self.centroid = self.centroid + self.sigma * z_w

        # -- store new centroid
        self.stats_new_centroids.append(copy.deepcopy(self.centroid))

        # Cumulation : update evolution path
        # Remember to use self.B, self.diagD wihch we store later
        # See line 142-145
        coeff = np.sqrt(1.0 - (1.0 - self.cs)**2)
        invsqrtC = self.B @ np.diag(1.0 / self.diagD) @ self.B.T
        self.ps = (1 - self.cs) * self.ps + coeff * self.mucov * (invsqrtC @ z_w)

        # -- store new evol path
        self.stats_ps.append(copy.deepcopy(self.ps))

        # Cumulation : update evolution path for centroid
        factor = np.sqrt(1.0 - ((1.0 - self.cc))**2)
        self.pc = (1.0 - self.cc) * self.pc + factor * self.mucov * z_w

        # Update covariance matrix
        Z = np.dot(self.weights, np.outer(z_i, z_i))

        self.C = """FILL : Code to calculate new covariance matrix """

        # -- store new covs
        self.stats_new_covs.append(copy.deepcopy(self.C))

        # Update new sigma in-place, can be done before too
        self.sigma *= np.exp(self.cs / self.ds * (np.linalg.norm(self.ps) / chiN) - 1.0)

        # Get the eigen decomposition for the covariance matrix to calculate inverse
        diagD_squared, self.B = np.linalg.eigh(self.C)

        self.diagD = np.sqrt(diagD_squared)
        self.B = self.B

    def reset(self):
        """Clears everything to rerun the problem"""
        pass_____no_output_____initial_centroid = np.random.randn(2, )
cma_es = CMAES(initial_centroid, 0.2, 10)_____no_output_____cma_es.run(current_problem)_____no_output_____

### Visualizing CMA-ES progress _____no_output_____First some setup code. This visualizes the progress of CMA based on the data we recorded in the class above and plots it in the objective function manifold._____no_output_____

normalizer = colors.Normalize(vmin=np.min(cma_es.weights), vmax=np.max(cma_es.weights))
sm = cm.ScalarMappable(norm=normalizer, cmap=plt.get_cmap('gray'))_____no_output_____from matplotlib import animation
from IPython.display import HTML_____no_output_____def animate_cma_es(gen):
    ax.cla()
    plot_problem_contour(current_problem, ((-11,-11), (11,11)), optimum=(0,0), ax=ax)

    plot_cov_ellipse(cma_es.stats_centroids[gen], cma_es.stats_covs[gen], volume=0.99, alpha=0.29,
                     fc='red', ec='darkred',
                     ax=ax)
    ax.plot(cma_es.stats_centroids[gen][0], cma_es.stats_centroids[gen][1], 'ro', markeredgecolor = 'none', ms=10)

    plot_cov_ellipse(cma_es.stats_new_centroids[gen], cma_es.stats_new_covs[gen], volume=0.99,
                     alpha=0.29, fc='green', ec='darkgreen', ax=ax)
    ax.plot(cma_es.stats_new_centroids[gen][0], cma_es.stats_new_centroids[gen][1], 'go', markeredgecolor = 'none', ms=10)

    for i in range(gen+1):
        if i == 0:
            ax.plot((0,cma_es.stats_ps[i][0]),
                     (0,cma_es.stats_ps[i][1]), 'b--')
        else:
            ax.plot((cma_es.stats_ps[i-1][0],cma_es.stats_ps[i][0]),
                     (cma_es.stats_ps[i-1][1],cma_es.stats_ps[i][1]),'b--')

    for i,ind in enumerate(cma_es.stats_offspring[gen]):
        if i < len(cma_es.weights):
            color = sm.to_rgba(cma_es.weights[i])
        else:
            color= sm.to_rgba(normalizer.vmin)
        ax.plot(ind[0], ind[1], 'o', color = color, ms=5, markeredgecolor = 'none')

    ax.set_ylim((-10,10))
    ax.set_xlim((-10,10))
    ax.set_title('$generation=$' +str(gen))
    return []_____no_output_____fig = plt.figure(figsize=(10,10))
ax = fig.gca()
anim = animation.FuncAnimation(fig, animate_cma_es, frames=cma_es.generations, interval=300, blit=True)
plt.close()_____no_output_____

In the animation below :
* Current centroid and covariance: **red**.
* Updated centroid and covariance: **green**.
* Sampled individuals: **shades of gray representing their corresponding weight**. (White is best)
* Evolution path: **blue line starting in (0,0)**. _____no_output_____

HTML(anim.to_html5_video())_____no_output_____