# Notebook from sylvain2002/CBM101
Path: D_Network_analysis/4_Connectome_C_elegans.ipynb

# Predicting behaviour of C. elegans from network principles _____no_output_____** WORK IN PROGRESS **

Regarding the brain as a network is also a perfectly valid application of graph theory. This idea of modelling the brain using network science is far from new, but having the computational tools to do anything interesting about it is. This notebook deals with one of such application on the much studied nematode C. *elegans*. The adult worm has only 959 cells. It has 302 neurons, 282 of which are play a role in controlling the body movement (the remaining 20 control the pharynx. A number of studies have mapped this connection diagram (*connectome*), and a complete map exists, freely available online.

A study in Nature (Yan et al., 2017) reveals how graph theory was applied to reveal a neuron, hitherto unknown to its role in the locomotion of C. *elegans*, as well as identifying previously established results. Here we will reproduce parts of the study.

Attempting to reproduce parts of https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5710776/

more:

http://barabasi.com/f/919.pdf
https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1001066

https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1995545/

data for the experiment (video): https://figshare.com/s/72716a92be1ab0f1e1d4_____no_output_____As usual, we start by importing necessary libraries._____no_output_____

import numpy as np
import pandas as pd
from pathlib import Path
import networkx as nx
import seaborn as sns

from matplotlib import pyplot as plt
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")_____no_output_____

Next we locate and import the connectome data set._____no_output_____

data_dir = Path.cwd() / 'data'
print(data_dir.exists())

data_fn = data_dir / 'NeuronConnect_wormatlas.xls'
print(data_fn.exists())True
True
data = pd.read_excel(data_fn)
data.head()_____no_output_____

The data has four fields. **Neuron 1** is the pre-synaptic neuron and **Neuron 2** is the post-synaptic. **Type** is the type of synapse, and **Nbr** is the number of synapses from Neuron1 to Neuron2. We are not interested in the Type column._____no_output_____#### Ex 1: a) remove the *Type* column. _____no_output_____

data.drop(['Type'], axis=1, inplace=True)
#comment: the "inplace=True" ensures that the data is dropped in place, without needing to write something
#along the lines of data = data.drop(...) _____no_output_____data.head()_____no_output_____

#### The current data only holds information about the neurons. We also need to know the connections to the muscles, as these are the cells that control motion._____no_output_____

muscles_fn = data_dir / 'Neuron_to_muscles_wormatlas.xls'
print(muscles_fn.exists())True

#### Ex. Read the excel file into a pandas dataframe called 'muscles'_____no_output_____

muscles = pd.read_excel(muscles_fn)_____no_output_____muscles.head()_____no_output_____muscles.describe()_____no_output_____

We don't care about the column "Landmark Position"._____no_output_____

muscles.drop(columns=['Landmark Position'], inplace=True)
muscles.head()_____no_output_____

As seen above, the data contains information about other connections too, like input from sensory cells, and muscle cells of no interest. We wish to only keep body wall muscles (those involved in controlling locomotion)._____no_output_____

## only keep body wall muscles - the ones with name prefixes MDR/L or MVR/L.
mscl = muscles.loc[:,"Landmark"].str.match(pat = "^M[DV][LR]")

#comment: this matching algorithm is known as a regular expression. This one lets you match for a string
#beginning with and M, followed by a D or V, followed by a L or R. In other words it will match anything
#starting with "MDL", "MDR", "MVL" or "MVR"_____no_output_____# number of body wall muscles
muscles[mscl].loc[:,"Landmark"].unique().shape_____no_output_____

#### Ex: Remove the rows in `muscles` which are not body wall muscles._____no_output_____

muscles = muscles[mscl]_____no_output_____

We chain the two dataframes together._____no_output_____

muscles.rename(columns = dict(zip(muscles.columns, data.columns)), inplace=True)
data = data.append(muscles, ignore_index=True)
data.head()_____no_output_____

Like they did in the study, we don't care about the number of synapses per connection, but signify each connection either as a **1** or a **0**. Secondly, we also don't care about the Type column. This is the last necessary step to complete our preprocessing. _____no_output_____#### Ex 2. Set all values in *Nbr* to 1.  Hint: when modifying a dataframe inplace, use the `.loc` method._____no_output_____

data.loc[:,'Nbr'] = np.ones(shape=(data.shape[0]),) # set all repeated synapses to 1

#comment: np.ones creates a vector of 1s, with a specified length (we set this to the size of the data frame)._____no_output_____data_____no_output_____mscl_____no_output_____

Networkx can read this format as into a Graph object using `from_pandas_edgelist`:_____no_output_____

# create graph object
G = nx.from_pandas_edgelist(data, 'Neuron 1', 'Neuron 2', ['Nbr'])_____no_output_____# remove dummy node NMJ; neuromuscular junction
G.remove_node("NMJ")
print(G.number_of_nodes())377

Currently, the network is disconnected (some subnetworks are unreachable from eachother), probably because of incomplete data. We can proceed by only keeping the largest one._____no_output_____

nx.connected.number_connected_components(G)_____no_output_____largest_cc = max(nx.connected_components(G), key=len)
print(len(largest_cc))
G = G.subgraph(largest_cc).copy() #our new graph374
G.has_node('PDB')_____no_output_____

The nodes sorted by degree indicate the more influential nodes (high degree) versus less influential._____no_output_____

sorted(G.degree(), key = lambda x: x[1], reverse=True)_____no_output_____

#### Ex 2. Make a list of the node degrees in G. Convert it to a numpy array._____no_output_____

node_degrees = np.array([e for v,e in list(G.degree())])_____no_output_____

#### Ex. Plot the network, set the node size to be proportional to the node degrees._____no_output_____

from matplotlib import cm_____no_output_____cm.Set2_____no_output_____pos = nx.kamada_kawai_layout(G)

figsize = (13,13)
plt.figure(figsize=figsize)

nx.draw_networkx_nodes(G, pos, node_color=-node_degrees,
                       node_size=node_degrees*13, cmap=cm.Set2)

nx.draw_networkx_edges(G, pos, edge_color='gray')

nx.draw_networkx_labels(G, pos)

plt.show()_____no_output_____

So already, simply by computing the node degree, we are directed to the most influential nodes. How do we know these are more important? Previous experiments have shown that the four top nodes, AVAR, AVAL, AVBR, AVBL, were already experimentally known to single-handedly cause motor dysfunction through laser ablation. This is not surprising, considering how many connections that are lost by removing one of them.

The neurons AVDR, PVCR, PVCL, AVDL which are ranked at 6th, 8th, 9th and 11th in the node degree hierarchy have also been expermentally validated as being crucial (by deactivating these neurons, the worm would have locomotive defects).
_____no_output_____

sorted(G.degree(), key = lambda x: x[1], reverse=True)[:11]_____no_output_____

**Conclusion**: Hubs are disproportionally more important for global network functioning._____no_output_____Thus we know that simple, straight forward network analysis can take us surprisingly far. However, we shouldn't stop here. First of all, there are many cases of high-degree nodes which *don't* exert any significant power over motion patterns. Secondly, there exists a range of low-degree which *do* play an important role. Identified by Barabasi et al., was the neurons DD04, DD05 and PDB.

Like the researchers in the original paper, we will take the analysis a step farther.

They identified another neuron with a key role in motor control, which was later confirmed experimentally. We will replicate the result here. _____no_output_____## Network architecture: optimizing signal transmission efficiency_____no_output_____Neuronal signal transmission is expensive. It requires more energy
to send a signal from A to B through multiple neurons than through
a single synapse (axonal transmission is less energy demanding). With that assumption in mind we expect evolution to have optimized the wiring costs in the brains of animals, like C elegans.
in other words: path length should be as small as possible. Lets test it!

(Watts and Strogatz, 1998)_____no_output_____To address this question we need to use a topological metric that encapsulates this property of having a minimal number of synapses between any 2 neurons.

**Characteristic path length :** the average shortest path between a node and all the other nodes in the network._____no_output_____

from networkx.convert_matrix import to_numpy_matrix_____no_output_____matrix = to_numpy_matrix(G)_____no_output_____plt.imshow(matrix)_____no_output_____muscles.loc[:,'Neuron 2'].values[:10]_____no_output_____#G.remove_nodes_from(muscles.loc[:,'Neuron 2'].values) # should you remove muscles?_____no_output_____L = nx.average_shortest_path_length(G)
C = nx.algorithms.cluster.average_clustering(G)
print(L, C)2.435625692993992 0.3371339990890196

The average number of "jumps" from one node to another is just below 3. This is surprisingly small, considering the network is around 300 neurons!_____no_output_____Let's compare it to a random graph of the same parameters (n,m): random graphs are known to have a very low hop count. (Random graphs are covered in another notebook)._____no_output_____

def get_nm(G):
    return G.number_of_nodes(), G.number_of_edges()_____no_output_____n,m = get_nm(G)

print(n, m)
R = nx.gnm_random_graph(n,m)
#R = nx.barabasi_albert_graph(n,7)

n_rand, m_rand = get_nm(R)
print(n_rand, m_rand)

plt.imshow(to_numpy_matrix(R))279 2290
279 2290
L_random = nx.average_shortest_path_length(R)
C_random = nx.algorithms.cluster.average_clustering(R)
print(L_random, C_random)2.305510430365385 0.06257181625937684

**Conclusion**: The connectome has an average path length almost as low as a random network (minimal). This is what we had expected, testifying to a near-optimal solution of reducing wiring cost. But why then, might you ask, wouldn't it be better for evolution to have gone for a random topology, to get the path length even a little lower?

The answer lies in the clustering coefficient, which is vastly higher in the C. elegans connectome. What is the meaning of the clustering coefficient? Does it have an evolutionary reason? Sporns (2004) argues that clustering facilitates segregation and thus specialization in subnetworks of the brain. Consider the primary visual cortex: it makes sense for it to have a higher intra-connectivity than inter-connectivity to any other arbitrary region, so it can focus on performing the task of early visual processing.

This combination of a low L and high C is the characteristic trait of a small-world network. The small-world network can thus be seen as a trade-off between high communication efficiency (L) and high specialization (C)._____no_output_____

# Perhaps add a point about robustness to percolation?_____no_output_____
#add network metrics, like pagerank, for exploratory fun
_____no_output_____

# Control theory (section not finished)
The research is rooted in control theory: based on the structural connectivity, how can one signal control the output? Can different outputs be controlled independently? By outputs here, we mean muscle cells, the actual effectors of behaviour. Each muscle cell is innervated by motor neuron - the interface from neuron and mucle. Just from the definitions it is obvious that obliterating a motor neuron has a crucial impact on motor function.

A basic result in control theory is that using k control units, at most k output units can be independently controllable. So for instance, if we wish to control 3 muscle cells through the bottleneck of 2 motor neurons, we can only control 2 of the muscles independently. (use figure for this).

This becomes important as each muscle needs to be innervated by motor neurons._____no_output_____To copmute this we need to introduce some values.

First, M is the number of control targets (muscles). N is the number of neurons (N). The connection diagram of the neurons and muscles is an N+M x N+M matrix, **A**. **B** is a N+MxS matrix signifying which nodes receive external control signals (e.g. sensory signals). **C** is a matrix, where each row signifies which units we attempt to control (i.e. the muscles).  _____no_output_____Kalman's criterion is a theoretical prerequisite for a system to be controllable:
$$ rank [CB, CAB, CA^2B, ..., CA^{N+M+1}B] = M$$

A problem with computing this is that there exist an incomprehensible amount of configurations for the edge weights, and we would need to test all of them. Instead the researchers used *structural controlability*, which is only dependent on the connectome *structure*, without regarding its exact synapse weights or the input signals._____no_output_____#### Simple example

we have N=2 neurons and M=2 muscles. We name the cells (neurons and muscles combined) from 1 to 4 (which is assigned as 1, which is 2... is not important). The connectivity diagram is represented as a matrix A, below. B is the matrix of which cells receive external input (i.e. sensory input). C is the matrix over nodes we wish to control (basically just 1 for muscle cells and 0 for neurons)._____no_output_____

# you are not expected to understand this code, but it simply checks whether the Kalman criterion is held.
def is_kalman(kal, M, random_state=1):
    """Whether kalman condition is satisfied.
    Non-zero weights are randomly set to avoid a Lebesque measure of zero."""
    kal_rnd = kal*np.random.rand(*kal.shape)
    rk = np.linalg.matrix_rank(kal*kal_rnd)
    print(f"Rank = {rk} \t M = {M}")
    return(rk == M)

def kalman(A,B,C,M,N):
    """produces the matrix [CB, CAB, CA^2B....CA^(M+N-1)B]"""
    ll = []
    for k in range(M+N):
        ll.append(_kalman(A,B,C,k))
    return np.concatenate(ll, axis=1)

def _kalman(A,B,C,k):
    """Helper function, computes matrix product C A^k B"""
    if k==0: return C.dot(B)
    return _kalman(A, B, C.dot(A), k-1)

rnk = np.linalg.matrix_rank_____no_output_____

The matrix entries are set to 1 just to signify a connection, but the actual weight is not necessarily 1 (in fact that is highly unlikely. _____no_output_____

A = np.array([[0,0,0,0],
              [1,0,0,0],
              [1,0,0,0],
              [0,0,1,0]])

B = np.array([[1,0,0,0]]).T

C = np.array([[0,1,0,0],
             [0,0,0,1]])

M = C.shape[0]
N = A.shape[0] - M
print(M, N)2 2
[print(m, end='\n\n') for m in [A,B,C]][[0 0 0 0]
 [1 0 0 0]
 [1 0 0 0]
 [0 0 1 0]]

[[1]
 [0]
 [0]
 [0]]

[[0 1 0 0]
 [0 0 0 1]]

kal = kalman(A,B,C,M,N)
print(kal)
is_kalman(kal, M)[[0 1 0 0]
 [0 0 1 0]]
Rank = 2 	 M = 2

In this case we change the desired control units from 2 and 4 to 2 and 3._____no_output_____

# another case
C = np.array([[0,1,0,0],
              [0,0,1,0]])

kal = kalman(A,B,C,M,N)
print(kal)
is_kalman(kal,M)[[0 1 0 0]
 [0 1 0 0]]
Rank = 1 	 M = 2
A = np.array([[0,0,0],
              [1,0,1],
              [1,1,0]])
B = np.array([[1,0,0]]).T
C = np.array([[0,1,0],
             [0,0,1]])

M = C.shape[0]
N = A.shape[0] - M

kal = kalman(A,B,C,M,N)
print(kal)
is_kalman(kal, M)[[0 1 1]
 [0 1 1]]
Rank = 2 	 M = 2
kalman(A,B,C,M,N)_____no_output_____# example from Liu et al (2011) https://www.nature.com/articles/nature10011.pdf
A = np.array([[0,0,0,0],
              [1,0,0,0],
              [1,0,0,1],
              [1,0,0,0]])

B = np.array([[1,0,0,0],
             [0,1,0,0]]).T

C = np.eye(*A.shape)

M = C.shape[0]; N=A.shape[0]-M
print(M, N)4 0
kal = kalman(A,B,C,M,N)
print(kal)[[1. 0. 0. 0. 0. 0. 0. 0.]
 [0. 1. 1. 0. 0. 0. 0. 0.]
 [0. 0. 1. 0. 1. 0. 0. 0.]
 [0. 0. 1. 0. 0. 0. 0. 0.]]
is_kalman(kal, M)Rank = 4 	 M = 4
nx.algorithms.bipartite.maximum_matching(G)_____no_output_____


# Notebook from YosefLab/scvi_tutorials
Path: model_user_guide.ipynb

# Constructing a high-level model_____no_output_____

!pip install --quiet scvi-colab
from scvi_colab import install
install()_____no_output_____import numpy as np
import scvi
from scvi import REGISTRY_KEYS
from scvi.data import AnnDataManager
from scvi.data.fields import (
    LayerField,
    CategoricalObsField,
    NumericalObsField,
    CategoricalJointObsField,
    NumericalJointObsField,
)
import torchGlobal seed set to 0

At this point we have covered

1. Data registration via `setup_anndata` and dataloaders via `AnnDataLoader`
2. Building a probabilistic model by subclassing `BaseModuleClass`

In this tutorial, we will cover the highest-level classes in `scvi-tools`: the model classes. The main purpose of these classes (e.g., `scvi.model.SCVI`) is to wrap the actions of module instantiation, training, and subsequent posterior queries of our module into a convenient interface. These model classes are the fundamental objects driving scientific analysis of data with `scvi-tools`. Out of convention, we will refer to these objects as "models" and the lower-level objects presented in the previous tutorial as "modules".

## A simple model class
Here we will walkthrough an example of building the `scvi.model.SCVI` class. We will progressively add functionality to the class.
_____no_output_____
### Sketch of `BaseModelClass`

Let us start by providing a high level overview of `BaseModelClass` that we will inherit. Note that this is pseudocode to provide intuition. We see that `BaseModelClass` contains some unverisally applicable methods, and some private methods (conventionally starting with `_` in Python) that will become useful after training the model._____no_output_____```python
class MyModel(UnsupervisedTrainingMixin, BaseModelClass)

    def __init__(self, adata):
        # sets some basic attributes like is_trained_
        # record the setup_dict registered in the adata
        self.adata = adata
        self.scvi_setup_dict_ = adata.uns["_scvi"]
        self.summary_stats = self.scvi_setup_dict_["summary_stats"]

    def _validate_anndata(self, adata):
        # check that anndata is equivalent by comparing
        # to the initial setup_dict

    def _make_dataloader(adata):
        # return a dataloader to iterate over adata

    def train(...):
        # Universal train method provided by UnsupservisedTrainingMixin
        # BaseModelClass does not come with train
        # In general train methods are straightforward to compose manually

    def save(...):
        # universal save method
        # saves modules, anndata setup dict, and attributes ending with _

    def load(...):
        # universal load method

    @classmethod
    def setup_anndata(cls, adata, ...)
        # anndata registration method
```_____no_output_____### Baseline version of `SCVI` class

Let's now create the simplest possible version of the `SCVI` class. We inherit the `BaseModelClass`, and write our `__init__` method.

We take care to do the following:

1. Set the `module` attribute to be equal to our `VAE` module, which here is the torch-level version of scVI.
2. Add a `_model_summary_string` attr, which will be used as a representation for the model.
3. Run `self.init_params_ = self._get_init_params(locals())`, which stores the arguments used to initialize the model, facilitating saving/loading of the model later.

To initialize the `VAE`, we can use the information in `self.summary_stats`, which is information that was stored in the anndata object at `setup_anndata()` time. In this example, we have only exposed `n_latent` to users through `SCVI`. In practice, we try to expose only the most relevant parameters, as all other parameters can be accessed by passing `model_kwargs`.

Finally, we write the `setup_anndata` class method, which is used to register the appropriate matrices inside the anndata that we will use to load data into the model. This method uses the `AnnDataManager` class to orchestrate the data registration process. More details about the `AnnDataManager` can be learned in our data handling tutorial._____no_output_____

from typing import Optional

from anndata import AnnData
from scvi.module import VAE
from scvi.model.base import BaseModelClass, UnsupervisedTrainingMixin

class SCVI(UnsupervisedTrainingMixin, BaseModelClass):
    """
    single-cell Variational Inference [Lopez18]_.
    """

    def __init__(
        self,
        adata: AnnData,
        n_latent: int = 10,
        **model_kwargs,
    ):
        super(SCVI, self).__init__(adata)

        self.module = VAE(
            n_input=self.summary_stats["n_vars"],
            n_batch=self.summary_stats["n_batch"],
            n_latent=n_latent,
            **model_kwargs,
        )
        self._model_summary_string = (
            "SCVI Model with the following params: \nn_latent: {}"
        ).format(
            n_latent,
        )
        self.init_params_ = self._get_init_params(locals())

    @classmethod
    def setup_anndata(
        cls,
        adata: AnnData,
        batch_key: Optional[str] = None,
        layer: Optional[str] = None,
        **kwargs,
    ) -> Optional[AnnData]:
        setup_method_args = cls._get_setup_method_args(**locals())
        anndata_fields = [
            LayerField(REGISTRY_KEYS.X_KEY, layer, is_count_data=True),
            CategoricalObsField(REGISTRY_KEYS.BATCH_KEY, batch_key),
            # Dummy fields required for VAE class.
            CategoricalObsField(REGISTRY_KEYS.LABELS_KEY, None),
            NumericalObsField(
                REGISTRY_KEYS.SIZE_FACTOR_KEY, None, required=False
            ),
            CategoricalJointObsField(
                REGISTRY_KEYS.CAT_COVS_KEY, None
            ),
            NumericalJointObsField(
                REGISTRY_KEYS.CONT_COVS_KEY, None
            ),
        ]
        adata_manager = AnnDataManager(
            fields=anndata_fields, setup_method_args=setup_method_args
        )
        adata_manager.register_fields(adata, **kwargs)
        cls.register_manager(adata_manager)_____no_output_____

Now we explore what we can and cannot do with this model. Let's get some data and initialize a `SCVI` instance. Of note, for testing purposes we like to use `scvi.data.synthetic_iid()` which returns a simple, small anndata object that was already run through `setup_anndata()`._____no_output_____

adata = scvi.data.synthetic_iid()
adata_____no_output_____

Above we saw in the `setup_anndata()` implementation that we ended the function with `cls.register_manager(adata_manager)`. This function stores the newly created `AnnDataManager` instance in a class-specific dictionary called `_setup_adata_manager_store`. Specifically, this maps from UUIDs (specific to each `AnnData` object; stored on `adata.uns["_scvi_uuid"]`) to `AnnDataManager` instances instantiated by that class's `setup_anndata()` function.

On model initialization, the model instance retrieves the `AnnDataManager` object specific to the passed in `adata`._____no_output_____

SCVI.setup_anndata(adata, batch_key="batch")
print(f"adata UUID (assigned by setup_anndata): {adata.uns['_scvi_uuid']}")
print(f"AnnDataManager: {SCVI._setup_adata_manager_store[adata.uns['_scvi_uuid']]}")
model = SCVI(adata)
modeladata UUID (assigned by setup_anndata): ae1459ee-966f-4f25-9ca0-48ae640be873
AnnDataManager: <scvi.data._manager.AnnDataManager object at 0x28e73b610>

### More `AnnDataManager` Details_____no_output_____The `AnnDataManager` class stores state on data registered with scvi-tools. Since each manager is specific to a single `AnnData`, each model instance has an `AnnDataManager` instance for every `AnnData` object it has interfaced with. In addition to `setup_anndata()`, new `AnnDataManager` objects are created via the `_validate_anndata()` method when called on new `AnnData` objects (not the `AnnData` the model instance was initialized with). `_validate_anndata()` should be called in any method that references data on the `AnnData` object, via the scvi-tools data handling strategy (e.g. `get_latent_representation()`). Any instance-specific `AnnDataManager` objects are stored in a separate class-specific manager store called `_per_instance_manager_store`, which maps model instance UUIDs (assigned on initialization) and `AnnData` UUIDs to `AnnDataManager` instances. This avoids the issue of incorrect `AnnDataManager` retrieval when working with two model instances working over the same `AnnData` object._____no_output_____

print(f"model instance UUID: {model.id}")
print(f"adata UUID: {adata.uns['_scvi_uuid']}")
print(f"AnnDataManager for adata: {SCVI._per_instance_manager_store[model.id][adata.uns['_scvi_uuid']]}") # { model instance UUID: { adata UUID: AnnDataManager } }model instance UUID: 7fdfe5d2-601a-492a-b9fa-9048a23c954b
adata UUID: ae1459ee-966f-4f25-9ca0-48ae640be873
AnnDataManager for adata: <scvi.data._manager.AnnDataManager object at 0x28e73b610>
adata2 = scvi.data.synthetic_iid()
model._validate_anndata(adata2)INFO     Input AnnData not setup with scvi-tools. attempting to transfer AnnData setup
print(f"adata2 UUID: {adata.uns['_scvi_uuid']}")
print(f"Model instance specific manager store: {SCVI._per_instance_manager_store[model.id]}") # { model instance UUID: { adata UUID: AnnDataManager } }adata2 UUID: ae1459ee-966f-4f25-9ca0-48ae640be873
Model instance specific manager store: {'ae1459ee-966f-4f25-9ca0-48ae640be873': <scvi.data._manager.AnnDataManager object at 0x28e73b610>, '42b64083-6b32-449e-9da3-5ccaca06cf04': <scvi.data._manager.AnnDataManager object at 0x28e73b9d0>}

Additionally, the data registration process can modify or add data on the `AnnData` object directly. As a result, if calls between two models are interleaved, it is possible that we refer to fields created by another model instance's data registration incorrectly. In order to avoid this, `_validate_anndata()` additionally checks the `AnnData` object for an `AnnDataManager`-specific UUID stored in `adata.uns['_scvi_manager_uuid']`. If this UUID is inconsistent with the `AnnDataManager` fetched from the manager store, this means the data registration must be replayed on the `AnnData` object before referencing any data on the `AnnData`. This is automatically done in `_validate_anndata()`.

As a result, we can interleave method calls on two model instances without worrying about this clobbering issue._____no_output_____

SCVI.setup_anndata(adata, batch_key=None) # No batch correction.
model2 = SCVI(adata)
print(f"Manager UUID: {model2.adata_manager.id}")
print(f"Last setup with manager UUID: {adata.uns['_scvi_manager_uuid']}")
print(f"Encoded batch obs field: {adata.obs['_scvi_batch']}")Manager UUID: 038cead6-6cc4-4fcb-93c2-845fb66eec7c
Last setup with manager UUID: 038cead6-6cc4-4fcb-93c2-845fb66eec7c
Encoded batch obs field: 0      0
1      0
2      0
3      0
4      0
      ..
395    0
396    0
397    0
398    0
399    0
Name: _scvi_batch, Length: 400, dtype: int8
model._validate_anndata(adata) # Replays registration on adata
print(f"Manager UUID: {model.adata_manager.id}")
print(f"Last setup with manager UUID: {adata.uns['_scvi_manager_uuid']}")
print(f"Encoded batch obs field: {adata.obs['_scvi_batch']}")Manager UUID: 0b11c46e-e008-4993-9f70-b8e521e9c05f
Last setup with manager UUID: 0b11c46e-e008-4993-9f70-b8e521e9c05f
Encoded batch obs field: 0      0
1      0
2      0
3      0
4      0
      ..
395    1
396    1
397    1
398    1
399    1
Name: _scvi_batch, Length: 400, dtype: int8

### The `train` method_____no_output_____A model can be trained simply by calling the `train` method._____no_output_____

model.train(max_epochs=20)GPU available: False, used: False
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
/Users/jhong/miniforge3/envs/scvi-env/lib/python3.9/site-packages/pytorch_lightning/trainer/data_loading.py:432: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
  rank_zero_warn(

We were able to train this model, as this method is inherited in the class. Let us now take a look at psedocode of the `train` method of `UnsupervisedTrainingMixin`. The function of each of these objects is described in the API reference.

```python
    def train(
        self,
        max_epochs: Optional[int] = 100,
        use_gpu: Optional[bool] = None,
        train_size: float = 0.9,
        **kwargs,
    ):
        """
        Train the model.
        """
        # object to make train/test/val dataloaders
        data_splitter = DataSplitter(
            self.adata,
            train_size=train_size,
            validation_size=validation_size,
            batch_size=batch_size,
            use_gpu=use_gpu,
        )
        # defines optimizers, training step, val step, logged metrics
        training_plan = TrainingPlan(
            self.module, len(data_splitter.train_idx),
        )
        # creates Trainer, pre and post training procedures (Trainer.fit())
        runner = TrainRunner(
            self,
            training_plan=training_plan,
            data_splitter=data_splitter,
            max_epochs=max_epochs,
            use_gpu=use_gpu,
            **kwargs,
        )
        return runner()
```_____no_output_____We notice two new things:

1. A training plan (`training_plan`)
2. A train runner (`runner`)

The `TrainRunner` is a lightweight wrapper of the PyTorch lightning's [`Trainer`](https://pytorch-lightning.readthedocs.io/en/stable/trainer.html#trainer-class-api), which is a completely black-box method once a `TrainingPlan` is defined. So what does the `TrainingPlan` do?

1. Configures optimizers (e.g., Adam), learning rate schedulers.
2. Defines the training step, which runs a minibatch of data through the model and records the loss.
3. Defines the validation step, same as training step, but for validation data.
4. Records relevant metrics, such as the ELBO.

In `scvi-tools` we have `scvi.lightning.TrainingPlan`, which should cover many use cases, from VAEs and VI, to MLE and MAP estimation. Developers may find that they need a custom `TrainingPlan` for e.g,. multiple optimizers and complex training scheme. These can be written and used by the model class.

Developers may also overwrite this train method to add custom functionality like Early Stopping (see TOTALVI's train method). In most cases the higher-level train method can call `super().train()`, which would be the `BaseModelClass` train method._____no_output_____### Save and load_____no_output_____We can also save and load this model object, as it follows the expected structure._____no_output_____

model.save("saved_model/", save_anndata=True)
model = SCVI.load("saved_model/")_____no_output_____

## Writing methods to query the model_____no_output_____So we have a model that wraps a module that has been trained. How can we get information out of the module and present in cleanly to our users? Let's implement a simple example: getting the latent representation out of the VAE.

This method has the following structure:

1. Validate the user-supplied data
2. Create a data loader
3. Iterate over the data loader and feed into the VAE, getting the tensor of interest out of the VAE._____no_output_____

from typing import Optional, Sequence

@torch.no_grad()
def get_latent_representation(
    self,
    adata: Optional[AnnData] = None,
    indices: Optional[Sequence[int]] = None,
    batch_size: Optional[int] = None,
) -> np.ndarray:
    r"""
    Return the latent representation for each cell.

    Parameters
    ----------
    adata
        AnnData object with equivalent structure to initial AnnData. If `None`, defaults to the
        AnnData object used to initialize the model.
    indices
        Indices of cells in adata to use. If `None`, all cells are used.
    batch_size
        Minibatch size for data loading into model. Defaults to `scvi.settings.batch_size`.

    Returns
    -------
    latent_representation : np.ndarray
        Low-dimensional representation for each cell
    """
    if self.is_trained_ is False:
        raise RuntimeError("Please train the model first.")

    adata = self._validate_anndata(adata)
    dataloader = self._make_dataloader(adata=adata, indices=indices, batch_size=batch_size)
    latent = []
    for tensors in dataloader:
        inference_inputs = self.module._get_inference_input(tensors)
        outputs = self.module.inference(**inference_inputs)
        qz_m = outputs["qz_m"]

        latent += [qz_m.cpu()]
    return torch.cat(latent).numpy()_____no_output_____

Note

Validating the anndata is critical to the user experience. If `None` is passed it just returns the anndata used to initialize the model, but if a different object is passed, it checks that this new object is equivalent in structure to the anndata passed to the model. We took great care in engineering this function so as to allow passing anndata objects with potentially missing categories (e.g., model was trained on batches `["A", "B", "C"]`, but the passed anndata only has `["B", "C"]`). These sorts of checks will ensure that your module will see data that it expects, and the user will get the results they expect without advanced data manipulations.

_____no_output_____As a convention, we like to keep the module code as bare as possible and leave all posterior manipulation of module tensors to the model class methods. However, it would have been possible to write a `get_z` method in the module, and just have the model class that method._____no_output_____## Mixing in pre-coded features

We have a number of Mixin classes that can add functionality to your model through inheritance. Here we demonstrate the [`VAEMixin`](https://www.scvi-tools.org/en/stable/api/reference/scvi.model.base.VAEMixin.html#scvi.model.base.VAEMixin) class.

Let's try to get the latent representation from the object we already created._____no_output_____

try:
    model.get_latent_representation()
except AttributeError:
    print("This function does not exist")This function does not exist

This method becomes avaialble once the `VAEMixin` is inherited. Here's an overview of the mixin methods, which are coded generally enough that they should be broadly useful to those building VAEs._____no_output_____```python
class VAEMixin:
    @torch.no_grad()
    def get_elbo(
        self,
        adata: Optional[AnnData] = None,
        indices: Optional[Sequence[int]] = None,
        batch_size: Optional[int] = None,
    ) -> float:
        pass

    @torch.no_grad()
    def get_marginal_ll(
        self,
        adata: Optional[AnnData] = None,
        indices: Optional[Sequence[int]] = None,
        n_mc_samples: int = 1000,
        batch_size: Optional[int] = None,
    ) -> float:
        pass

    @torch.no_grad()
    def get_reconstruction_error(
        self,
        adata: Optional[AnnData] = None,
        indices: Optional[Sequence[int]] = None,
        batch_size: Optional[int] = None,
    ) -> Union[float, Dict[str, float]]:
        pass

    @torch.no_grad()
    def get_latent_representation(
        self,
        adata: Optional[AnnData] = None,
        indices: Optional[Sequence[int]] = None,
        give_mean: bool = True,
        mc_samples: int = 5000,
        batch_size: Optional[int] = None,
    ) -> np.ndarray:
        pass

```_____no_output_____Let's now inherit the mixin into our SCVI class._____no_output_____

from scvi.model.base import VAEMixin, UnsupervisedTrainingMixin

class SCVI(VAEMixin, UnsupervisedTrainingMixin, BaseModelClass):
    """
    single-cell Variational Inference [Lopez18]_.
    """

    def __init__(
        self,
        adata: AnnData,
        n_latent: int = 10,
        **model_kwargs,
    ):
        super(SCVI, self).__init__(adata)

        self.module = VAE(
            n_input=self.summary_stats["n_vars"],
            n_batch=self.summary_stats["n_batch"],
            n_latent=n_latent,
            **model_kwargs,
        )
        self._model_summary_string = (
            "SCVI Model with the following params: \nn_latent: {}"
        ).format(
            n_latent,
        )
        self.init_params_ = self._get_init_params(locals())

    @classmethod
    def setup_anndata(
        cls,
        adata: AnnData,
        batch_key: Optional[str] = None,
        layer: Optional[str] = None,
        **kwargs,
    ) -> Optional[AnnData]:
        setup_method_args = cls._get_setup_method_args(**locals())
        anndata_fields = [
            LayerField(REGISTRY_KEYS.X_KEY, layer, is_count_data=True),
            CategoricalObsField(REGISTRY_KEYS.BATCH_KEY, batch_key),
            # Dummy fields required for VAE class.
            CategoricalObsField(REGISTRY_KEYS.LABELS_KEY, None),
            NumericalObsField(
                REGISTRY_KEYS.SIZE_FACTOR_KEY, None, required=False
            ),
            CategoricalJointObsField(
                REGISTRY_KEYS.CAT_COVS_KEY, None
            ),
            NumericalJointObsField(
                REGISTRY_KEYS.CONT_COVS_KEY, None
            ),
        ]
        adata_manager = AnnDataManager(
            fields=anndata_fields, setup_method_args=setup_method_args
        )
        adata_manager.register_fields(adata, **kwargs)
        cls.register_manager(adata_manager)_____no_output_____SCVI.setup_anndata(adata, batch_key="batch")
model = SCVI(adata)
model.train(10)
model.get_latent_representation()GPU available: False, used: False
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
/Users/jhong/miniforge3/envs/scvi-env/lib/python3.9/site-packages/pytorch_lightning/trainer/data_loading.py:432: UserWarning: The number of training samples (3) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.
  rank_zero_warn(

## Summary_____no_output_____We learned the structure of the high-level model classes in scvi-tools, and learned how a simple version of `SCVI` is implemented.

Questions? Comments? Keep the discussion going on our [forum](https://discourse.scvi-tools.org/)_____no_output_____


# Notebook from anavaldi/digital-methods-text-mining
Path: notebooks/02_analysing-text_solutions.ipynb

# 2. Analysing text_____no_output_____This notebook will introduce you to the basics of analysing text.
You'll learn how to preprocess and represent text.

_____no_output_____Legend of symbols:

- 🤓: Tips

- 🤖📝: Your turn

- ❓: Question

- 💫: Extra exercise _____no_output_____## 2.1. Corpus preprocessing_____no_output_____Let's import the news dataset using pandas:_____no_output_____

! pip install pandas
import pandas as pd
df = pd.read_csv('../data/news.csv')Requirement already satisfied: pandas in /home/avaldivia/env37/lib/python3.7/site-packages (1.0.5)
Requirement already satisfied: numpy>=1.13.3 in /home/avaldivia/env37/lib/python3.7/site-packages (from pandas) (1.19.0)
Requirement already satisfied: python-dateutil>=2.6.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from pandas) (2.8.1)
Requirement already satisfied: pytz>=2017.2 in /home/avaldivia/env37/lib/python3.7/site-packages (from pandas) (2020.1)
Requirement already satisfied: six>=1.5 in /home/avaldivia/env37/lib/python3.7/site-packages (from python-dateutil>=2.6.1->pandas) (1.15.0)
WARNING: You are using pip version 20.2b1; however, version 21.0.1 is available.
You should consider upgrading via the '/home/avaldivia/env37/bin/python3.7 -m pip install --upgrade pip' command.
df.head()_____no_output_____

Analyse the corpus of the first row:_____no_output_____

df['corpus'][0]_____no_output_____print(df['corpus'][0])The reindeer is the emblematic Christmas animal and, while not exactly magical, it is among the best adapted to snowy conditions.For a start, a reindeer’s feet have four toes with dewclaws that spread out to distribute its weight like snowshoes, and are equipped with sharp hooves for digging in snow.A reindeer’s nose warms the air on its way to the lungs, cooling it again before it is exhaled. As well as retaining heat, this helps prevent water from being lost as vapour. This is why reindeer breath does not steam like human and horse breath.A reindeer’s thick double-layered coat is so efficient that it is more likely to overheat than get too cold, especially when running. When this happens, reindeer pant like dogs to cool down, bypassing the nasal heat exchanger.Snowfields may be featureless to human eyes, but reindeer are sensitive to ultraviolet light, an evolutionary development that only occurred after the animals moved to Arctic regions. Snow reflects ultraviolet, so this ultravision allows reindeer to spot anything lying on it, in particular lichen, which they eat, and traces of urine showing where other reindeer have passed.But while reindeer thrive in Christmas-card weather, they are increasingly challenged by climate change and the freeze-thaw conditions that produce poor grazing.

❓ What's the difference between the first output and the second?_____no_output_____### 🤖📝 **Your turn**_____no_output_____Now, print the 5th and 45th row:_____no_output_____

print(df['corpus'][4])Good morning, we’re now exactly two weeks out from the vote. Today the climate crisis is high on the agenda – Labour will unveil its environment policies, there is the leaders’ TV climate debate tonight, and all of this comes after a warning from scientists that the world may already have crossed a series of tipping points meaning “we are in a state of planetary emergency”. Make sure to keep on top of politics at our live blog.What’s going on?Jeremy Corbyn will be in Southampton where he will set out the party’s environment policies, including the planting of 2bn trees by 2040. Labour also plans to create 10 new national parks as part of a rewilding policy to tackle the climate emergency. The proposals also include an investment of £1.2bn to restore habitats such as woodlands and peat bogs in England, and extra funding for national park authorities.Tonight the leaders of most parties – as far as we know, Boris Johnson has not accepted the invitation – will participate in a Channel 4 debate on climate issues. At this stage Nigel Farage also appears to be a non-starter.One plank of Labour’s energy policy is under fire today, as lawyers have warned that the party’s plans to take large parts of the energy industry back under public control puts it on a collision course with EU laws that guard Europe-owned companies against government takeovers.There may be further fallout from Corbyn’s deadliest line of attack in this election campaign: his claims that leaked documents show Johnson wants to sell off the NHS in a trade deal with the US. The official papers reveal US and UK officials have repeatedly discussed dismantling protections that keep NHS drug prices down as part of their negotiations about a post-Brexit trade deal. Dennis Campbell and Jamie Grierson have unpacked what the dossier says and what it means. Heather Stewart writes that “Corbyn had one central aim as he brandished the 451-page ‘secret’ NHS document at a hastily-arranged press conference on Wednesday: to drag the general election debate safely back into Labour’s comfort zone” after the antisemitism accusations”.A new poll for the Times points to a thumping Conservative victory. While the Guardian is treating all polls with suspicion, read why here, the MRP poll from YouGov came closest to calling the unexpected result of the 2017 general election.The poll predicts the Conservatives will win 359 seats (42 gains), leaving Johnson with a majority of 68. Labour, meanwhile, would fall back to 211 seats – a result that would be in line with the disaster of 1983. Not everyone is happy with that message, including Johnson’s adviser Dominic Cummings, who has told Brexit supporters that the general election is “much tighter” than polls might suggest and urged them to persuade their friends to vote Tory.At a glanceThe day aheadJo Swinson starts the day with a roundtable on homelessness in London and Corbyn will be in Southampton to announce the party’s environment policies.A foreign policy debate, featuring foreign secretary Dominic Raab, shadow foreign secretary Emily Thornberry as well as the foreign affairs spokespeople for the Lib Dems and SNP. If you’re interested, that will be broadcast on Radio 4 tonight.Tonight, party leaders will participate in a debate on the climate crisis, broadcast on Channel 4. Johnson has not confirmed whether he will be there and Farage has rejected the invite, but all other parties will be attending.A quiet day for the Tories, who have no major events or announcements scheduled.Sign up here to receive Andrew Sparrow’s afternoon round-up from the campaign trail – direct to your email inbox.Best of the rest> The world may already have crossed a series of climate tipping points that mean “we are in a state of planetary emergency”, scientists say. A 5C rise was previously thought necessary to cause things like runaway loss of ice sheets or forests. But the latest evidence suggests this could start upwards of 1C, which we have already passed. Writing in Nature, they acknowledge the science of tipping points is complex but “to err on the side of danger is not a responsible option” and the rate of the damage could still be partly under our control if urgent international action is taken.> Today’s school-leavers are enjoying “the best time ever” to take their pick of university courses amid fierce competition between institutions recruiting from a shrinking pool of 18-year-olds, according to the Ucas. Nearly 98% of applicants received offers of a place to study for a degree this year and that could be even higher next year. A record 34% of UK 18-year-olds entered higher education in 2019. Of the remaining 66%, many are likely to enter higher education before they turn 30.> Donald Trump has signed into law legislation backing pro-democracy protesters in Hong Kong, angering Beijing which has condemned the bill as “full of prejudice and arrogance”. The legislation, approved unanimously by the US Senate and by all but one lawmaker in the House of Representatives last week, threatens sanctions on Chinese and Hong Kong officials deemed responsible for human rights abuses in Hong Kong. “This is a pure interference in China’s internal affairs,” thundered China’s ministry of foreign affairs, hours after the bill was signed by Trump.Today in Focus podcast: Swinging times in PeterboroughRobert Booth on the battle for one of the key swing seats set to determine the election. Plus: Can Dündar on press freedoms under threat in Turkey.Today in Focus Swinging times in Peterborough Sorry your browser does not support audio - but you can download here and listen https://audio.guim.co.uk/2019/11/27-65672-191128.TIF_Peterborough.mp3 00:00:00 00:32:07Lunchtime read: ‘Either working or tormenting myself’The Pussycat Dolls are back and their main singer, Nicole Scherzinger, is determined to enjoy it. She discusses body image, online abuse, success and Simon Cowell with Elle Hunt.Facebook Twitter Pinterest Nicole Scherzinger rose to fame as lead singer of the Pussycat Dolls. Photograph: Pal Hansen/The GuardianSportJürgen Klopp lamented Liverpool’s failure to book a Champions League “holiday” in Salzburg next month but insisted his European champions would thrive on the pressure to reach the knockout stages in Austria. Daniel Wass’s freak goal earned Valencia a deserved draw with Chelsea at the Mestalla, where Kepa Arrizabalaga slipped from Blues hero to villain and the hosts muddled their lines. Ollie Pope has been placed on standby to keep wicket for England during their must-win second Test against New Zealand after Jos Buttler emerged as an injury doubt 24 hours out from the match. Captain Joe Root, in need of runs, has said he is not far away from making a good score.N’Golo Kanté has denied he was threatened by someone carrying a gun months after his transfer to Chelsea amid an extraordinary dispute between his advisers. Chelsea’s signing of Sam Kerr is more evidence of a power shift away from the US in women’s football, and clubs and leagues outside England are taking action, writes Suzanne Wrack. And the former MLB player Aubrey Huff is taking Bernie Sanders’ run for the White House very seriously, teaching his sons to shoot in case the Democrat wins the presidency in 2020 and introduces socialism to the US.BusinessTwo former Bank of England policymakers discuss the prospects for the economy as our monthly “Brexit watch” analysis of data reveals October saw wages remain in the doldrums. One of the former bank experts, David Blanchflower, says Britain’s growing army of underemployed people (those who want more work than they get) is the reason wages are remaining so stubbornly low. The FTSE100 is on track to open down after Asian markets took fright at Donald Trump’s signing of a bill backing Hong Kong’s protesters. The pound was flat overnight at $1.293 and €1.174.The papersMourning on the front pages today for, as the Times aptly puts it, some “giants of culture”: the Australian-born writer and broadcaster Clive James, the legendary writer and director Jonathan Miller, and the TV chef Gary Rhodes who has died suddenly, at just 59 years of age, in the middle of filming a new series. The splash in the Times is “Johnson heads for big majority”, based on the only published poll that predicted a hung parliament in the last election.Facebook Twitter Pinterest Guardian front page, Thursday 28 November 2019.On the same topic, “Poll boost for Tories”, says the Metro and in the i it’s “Johnson on course to break deadlock”, meaning the Brexit impasse in parliament. The Telegraph would prefer to tone that down a bit – “Election too close to call, warns Cummings” – as Johnson’s adviser deploys the age-old tactic of lowballing your prospects to keep voters on board.The Express gives Johnson a hand in trying to banish the spectre of austerity, reporting his comments that it was “wrong for Britain”. The Mirror’s counterpunch is “THE PROOF” as Corbyn reveals papers that Labour says “show our health service is on table in Brexit trade talks with US”. The Guardian has “Secret papers prove Tories want to sell NHS – Corbyn”. The FT is out on its own with “Lagarde bid to put climate change at the heart of ECB monetary policy” – the downpage lead is a thinktank’s warning about the big parties’ spending plans busting budget rules. The Sun splashes on WhatsApp messages allegedly sent to a woman by David Pemsel, the incoming Premier League chief executive, who is currently the chief executive of Guardian Media Group.Sign upThe Guardian Morning Briefing is delivered to thousands of inboxes bright and early every weekday. If you are not already receiving it by email, you can sign up here.For more news: www.theguardian.com
print(df['corpus'][44])The world may have hit a hopeful “turning point” in the struggle to tackle the climate crisis despite escalating greenhouse gas emissions and the recalcitrance of major emitters Brazil and the US, according to the United Nations secretary general.On Sunday, ahead of a key UN climate summit in New York, the World Meteorological Organisation published new data showing 2014-19 to be the warmest five-year period on record.But the UN secretary general, António Guterres, said recent action by some countries and businesses, as well as the stunning rise of the youth climate movement, gave him hope that international goals to avoid catastrophic global heating could be met.“I see a new momentum,” Guterres said. “I believe in these last few months [there has been] a turning point. Six months ago, I must tell you, I was quite pessimistic about everything. I would see no movement, now I see a lot of movement and we need to boost that movement.”Guterres said he expects “very meaningful” climate commitments to be made by countries during the summit on Monday, which was thrown open to world leaders who had new initiatives on cutting planet-warming gases to announce.Quick guide Covering Climate Now: how more than 250 newsrooms are joining forces this week to spotlight the climate crisis Show Hide Hundreds of newsrooms around the world are banding together this week to commit their pages and air time to what may be the most consequential story of our time. As world leaders descend on New York for the UN Climate Action Summit on 23 September – and millions of activists prepare for a global climate strike on 20 September – the media partnership Covering Climate Now is launching its first large-scale collaboration to increase climate coverage in the global media and focus public attention on this emergency. The Guardian is the lead partner in Covering Climate Now, which was founded earlier this year by the Columbia Journalism Review and the Nation. The partnership currently includes 250 newsrooms representing 32 countries with a combined monthly reach of more than a billion people. The network represents every corner of the media including TV networks (CBS News, Al Jazeera), newspapers (El País, the Toronto Star), digital players (BuzzFeed, HuffPost, Vox), wire services (Getty Images, Bloomberg), magazines (Nature, Science), and dozens of podcasts, local publishers, radio and TV stations. You can learn more about the initiative here.Representatives from about 60 countries will speak, including Angela Merkel, chancellor of Germany, India’s prime minister, Narendra Modi, and Boris Johnson, the British prime minister.The summit is designed to sharply prod countries to do more to meet the commitments of the Paris climate accords, where governments agreed to limit the average global temperature rise to 2C above the pre-industrial era. The world has already warmed by around 1C since this time.Progress has been limited. Global emissions hit a record high last year, Donald Trump has said the US, the world’s second-largest emitter, will quit the Paris deal and the Brazilian president, Jair Bolsonaro, has opened the way to raze vast tracts of the Amazon rainforest.Guterres said neither Brazil nor the US have “turned up” with any new commitments for the UN summit.“Clearly, we are lagging behind and that there are many resistances in many areas,” he said. “I feel that we are still running late and we need to accelerate. The next few years are absolutely crucial to reverse the present trend, that is still a negative trend.“Emissions are still growing and the situation is getting worse, clearly. I am deeply convinced that climate change is the defining issue of our time.”The daunting task of staving off disastrous climate change has been laid bare in a new analysis released by the UN that shows global emissions are not estimated to peak by 2030, the year when scientists say emissions must be slashed by nearly half from now to avoid devastating heatwaves, flooding and loss of species.This dire forecast, driven by the continued dominance of fossil fuels despite the growth of alternatives such as solar and wind power, means countries will need to at least triple their current emissions reduction commitments to meet the Paris target, the UN report states.The G20, the world’s wealthiest countries, represent 80% of emissions but are according to the UN doing “nowhere near enough” to scale up their ambition, eclipsed by developing countries that in the case of low-lying countries in Asia and the Pacific are at most peril from rising seas and drought.Guterres, who has called for net zero emissions by 2050 and the end of new coal projects from next year, said he expects countries will do more, although he admitted he was concerned by a spate of new coalmines backed by China.Play Video 2:16 How the climate strike travelled around the world - videoOn the Trump administration, the secretary general said: “Of course we understand that the policy that exists today is not in line with this concern [of climate change].”The UN summit is set to provide a sobering counterpoint to Friday’s global climate strikes, where millions across 150 countries skipped work or school in order to protest against inaction over the climate crisis.Guterres said pressure from young people, galvanized by the Swedish teen activist Greta Thunberg, will be “fundamental” in shaping the response from countries.It appears however to have made little impression on Trump, who will pointedly be at the UN on Monday to attend a gathering on religious freedom rather than the climate summit.Climate strikes: hoax photo accusing Australian protesters of leaving rubbish behind goes viral Read more“I don’t think other countries feel there’s much point putting pressure on President Trump because he hasn’t put off any signal in changing,” said Todd Stern, who was US special envoy for climate change during the Paris negotiations.“In the absence of the US there is some lack of energy in the overall system so it’s good that the secretary general is convening people and pushing them hard.”Some major businesses have set out new goals, with a group of 87 companies – including Nestlé, Nokia and L’Oréal – on Sunday committing to net zero emissions by 2050. But the abdication of the US looms large.“The EU and China are the parties that can take this forward,” said Frank Rijsberman, director general of the Global Green Growth Institute. “It looks like we will have to wait until there’s another US president.”The summit will be considered a success with a raft of new national commitments but it’s clear that even that may not be enough.“I’m looking for something that will begin to bend that curve,” said Mary Robinson, former UN high commissioner for human rights. “And until it begins to bend, we’re not there at all.”

### 2.1.1. Encodings_____no_output_____🤓 The string **<tt>encode()<tt>** method returns encoded version of the given string. Using string's **<tt>encode()<tt>** method, you can convert unicoded strings into any encodings supported by Python. By default, Python uses UTF-8 encoding._____no_output_____

df['corpus'][1]_____no_output_____df['corpus'][1].encode()_____no_output_____

❓ Which are the symbols that are encoded?_____no_output_____### 2.1.2. Regular expressions_____no_output_____Regular expressions (called REs, or regexes, or regex patterns) are essentially a tiny, highly specialized programming language embedded inside Python and made available through the **<tt>re()<tt>** module._____no_output_____

import reERROR: Could not find a version that satisfies the requirement re (from versions: none)
ERROR: No matching distribution found for re
WARNING: You are using pip version 20.2b1; however, version 21.0.1 is available.
You should consider upgrading via the '/home/avaldivia/env37/bin/python3.7 -m pip install --upgrade pip' command.

Using this little language, you specify the rules for the set of possible strings that you want to match._____no_output_____🤓 The first metacharacters we’ll look at are **<tt>[ and ]<tt>**. They’re used for specifying a character class, which is a set of characters that you wish to match. Characters can be listed individually, or a range of characters can be indicated by giving two characters and separating them by a **<tt>'-'<tt>**. For example, **<tt>[abc]<tt>** will match any of the characters **<tt>a<tt>**, **<tt>b<tt>**, or **<tt>c<tt>**; this is the same as **<tt>[a-c]<tt>**, which uses a range to express the same set of characters. If you wanted to match only lowercase letters, your RE would be **<tt>[a-z]<tt>**._____no_output_____

p = re.compile('[a-z]+')
p_____no_output_____

Now, you can try matching various strings against the RE **<tt> [a-z]+<tt>**:_____no_output_____An empty string shouldn’t match at all, since **+** means *one or more repetitions*. _____no_output_____

p.match("")
print(p.match(""))None

Now, let’s try it on a string that it should match, such as **<tt>tempo<tt>**._____no_output_____

m = p.match("tempo")
print(m)<_sre.SRE_Match object; span=(0, 5), match='tempo'>
m = p.match("asgasgasg?asgagsa?")
print(m)<_sre.SRE_Match object; span=(0, 9), match='asgasgasg'>

In this case, **<tt>match<tt>** will return a match object. This object contains the following features:_____no_output_____![image.png](attachment:image.png)_____no_output_____**<tt>group()<tt>** returns the substring that was matched by the RE. **<tt>start()<tt>**  and **<tt>end()<tt>**  return the starting and ending index of the match. **<tt>span()<tt>**  returns both start and end indexes in a single tuple. Since the **<tt>match()<tt>**  method only checks if the RE matches at the start of a string, **<tt>start()<tt>**  will always be zero. However, the **<tt>search()<tt>**  method of patterns scans through the string, so the match may not start at zero in that case._____no_output_____

m.group()_____no_output_____m.start(), m.end()_____no_output_____m.span()_____no_output_____

**Metacharacters** are characters with a special meaning:_____no_output_____![image-5.png](attachment:image-5.png)
![image-4.png](attachment:image-4.png)_____no_output_____A **set** is a set of characters inside a pair of square brackets **<tt>[]<tt>** with a special meaning:_____no_output_____![image.png](attachment:image.png)_____no_output_____#### 💫 Let's get more familiarised with regular expressions.
Open: https://regex101.com/_____no_output_____Two pattern methods return all of the matches for a pattern. **<tt>findall()<tt>** returns a list of matching strings:_____no_output_____

p = re.compile('\d+')
p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping')_____no_output_____

The **<tt>search()<tt>** function searches the string for a match, and returns a Match object if there is a match._____no_output_____Search for the first white-space character in the string:_____no_output_____

txt = "The rain in Spain"
x = re.search("\s", txt)
x_____no_output_____print("The first white-space character is located in position:", x.start()) The first white-space character is located in position: 3

### 🤖📝 **Your turn**_____no_output_____Search for the word **Portugal** in the next sentence:_____no_output_____

txt = "The rain in Portugal"
x = re.search("Portugal", txt)
print(x) <_sre.SRE_Match object; span=(12, 20), match='Portugal'>

Search for the expression **ai** in the next sentence and extract the position:_____no_output_____

txt = "The rain in Spain"
x = re.search("ai", txt)
print(x)
print(x.span())<_sre.SRE_Match object; span=(5, 7), match='ai'>
(5, 7)
# Anthony's solution
x = re.finditer('ai', txt, flags=re.IGNORECASE)
for match in x:
    print(match)
<_sre.SRE_Match object; span=(5, 7), match='ai'>
<_sre.SRE_Match object; span=(14, 16), match='ai'>

Now, let's find the word **Macron** in the 46th row of the news' corpus and extract its position:_____no_output_____

print(re.search("Macron", df['corpus'][45]))<_sre.SRE_Match object; span=(108, 114), match='Macron'>

🤓 You can read more about regex syntaxis at: https://docs.python.org/3/howto/regex.html_____no_output_____## 2.2. Representing text_____no_output_____In this new section, we will learn how to buil term-document matrices in Python._____no_output_____To do so, we will import **<tt>scikit-learn()<tt>** library. It features various classification, regression and clustering algorithms including support vector machines, random forests, gradient boosting, k-means and DBSCAN, and is designed to interoperate with the Python numerical and scientific libraries NumPy and SciPy.

**<tt>CountVectorizer<tt>** converts a collection of raw documents to a matrix of vocabulary count features.
**<tt>TfidfVectorizer<tt>** converts a collection of raw documents to a matrix of TF-IDF features._____no_output_____🤓 Let's read the documentation:
- https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer
- https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html_____no_output_____

! pip install sklearn
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizerRequirement already satisfied: sklearn in /home/avaldivia/env37/lib/python3.7/site-packages (0.0)
Requirement already satisfied: scikit-learn in /home/avaldivia/env37/lib/python3.7/site-packages (from sklearn) (0.23.1)
Requirement already satisfied: threadpoolctl>=2.0.0 in /home/avaldivia/env37/lib/python3.7/site-packages (from scikit-learn->sklearn) (2.1.0)
Requirement already satisfied: scipy>=0.19.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.5.0)
Requirement already satisfied: joblib>=0.11 in /home/avaldivia/env37/lib/python3.7/site-packages (from scikit-learn->sklearn) (0.15.1)
Requirement already satisfied: numpy>=1.13.3 in /home/avaldivia/env37/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.19.0)
WARNING: You are using pip version 20.2b1; however, version 21.0.1 is available.
You should consider upgrading via the '/home/avaldivia/env37/bin/python3.7 -m pip install --upgrade pip' command.
# Sample data for analysis
df_sentences = df[0:3]_____no_output_____df_sentences_____no_output_____df_sentences.iloc[0]_____no_output_____# Initialize
countvoc_vectorizer = CountVectorizer()
tf_idf_vectorizer = TfidfVectorizer()

doc_countvoc = countvoc_vectorizer.fit_transform(df_sentences['corpus'])
doc_tf_idf = tf_idf_vectorizer.fit_transform(df_sentences['corpus'])

_____no_output_____doc_countvoc_____no_output_____doc_tf_idf_____no_output_____# Create vocabulary count matrix
matrix_countvoc = pd.DataFrame(doc_countvoc.toarray().transpose(),
                   index=countvoc_vectorizer.get_feature_names())

# Create term-document matrix
matrix_tfidf = pd.DataFrame(doc_tf_idf.toarray().transpose(),
                   index=tf_idf_vectorizer.get_feature_names())_____no_output_____matrix_countvoc_____no_output_____# Vocabulary count matrix
matrix_countvoc.columns = ["new 1", "new 2", "new 3"]
print(matrix_countvoc)             new 1  new 2  new 3
100              0      1      0
12th             0      0      1
13               0      0      1
1980s            0      0      1
1990             0      1      0
2015             0      1      1
2016             0      0      1
2030             0      6      0
2050             0      3      0
2c               0      2      0
40               0      1      0
50               0      1      0
55               0      2      0
5c               0      5      0
65               0      2      0
700              0      1      0
abiri            0      0      1
about            0      2      3
above            0      1      0
abuja            0      0      1
access           0      0      1
according        0      0      1
accordingly      0      2      0
accusation       0      0      1
act              0      1      0
action           0      4      0
activist         0      0      1
activists        0      0      2
adapted          1      0      0
advice           0      1      0
...            ...    ...    ...
weber            0      1      0
week             0      1      0
weekly           0      0      1
weight           1      0      0
well             1      3      0
went             0      0      2
were             0      0      1
when             2      2      2
where            1      0      0
whether          0      2      0
which            1      3      3
while            2      3      0
who              0      1      4
why              1      0      0
will             0      2      0
with             2      5      5
withdrawal       0      1      0
within           0      1      0
without          0      0      1
word             0      2      0
work             0      0      1
worst            0      0      1
would            0      3      1
year             0      2      1
years            0      2      1
yes              0      1      0
york             0      0      1
you              0      2      0
young            0      1      0
zero             0      2      0

[725 rows x 3 columns]
# Term-document matrix
matrix_tfidf.columns = df_sentences.columns
print(matrix_tfidf)_____no_output_____

❓ How would you describe this matrix?_____no_output_____🤓 In numerical analysis and scientific computing, a **sparse matrix** or sparse array is a matrix in which most of the elements are zero. There is no strict definition how many elements need to be zero for a matrix to be considered sparse but a common criterion is that the number of non-zero elements is roughly the number of rows or columns.

By contrast, if most of the elements are nonzero, then the matrix is considered **dense**. The number of zero-valued elements divided by the total number of elements (e.g., m × n for an m × n matrix) is sometimes referred to as the sparsity of the matrix. _____no_output_____Let's select only 4 features:_____no_output_____

vectorizer = TfidfVectorizer(max_features=4)
doc_vec = vectorizer.fit_transform(df_sentences.iloc[0])
matrix_2 = pd.DataFrame(doc_vec.toarray().transpose(),
                   index=vectorizer.get_feature_names())_____no_output_____print(matrix_2)_____no_output_____

And now, let's remove **stop_words**:_____no_output_____

vectorizer = TfidfVectorizer(stop_words='english')
doc_vec = vectorizer.fit_transform(df_sentences['corpus'])
matrix_3 = pd.DataFrame(doc_vec.toarray().transpose(),
                   index=vectorizer.get_feature_names())
print(matrix_3)                    0         1         2
100          0.000000  0.029605  0.000000
12th         0.000000  0.000000  0.042791
13           0.000000  0.000000  0.042791
1980s        0.000000  0.000000  0.042791
1990         0.000000  0.029605  0.000000
2015         0.000000  0.022515  0.032543
2016         0.000000  0.000000  0.042791
2030         0.000000  0.177628  0.000000
2050         0.000000  0.088814  0.000000
2c           0.000000  0.059209  0.000000
40           0.000000  0.029605  0.000000
50           0.000000  0.029605  0.000000
55           0.000000  0.059209  0.000000
5c           0.000000  0.148023  0.000000
65           0.000000  0.059209  0.000000
700          0.000000  0.029605  0.000000
abiri        0.000000  0.000000  0.042791
abuja        0.000000  0.000000  0.042791
access       0.000000  0.000000  0.042791
according    0.000000  0.000000  0.042791
accordingly  0.000000  0.059209  0.000000
accusation   0.000000  0.000000  0.042791
act          0.000000  0.029605  0.000000
action       0.000000  0.118419  0.000000
activist     0.000000  0.000000  0.042791
activists    0.000000  0.000000  0.085581
adapted      0.067231  0.000000  0.000000
advice       0.000000  0.029605  0.000000
advised      0.000000  0.029605  0.000000
agba         0.000000  0.000000  0.042791
...               ...       ...       ...
vapour       0.067231  0.000000  0.000000
victim       0.000000  0.000000  0.042791
von          0.000000  0.029605  0.000000
vote         0.000000  0.029605  0.000000
voted        0.000000  0.029605  0.000000
want         0.000000  0.029605  0.000000
wanted       0.000000  0.000000  0.042791
warming      0.000000  0.029605  0.000000
warms        0.067231  0.000000  0.000000
warn         0.000000  0.000000  0.042791
watch        0.000000  0.000000  0.085581
water        0.067231  0.000000  0.000000
way          0.067231  0.000000  0.000000
weather      0.067231  0.000000  0.000000
weatherman   0.000000  0.029605  0.000000
weber        0.000000  0.029605  0.000000
week         0.000000  0.029605  0.000000
weekly       0.000000  0.000000  0.042791
weight       0.067231  0.000000  0.000000
went         0.000000  0.000000  0.085581
withdrawal   0.000000  0.029605  0.000000
word         0.000000  0.059209  0.000000
work         0.000000  0.000000  0.042791
worst        0.000000  0.000000  0.042791
year         0.000000  0.045030  0.032543
years        0.000000  0.045030  0.032543
yes          0.000000  0.029605  0.000000
york         0.000000  0.000000  0.042791
young        0.000000  0.029605  0.000000
zero         0.000000  0.059209  0.000000

[599 rows x 3 columns]

### 2.2.1. Word clouds_____no_output_____And finally, one of the most cool visualisations in text mining: word clouds!_____no_output_____

!pip install matplotlib
import matplotlib.pyplot as plt

!pip install wordcloud
from wordcloud import WordCloudRequirement already satisfied: matplotlib in /home/avaldivia/env37/lib/python3.7/site-packages (3.2.2)
Requirement already satisfied: cycler>=0.10 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib) (0.10.0)
Requirement already satisfied: python-dateutil>=2.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib) (2.8.1)
Requirement already satisfied: kiwisolver>=1.0.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib) (1.2.0)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib) (2.4.7)
Requirement already satisfied: numpy>=1.11 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib) (1.19.0)
Requirement already satisfied: six in /home/avaldivia/env37/lib/python3.7/site-packages (from cycler>=0.10->matplotlib) (1.15.0)
WARNING: You are using pip version 20.2b1; however, version 21.0.1 is available.
You should consider upgrading via the '/home/avaldivia/env37/bin/python3.7 -m pip install --upgrade pip' command.
Requirement already satisfied: wordcloud in /home/avaldivia/env37/lib/python3.7/site-packages (1.8.1)
Requirement already satisfied: numpy>=1.6.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from wordcloud) (1.19.0)
Requirement already satisfied: pillow in /home/avaldivia/env37/lib/python3.7/site-packages (from wordcloud) (7.1.2)
Requirement already satisfied: matplotlib in /home/avaldivia/env37/lib/python3.7/site-packages (from wordcloud) (3.2.2)
Requirement already satisfied: cycler>=0.10 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib->wordcloud) (0.10.0)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib->wordcloud) (2.4.7)
Requirement already satisfied: python-dateutil>=2.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib->wordcloud) (2.8.1)
Requirement already satisfied: kiwisolver>=1.0.1 in /home/avaldivia/env37/lib/python3.7/site-packages (from matplotlib->wordcloud) (1.2.0)
Requirement already satisfied: six in /home/avaldivia/env37/lib/python3.7/site-packages (from cycler>=0.10->matplotlib->wordcloud) (1.15.0)
WARNING: You are using pip version 20.2b1; however, version 21.0.1 is available.
You should consider upgrading via the '/home/avaldivia/env37/bin/python3.7 -m pip install --upgrade pip' command.
word_cloud = WordCloud(background_color="white", repeat=True, stopwords=['the','that'])
word_cloud.generate(' '.join(df[0:100]['corpus']))_____no_output_____' '.join(df[0:3]['corpus'])_____no_output_____plt.axis("off")
plt.imshow(word_cloud, interpolation="bilinear")
plt.show()_____no_output_____

If you want to customise your word cloud, check: https://amueller.github.io/word_cloud/auto_examples/single_word.html#sphx-glr-auto-examples-single-word-py._____no_output_____### 💫 **Extra exercises**_____no_output_____1. Create a wordcloud for one topic on the news dataset.
2. Create a wordcloud for all news that mentioned Macron.
3. Find the word 'violence' in the news dataset._____no_output_____

df_climatic = df.loc[df['topic'] == 'climatic']_____no_output_____df_climatic.head()_____no_output_____word_cloud_climatic = WordCloud(background_color="white", repeat=True, stopwords=['the','that', 'of', 'to', 'and', 'with', 'for', 'it', 'in', 'by'])
word_cloud_climatic.generate(' '.join(df_climatic['corpus']))_____no_output_____plt.axis("off")
plt.imshow(word_cloud_climatic, interpolation="bilinear")
plt.show()_____no_output_____

### Resources_____no_output_____📕 Bengfort, B., Bilbro, R., & Ojeda, T. (2018). *Applied text analysis with python: Enabling language-aware data products with machine learning.* O'Reilly Media, Inc.

📕 Bird, S., Klein, E., & Loper, E. (2009). *Natural language processing with Python: analyzing text with the natural language toolkit.* " O'Reilly Media, Inc.".

🌍 https://docs.python.org/3/howto/regex.html

🌍 https://www.w3schools.com/python/python_regex.asp_____no_output_____


# Notebook from gmstanle/msc_aging
Path: 2_aging_signature/archive_initial_submission/filter_droplet.ipynb

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import scanpy.api as sc
# from anndata import read_loom
from anndata import read_h5ad
sc.logging.print_versions()scanpy==1.4.3 anndata==0.6.20 umap==0.3.8 numpy==1.16.4 scipy==1.2.1 pandas==0.25.0 scikit-learn==0.21.1 statsmodels==0.9.0 python-igraph==0.7.1 louvain==0.6.1
# Data path
data_path = '/data3/martin/tms_gene_data'_____no_output_____# Raw data
adata = read_h5ad(f"{data_path}/maca-droplet-raw-data-annotated.h5ad")_____no_output_____# Mean depth
print('Mean sequencing depth', adata.X.sum(axis=1).mean())Mean sequencing depth 6109.0303

## QC filtering_____no_output_____

# Filter cells and genes
print(adata)
print(f'\n# remove genes expressed in less than 5 cells')
sc.pp.filter_genes(adata, min_cells=5)
print(f'# remove cell expressing less than 500 genes')
sc.pp.filter_cells(adata, min_genes=500)
print(adata)
print('\n# remove cells whose \n\t1. Tissue annotation is nan'
      + '\n\t2. n_counts is smaller than 1000')
adata.obs['n_counts'] = np.sum(adata.X, axis=1).A1
ind_select = (adata.obs['tissue']!='nan') & (adata.obs['age']!='nan') & \
            (adata.obs['n_counts']>=3000)
adata = adata[ind_select,]
print(adata)AnnData object with n_obs × n_vars = 418908 × 23433
    obs: 'age', 'batch', 'cell', 'cell_ontology_class', 'cell_ontology_id', 'free_annotation', 'method', 'mouse.id', 'n_genes', 'sex', 'subtissue', 'tissue', 'tissue_free_annotation'

# remove genes expressed in less than 5 cells
# remove cell expressing less than 500 genes
AnnData object with n_obs × n_vars = 345454 × 19861
    obs: 'age', 'batch', 'cell', 'cell_ontology_class', 'cell_ontology_id', 'free_annotation', 'method', 'mouse.id', 'n_genes', 'sex', 'subtissue', 'tissue', 'tissue_free_annotation'
    var: 'n_cells'

# remove cells whose
	1. Tissue annotation is nan
	2. n_counts is smaller than 1000
View of AnnData object with n_obs × n_vars = 235325 × 19861
    obs: 'age', 'batch', 'cell', 'cell_ontology_class', 'cell_ontology_id', 'free_annotation', 'method', 'mouse.id', 'n_genes', 'sex', 'subtissue', 'tissue', 'tissue_free_annotation', 'n_counts'
    var: 'n_cells'

## Write the data_____no_output_____

adata.write(f'{data_path}/droplet_filtered.h5ad')_____no_output_____


# Notebook from sirusb/DL_genomics_example
Path: preprocessing/PrepareData.ipynb

# Install needed libraries_____no_output_____Many bioinformatics related R packages are available at Bioconductor, so we will just use that to install.
BioconductorManager can also download from CRAN if the library is not a Bioconductor library_____no_output_____

## Install some required packages
if (!requireNamespace("BiocManager", quietly = TRUE))
  install.packages("BiocManager")

requiredPackages <- c("rtracklayer","EnrichedHeatmap","argparse","ggsci",
                      "BSgenome.Mmusculus.UCSC.mm10","readr","ggplot2","furrr",
                      "circlize","glue",'tidyverse',"genomation",'logger')

for(pkg in requiredPackages){
  if( !is.element(pkg, .packages(all.available = TRUE)) ) {
    message(paste0(pkg," missing, installing it ..."))
    BiocManager::install(pkg)
  }
}

suppressPackageStartupMessages(require('logger'))
options(readr.num_columns = 0)_____no_output_____log_info("Loading packages ...")
## Load required packages
for(pkg in requiredPackages){
  suppressPackageStartupMessages(require(pkg,character.only = TRUE))
}Warning message:
“package ‘GenomicRanges’ was built under R version 4.0.3”
Warning message:
“package ‘BiocGenerics’ was built under R version 4.0.3”
Warning message:
“package ‘S4Vectors’ was built under R version 4.0.3”
Warning message:
“package ‘IRanges’ was built under R version 4.0.3”
Warning message:
“package ‘GenomeInfoDb’ was built under R version 4.0.3”
Warning message:
“package ‘ComplexHeatmap’ was built under R version 4.0.3”
Warning message:
“package ‘XVector’ was built under R version 4.0.3”
Warning message:
“package ‘readr’ was built under R version 4.0.3”
Warning message:
“package ‘future’ was built under R version 4.0.3”
Warning message:
“package ‘circlize’ was built under R version 4.0.3”
Warning message:
“package ‘glue’ was built under R version 4.0.3”
Warning message:
“package ‘tidyr’ was built under R version 4.0.3”
Warning message:
“package ‘forcats’ was built under R version 4.0.3”

# Download a toy example_____no_output_____We will download data from this study: https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE169534
Here we have some ATAC-seq data, that tell us what the open chromatin regions in the genome in the immortalized mouse embryonic fibroblasts._____no_output_____![ATACseq](img/ATACseq.png)_____no_output_____We download the data_____no_output_____

#system("wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5208nnn/GSM5208935/suppl/GSM5208935_ATAC_C2_replicate_1.bw -P ../raw_data/")
#system("wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5208nnn/GSM5208938/suppl/GSM5208938_ATAC_C2_replicate_2.bw -P ../raw_data/")
system("wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE169nnn/GSE169534/suppl/GSE169534_SEACR-ATACseq-C2-Ezh2KO-peaks.bed.gz -P ../raw_data/")
system("wget https://raw.githubusercontent.com/sirusb/HMMDomainCaller/master/mm10-blacklist.v2.bed -P ../raw_data/")_____no_output_____system("wget https://meme-suite.org/meme/meme-software/Databases/motifs/motif_databases.12.21.tgz -P ../raw_data/")_____no_output_____list.files("../raw_data/",full.names = TRUE)_____no_output_____

## Read the data_____no_output_____

Peaks = readr::read_tsv("../raw_data//GSE169534_SEACR-ATACseq-C2-Ezh2KO-peaks.bed",col_names = FALSE)
head(Peaks)_____no_output_____

Convert to GenomicRanges object for easy manipulation_____no_output_____

colnames(Peaks) = c("seqnames","start","end","total_signal","max_signal","max signal region")
Peaks.GR = GRanges(Peaks)
Peaks.GR_____no_output_____hist(log10(width(Peaks.GR)))_____no_output_____

Let's remove the very short regions and regions located in sex chromosomes (X and Y chromosomes)_____no_output_____

table(seqnames(Peaks.GR))_____no_output_____length(Peaks.GR)_____no_output_____filtered_Peaks.GR = subset(Peaks.GR, width >= 500 & seqnames %in% glue("chr{c(1:19)}"))
filtered_Peaks.GR = keepSeqlevels(filtered_Peaks.GR, value = seqlevelsInUse(filtered_Peaks.GR))
length(filtered_Peaks.GR)_____no_output_____

## Remove black listed regions_____no_output_____There are some regions of the genome that always have signal, but it is a false positive, so we filter any region that falls into that region_____no_output_____

mm10_blackRegions = rtracklayer::import.bed("../raw_data/mm10-blacklist.v2.bed")_____no_output_____filtered_Peaks.GR = subsetByOverlaps(filtered_Peaks.GR, mm10_blackRegions,invert = TRUE)
length(filtered_Peaks.GR)_____no_output_____

# Check the distribution of the number peaks per-chromosome_____no_output_____

filtered_Peaks.df = as.data.frame(filtered_Peaks.GR)[,1:3]

options(repr.plot.width = 10, repr.plot.height = 6)
ggplot(filtered_Peaks.df, aes(x=seqnames)) +
   geom_bar(width=0.8, fill="black") +
   theme_bw() +
   xlab("Chromosome") +
   ylab("Number of open chromatin regions") +
   theme(axis.text.x = element_text(angle = 90,hjust = 1,size = 12),
         axis.text = element_text(size = 12))_____no_output_____

We can use chromosomes chr1 to chr5 for training and the rest for testing.

### Generate 200bp windows and group them into positive and label them_____no_output_____We use a sliding window of size 200bp to generated windows_____no_output_____

seqlen = seqlengths(BSgenome.Mmusculus.UCSC.mm10)[1:19]
winsize = 500

all_bins = GRangesList()
for(ch in names(seqlen)){
    print(ch)
    region <- GRanges(ch, IRanges(1,seqlen[ch]))
    bins = slidingWindows(region,width = winsize,step = winsize)[[1]]
    # exclude blak listed regions
    bins = subsetByOverlaps(bins,mm10_blackRegions,invert = T)
    all_bins[[ch]] = bins
  }[1] "chr1"
[1] "chr2"
sapply(all_bins,length)_____no_output_____all_bins = unlist(all_bins)_____no_output_____prettyNum(length(all_bins),big.mark = ",")_____no_output_____

## Mark positive and negative samples_____no_output_____

all_bins$Class = rep(0,length(all_bins))
positive = findOverlaps(all_bins, filtered_Peaks.GR) %>% queryHits()
all_bins$Class[positive]=1_____no_output_____table(all_bins$Class)/length(all_bins)_____no_output_____

## Subsample data_____no_output_____

require(tidymodels)Loading required package: tidymodels

── Attaching packages ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── tidymodels 0.1.2 ──

✔ broom     0.7.5      ✔ recipes   0.1.15
✔ dials     0.0.9      ✔ rsample   0.0.9
✔ infer     0.5.4      ✔ tune      0.1.3
✔ modeldata 0.1.0      ✔ workflows 0.2.2
✔ parsnip   0.1.5      ✔ yardstick 0.0.8

Warning message:
“package ‘broom’ was built under R version 4.0.3”
Warning message:
“package ‘recipes’ was built under R version 4.0.3”
── Conflicts ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── tidymodels_conflicts() ──
✖ dplyr::collapse()     masks glue::collapse(), Biostrings::collapse(), IRanges::collapse()
✖ dplyr::combine()      masks BiocGenerics::combine()
✖ purrr::compact()      masks XVector::compact()
✖ rsample::complement() masks Biostrings::complement()
✖ dials::degree()       masks circlize::degree()
✖ dplyr::desc()         masks IRanges::desc()
✖ scales::discard()     masks purrr::discard()
✖ recipes::discretize() masks EnrichedHeatmap::discretize()
✖ tidyr::expand()       masks S4Vectors::expand()
✖ dplyr::filter()       masks stats::filter()
✖ dplyr::first()        masks S4Vectors::first()
✖ recipes::fixed()      masks stringr::fixed()
✖ dplyr::lag()          masks stats::lag()
✖ ggplot2::Position()   masks BiocGenerics::Position(), base::Position()
✖ recipes::prepare()    masks ComplexHeatmap::prepare()
✖ purrr::reduce()       masks GenomicRanges::reduce(), IRanges::reduce()
✖ dplyr::rename()       masks S4Vectors::rename()
✖ dplyr::slice()        masks XVector::slice(), IRanges::slice()
✖ yardstick::spec()     masks readr::spec()
✖ recipes::step()       masks stats::step()
✖ parsnip::translate()  masks Biostrings::translate()
✖ recipes::update()     masks GenomicRanges::update(), stats4::update(), stats::update()

all_bins.df = as_tibble(all_bins)
all_bins.df$width = NULL
all_bins.df$strand = NULL
all_bins.df$Class = factor(all_bins.df$Class)
all_bins.df$seqnames = factor(all_bins.df$seqnames)
head(all_bins.df)_____no_output_____class(split(all_bins.df, all_bins.df$seqnames)[[1]])_____no_output_____set.seed(08222021)
all_bins_subsampled <-  split(all_bins.df, all_bins.df$seqnames) %>%
                          map(.f = function(df) {

                           recipe( ~ .,data = df) %>%
                           themis::step_downsample(Class, under_ratio = 1.5) %>%
                           prep() %>%
                           bake(new_data = NULL)

                          }) %>%
                         do.call(what = "rbind")_____no_output_____class(all_bins_subsampled)_____no_output_____options(repr.plot.width = 6, repr.plot.height = 5)
ggplot(all_bins_subsampled, aes(x=Class)) + geom_bar() + theme_bw()_____no_output_____options(repr.plot.width = 15, repr.plot.height = 5)
ggplot(all_bins_subsampled, aes(x=seqnames, fill=Class)) + geom_bar(position = position_dodge()) + theme_bw()_____no_output_____head(all_bins_subsampled)_____no_output_____

## Get DNA Sequences of the subsampled regions_____no_output_____

selected_bins_Seqs = getSeq(x = BSgenome.Mmusculus.UCSC.mm10,
              names= all_bins_subsampled$seqnames,
              start = all_bins_subsampled$start,
              end = all_bins_subsampled$end
              )
selected_bins_Seqs_____no_output_____selected_bins_Seqs.df = as.data.frame(selected_bins_Seqs)
selected_bins_Seqs.df$chromosome = all_bins_subsampled$seqnames
selected_bins_Seqs.df$Class = all_bins_subsampled$Class
head(selected_bins_Seqs.df)_____no_output_____table(nchar(selected_bins_Seqs.df$x))_____no_output_____

## Save results_____no_output_____

colnames(selected_bins_Seqs.df)[1] = "sequences"
readr::write_tsv(selected_bins_Seqs.df, file="../data/example_ATACseq_data.tsv")_____no_output_____