# Notebook from Vizzuality/data_sci_tutorials
Path: work/Landsat 7 annual basemaps - Pt 1.ipynb

# Natural colour composites using Lansdat 7 imagery_____no_output_____Annual basemap layers for the years 2013-2016 were produced with Landsat 8 imagery and are already available in the Global Forest Watch platform. In this notebook we'll explore different approaches to generate annual basemaps for the years 2000-2014 using Landsat 7. We will aim to replicate the natural-color tones of the 2013-2016 composites, and also explore ways to improve the rendering speed for high zoom level tiles.

The main purpose of this notebook is just to give a preview of the look and feel of the resulting composites. A separate document will be prepared to describe the workflow for exporting tiles to GCS and into the microservice.

If you're viewing this notebook form GH you'll need to wait a few minutes for all the images to show up.

You can skip this introduction and head over sections 3, 4 and 5, which show the most promising results for rendering tiles at different z-resolutions_____no_output_____

import os
import ee
import json
import ipywidgets
from IPython.display import Image
import folium
import pprint
import datetime
from dateutil.relativedelta import relativedelta

ee.Initialize()_____no_output_____

# Setup visualisation tools
We will first define a few functions to help us visualise the results, these include:
-  `tile_url`: generates tile urls from images computed in Earth Engine
-  `dateRange`: returns a custom date range calculated around a target year
-  `toImage`: retrieves images from composites computed in Earth Engine and plots them in 3 different zoom levels._____no_output_____

# Function to plot tiles (Images)
def tile_url(image, viz_params=None):
    """Create a target url for tiles for an image.
    e.g.
    im = ee.Image("LE7_TOA_1YEAR/" + year).select("B3","B2","B1")
    viz = {'opacity': 1, 'gain':3.5, 'bias':4, 'gamma':1.5}
    url = tile_url(image=im,viz_params=viz)
    """
    if viz_params:
        d = image.getMapId(viz_params)
    else:
        d = image.getMapId()
    base_url = 'https://earthengine.googleapis.com'
    url = (base_url + '/map/' + d['mapid'] + '/{z}/{x}/{y}?token=' + d['token'])
    return url_____no_output_____# Function to generate a date range
def dateRange(mid_year,period):
    '''
    Params:
        mid_year (assumes yyyy): Mid point of the time range
        period (int): number of years
    Returns:
        dictionary with a 'start' and an 'end' date in a `yyyy-mm-dd` format.
        The reference year is also provided in the 'central' key
    '''
    # define central year, commencing in JAN-01
    mid = datetime.datetime.strptime(str(mid_year) + '-01-01', '%Y-%m-%d')

    # get half period
    half_period = int(period/2)

    # compute date range
    if period%2 ==1:
        end = mid + relativedelta(years= half_period,months=6)
        start = mid - relativedelta(years= half_period,months=6)
    else:
        end = mid + relativedelta(years= half_period)
        start = mid - relativedelta(years= half_period)

    dates = {'start':start.strftime('%Y-%m-%d'),'central':str(mid_year) ,'end':end.strftime('%Y-%m-%d')}

    print('Returning '+ str(period) + ' years range: from '+ dates['start'] + ' to ' + dates['end'])

    return(dates)_____no_output_____# Function to produce image
def toImage(composite,visParam,title):
    '''
    Retrieves Google Earth image's thumbUrls and plot the results at 3 different zoom levels. Note that the `region_z` objects are pulled from the global environment.
    composite: ee.Image (assumes 3 bands)
    visParam: image.vizualisation params e.g. {'opacity': 1, 'gain':3.5, 'bias':4, 'gamma':1.5}
    title: plot
    '''
    visParam['region'] = region_z10
    print('Z10: '+ title + ': ' + dates['start'] + ' - ' + dates['end'])
    image1 = Image(url=composite.getThumbUrl(visParam))
    display(image1)

    visParam['region'] = region_z6
    print('Z6: '+ title + ': ' + dates['start'] + ' - ' + dates['end'])
    image2= Image(url=composite.getThumbUrl(visParam))
    display(image2)

    visParam['region'] = region_z4
    print('Z4: '+ title + ': ' + dates['start'] + ' - ' + dates['end'])
    image3= Image(url=composite.getThumbUrl(visParam))
    display(image3)

    _____no_output_____

## Define ROIs

Below we will calculate buffer zones from a given lon-lat coordinate and generate 3 ROI objects. These will then be used in the `toImage` function to print Lansat images in the notebook. These ROIs are roughly equivalent to zoom levels 4, 6 and 10.

Some sites of interest:

|site|ISO|lon|lat|
|-----------|-------|---|---|
|El Triunfo|MEX|-92.7548|15.6151|
|Chimalapas|MEX|-94.3835|16.7604|
|Leticia|COL|-70.2727|-3.8094|
|Oslo|NOR|10.679|59.955|
|Okavango Delta|BWA|22.346|-18.792|
|Gran Canaria|ESP|-15.5786|27.9507|_____no_output_____

# --- Coordinates
lon = -15.5786
lat = 27.9507
zstart = 10_____no_output_____# -- Set ROI for IPython.display
point = ee.Geometry.Point(lon,lat)

# set bufffers:units is meters
# resulting zoom levels are an approximation
buffer_z10 = point.buffer(40000)
buffer_z6 = point.buffer(650000)
buffer_z4 = point.buffer(2500000)

# convert to FeatureCollection
bufferFeat_z10 = ee.FeatureCollection(buffer_z10).set('area',1)
bufferFeat_z6 = ee.FeatureCollection(buffer_z6).set('area',1)
bufferFeat_z4 = ee.FeatureCollection(buffer_z4).set('area',1)

# extract bounds
region_z10 = buffer_z10.bounds().getInfo()['coordinates']
region_z6 = buffer_z6.bounds().getInfo()['coordinates']
region_z4 = buffer_z4.bounds().getInfo()['coordinates']

#  -- preview ROI
# roi = tile_url(bufferFeat_z10,{'color': '#FF5733'})
# map = folium.Map(location=[lat,lon],zoom_start=zstart)
# map.add_tile_layer(tiles=roi, attr="na")
# map_____no_output_____

## Landsat products

Earth Engine provides a variety of [Landsat products](https://developers.google.com/earth-engine/landsat) which are cataloged according to their data quality and processing methods. Some of these proved to be very useful at certain zoom levels when producing the basemaps for the 2013-2016 period.

In the sections below we'll apply a similar compositing methodology with the equivalent Landsat 7 products. These include the following collections:

## Orthorectified

**USGS Landsat 7 Raw Scenes (Orthorectified)**

- id: LANDSAT/LE7_L1T
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE7_L1T)
- dates: Jan 1, 1999 - Apr 30, 2017

## Collection 1 Tier 1

Landsat scenes with the highest available data quality are placed into Tier 1 and are considered suitable for time-series processing analysis. The T1_RT collection contains both Tier 1 and Real-Time (RT) assets.

**USGS Landsat 7 Collection 1 Tier 1 Raw Scenes**

- id: LANDSAT/LE07/C01/T1
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE07/C01/T1)
- dates: Jan 1, 1999 - Present

**USGS Landsat 7 Collection 1 Tier 1 and Real-Time data aw Scenes]**

- id: LANDSAT/LE07/C01/T1_RT
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE07/C01/T1_RT)
- dates: Jan 1, 1999 - Present

## Calibrated TOA

These Landsat 7 composites are made from Level L1T orthorectified scenes, using the computed top-of-atmosphere (TOA) reflectance. As of May 1, 2017, the USGS is no longer producing Pre-Collection Landsat, and therefore these collections are on longer being updated.

**USGS Landsat 7 Collection 1 Tier 1 and Real-Time data OA Reflectance]**

- id: LANDSAT/LE07/C01/T1_RT_TOA
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE07/C01/T1_RT_TOA)
- dates: Jan 1, 1999 - Present

**USGS Landsat 7 TOA Reflectance (Orthorectified)**

- id: LANDSAT/LE7_L1T_TOA
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE7_L1T_TOA)
- dates: Jan 1, 1999 - Apr 30, 2017

**Landsat 7 Annual Greenest-Pixel TOA Reflectance Composite**

- id: LE7_L1T_ANNUAL_GREENEST_TOA
- [description](https://code.earthengine.google.com/dataset/LANDSAT/LE7_L1T_ANNUAL_GREENEST_TOA)
- dates: Jan 1, 1999 - Jan 1, 2018_____no_output_____---------_____no_output_____## 1. USGS Landsat 7 Raw Scenes (Orthorectified)

 ` LANDSAT/LE7_L1T`_____no_output_____

dates = dateRange(2010,3)

# load collection and filter by dates
l7_l1t = ee.ImageCollection('LANDSAT/LE7_L1T').filterDate(dates['start'], dates['end'])Returning 3 years range: from 2008-07-01 to 2011-07-01

### Cloud free composite

Returns a cloud-free composite computed with the `ee.Algorithms.Landsat.simpleComposite()`. This function converts raw scenes to TOA reflectance, applies the simple cloud score and takes a given percentile of the least cloudy pixels.

[Sample code](https://code.earthengine.google.com/e8832e9bf4d073cda0dcdb6638a065ae):
```
var collection = ee.ImageCollection('LANDSAT/LC8_L1T').filterDate('2015-01-01T00:00','2015-12-31T00:00');
var composite = ee.Algorithms.Landsat.simpleComposite({collection: collection, percentile: 50, maxDepth:80, cloudScoreRange: 1, asFloat:true});
var hsv2 = composite.select(['B4', 'B3', 'B2']).rgbToHsv();
var median = hsv2.hsvToRgb().visualize({gain:1000, gamma: [1.15, 1.4, 1.15]});
```_____no_output_____

# cutsom composite
## - Playing with the percentile values results in different brightness intensities.
l7_l1t_composite = ee.Algorithms.Landsat.simpleComposite(
    collection= l7_l1t,
    percentile= 30,
    maxDepth=80,
    cloudScoreRange= 1, asFloat=True)

# to HSV
hsv_l1t = l7_l1t_composite.select(['B3', 'B2', 'B1']).rgbToHsv()

# make image
rgb_l1t= hsv_l1t.hsvToRgb().visualize()_____no_output_____visParam = {
    'gain':3,
    'gamma':'1.15,1.4,1.15',
}

# plot
toImage(rgb_l1t,visParam,'simpleComposite (custom parameters)')Z10: simpleComposite (custom parameters): 2008-07-01 - 2011-07-01
# Map tiles
# ee_tiles = tile_url(rgb_l1t,visParam)
# map = folium.Map(location=[lat,lon], zoom_start=5)
# map.add_tile_layer(tiles=ee_tiles, attr="NASA")
# map_____no_output_____

### Cloud free composite + pan-sharpen

Now we'll pansharpen the above composite.

[Sample code](https://code.earthengine.google.com/cbe5822c7166f99fc414d59d8e70c7fc)_____no_output_____

# Sharpen above compostie
l7_l1t_sharpened = ee.Image.cat([hsv_l1t.select('hue'), hsv_l1t.select('saturation'),
                          l7_l1t_composite.select('B8')]).hsvToRgb() # .visualize(gain=600,gamma= [1.15, 1.4, 1.15])_____no_output_____visParam = {
    'gain':1000,
    'gamma':'1.15,1.4,1.15',
}

# plot
toImage(l7_l1t_sharpened,visParam,'simpleComposite (custom parameters) + pansharpen')Z10: simpleComposite (custom parameters) + pansharpen: 2008-07-01 - 2011-07-01
# Map tiles
# ee_tiles = tile_url(l7_l1t_sharpened,visParam)
# map = folium.Map(location=[lat,lon], zoom_start=11)
# map.add_tile_layer(tiles=ee_tiles, attr="NASA")
# map_____no_output_____

## 2. USGS Landsat 7 TOA Reflectance (Orthorectified)
`LANDSAT/LE7_L1T_TOA`_____no_output_____

dates = dateRange(2010,4)

# load collection and filter by dates
l7_l1t_toa = ee.ImageCollection('LANDSAT/LE7_L1T_TOA').filterDate(dates['start'], dates['end'])Returning 4 years range: from 2008-01-01 to 2012-01-01

###  Filter by cloud cover and median over time + pan-sharpen

This method selects a subset of scenes at a given location, applies a cloud cover filter and takes the median value of the resulting pixels. Since we're working with a TOA reflectance data we don't need to calculate this parameter. We'll also pan-sharpen these images so results can be used for high resolution tiles.

The original workflow for processing Landsat 8 data included a `CLOUD_COVER_LAND` filter, which is used to calculate could scores of the [land pixels in a scene](https://landsat.usgs.gov/what-difference-between-scene-cloud-cover-and-land-cloud-cover-scores). This parameter is not available in the Landsat 7, so instead I'm applying a simple `CLOUD_COVER` score.

The results are not of much use though...

[Source](https://code.earthengine.google.com/2f72c2a15bd46646329e28c13ba2a558)

```
// Get the median over time, in each band, in each pixel.
var median = ee.ImageCollection('LANDSAT/LC8_L1T_TOA')
                  .filterDate('2013-04-01', '2016-12-31').filter(ee.Filter.lt('CLOUD_COVER_LAND', 35)).median();

var hsv = median.select(['B4', 'B3', 'B2']).rgbToHsv();
var sharpened = ee.Image.cat([hsv.select('hue'), hsv.select('saturation'), median.select('B8')]).hsvToRgb().visualize({gain:1000, gamma: [1.15, 1.4, 1.15]});

```_____no_output_____

l7_l1t_toa_med = l7_l1t_toa.filter(ee.Filter.lt('CLOUD_COVER', 7)).median()
hsv_l1t_toa = l7_l1t_toa_med.select(['B3', 'B2', 'B1']).rgbToHsv()_____no_output_____# Sharpen above compostie
l7_l1t_toa_sharpened = ee.Image.cat([hsv_l1t_toa.select('hue'), hsv_l1t_toa.select('saturation'),
                                     l7_l1t_toa_med.select('B8')]).hsvToRgb()_____no_output_____visParam= {
    'gain': 800,
    'gamma':'1.15, 1.4, 1.15'
}

# plot
toImage(l7_l1t_toa_sharpened,visParam,'Filter by cloud & median values + Pansharpened')Z10: Filter by cloud & median values + Pansharpened: 2008-01-01 - 2012-01-01

## 3. USGS Landsat 7 Collection 1 Tier 1 Raw Scenes
`LANDSAT/LE07/C01/T1`_____no_output_____

dates = dateRange(2010,3)

# load collection and filter by dates
l7_t1 = ee.ImageCollection('LANDSAT/LE07/C01/T1').filterDate(dates['start'], dates['end'])Returning 3 years range: from 2008-07-01 to 2011-07-01

### Cloud free composite + pan-sharpen

In this example we will generate a composite with the `ee.Algorithms.Landsat.simpleComposite()` module. We'll provide with a custom cloud cover threshold and percentile value to filter the resulting pixels.

This particular combination looks very useful for lower resolution tiles (Z6), and the color palette is similar to the Landsat basemaps in GFW for recent years

[Source](https://developers.google.com/earth-engine/landsat#simple-composite)_____no_output_____

# Create a cloud-free composite with custom parameters for
customComposite = ee.Algorithms.Landsat.simpleComposite(
  collection= l7_t1,
    percentile= 30,
    cloudScoreRange= 8,
    asFloat= True)_____no_output_____# Convert the RGB bands to the HSV color space.
hsv_custom = customComposite.select(['B3', 'B2', 'B1']).rgbToHsv()

# now from HSV to RGB
rgb_custom = hsv_custom.hsvToRgb()_____no_output_____# Define visualisation parameters
visParam= {
    'gain': 700,
    'gamma':'1.15, 1.4, 1.15'
}

# plot
toImage(rgb_custom,visParam,'Simple composite (custom parameters)')Z10: Simple composite (custom parameters): 2008-07-01 - 2011-07-01
# Map tiles
# ee_tiles = tile_url(rgb_custom,visParam)  # hsv_custom looks cool
# map = folium.Map(location=[lat,lon], zoom_start=7)
# map.add_tile_layer(tiles=ee_tiles, attr="NASA")
# map_____no_output_____

Increasing the date query to 5 years and changing the visualisation parameters (see below) returns composites that could also be used for higher zoom levels._____no_output_____

dates = dateRange(2010,5)  # <- increase date range

# load collection and filter by dates
l7_t1 = ee.ImageCollection('LANDSAT/LE07/C01/T1').filterDate(dates['start'], dates['end'])

# Create a cloud-free composite with custom parameters for
customComposite = ee.Algorithms.Landsat.simpleComposite(
  collection= l7_t1,
    percentile= 60,
    cloudScoreRange= 1,
    asFloat= True)

# Define visualisation parameters
visParam= {
    'gain': 800,
    'gamma':'1.15, 1.40, 1.15'
}

# plot results
toImage(rgb_custom,visParam,'Simple composite (custom parameters)')Returning 5 years range: from 2007-07-01 to 2012-07-01
Z10: Simple composite (custom parameters): 2007-07-01 - 2012-07-01

## 4. USGS Landsat 7 Collection 1 Tier 1 and Real-Time data Raw Scenes
`LE07/C01/T1_RT/`

Tier 1 collections contain images with the highest quality available. In this section we'll compute 2 composites using the  `simpleComposite` algorithm, one with default parameters and the other with a custom cloud score and percentile value

[Source](https://code.earthengine.google.com/9c32b62c959c1f4121cb3218380afb7d) _____no_output_____

dates = dateRange(2010,4)

# load collection and filter by dates
l7_t1_rt = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT').filterDate(dates['start'], dates['end'])Returning 4 years range: from 2008-01-01 to 2012-01-01
l7_t1_rt_simpleComp = ee.Algorithms.Landsat.simpleComposite(l7_t1_rt)_____no_output_____# Define visParams
visParam = {
    'bands':'B3,B2,B1',
    'gain':2.5,
    'gamma':'1.3, 1.3, 1.3',
#     'dimensions': 800
}

# plot
toImage(l7_t1_rt_simpleComp,visParam,'Simple composite (default params)')Z10: Simple composite (default params): 2008-01-01 - 2012-01-01
# Create a cloud-free composite with custom parameters for
customComposite = ee.Algorithms.Landsat.simpleComposite(
  collection= l7_t1_rt,
    percentile= 50,
    cloudScoreRange= 1,
    asFloat= True)_____no_output_____# Convert the RGB bands to the HSV color space.
hsv_custom = customComposite.select(['B3', 'B2', 'B1']).rgbToHsv()

# now from HSV to RGB
rgb_custom = hsv_custom.hsvToRgb()_____no_output_____# Define visParams
visParam = {
    'gain':700,
    'gamma':'1.15, 1.4, 1.15'
}

# plot
toImage(rgb_custom,visParam,'C1 Tier1 Pansharepened')Z10: C1 Tier1 Pansharepened: 2008-01-01 - 2012-01-01
# Map tiles
# ee_tiles = tile_url(rgb_custom,visParam)
# map = folium.Map(location=[lat,lon], zoom_start=9)
# map.add_tile_layer(tiles=ee_tiles, attr="NASA")
# map.add_tile_layer(tiles=feat_tiles, attr="na")
# map_____no_output_____

## 5. USGS Landsat 7 Collection 1 Tier 1 and Real-Time data TOA Reflectance
` LANDSAT/LE07/C01/T1_RT_TOA`_____no_output_____

dates = dateRange(2010,4)
# load collection and filter by dates
l7_t1_rt_toa = ee.ImageCollection('LANDSAT/LE07/C01/T1_RT_TOA').filterDate(dates['start'], dates['end'])Returning 4 years range: from 2008-01-01 to 2012-01-01

###  Filter by cloud cover and median over time + pan-sharpen

This method selects a subset of scenes at a given location, applies a cloud cover filter and takes the median value of the resulting pixels. Since we're working with a TOA reflectance data we don't need to calculate this parameter. Finally we'll also pan-sharpen these images

The results could be of potential use for high resolution tiles
_____no_output_____

l7_t1_rt_toa_med = l7_t1_rt_toa.filter(ee.Filter.lt('CLOUD_COVER', 10)).median()_____no_output_____# Define visParams
visParam = {
    'bands':'B3,B2,B1',
    'min':0,
    'max':0.25,
#     'dimensions': 800
}

# plot
toImage(l7_t1_rt_toa_med,visParam,'Filter by cloud cover | median')Z10: Filter by cloud cover | median: 2008-01-01 - 2012-01-01

## 6. Landsat 7 Annual Greenest-Pixel TOA Reflectance Composite
`LE7_L1T_ANNUAL_GREENEST_TOA`_____no_output_____

dates = dateRange(2010,5)

# load collection and filter by dates
l7_toa_greenest_annual = ee.ImageCollection('LE7_L1T_ANNUAL_GREENEST_TOA').filterDate(dates['start'], dates['end'])Returning 5 years range: from 2007-07-01 to 2012-07-01

#### Greenest pixel, and Greenest + Pansharp

Lastly we'll experiment with the greenest-pixel collection. We'll compute a simple composite with the median values of a given period, and will also generate a pan-sharpened version of this composite

[Source](https://code.earthengine.google.com/3b230f4c217c589ab267e438798b5e28)

```
//--- Greenest pixel
// NDVI Function
var addNDVI = function(image) {
var ndvi = image.normalizedDifference(['B5', 'B4']).rename('NDVI');
  return image.addBands(ndvi);
};

var withNDVI = l8.map(addNDVI);

// Make a "greenest" pixel composite.
var greenest = withNDVI.qualityMosaic('NDVI');

// Display the result.
var visParams = {bands: ['B3', 'B2', 'B1'], max: 0.3};
Map.addLayer(greenest, visParams, 'Greenest pixel composite');

// ---- Greenest + Pansharp
var hsv = greenest.select(['B3', 'B2', 'B1']).rgbToHsv();
var sharpened = ee.Image.cat([hsv.select('hue'), hsv.select('saturation'), greenest.select('B8')]).hsvToRgb().visualize({gain:1000, gamma: [1.15, 1.4, 1.15]});

Map.addLayer(sharpened,null,'pan+green');

``` _____no_output_____

# get media (filtering by CLOUD_COVER is not available in this collection)
l7_toa_greenest_annual = l7_toa_greenest_annual.median()

# Plot
visParam = {
    'bands':'B3,B2,B1',
    'max':0.3
}

# plot
toImage(l7_toa_greenest_annual,visParam,'Greenest-pixel')Z10: Greenest-pixel: 2007-07-01 - 2012-07-01
# calculate HSV
hsv_greenest = l7_toa_greenest_annual.select(['B3', 'B2', 'B1']).rgbToHsv()

# Sharpen above compostie
l7_toa_greenest_sharpened = ee.Image.cat([hsv_greenest.select('hue'), hsv_greenest.select('saturation'),
                                     l7_toa_greenest_annual.select('B8')]).hsvToRgb()_____no_output_____# Plot
visParam = {
    'gain':1000,
    'gamma': '1.15, 1.4, 1.15'
}

# plot
toImage(l7_toa_greenest_sharpened,visParam,'Greenest-pixel + pansharpen')Z10: Greenest-pixel + pansharpen: 2007-07-01 - 2012-07-01


# Notebook from genepattern/TCGAImporter-notebooks
Path: TCGA_HTSeq_counts/LGG/Brain Lower Grade Glioma (LGG).ipynb

# Brain Lower Grade Glioma (LGG)
[Jump to the urls to download the GCT and CLS files](#Downloads)_____no_output_____**Authors:** Alejandra Ramos, Marylu Villa and Edwin Juarez
**Is this what you want your scientific identity to be?**
**Contact info:** Email Edwin at [ejuarez@cloud.ucsd.edu](mailto:ejuarez@cloud.ucsd.edu) or post a question in http://www.genepattern.org/help_____no_output_____This notebook provides the steps to download all the LGG samples from The Cancer Genome Atlas (TCGA) contained in the Genomic Data Commons (GDC) Data portal. These samples can be downloaded as a GCT file and phenotype labels (primary tumor vs normal samples) can be downloaded as a CLS file. These files are compatible with other GenePattern Analyses._____no_output_____![image.png](attachment:image.png)_____no_output_____# Overview
_____no_output_____
Lower grade glioma is a type of cancer that develops in the glial cells of the brain. Glial cells support the brain’s nerve cells and keep them healthy. Tumors are classified into grades I, II, III or IV based on standards set by the World Health Organization.

TCGA is including grades II and III in this specific study, as opposed to the more common and higher grade brain tumor, gliobastoma multiforme (GBM) which was studied in a separate project.

_____no_output_____

_____no_output_____# LGG Statistics_____no_output_____In 2010, more than 22,000 Americans were estimated to have been diagnosed and 13,140 were estimated to have died from brain and other cancers of the nervous system.

_____no_output_____

http://journals.plos.org/plosmedicine/article?id=10.1371/journal.pmed.1002192
_____no_output_____# Dataset's Demographic information_____no_output_____
TCGA contained 529 LGG samples (511 primary cancer samples, and 0 normal tissue samples and the rest are ignored) from 511 people. Below is a summary of the demographic information represented in this dataset. If you are interested in viewing the complete study, as well as the files on the GDC Data Portal, you can follow <a href="https://portal.gdc.cancer.gov/repository?facetTab=cases&filters=%7B%22op%22%3A%22and%22%2C%22content%22%3A%5B%7B%22op%22%3A%22in%22%2C%22content%22%3A%7B%22field%22%3A%22cases.project.project_id%22%2C%22value%22%3A%5B%22TCGA-UVM%22%5D%7D%7D%2C%7B%22op%22%3A%22in%22%2C%22content%22%3A%7B%22field%22%3A%22files.analysis.workflow_type%22%2C%22value%22%3A%5B%22HTSeq%20-%20Counts%22%5D%7D%7D%2C%7B%22op%22%3A%22in%22%2C%22content%22%3A%7B%22field%22%3A%22files.experimental_strategy%22%2C%22value%22%3A%5B%22RNA-Seq%22%5D%7D%7D%5D%7D&searchTableTab=cases" target="_blank">this link.(these data were gathered on July 17th, 2018)

_____no_output_____![image.png](attachment:image.png)_____no_output_____# Login to GenePattern
_____no_output_____

 Instructions

Login to the *GenePattern Cloud* server.

_____no_output_____

# Requires GenePattern Notebook: pip install genepattern-notebook
import gp
import genepattern

# Username and password removed for security reasons.
genepattern.display(genepattern.session.register("https://gp-beta-ami.genepattern.org/gp", "", ""))_____no_output_____

# Downloading RNA-Seq HTSeq Counts Using TCGAImporter_____no_output_____Use the TCGAImporter module to download RNA-Seq HTSeq counts from the GDC Data Portal using a Manifest file and a Metadata file_____no_output_____
Input files

Manifest file: a file containing the list of RNA-Seq samples to be downloaded.

Metadata file: a file containing information about the files present at the GDC Data Portal. Instructions for downloading the Manifest and Metadata files can be found here: https://github.com/genepattern/TCGAImporter/blob/master/how_to_download_a_manifest_and_metadata.pdf

Output files

LGG_TCGA.gct - This is a tab delimited file that contains the gene expression (HTSeq counts) from the samples listed on the Manifest file. For more info on GCT files, look at reference 1

LGG_TCGA.cls - The CLS file defines phenotype labels (in this case Primary Tumor and Normal Sample) and associates each sample in the GCT file with a label. For more info on CLS files, look at reference 2

_____no_output_____

 Instructions

Load the manifest file in **Manifest** parameter.

Load the metadata file in **Metadata** parameter.

Click **run**.

_____no_output_____
Estimated run time for TCGAImporter : ~ 7 minutes

_____no_output_____

tcgaimporter_task = gp.GPTask(genepattern.session.get(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.analysis:00369')
tcgaimporter_job_spec = tcgaimporter_task.make_job_spec()
tcgaimporter_job_spec.set_parameter("manifest", "https://cloud.genepattern.org/gp/users/marylu257/tmp/run6574281092269727853.tmp/LGG_MANIFEST.txt")
tcgaimporter_job_spec.set_parameter("metadata", "https://cloud.genepattern.org/gp/users/marylu257/tmp/run2383346596088480063.tmp/LGG_METADATA.json")
tcgaimporter_job_spec.set_parameter("output_file_name", "TCGA_dataset")
tcgaimporter_job_spec.set_parameter("gct", "True")
tcgaimporter_job_spec.set_parameter("translate_gene_id", "False")
tcgaimporter_job_spec.set_parameter("cls", "True")
genepattern.display(tcgaimporter_task)

job35224 = gp.GPJob(genepattern.session.get(0), 35224)
genepattern.display(job35224)_____no_output_____collapsedataset_task = gp.GPTask(genepattern.session.get(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.analysis:00134')
collapsedataset_job_spec = collapsedataset_task.make_job_spec()
collapsedataset_job_spec.set_parameter("dataset.file", "https://cloud.genepattern.org/gp/jobResults/32334/TCGA_dataset.gct")
collapsedataset_job_spec.set_parameter("chip.platform", "ftp://ftp.broadinstitute.org/pub/gsea/annotations/ENSEMBL_human_gene.chip")
collapsedataset_job_spec.set_parameter("collapse.mode", "Maximum")
collapsedataset_job_spec.set_parameter("output.file.name", "<dataset.file_basename>.collapsed")
genepattern.display(collapsedataset_task)

job32411 = gp.GPJob(genepattern.session.get(0), 32411)
genepattern.display(job32411)_____no_output_____

# Downloads
_____no_output_____
You can download the input and output files of TCGAImporter for this cancer type here:

Inputs:

https://datasets.genepattern.org/data/TCGA_HTSeq_counts/LGG/LGG_MANIFEST.txt

https://datasets.genepattern.org/data/TCGA_HTSeq_counts/LGG/LGG_METADATA.json

Outputs:

https://datasets.genepattern.org/data/TCGA_HTSeq_counts/LGG/LGG_TCGA.gct

https://datasets.genepattern.org/data/TCGA_HTSeq_counts/LGG/LGG_TCGA.cls

_____no_output_____If you'd like to download similar files for other TCGA datasets, visit this link:
- https://datasets.genepattern.org/?prefix=data/TCGA_HTSeq_counts/_____no_output_____# References_____no_output_____[1] http://software.broadinstitute.org/cancer/software/genepattern/file-formats-guide#GCT_____no_output_____[2] http://software.broadinstitute.org/cancer/software/genepattern/file-formats-guide#CLS_____no_output_____[3] https://cancergenome.nih.gov/cancersselected/lowergradeglioma

[4] https://es.wikipedia.org/wiki/Glioblastoma

[5] http://journals.plos.org/plosmedicine/article?id=10.1371/journal.pmed.1002192

_____no_output_____


# Notebook from LD-Lab/16S-Taxa-Phlyo
Path: process/Bowtie2-test.ipynb

bowtie2-build example/reference/lambda_virus.fa example/index/lambda_virus_____no_output_____bowtie2-build /mnt/d/Lab/16S-Taxa-Phlyo/database/SILVA_138.1_SSURef_NR99_tax_silva.fasta /mnt/d/Lab/16S-Taxa-Phlyo/database/SILVA_138.1_SSURef_NR99_____no_output_____bowtie2 -x ecoli -1 SAMPLE_r1.fastq -2 SAMPLE_r2.fastq -U SAMPLE_single_reads.fastq --no-unal -p 6 -S merged.sam_____no_output_____

bowtie2 -x /mnt/d/Lab/16S-Taxa-Phlyo/database/SILVA_138.1_SSURef_NR99 -U /mnt/d/Lab/16S-Taxa-Phlyo/result/M15/merged.fasta --no-unal -p 6 -S merged.sam_____no_output_____

/mnt/d/Lab/16S-Taxa-Phlyo/database/SILVA_138.1_SSURef_NR99_____no_output_____/mnt/d/Lab/16S-Taxa-Phlyo/result/M15/merged.fasta_____no_output_____perl: warning: Setting locale failed.
perl: warning: Please check that your locale settings:
        LANGUAGE = "en.UTF-8",
        LC_ALL = (unset),
        LANG = "en.UTF-8"
    are supported and installed on your system.
perl: warning: Falling back to the standard locale ("C")._____no_output_____


# Notebook from nx-nextension/nexus
Path: docs/src/main/paradox/docs/getting-started/notebooks/dataset_from_different_sources.ipynb

# Tutorial: Integrate Neuroscience Datasets from Multiple Sources using MINDS_____no_output_____## Initialize and configure_____no_output_____

!pip install nexusforge==0.6.2_____no_output_____!pip install allensdk_____no_output_____!pip install neurom[plotly]==3.0.1_____no_output_____!pip install --upgrade nest-asyncio==1.5.1_____no_output_____

### Get an authentication token_____no_output_____The [Nexus sandbox application](https://sandbox.bluebrainnexus.io/web) can be used to get a token:

- Step 1: From the [web page](https://sandbox.bluebrainnexus.io/web), click on the login button in the top right corner and follow the instructions on screen.

- Step 2: You will then see a `Copy token` button in the top right corner. Click on it to copy the token to the clipboard.
_____no_output_____Once a token is obtained, proceed to paste it as the value of the `TOKEN` variable below.

__Important__: A Nexus token is valid for 8 hours, if your working session is open for more than 8 hours, you may need to refresh the value of the token and reintialize the forge client in the _'Configure a forge client to store, manage and access datasets'_ section below._____no_output_____

import getpass_____no_output_____TOKEN = getpass.getpass()_____no_output_____

### Configure a forge client to store, manage and access datasets_____no_output_____

import uuid
import base64
import requests
import json
from pathlib import Path

from kgforge.core import KnowledgeGraphForge
from kgforge.specializations.mappings import DictionaryMapping

from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.core.cell_types_cache import CellTypesCache_____no_output_____r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')
dirpath = './rdfmodel'
Path(dirpath).mkdir(parents=True, exist_ok=True)
with open(f'{dirpath}/jsonldcontext.json', 'w') as outfile:
    json.dump(r.json(), outfile)_____no_output_____ORG = "github-users"
PROJECT = ""  # Provide here the automatically created project name created when you logged into the Nexus sandbox instance._____no_output_____forge = KnowledgeGraphForge("https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml",
                            bucket=f"{ORG}/{PROJECT}",
                            endpoint="https://sandbox.bluebrainnexus.io/v1",
                            token=TOKEN)_____no_output_____

## Download datasets from Allen Cell Types Database and MouseLight_____no_output_____### Download mouse neuron morphologies from the Allen Cell Types Database_____no_output_____We will be downloading mouse neuron morphology data from the [Allen Cell Types Database](https://celltypes.brain-map.org/). The [AllenSDK](https://allensdk.readthedocs.io/en/latest/) can be used for data download._____no_output_____

ALLEN_DIR = "allen_cell_types_database"_____no_output_____ctc = CellTypesCache(manifest_file=f"{ALLEN_DIR}/manifest.json")_____no_output_____MAX_CELLS = 10 # Increase to include more cells
SPECIES = CellTypesApi.MOUSE_____no_output_____nm_allen_identifiers = [cell["id"] for cell in ctc.get_cells(species=[SPECIES], require_reconstruction = True)][:MAX_CELLS]
print(f"Selected a mouse neuron with identifier: {nm_allen_identifiers}")_____no_output_____

Select metadata_____no_output_____

with open(f"{ALLEN_DIR}/cells.json") as f:
    allen_cell_types_metadata = json.load(f)_____no_output_____nm_allen_metadata = [neuron for neuron in allen_cell_types_metadata if neuron["specimen__id"] in nm_allen_identifiers]_____no_output_____

Download reconstruction files_____no_output_____

for identifier in nm_allen_identifiers:
    ctc.get_reconstruction(identifier)_____no_output_____

### Download mouse neuron electrophysiology recordings from the Allen Cell Types Database_____no_output_____Download Electrophysiology recordings_____no_output_____

for identifier in nm_allen_identifiers:
    ctc.get_ephys_data(identifier)_____no_output_____

### Download mouse neuron morphologies from MouseLight project_____no_output_____We will be downloading mouse neuron morphology data from the [MouseLight project](https://www.janelia.org/project-team/mouselight)._____no_output_____

URL_GRAPHQL = "http://ml-neuronbrowser.janelia.org/graphql/"_____no_output_____URL_JSON = "http://ml-neuronbrowser.janelia.org/json/"_____no_output_____URL_SWC = "http://ml-neuronbrowser.janelia.org/swc/"_____no_output_____

Select metadata_____no_output_____

nm_request = requests.post(URL_GRAPHQL, json={"operationName":"SearchNeurons",
                                "variables":{
                                    "context":{
                                        "scope":6,
                                        "nonce":"cjyo7xu7k00033h5yrj9jfpoy",
                                        "predicates":[{
                                            "predicateType":3,
                                            "tracingIdsOrDOIs":["1"],
                                            "tracingIdsOrDOIsExactMatch":False,
                                            "tracingStructureIds":["68e76074-1777-42b6-bbf9-93a6a5f02fa4"],
                                            "nodeStructureIds":["c37953e1-a1e9-4b9a-847e-08d9566ced65"],
                                            "operatorId":None,
                                            "amount":0,
                                            "brainAreaIds":[],
                                            "arbCenter":{
                                                "x":None,
                                                "y":None,
                                                "z":None},
                                            "arbSize":None,
                                            "invert":False,
                                            "composition":3
                                        }]
                                    }
                                },
                                "query":"""query SearchNeurons($context: SearchContext) {\n  searchNeurons(context: $context)
                                            {\n    totalCount\n    queryTime\n    nonce\n    \n    neurons {\n      id\n
                                            idString\n      tracings {\n        id\n        tracingStructure {\n          id\n
                                            name\n          value\n          __typename\n        }\n        soma {\n          id\n
                                            x\n          y\n          z\n          radius\n          parentNumber\n
                                            sampleNumber\n          brainAreaIdCcfV30\n          structureIdentifierId\n
                                            __typename\n        }\n        __typename\n      }\n      __typename\n    }\n
                                            __typename\n  }\n}\n"""
                               })
nm_mouselight_graphql = json.loads(nm_request.text)["data"]["searchNeurons"]["neurons"]
nm_mouselight_names = [x["idString"] for x in nm_mouselight_graphql]_____no_output_____nm_mouselight_metadata = list()
for name in nm_mouselight_names[0:MAX_CELLS]:
    a = requests.post(URL_JSON,
                          json={"ids": [name]},
                          headers={"Accept": "*/*", "Content-Type": "application/json"})
    c = json.loads(a.content.decode('utf-8'))
    nm_mouselight_metadata.append(c["contents"])_____no_output_____for i, nm in enumerate(nm_mouselight_metadata):
    allenId = nm["neurons"][0]["soma"]["allenId"]
    allenInfo = nm["neurons"][0]["allenInformation"]
    for info in allenInfo:
        if info["allenId"] == allenId:
            allenLabel = info["name"]
    nm_mouselight_metadata[i]["neurons"][0]["allenLabel"] = allenLabel_____no_output_____

Download reconstruction files_____no_output_____

for name in nm_mouselight_names[0:MAX_CELLS]:
    a = requests.post(URL_SWC,
                      json={"ids": [name]},
                      headers={"Accept": "*/*", "Content-Type": "application/json"})
    c = eval(a.content)
    base64_message = c["contents"]
    base64_bytes = base64_message.encode('ascii')
    message_bytes = base64.b64decode(base64_bytes)
    dirpath = './mouselight'
    Path(dirpath).mkdir(parents=True, exist_ok=True)
    with open(f"{dirpath}/{name}.swc", "wb") as f:
        f.write(message_bytes)_____no_output_____

## Mapping_____no_output_____### Map the Allen Cell Types Database neuron morphologies to Neuroshapes_____no_output_____

allen_nm_mapping = DictionaryMapping.load("https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_morphology_dataset.hjson") # TODO
nm_allen_resources = forge.map(nm_allen_metadata, allen_nm_mapping)_____no_output_____

### Map the Allen Cell Types Database neuron electrophysiology recordings to Neuroshapes_____no_output_____

allen_ephys_mapping = DictionaryMapping.load("https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_ephys_dataset.hjson") # TODO
nephys_allen_resources = forge.map(nm_allen_metadata, allen_ephys_mapping)_____no_output_____

### Map the MouseLight neuron morphologies to Neuroshapes_____no_output_____

mouselight_nm_mapping = DictionaryMapping.load("https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/mouselight_dataset.hjson") # TODO
nm_mouselight_resources = forge.map(nm_mouselight_metadata, mouselight_nm_mapping)_____no_output_____

## Register

If the registration fails, try refreshing the access token and reinitializing the forge client in the _'Configure a forge client to store, manage and access datasets'_ section._____no_output_____### Register the Allen Cell Types Database neuron morphologies_____no_output_____

for resource in nm_allen_resources:
    resource.id = forge.format("identifier", "neuronmorphologies", str(uuid.uuid4()))_____no_output_____forge.register(nm_allen_resources)_____no_output_____

### Register the Allen Cell Types Database neuron electrophysiology recordings_____no_output_____

for resource in nephys_allen_resources:
    resource.id = forge.format("identifier", "traces", str(uuid.uuid4()))_____no_output_____forge.register(nephys_allen_resources)_____no_output_____

### Register the MouseLight neuron morphologies_____no_output_____

for resource in nm_mouselight_resources:
    resource.id = forge.format("identifier", "neuronmorphologies", str(uuid.uuid4()))_____no_output_____forge.register(nm_mouselight_resources)_____no_output_____

### Save the created resources in JSON files_____no_output_____

dirpath = './database'
Path(dirpath).mkdir(parents=True, exist_ok=True)
with open(f"{dirpath}/mouselight-protocols.json","w") as f:
    json.dump(forge.as_jsonld(nm_mouselight_resources, form="expanded"),f)_____no_output_____with open(f"{dirpath}/allen-morphologies-protocols.json","w") as f:
    json.dump(forge.as_jsonld(nm_allen_resources, form="expanded"),f)_____no_output_____with open(f"{dirpath}/allen-ephys-protocols.json","w") as f:
    json.dump(forge.as_jsonld(nephys_allen_resources, form="expanded"),f)_____no_output_____

## Access_____no_output_____### Set filters_____no_output_____

_type = "NeuronMorphology"

filters = {"type": _type}_____no_output_____

### Run Query_____no_output_____

number_of_results = 10  # You can limit the number of results, pass `None` to fetch all the results

data = forge.search(filters, limit=number_of_results)

print(f"{str(len(data))} dataset(s) of type {_type} found")_____no_output_____

### Display the results as pandas dataframe_____no_output_____

property_to_display = ["id","name","subject","brainLocation.brainRegion.id","brainLocation.brainRegion.label","brainLocation.layer.id","brainLocation.layer.label", "contribution","brainLocation.layer.id","brainLocation.layer.label","distribution.name","distribution.contentUrl","distribution.encodingFormat"]
reshaped_data = forge.reshape(data, keep=property_to_display)

forge.as_dataframe(reshaped_data)_____no_output_____

### Download_____no_output_____

dirpath = "./downloaded/"
forge.download(data, "distribution.contentUrl", dirpath, overwrite=True)_____no_output_____ls ./downloaded/_____no_output_____

### Display a result as 3D Neuron Morphology_____no_output_____

from neurom import load_morphology
from neurom.view.plotly_impl import plot_morph3d
import IPython_____no_output_____neuron = load_morphology(f"{dirpath}/{data[0].distribution.name}")
plot_morph3d(neuron, inline=False)
IPython.display.HTML(filename='./morphology-3D.html')_____no_output_____

## Version the dataset
Tagging a dataset is equivalent to `git tag`. It allows to version a dataset._____no_output_____

forge.tag(data, value="releaseV112")_____no_output_____# The version argument can be specified to retrieve the dataset at a given tag.

tagged_data = forge.retrieve(id=data[0].id, version="releaseV112")_____no_output_____forge.as_dataframe(tagged_data)_____no_output_____data[0].description="Neuron Morphology from Allen"_____no_output_____forge.update(data[0])_____no_output_____non_tagged_data = forge.retrieve(id=data[0].id)_____no_output_____forge.as_dataframe(non_tagged_data)_____no_output_____