# Notebook from danledinh/human_melanocytes
Path: DL20200328_primary_S2.ipynb

# S2: All cells QC_____no_output_____

from notebook_utils import *

# print modules and versions
imports()/home/ubuntu/py_36_env/lib/python3.6/site-packages/anndata/core/anndata.py:17: FutureWarning: pandas.core.index is deprecated and will be removed in a future version.  The public classes are available in the top-level namespace.
  from pandas.core.index import RangeIndex
/home/ubuntu/py_36_env/lib/python3.6/site-packages/scanpy/api/__init__.py:6: FutureWarning:

In a future version of Scanpy, `scanpy.api` will be removed.
Simply use `import scanpy as sc` and `import scanpy.external as sce` instead.

  FutureWarning
# settings
wkdir = '/home/ubuntu/data/DL20181011_melanocyte_test_data/local_data'

# read raw data (expression data) to pandas df
pre_adata = (pd.read_csv(f'{wkdir}/DL20190115_big_counts.txt',
                      index_col=[0])
                .rename(columns = {'gene':'gene_name'})
                .set_index('gene_name'))
cell_names = ['_'.join(x.split('_')[:2]) for x in pre_adata.columns]
pre_adata.columns = cell_names

# read fetal supp data
pre_adata_supp1 = (pd.read_csv(f'{wkdir}/DL20190319_fetal_v2.csv')
                  .rename(columns = {'gene':'gene_name'})
                   .set_index('gene_name')
                  )

# read adult supp data
pre_adata_supp2 = (pd.read_csv(f'{wkdir}/DL20190417_adult_v2.csv')
                  .rename(columns = {'gene':'gene_name'})
                   .set_index('gene_name')
                  )

# merge with raw
pre_adata = pd.concat([pre_adata, pre_adata_supp1,pre_adata_supp2], axis=1)
cell_names = pre_adata.columns.tolist()

# metadata:
## download lastest from google drive and push to s3
meta_df = pd.read_csv(f'{wkdir}/DL20190417_mel_metadata.csv')
meta_df = meta_df.drop(['GWAS_consent',
                        'sort_info_exported',
                       'sort_date',
                       'plate_order',
                       'sample_number',
                       'plate_status'], axis = 1)
possible_wells = [f'{y}{x}'for y in string.ascii_uppercase[:-10] for x in range(1,25)]
all_wells_df = pd.DataFrame()
for plate in meta_df.plate_barcode:
    well_df = pd.DataFrame({'well':possible_wells})
    well_df['plate_barcode'] = plate
    all_wells_df = pd.concat([all_wells_df, well_df], axis = 0)

meta_df = pd.merge(meta_df, all_wells_df, 'left', 'plate_barcode')
meta_df['cell_name'] = [f'{x}_{y}' for x,y in zip(meta_df.well, meta_df.plate_barcode)]

merged_anno = pd.merge(pd.DataFrame({'cell_name':cell_names}), meta_df, 'left', 'cell_name')
merged_anno = merged_anno.set_index('cell_name')

_____no_output_____# nreads
dist_df = pd.DataFrame({'nreads':pre_adata.astype(int).sum().values,
                        'ngenes':(pre_adata > 0).sum().values,
                        'cell_id':pre_adata.columns
                       }).set_index('cell_id')
plotnine.options.figure_size = (3,3)

plot = (ggplot(dist_df)
        + theme_bw()
        + geom_density(aes('nreads'), fill='grey')
        + geom_vline(aes(xintercept=50000), linetype='--')
        + scale_x_log10()
       )
print(plot)

# ngenes
plot = (ggplot(dist_df)
        + theme_bw()
        + geom_density(aes('ngenes'), fill='grey')
        + geom_vline(aes(xintercept=1250), linetype='--')
        + scale_x_log10()
       )
print(plot)

passing_cells = (dist_df
                 .query('nreads >= 50000')
                 .query('ngenes >= 1250')
                 .reset_index()['cell_id']
                 .tolist()
                )
filtered_pre_adata = pre_adata.loc[:, passing_cells]
dist_df2 = pd.DataFrame({'cell_count_per_gene':(filtered_pre_adata > 0).sum(axis=1)
                       })

# n cells per gene
plot = (ggplot(dist_df2)
        + theme_bw()
        + geom_density(aes('cell_count_per_gene'), fill='grey')
        + geom_vline(aes(xintercept=3), linetype='--')
        + scale_x_log10()
       )
print(plot)/home/ubuntu/py_36_env/lib/python3.6/site-packages/pandas/core/series.py:679: RuntimeWarning: divide by zero encountered in log10
  result = getattr(ufunc, method)(*inputs, **kwargs)
/home/ubuntu/py_36_env/lib/python3.6/site-packages/plotnine/layer.py:360: PlotnineWarning: stat_density : Removed 13 rows containing non-finite values.
  data = self.stat.compute_layer(data, params, layout)

### SRA upload_____no_output_____

# source_dict = {'10WK03':'Laurence Baskin',
#              '12WK05':'Laurence Baskin',
#              '12WKM01':'Laurence Baskin',
#              '16WKM04':'Laurence Baskin',
#              '18WKM06':'Laurence Baskin',
#              '9.5WK02':'Laurence Baskin',
#              'A1011L':'Aaron D. Tward',
#              'A1012M':'Aaron D. Tward',
#              'A1014L':'Scott VandenBerg (UCSF BIOS)',
#              'A1015LM':'Aaron D. Tward',
#              'A1016LM':'Scott VandenBerg (UCSF BIOS)',
#              'A1017LM':'Scott VandenBerg (UCSF BIOS)',
#              'A1020LM':'Aaron D. Tward',
#              'A1021M':'Scott VandenBerg (UCSF BIOS)',
#              'A1022M':'Aaron D. Tward',
#              'A1025L':'Scott VandenBerg (UCSF BIOS)',
#              'A1026L':'Scott VandenBerg (UCSF BIOS)',
#              'A1033M':'Scott VandenBerg (UCSF BIOS)',
#              'A1038LM':'Scott VandenBerg (UCSF BIOS)',
#              'A1046M':'Scott VandenBerg (UCSF BIOS)',
#              'FS030_LM':'Thea Mauro',
#              'FS043_LM':'Thea Mauro',}
_____no_output_____# # biosample attributes TSV
# biosampledf = (merged_anno
#                .loc[:,['patient_id','age','sex']]
#                .reset_index()
#                .rename(columns={'cell_name':'sample_name',
#                                 'patient_id':'isolate'
#                                })
#               )
# biosampledf['true_year'] = [float(x.split('WK')[0])/52 if len(x.split('WK')[0]) <= 3 \
#                       else np.nan \
#                       for x in biosampledf.isolate]
# biosampledf['age'] = [y+(9/12) if np.isnan(x) \
#                       else x \
#                       for x,y in zip(biosampledf['true_year'],
#                                      biosampledf['age'])]
# biosampledf['age'] = [f'{x} yr' for x in biosampledf['age'].round(2)]
# biosampledf = biosampledf.drop('true_year', axis = 1)
# biosampledf['biomaterial_provider'] = [source_dict.get(x) for x in biosampledf['isolate']]
# biosampledf['organism'] = 'Homo sapiens'
# biosampledf['tissue'] = 'Skin'
# biosampledf = biosampledf.loc[:, ['sample_name',
#                                   'organism',
#                                   'isolate',
#                                   'age',
#                                   'biomaterial_provider',
#                                   'sex',
#                                   'tissue',
#                                  ]
#                              ]
# biosampledf['sex'] = biosampledf['sex'].replace('Ukn','Unknown')

# # minimal biosample matrix
# mini_biosampledf = (biosampledf
#                     .loc[:, ['organism',
#                               'isolate',
#                               'age',
#                               'biomaterial_provider',
#                               'sex',
#                               'tissue',]]
#                     .drop_duplicates()
#                    )
# mini_biosampledf['sample_name'] = mini_biosampledf['isolate'].values
# sample2isolate = {'10WK03':'fetus, 10 wk, i03',
#              '12WK05':'fetus, i05',
#              '12WKM01':'fetus, iM01',
#              '16WKM04':'fetus, iM04',
#              '18WKM06':'fetus, iM06',
#              '9.5WK02':'fetus, i02',
#              'A1011L':'adult, i1011',
#              'A1012M':'adult, i1012',
#              'A1014L':'adult, i1014',
#              'A1015LM':'adult, i1015',
#              'A1016LM':'adult, i1016',
#              'A1017LM':'adult, i1017',
#              'A1020LM':'adult, i1020',
#              'A1021M':'adult, i1021',
#              'A1022M':'adult, i1022',
#              'A1025L':'adult, i1025',
#              'A1026L':'adult, i1026',
#              'A1033M':'adult, i1033',
#              'A1038LM':'adult, i1038',
#              'A1046M':'adult, i1046',
#              'FS030_LM':'neonate, i030',
#              'FS043_LM':'neonate, i043',
#                  }

# mini_biosampledf['isolate'] = [sample2isolate.get(x) for x in mini_biosampledf['sample_name']]
# mini_biosampledf = mini_biosampledf.loc[:, ['sample_name',
#                                   'organism',
#                                   'isolate',
#                                   'age',
#                                   'biomaterial_provider',
#                                   'sex',
#                                   'tissue',
#                                  ]
#                              ]
# mini_biosampledf.to_csv(f'/home/ubuntu/s3_dump/biosample_mini.tsv',
#                    index=False,
#                    sep='\t'
#                   )
# mini_biosampledf_____no_output_____# # create patient-specific bam list to be merged
# for patient_val in set(biosampledf.isolate):
#     bam_dir = '/home/ubuntu/s3_dump/bams'
#     outbamlist = f'/home/ubuntu/s3_dump/merged_aux/{patient_val}.txt'

#     with open(outbamlist, 'w') as outf:
#         for idx, cell_id in enumerate(biosampledf.query(f'isolate == "{patient_val}"')['sample_name']):
#             inbam = f'{bam_dir}/{cell_id}.bam'
#             outf.write(inbam + '\n')

# # create patient-specific header
# for patient_val in set(biosampledf.isolate):
#     out_dir = '/home/ubuntu/s3_dump/merged_aux'
#     out_fn = f'{out_dir}/{patient_val}.header'
#     with open(out_fn, 'w') as outf:

#         ### create base header with samtools ###
#         input_header_fn = '/home/ubuntu/s3_dump/base.header'
#         with open(input_header_fn, 'r') as base_header:
#             for line in base_header:
#                 if line.startswith('@PG') or line.startswith('@CO'):
#                     pass
#                 else:
#                     outf.write(line)

#             for idx, cell_id in enumerate(biosampledf.query(f'isolate == "{patient_val}"')['sample_name']):
#                 outf.write('@RG' + '\t' + f'ID:{cell_id}' + '\n')_____no_output_____# # metadata tsv
# md5_output = '/home/ubuntu/s3_dump/merged_bams//files.md5'
# md5_df = pd.read_csv(md5_output, sep ='  ', header=None)
# md5_df.columns=['MD5_checksum','filename']

# md5_df['sample_name'] = [x.split('.rehead.bam')[0] for x in md5_df['filename']]
# md5_df['library_ID'] = md5_df['sample_name'].values
# md5_df['filetype'] = 'bam'
# md5_df['assembly'] = 'hg38'
# md5_df['design_description'] = 'smart-seq2'
# md5_df['instrument_model'] = 'Illumina NovaSeq 6000'
# md5_df['platform'] = 'ILLUMINA'
# md5_df['library_layout'] = 'paired'
# md5_df['library_selection'] = 'other'
# md5_df['library_source'] = 'TRANSCRIPTOMIC'
# md5_df['library_strategy'] = 'RNA-Seq'
# md5_df['title'] = 'single-cell RNAseq of human melanocytes'

# #### TBD####
# md5_df['biosample_accession'] = ''
# md5_df['bioproject_accession'] = ''
# #### TBD####

# md5_df = md5_df.loc[:, ['bioproject_accession',
#                         'biosample_accession',
#                         'sample_name',
#                         'library_ID',
#                         'title',
#                         'library_strategy',
#                         'library_source',
#                         'library_selection',
#                         'library_layout',
#                         'platform',
#                         'instrument_model',
#                         'design_description',
#                         'assembly',
#                         'filetype',
#                         'filename',
#                         'MD5_checksum',]]
# # starts, ends = create_ranges(len(md5_df), 1000)
# # for start,end in zip(starts, ends):
# #     print(start, end)
# #     (md5_df
# #      .iloc[start:end,:]
# #      .to_csv(f'/home/ubuntu/s3_dump/SRA_metadata_{start}-{end}.tsv',
# #                    index=False,
# #                    sep='\t',
# #                   )
# #     )
# md5_df.to_csv(f'/home/ubuntu/s3_dump/SRA_metadata.tsv',
#                    index=False,
#                    sep='\t',
#                   )
# md5_df_____no_output_____# bam_dir = '/home/ubuntu/s3_dump/bams'
# input_header_fn = '/home/ubuntu/s3_dump/trimmed_header.txt'

# # create complete header
# for patient_val in set(biosampledf.isolate):
#     bam_dir = '/home/ubuntu/s3_dump/bams'
#     header_fn = f'/home/ubuntu/s3_dump/merged_bams/{patient_val}.header'

#     with open(header_fn, 'w') as outf:
#         with open(input_header_fn, 'r') as header_text:
#             for line in header_text:
#                 outf.write(line)

#             for idx, cell_id in enumerate(biosampledf.query(f'isolate == "{patient_val}"')['sample_name']):
#                 outf.write('@RG' + '\t' + f'ID:{cell_id}' + '\n')
_____no_output_____


# Notebook from NishaNagasimha/egeria
Path: open-metadata-resources/open-metadata-labs/asset-management-labs/automated-curation.ipynb

![Egeria Logo](https://raw.githubusercontent.com/odpi/egeria/master/assets/img/ODPi_Egeria_Logo_color.png)

### Egeria Hands-On Lab
# Welcome to the Automated Curation Lab

**NOTE - this lab is still under construction and should not be used**_____no_output_____## Introduction

Egeria is an open source project that provides open standards and implementation libraries to connect tools, catalogs and platforms together so they can share information about data and technology (called metadata).

In the [Building a Data Catalog](building-a-data-catalog.ipynb) lab, Peter Profile and Erin Overivew
manually catalogued the weekly measurement files for the Drop Foot clinical trial.

In this hands-on lab you will get a chance to work with Egeria's governance servers to
automate this onboarding process._____no_output_____## The scenario

[Coco Pharmaceuticals](https://opengovernance.odpi.org/coco-pharmaceuticals/)
is conducting a clinical trial with two hospitals: Oak Dean Hospital and Old Market Hospital.
Each week the two hospitals send Coco Pharmaceuticals a set of measurements from the patients
involved in the trial.  These measurements are located in a CSV file that the hospital sends through
secure file transfer to a folder in Coco Pharmaceutical's landing area.

These files need to be copied into the data lake and catalogued so that they are only visible to the
staff involved in the clinical trial.  It is also important that the lineage of these files is
maintained so the source of the data can be traced.  This process is shown in Figure 1.

![Scenario](../images/automated-curation-scenario.png)
> **Figure 1:** Clinical trial weekly measurements onboarding process

Peter Profile and Erin Overview are responsible for this onboarding process.
![Peter and Erin](../images/peter-and-erin.png)

They have defined a list of requirements for the process:

* Files must be in the landing area for a minimum amount of time.
* As a new file is received, it needs to be catalogued, including:
   * Description
   * Connection details to enable the data scientists to access the contents
   * Column details
   * Governance zones defining the files' visibility
   * Owner
* A file is not accessible by any of the data lake users until the cataloguing process is complete.
* They must record lineage of each measurements file so they know which hospital it came from.

They have been [manually cataloguing the measurements files](building-a-data-catalog.ipynb) for
the first few weeks to prove the approach but now it is time to automate the process since:
* This clinical trial is planned to run for two years.
* There is a expected to be a ramp up of other clinical trials running simultaneously and the
  file onboarding workload would soon become overwhelming if they continued with the manual approach.

They plan to use an
[Integration Daemon](https://egeria.odpi.org/open-metadata-implementation/admin-services/docs/concepts/integration-daemon.html)
called **exchangeDL01** to capture the technical metadata of the files.
Then the
[Engine Host](https://egeria.odpi.org/open-metadata-implementation/admin-services/docs/concepts/engine-host.html)
server called **governDL01** will manage the move of the file into the data lake,
the augmentation of the metadata properties of the files and the creation of the lineage._____no_output_____## Setting up

Coco Pharmaceuticals make widespread use of Egeria for tracking and managing their data and related assets.
Figure 2 below shows their servers and the Open Metadata and Governance (OMAG) Server Platforms that are hosting them.

![Figure 2](../images/coco-pharmaceuticals-systems-omag-server-platforms.png)
> **Figure 2:** Coco Pharmaceuticals' OMAG Server Platforms

The code below checks that the platforms are running.  It checks that the servers are configured and then if they are running on the platform.  If a server is configured, but not running, it will start it.

Look for the "Done." message that is displayed after the governance servers have started._____no_output_____

# Start up the metadata servers and the view server
%run ../common/environment-check.ipynb

print("Start up the Governance Servers")
activatePlatform(dataLakePlatformName, dataLakePlatformURL, [governDL01Name, exchangeDL01Name])

print("Done. ")_____no_output_____

----
You should see that both the metadata servers `cocoMDS1` and `cocoMDS2` along with the integration daemon `exchangeDL01` and the engine host server `governDL01` have started.

If any of the platforms are not running, follow [this link to set up and run the platform](https://egeria.odpi.org/open-metadata-resources/open-metadata-labs/).  If any server is reporting that it is not configured then
run the steps in the [Server Configuration](../egeria-server-config.ipynb) lab to configure
the servers.  Then re-run the previous step to ensure all of the servers are started.

## Review the status of the integration daemon

At this point, even though both `exchangeDL01` and `governDL01` are running, there is still work to set up the full data pipeline.  The command below queries the status of `exchangeDL01`._____no_output_____

getIntegrationDaemonStatus(exchangeDL01Name, exchangeDL01PlatformName, exchangeDL01PlatformURL, petersUserId)_____no_output_____

----
Notice that the Files Integrator Open Metadata Integration Service (OMIS) is running three connectors and they are all failing because the directories (folders) that they are supposed to be monitoring do not exist.  This is because no data files
have arrived from either hospital.  As the data pipeline is set up in this lab, the directories will get created and we will be able to restart the connectors to get them working._____no_output_____

OakDeneConnectorFolder   = fileSystemRoot + '/landing-area/hospitals/oak-dene/clinical-trials/drop-foot'
OldMarketConnectorFolder = fileSystemRoot + '/landing-area/hospitals/old-market/clinical-trials/drop-foot'
folderConnectorFolder    = fileSystemRoot + '/data-lake/research/clinical-trials/drop-foot/weekly-measurements'
_____no_output_____

If you know the class name of an integration connector's provider, it is possible to check if the connector is of the right type for an integration service.  This function also returns full details of the connector type, which often includes descriptive information as well as the configuration properties that it supports.

The connectors configured in the Files Integrator OMIS are shown in figure 3:

![Figure 3](../images/integration-daemon.png)
> **Figure 3:** exchangeDL01 with its connectors

The class names of these integration connectors' providers can be seen in the connection object embedded in the error message displayed with the connectors' status.

The commands below request that the Files Integrator OMIS service validate and return the connector type for each of these connectors._____no_output_____

dataFilesMonitorProviderClassName = "org.odpi.openmetadata.adapters.connectors.integration.basicfiles.DataFilesMonitorIntegrationProvider"
dataFolderMonitorProviderClassName = "org.odpi.openmetadata.adapters.connectors.integration.basicfiles.DataFolderMonitorIntegrationProvider"

print("Data Files Monitor Integration Connector Type:")
validateIntegrationConnector(exchangeDL01Name, exchangeDL01PlatformName, exchangeDL01PlatformURL, "files-integrator", petersUserId, dataFilesMonitorProviderClassName)

print("")
print("Data Folder Monitor Integration Connector Type:")
validateIntegrationConnector(exchangeDL01Name, exchangeDL01PlatformName, exchangeDL01PlatformURL, "files-integrator", petersUserId, dataFolderMonitorProviderClassName)_____no_output_____

----
Both connectors supports the `templateQualifiedName` and the `allowCatalogDelete` configuration properties.  If you are curious
about their meaning, review the definitions in the connector catalog:

* [Data Files Monitor Integration Connector](https://egeria.odpi.org/open-metadata-publication/website/connector-catalog/data-files-monitor-integration-connector.html)
* [Data Folder Monitor Integration Connector](https://egeria.odpi.org/open-metadata-publication/website/connector-catalog/data-folder-monitor-integration-connector.html)

Later in the lab, we will be setting up the `templateQualifiedName`.

----
The command below tries to validate a connector that does not exist.  You can see that the request fails with
a class not found exception._____no_output_____

invalidProviderClassName = "org.odpi.openmetadata.adapters.connectors.integration.basicfiles.DummyProvider"

print("Invalid Integration Connector:")
validateIntegrationConnector(exchangeDL01Name, exchangeDL01PlatformName, exchangeDL01PlatformURL, "files-integrator", petersUserId, invalidProviderClassName)_____no_output_____

----
The next command tries to validate the data files connector with the Database Integrator OMIS.  This
request also fails since this connector is not compatible with the Database Integrator OMIS._____no_output_____

print("Data Files Monitor Integration Connector:")
validateIntegrationConnector(exchangeDL01Name, exchangeDL01PlatformName, exchangeDL01PlatformURL, "database-integrator", petersUserId, dataFilesMonitorProviderClassName)
_____no_output_____

----
## Setting up the file transfer into the landing area

The directories (folders) that the integration connector are configured to monitor could be created by using a file system command.
In this lab, however, the creation of a landing area folder will occur when the first file is received from the corresponding hospital.
This is to avoid needing to provide this notebook with access to the file system.

We are going to use a provisioning governance action service called `Move/Copy File Governance Action Service` to simulate the
file transfer from a hospital to its folder in the landing zone.  This service
runs in a governance engine that is supported by the Governance Action Open Metadata Engine Service (OMES).  As with the integration services,
it is possible to validate and understand the connector through a call to the appropriate server.
_____no_output_____

moveCopyFileProviderClassName = "org.odpi.openmetadata.adapters.connectors.governanceactions.provisioning.MoveCopyFileGovernanceActionProvider"

print("Move/Copy File Governance Action Service:")
validateGovernanceActionEngineConnector(governDL01Name, governDL01PlatformName, governDL01PlatformURL, petersUserId, moveCopyFileProviderClassName)
_____no_output_____

## Configuring the governance engine

Figure 4 shows the three governance engines configured in the `governDL01` engine host server.

![Figure 4](../images/engine-host.png)
> **Figure 4:** Governance Engines for governDL01

The command below queries the status of each governance engine running in `governDL01`.
The governance action services that will support the onboarding of files for clinical trials will run in the `AssetGovernance`
governance engine.  The other two governance engines are the subject of the [Open Discovery Lab](../administration-labs/open-discovery-config.ipynb)._____no_output_____

getGovernanceEngineStatuses(governDL01Name, governDL01PlatformName, governDL01PlatformURL, petersUserId)
_____no_output_____

----
The status code `ASSIGNED` means that the governance engine was listed in Engine Host's configuration
document - ie the governance engine was assigned to this server - but Engine Host has not been
able to retrieve the configuration for the governance engine from the metadata server (`cocoMDS1`).

When the basic governance engine properties have been retrieved from the metadata server then the status code
becomes `CONFIGURING` and more decriptive information is returned with the status.

When governance services are registered with the governance engine, the status moves to `RUNNING` and it is possible to see the list of supported request types for the governance engine.

The next step in the lab is to add configuration for the governance engine to `cocoMDS2` until the
`AssetGovernance` governance engine is running._____no_output_____

assetGovernanceEngineName = "AssetGovernance"
assetGovernanceEngineDisplayName = "Asset Governance Action Engine"
assetGovernanceEngineDescription = "Monitors, validates and enriches metadata relating to assets."

assetGovernanceEngineGUID = createGovernanceEngine(cocoMDS2Name,
                                                   cocoMDS2PlatformName,
                                                   cocoMDS2PlatformURL,
                                                   erinsUserId,
                                                   "GovernanceActionEngine",
                                                   assetGovernanceEngineName,
                                                   assetGovernanceEngineDisplayName,
                                                   assetGovernanceEngineDescription)

if (assetGovernanceEngineGUID):
    print (" ")
    print ("The guid for the " + assetGovernanceEngineName + " governance engine is: " + assetGovernanceEngineGUID)
    print (" ")

print ("Done. ")    _____no_output_____

----

Now the governance engine is defined, its status moves to `CONFIGURING`.

----_____no_output_____

getGovernanceEngineStatuses(governDL01Name, governDL01PlatformName, governDL01PlatformURL, petersUserId)
_____no_output_____

----

Next, the description of the `Move/Copy File Governance Action Service` is added to cocoMDS2.  There are two parts to
registering a governance action service.  The first is to create a GovernanceService definition that identifies
the implementation of the service and the second part registers this GovernanceService definition with the governance engine.
This registration maps one or more of the governance engine's request types, along with the default request parameters to
the GovernanceService definition.  This mapping is used to translate a request to the governance engine into an
invocation of a governance action service.

Figure 5 shows the structure of the resulting definitions for a governance engine.  A governance action service may be
registered with multiple governance engines, using the same or different request types.

![Figure 5](../images/governance-action-request-type.png)
> **Figure 5:** Structure of the governance services within a governance engine

Since a governance action service is implemented as a connector, part of the GovernanceService definition includes the
connection object used to initialize the service.
`Move/Copy File Governance Action Service` is highly
configurable through the configuration properties supplied in its connection object and so we can use it in two modes.

The first instance will simulate the secure file transfer from the hospital to the
landing area. In this case, it should not make use of any metadata and not create lineage._____no_output_____

ftpGovernanceServiceName = "ftp-governance-action-service"
ftpGovernanceServiceDisplayName = "FTP Governance Action Service"
ftpGovernanceServiceDescription = "Simulates FTP from an external party."
ftpGovernanceServiceProviderClassName = moveCopyFileProviderClassName
ftpGovernanceServiceConfigurationProperties = {
        "provisionUncataloguedFiles" : "",
        "noLineage" : ""
    }
ftpGovernanceServiceRequestType = "ftp-to-landing-area"
_____no_output_____

The second instance is part of Coco Pharmaceuticals onboarding process and will be driven from the appearance of the
Asset created by the integration daemon when a file arrives in the landing area.
This instance will also produce lineage and change the resulting filename so that the files are sequenced according to their
arrival.  For example:
 * DropFoot_000001.csv
 * DropFoot_000002.csv

This aids the time-based loading of the files into a database by ensuring any corrections to the readings are applied in the
correct order._____no_output_____

dlGovernanceServiceName = "provision-weekly-measurements-governance-action-service"
dlGovernanceServiceDisplayName = "FTP Governance Action Service"
dlGovernanceServiceDescription = "Provisions weekly measurment files from the landing area to the "
dlGovernanceServiceProviderClassName = moveCopyFileProviderClassName
dlGovernanceServiceConfigurationProperties = {
        "targetFileNamePattern" : "DropFoot_{1, number,000000}.csv"
    }
dlGovernanceServiceRequestType = "provision-to-data-lake"
_____no_output_____

This next code issues the calls to create the two versions of the governance action service and add them to the AssetGovernance
governance engine
_____no_output_____

governanceServiceGUID = createGovernanceService(cocoMDS2Name,
                                                cocoMDS2PlatformName,
                                                cocoMDS2PlatformURL,
                                                erinsUserId,
                                                "GovernanceActionService",
                                                ftpGovernanceServiceName,
                                                ftpGovernanceServiceDisplayName,
                                                ftpGovernanceServiceDescription,
                                                ftpGovernanceServiceProviderClassName,
                                                ftpGovernanceServiceConfigurationProperties)

if (governanceServiceGUID):
    print (" ")
    print ("The guid for the " + governanceServiceName + " governance service is: " + governanceServiceGUID)
    registerGovernanceServiceWithEngine(cocoMDS1Name,
                                        cocoMDS1PlatformName,
                                        cocoMDS1PlatformURL,
                                        petersUserId,
                                        assetGovernanceEngineGUID,
                                        governanceServiceGUID,
                                        ftpGovernanceServiceRequestType)
    print ("Service registered as: " + ftpGovernanceServiceRequestType)
    print (" ")

governanceServiceGUID = createGovernanceService(cocoMDS2Name,
                                                cocoMDS2PlatformName,
                                                cocoMDS2PlatformURL,
                                                erinsUserId,
                                                "GovernanceActionService",
                                                dlGovernanceServiceName,
                                                dlGovernanceServiceDisplayName,
                                                dlGovernanceServiceDescription,
                                                dlGovernanceServiceProviderClassName,
                                                dlGovernanceServiceConfigurationProperties)

if (governanceServiceGUID):
    print (" ")
    print ("The guid for the " + governanceServiceName + " governance service is: " + governanceServiceGUID)
    registerGovernanceServiceWithEngine(cocoMDS1Name,
                                        cocoMDS1PlatformName,
                                        cocoMDS1PlatformURL,
                                        petersUserId,
                                        assetGovernanceEngineGUID,
                                        governanceServiceGUID,
                                        dlGovernanceServiceRequestType)
    print ("Service registered as: " + dlGovernanceServiceRequestType)
    print (" ")
  _____no_output_____

Next we need a governance action service to listen for the
    _____no_output_____

def initiateGovernanceAction(serverName, serverPlatformName, serverPlatformURL, userId, governanceEngineName, qualifiedName, requestType, requestProperties):
    commandURLRoot = serverPlatformURL + "/servers/" + serverName + "/open-metadata/access-services/governance-engine/users/" + userId
    initiateGovernanceActionURL = configCommandURLRoot + '/governance-engines/' + governanceEngineName + '/governance-actions/initiate'
    governanceActionGUID = None
    try:
        body = {
            "class" : "GovernanceActionRequestBody ",
            "qualifiedName" : qualifiedName,
            "requestType" : requestType,
            "requestProperties" : requestProperties
        }
        response=issuePost(initiateGovernanceActionURL, body)
        if response.status_code == 200:
            relatedHTTPCode = response.json().get('relatedHTTPCode')
            if relatedHTTPCode == 200:
                governanceActionGUID = response.json().get('guid')
            else:
                printUnexpectedResponse(serverName, serverPlatformName, serverPlatformURL, response)
        else:
            printUnexpectedResponse(serverName, serverPlatformName, serverPlatformURL, response)
    except Exception as error:
        print("Exception: %s" % error )
        print("Platform " + serverPlatformURL + " is returning an error")
    return governanceActionGUID

def getGovernanceAction(serverName, serverPlatformName, serverPlatformURL, userId, governanceActionGUID):
    commandURLRoot = serverPlatformURL + "/servers/" + serverName + "/open-metadata/access-services/governance-engine/users/" + userId
    getGovernanceActionURL = configCommandURLRoot + '/governance-actions/' + governanceActionGUID
    try:
        response=issueGet(getGovernanceActionURL)
        if response.status_code == 200:
            relatedHTTPCode = response.json().get('relatedHTTPCode')
            if relatedHTTPCode == 200:
                element = response.json().get('element')
                if element:
                    print(element)
            else:
                printUnexpectedResponse(serverName, serverPlatformName, serverPlatformURL, response)
        else:
            printUnexpectedResponse(serverName, serverPlatformName, serverPlatformURL, response)
    except Exception as error:
        print("Exception: %s" % error )
        print("Platform " + serverPlatformURL + " is returning an error")
_____no_output_____ftpRequestType = "copy-file"

if governanceServiceGUID:
    registerGovernanceServiceWithEngine(cocoMDS2Name,
                                        cocoMDS2PlatformName,
                                        cocoMDS2PlatformURL,
                                        erinsUserId,
                                        assetGovernanceEngineGUID,
                                        governanceServiceGUID,
                                        ftpRequestType)
    print (" ")
    print ("Service registered as: " + ftpRequestType)
    print (" ")

print ("Done. ")_____no_output_____

----
**Stop here**

Plan of the lab

* Review the status of the integration daemon and engine host - show that they are both in need to work
* Create the folders for the integration daemon connectors and restart them - validate that they are working
* Set up the FTP provisioning in the engine host
* Add the first file into Oak Dene's landing area
* Validate that the file is created in open metadata by the integration connector
* Set up the provisioning ga service and test it by creating a governance action to drive it.
* Validate that the file is transferred and the new asset is catalogued and the folder is updated
* View the lineage
* Set up the provisioning service in a simple governance action process and test it
* Set up the watch dog service and check it drives the governance action process
* Set up the origin and template - then configure the landing area integration connector to use a template - show the template affecting the set up of the landing area asset
* Extend the governance action process to use the lineage to set up the origin. Show the origin on the new file asset.
* Explain the use of triage if the origin can not be set.
* Set up a template the owner and reconfigure the data lake folder
* Extend the governance action to create a verification point and set the zone - with a triage alternate.
----
_____no_output_____

# Test the status of all of the connectors

csvDiscoveryProviderClassName = "org.odpi.openmetadata.adapters.connectors.discoveryservices.CSVDiscoveryServiceProvider"

print("CSV Discovery Service:")
validateAssetAnalysisEngineConnector(governDL01Name, governDL01PlatformName, governDL01PlatformURL, petersUserId, csvDiscoveryProviderClassName)

_____no_output_____

----
_____no_output_____---_____no_output_____----_____no_output_____----_____no_output_____----
_____no_output_____----
_____no_output_____----_____no_output_____----_____no_output_____


# Notebook from FlorentCLMichel/learning_data_science
Path: Projects_Jupyter/Jeopardy_questions.ipynb

# Patterns of Jeopardy! questions (Work in progress)

Jeopardy! is a popular American televised quizz competition.
It has now aired more than 8000 episodes, was adapted in many other countries, and was ranked in 2013 among the 60 greatest shows in American television history by [TV Guide](https://www.tvguidemagazine.com/).
More information can be found on [this wikipedia page](https://en.wikipedia.org/wiki/Jeopardy!).
In this project, we will study a set of questions and answers which appeared in the show and try to find patterns which may help a candidate prepare for the game.

## Data Cleaning

The dataset, `JEOPARDY_CSV.csv`, can be found on [this reddit thread](https://www.reddit.com/r/datasets/comments/1uyd0t/200000_jeopardy_questions_in_a_json_file/).
It contains 216,930 jeopardy questions and answers among the 252,583 which had been asked at the time of posting.
Its columns are:
* `category`: the question category,
* `value`: the amount of money the question is worth (“None” for Final Jeopardy! and Tiebreaker questions),
* `question`: text of the question,
* `answer`: text of the answer,
* `round`: the round of Jeopardy that the question was asked in: “Jeopardy!”, “Double Jeopardy!”, “Final Jeopardy!”, or “Tiebreaker”,
* `show_number` : the episode number the question was asked in,
* `air_date`: the show air date in format YYYY-MM-DD.

Let us first import the modules we will need: _____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt_____no_output_____

We then read the dataset in the dataframe `jeopardy` and print its first five rows:_____no_output_____

jeopardy = pd.read_csv('../Data/JEOPARDY_CSV.csv')
jeopardy.head()_____no_output_____

Let us check the column names:_____no_output_____

jeopardy.columns_____no_output_____

Some of them have a leading space.
Let us get rid of it:_____no_output_____

def delete_leading_spaces(s):
    '''
    Returns the string s stripped of leading spaces
    '''
    while s[0] == ' ':
        s = s[1:]
    return s

jeopardy.columns = map(delete_leading_spaces, jeopardy.columns)
jeopardy.columns_____no_output_____

Let us now check for missing values: _____no_output_____

jeopardy.isnull().sum()_____no_output_____

There are two missing values in the `Answer` column.
Let us see where they are:_____no_output_____

jeopardy[jeopardy['Answer'].isnull()]_____no_output_____

The Answer to these two questions is “Null”, which seems to have confused Pandas when importaing the file.
Let us correct this error: _____no_output_____

jeopardy.fillna('Null', inplace=True)
jeopardy.isnull().sum()_____no_output_____

We now convert the `Show Number` and `Value` column to integers:_____no_output_____

jeopardy['Show Number'] = jeopardy['Show Number'].astype(int)
jeopardy['Value'] = jeopardy['Value'].str.replace('$','').str.replace(',','').replace('None',np.nan).astype(float)_____no_output_____

Finally, let us convert the `Air Date` column to datetime format:_____no_output_____

jeopardy['Air Date'] = pd.to_datetime(jeopardy['Air Date'], format = '%Y-%m-%d')_____no_output_____

We can use this to see in which year the questions present in the dataset were asked._____no_output_____

jeopardy['Air Date'].dt.year.value_counts(sort=False).plot(kind='bar')
plt.xlabel('year')
plt.ylabel('number of questions')
plt.grid()
plt.show()_____no_output_____

Most of the questions present in the dataset were asked between 1997 and 2011, with a few questions asked between 1984 and 1995.
It would be interesting to see if the increase in the number of questions in and after 1996 as well as the drop in 2002 can be explained.
But this is beyond the scope of the present study.

The small number of questions for 2012 may be due to the fact that the dataset covers only the beginning of that year.
Let us check this:_____no_output_____

jeopardy[jeopardy['Air Date'].dt.year == 2012]['Air Date'].dt.month.value_counts()_____no_output_____

The dataset extends only to January 2012.
It is thus not surprizing to have only a few questions for that year.

Let us now normalize the columns `Question` and `Answer` by converting them to lowercase and replacing punctuation marks with spaces:_____no_output_____

jeopardy['Question'] = jeopardy['Question'].str.replace('\W',' ').str.lower()
jeopardy['Answer'] = jeopardy['Answer'].str.replace('\W',' ').str.lower()_____no_output_____

## Data Analysis

### How often is the answer in the question?

One possible tactic to find for the answer to a question is to look at question itself and see if it may contain part of the answer.
Let us estimate how frequenctly it may succeed by counting the average fraction of words of the answer which are included in the question, excluding the word “the” which appear commonly in answers but does not change their meaning. _____no_output_____

def fraction_answer_in_question(row):
    '''
    Returns the fraction of the number of words in the answer which are present in the question
    '''
    words_answer = row['Answer'].split(' ')
    if 'the' in words_answer:
        words_answer.remove('the')
    # if the answer is empty, return 0
    if len(words_answer) == 0:
        return 0
    words_question = row['Question'].split()
    match_word = 0
    for word in words_answer:
        if word in words_question:
            match_word += 1
    return match_word / len(words_answer)

jeopardy['Answer in question'] = jeopardy.apply(fraction_answer_in_question, axis=1)
print('Fraction of words in answers appearing in questions: {:.2f}%'.format(100*jeopardy['Answer in question'].mean()))Fraction of words in answers appearing in questions: 5.84%

The fraction of words in the answers appearing in the corresponding questions is close to 6%, which is not a lot.
Looking for the answer in the question can thus give a candidate only a limited boost (although it may be useful for winning close games).

### How often are questions recycled?

Let us now estimate the fraction of words in each question which has been recycled.
We exclude words with 5 letters of less, as many of them are common words which contribute little to the actual meaning of the question.
A sufficiently long word will be called recycled if it has already appeared in an earlier question.

Let us first sort the `jeopardy` dataframe by air date:_____no_output_____

jeopardy = jeopardy.sort_values('Air Date')_____no_output_____

We now compute the fraction of each question which has been recycled, and save it in the `Fraction of Question Recycled` column:_____no_output_____

vocabulary = set([])
fraction_recycled = []
for question in jeopardy['Question']:
    words_question = question.split(' ')
    match_count = 0
    word_count = 0
    for word in words_question:
        if len(word) >= 6:
            word_count += 1
            if word in vocabulary:
                match_count += 1
            else:
                vocabulary.add(word)
    # if there is no long enough word, return 0
    if word_count == 0:
        fraction_recycled.append(0)
    else:
        fraction_recycled.append(match_count / word_count)

jeopardy['Fraction of Question Recycled'] = fraction_recycled
print('Fraction of recycled questions: {:.2f}%'.format(100*jeopardy['Fraction of Question Recycled'].mean()))Fraction of recycled questions: 89.92%

The fraction of recycled long words in questions is close to 90%.
It thus seems that studying previous questions can give a good idea of which questions may be asked.

Let us plot the evolution of the fraction of recycled questions with years:_____no_output_____

jeopardy['Air Year'] = jeopardy['Air Date'].dt.year
jeopardy.groupby('Air Year').mean()['Fraction of Question Recycled'].plot()
plt.grid()
plt.ylabel('Fraction of Question Recycled')
plt.show()_____no_output_____

The fraction of questions recycled has increased fast before 1990, before going above 90% in 2000 and staying nearly stable since then.

### Most-used words in high-value questions

Let us now determine the most-used words, focusing on questions with high values.
Let us first plot a histogram of the question values to determine which of them may count as high-value:_____no_output_____

(np.log(jeopardy['Value'])/np.log(10)).hist()
plt.xlabel('$\log_{10}(value ($))$')
plt.show()_____no_output_____

The peak of the distribution seems to be somewhere between \\$300 and \\$1000.
Let us take \\$800 as a threshold._____no_output_____

nwords = 10
threshold = 800
freq_words_questions = {}
for _, row in jeopardy.iterrows():
    if row['Value'] >= threshold:
        words_question = row['Question'].split(' ')
        for word in words_question:
            if len(word) >= 6:
                if word in freq_words_questions:
                    freq_words_questions[word] += 1
                else:
                    freq_words_questions[word] = 1
sorted(freq_words_questions.items(), key=lambda item: -item[1])[:nwords]_____no_output_____

The words “archive”, “target”, and “_blank” do not provide much information (and the first and third seem to mostly appear in media contents of questions, such as urls).
Discarding them, we find that the most used words in high-value questions are “country”, “called”, “american”, and “president”.
A candidate who wants to earn a lot of money at Jeopardy! should thus study the world geography and American history.

### Most frequenct categories

To help a future candidate prepare for the game, it is useful to know which categories are most represented.
Let us first see which of them appear most in all questions: _____no_output_____

res = jeopardy['Category'].value_counts()
print(res.head(5))
print('Percentage of questions covered: {:.2f}'.format(100*(res.head(5).sum()/res.sum())))BEFORE & AFTER      547
SCIENCE             519
LITERATURE          496
AMERICAN HISTORY    418
POTPOURRI           401
Name: Category, dtype: int64
Percentage of questions covered: 1.10

These five categories taken together account for just over 1% of all questions asked.
To be succesful, a candidate should thus study a broad range of topics instead of focusing on a few questions.

Let us now restrict ourselves to high-value questions, with a thresholf of 800USD:_____no_output_____

threshold = 800
res = jeopardy[jeopardy['Value'] >= threshold]['Category'].value_counts()
print(res.head(5))
print('Percentage of questions covered: {:.2f}'.format(100*(res.head(5).sum()/res.sum())))BEFORE & AFTER             304
LITERATURE                 201
SCIENCE                    198
OPERA                      150
COLLEGES & UNIVERSITIES    141
Name: Category, dtype: int64
Percentage of questions covered: 1.07

The first three categories are the same as for all questions, with the ranks of “Literature” and “Science” exchanged.
The five most frequenct categories again cover only a small fraction of all questions.

Finally, let us see which categories have the highest total values:_____no_output_____

n_categories = 5
res = jeopardy.groupby('Category')['Value'].sum().sort_values(ascending=False)
print('Highest value categories')
print(res.head(n_categories))
print('Fraction of the total value covered: {:.2f}%'.format(100*res.head(n_categories).sum()/res.sum()))Highest value categories
Category
BEFORE & AFTER    491900.0
SCIENCE           361221.0
LITERATURE        344900.0
OPERA             254300.0
WORD ORIGINS      251600.0
Name: Value, dtype: float64
Fraction of the total value covered: 1.06%

The three categories with a highest value are the same as the most frequent ones.
We also find that they account for a small fraction of the total value only.

## Conclusion

In this short project, we looked at some statistics of the Jeopardy! questions and answers.
We found that:
* On average, a bit less that 6% of the answer is already in the question.
Looking for the answer in the question can thus give a candidate a small advantage only.
* More than 90% of questions asked after the year 2000 had already been asked before.
A good strategy a future candidate can follow to be successful is thus to study previously-asked questions.
* The three categories which appear most frecuently are “Before & after”, “Science”, and “Literature”.
However, they account for only just over 1% of all questions.
A future candidate should thus study a broader range of topics to be successful. _____no_output_____


# Notebook from Mashfiq137/Thesis
Path: Notebooks/All notebooks/VGG16/VGG16.ipynb

import glob
import random as rn
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import cv2

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.optimizers import RMSprop, Adam
from tensorflow.keras.utils import to_categorical
from sklearn.metrics import classification_report, confusion_matrix

%matplotlib inline
import warnings
warnings.filterwarnings('ignore')

import numpy as np
import os
import cv2
import matplotlib.pyplot as plt
%matplotlib inline
import operator
import tensorflow as tf
import random
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model

import numpy as np
import os
import cv2
import matplotlib.pyplot as plt
from keras.models import load_model
import numpy as np
import pandas as pd
import os
import glob
import cv2
import random
import matplotlib.pyplot as plt
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.optimizers import RMSprop
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import regularizers
from keras.callbacks import CSVLogger
#from livelossplot import PlotLossesKeras
import os
import numpy as np
#from imgaug import augmenters as iaa
#import cv2
from keras.layers.normalization import BatchNormalization
#import seaborn as sns
import pandas as pd
from keras import initializers
from keras import optimizers
import keras.backend as K
import tensorflow as tf
from keras.models import Model_____no_output_____path = '../input/d/mashfiqrizvee/sampledataset2/'

# define paths
train_air_trapping_dir = path + 'Train/Air_trapping/'
train_aortic_elongation_dir = path + 'Train/Aortic_elongation/'
train_calcified_granuloma_dir = path + 'Train/Air_trapping/'
train_calcified_granuloma_dir = path + 'Train/Calcified_granuloma/'
train_callus_rib_fracture_dir = path + 'Train/Callus_rib_fracture/'
train_hiatal_hernia_dir = path + 'Train/Hiatal_hernia/'
train_laminar_atelectasis_fracture_dir = path + 'Train/Laminar_atelectasis/'
train_pleural_effusion_dir = path + 'Train/Pleural_effusion/'

test_air_trapping_dir = path + 'Test/Air_trapping/'
test_aortic_elongation_dir = path + 'Test/Aortic_elongation/'
test_calcified_granuloma_dir = path + 'Test/Calcified_granuloma/'
test_callus_rib_fracture_dir = path + 'Test/Callus_rib_fracture/'
test_hiatal_hernia_dir = path + 'Test/Hiatal_hernia/'
test_laminar_atelectasis_fracture_dir = path + 'Test/Laminar_atelectasis/'
test_pleural_effusion_dir = path + 'Test/Pleural_effusion/'

val_air_trapping_dir = path + 'val/Air_trapping/'
val_aortic_elongation_dir = path + 'val/Aortic_elongation/'
val_calcified_granuloma_dir = path + 'val/Calcified_granuloma/'
val_callus_rib_fracture_dir = path + 'val/Callus_rib_fracture/'
val_hiatal_hernia_dir = path + 'val/Hiatal_hernia/'
val_laminar_atelectasis_fracture_dir = path + 'val/Laminar_atelectasis/'
val_pleural_effusion_dir = path + 'val/Pleural_effusion/'

# find all files, our files has extension jpeg
#train_normal_cases = glob.glob(train_normal_dir + '*png')
train_air_trapping_cases = glob.glob(train_air_trapping_dir + '*png')
train_aortic_elongation_cases = glob.glob(train_aortic_elongation_dir + '*png')
train_calcified_granuloma_cases = glob.glob(train_calcified_granuloma_dir + '*png')
train_callus_rib_fracture_cases = glob.glob(train_callus_rib_fracture_dir + '*png')
train_hiatal_hernia_cases = glob.glob(train_hiatal_hernia_dir + '*png')
train_laminar_atelectasis_fracture_cases = glob.glob(train_laminar_atelectasis_fracture_dir + '*png')
train_pleural_effusion_cases = glob.glob(train_pleural_effusion_dir + '*png')

test_air_trapping_cases = glob.glob(test_air_trapping_dir + '*png')
test_aortic_elongation_cases = glob.glob(test_aortic_elongation_dir + '*png')
test_calcified_granuloma_cases = glob.glob(test_calcified_granuloma_dir + '*png')
test_callus_rib_fracture_cases = glob.glob(test_callus_rib_fracture_dir + '*png')
test_hiatal_hernia_cases = glob.glob(test_hiatal_hernia_dir + '*png')
test_laminar_atelectasis_fracture_cases = glob.glob(test_laminar_atelectasis_fracture_dir + '*png')
test_pleural_effusion_cases = glob.glob(test_pleural_effusion_dir + '*png')

val_air_trapping_cases = glob.glob(val_air_trapping_dir + '*png')
val_aortic_elongation_cases = glob.glob(val_aortic_elongation_dir + '*png')
val_calcified_granuloma_cases = glob.glob(val_calcified_granuloma_dir + '*png')
val_callus_rib_fracture_cases = glob.glob(val_callus_rib_fracture_dir + '*png')
val_hiatal_hernia_cases = glob.glob(val_hiatal_hernia_dir + '*png')
val_laminar_atelectasis_fracture_cases = glob.glob(val_laminar_atelectasis_fracture_dir + '*png')
val_pleural_effusion_cases = glob.glob(val_pleural_effusion_dir + '*png')

# make path using / instead of \\ ... this may be redudant step
'''
train_normal_cases = [x.replace('\\', '/') for x in train_normal_cases]
train_pneu_cases = [x.replace('\\', '/') for x in train_pneu_cases]

test_normal_cases = [x.replace('\\', '/') for x in test_normal_cases]
test_pneu_cases = [x.replace('\\', '/') for x in test_pneu_cases]
val_normal_cases = [x.replace('\\', '/') for x in val_normal_cases]
val_pneu_cases = [x.replace('\\', '/') for x in val_pneu_cases]

'''

# create lists for train, test & validation cases, create labels as well
train_list = []
test_list = []
val_list = []

for x in train_air_trapping_cases:
    train_list.append([x, 0])

for x in train_aortic_elongation_cases:
    train_list.append([x, 1])

for x in train_calcified_granuloma_cases:
    train_list.append([x, 2])

for x in train_callus_rib_fracture_cases:
    train_list.append([x, 3])

for x in train_hiatal_hernia_cases:
    train_list.append([x,4])

for x in train_laminar_atelectasis_fracture_cases:
    train_list.append([x,5])

for x in train_pleural_effusion_cases:
    train_list.append([x,6])

for x in test_air_trapping_cases:
    test_list.append([x, 0])

for x in test_aortic_elongation_cases:
    test_list.append([x, 1])

for x in test_calcified_granuloma_cases:
    test_list.append([x, 2])

for x in test_callus_rib_fracture_cases:
    test_list.append([x, 3])

for x in test_hiatal_hernia_cases:
    test_list.append([x,4])

for x in test_laminar_atelectasis_fracture_cases:
    test_list.append([x,5])

for x in test_pleural_effusion_cases:
    test_list.append([x,6])

for x in val_air_trapping_cases:
    val_list.append([x, 0])

for x in val_aortic_elongation_cases:
    val_list.append([x, 1])

for x in val_calcified_granuloma_cases:
    val_list.append([x, 2])

for x in val_callus_rib_fracture_cases:
    val_list.append([x, 3])

for x in val_hiatal_hernia_cases:
    val_list.append([x,4])

for x in val_laminar_atelectasis_fracture_cases:
    val_list.append([x,5])

for x in val_pleural_effusion_cases:
    val_list.append([x,6])

# shuffle/randomize data as they were loaded in order: normal cases, then pneumonia cases
rn.shuffle(train_list)
rn.shuffle(test_list)
rn.shuffle(val_list)

# create dataframes
train_df = pd.DataFrame(train_list, columns=['image', 'label'])
test_df = pd.DataFrame(test_list, columns=['image', 'label'])
val_df = pd.DataFrame(val_list, columns=['image', 'label'])_____no_output_____plt.figure(figsize=(20,5))

plt.subplot(1,3,1)
sns.countplot(train_df['label'])
plt.title('Train data')

plt.subplot(1,3,2)
sns.countplot(test_df['label'])
plt.title('Test data')

plt.subplot(1,3,3)
sns.countplot(val_df['label'])
plt.title('Validation data')

plt.show()_____no_output_____def process_data(img_path):
    img = cv2.imread(img_path)
    img = cv2.resize(img, (196, 196))
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = img/255.0
    img = np.reshape(img, (196,196,1))

    return img

def compose_dataset(df):
    data = []
    labels = []

    for img_path, label in df.values:
        data.append(process_data(img_path))
        labels.append(label)

    return np.array(data), np.array(labels)_____no_output_____X_train, y_train = compose_dataset(train_df)
X_test, y_test = compose_dataset(test_df)
X_val, y_val = compose_dataset(val_df)

print('Train data shape: {}, Labels shape: {}'.format(X_train.shape, y_train.shape))
print('Test data shape: {}, Labels shape: {}'.format(X_test.shape, y_test.shape))
print('Validation data shape: {}, Labels shape: {}'.format(X_val.shape, y_val.shape))_____no_output_____# define generator

datagen = ImageDataGenerator(
    rotation_range=10,
    zoom_range = 0.1,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=False,
    vertical_flip=False
)

# fit generator on our train features
datagen.fit(X_train)_____no_output_____y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
y_val = to_categorical(y_val)_____no_output_____from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19

_____no_output_____'''
def vgg():
    base_model = ResNet50(weights=None,include_top=False,pooling='avg',input_shape=(196,196,1))
    predictions=Dense(7,activation='softmax',trainable=True)(base_model.output)
    for layer in base_model.layers:
        layer.trainable=True
    model=Model(inputs=[base_model.input], outputs=[predictions])

    optim = tf.keras.optimizers.Adam(lr=1e-5,decay=1e-5)
    #loss_func = 'mae'

    model.compile(loss='categorical_crossentropy', optimizer=optim, metrics=['accuracy'])
    return model

model=None
model = vgg()
model.summary()
'''_____no_output_____
from keras.optimizers import Adam, RMSprop_____no_output_____

def build_model(pretrained):
    model = Sequential([
        pretrained,
        layers.GlobalAveragePooling2D(),
        layers.Dense(7, activation='softmax')
    ])

    model.compile(
        loss='caegorical_crossentropy',
        optimizer=Adam(lr=1e-5),
        metrics=['accuracy']
    )

    return model_____no_output_____

from tensorflow.keras import layers_____no_output_____model=tf.keras.applications.VGG16(include_top=False,weights='imagenet',input_tensor=None,input_shape=(196,196,3),pooling=None,classes=7,)
x=base_model.output
x = Flatten()(x)
x = Dense(256, activation=softmax)(x)
x = Dense(512, activation=softmax)(x)
x = Dense(1024,activation=softmax)(x)
prediction = Dense(7, activation=tf.nn.softmax)(x)
optimizer = Adam(lr=0.0001,decay=1e-5)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])

callback =EarlyStopping(monitor='loss', patience=10)
history = model.fit(datagen.flow(X_train,y_train, batch_size=64),validation_data=(X_test, y_test), epochs = 100, verbose = 2, callbacks=[callback],class_weight={0:0.677839851, 1: 0.6265060241,2:1.373584906,3:1.394636015,4:1.662100457,5:0.9680851064,6:1.177993528})
#, _____no_output_____print('Lowest training loss: ', min(history.history['loss']))
print('Lowest validation loss: ', min(history.history['val_loss']))
print('Highest training accuracy: ', max(history.history['accuracy']))
print('Highest validation accuracy: ', max(history.history['val_accuracy']))_____no_output_____model.save("VGG16.h5")_____no_output_____

# Evaluation
First, we will quickly check evolution of loss and accuracy over epochs and then draw confusion matrix on test dataa. Then how our validation set (16 cases) will work with trained model and compare real vs predicted label_____no_output_____## Draw loss on train vs test data evolution_____no_output_____

plt.figure(figsize=(20,5))

# plot loss & val loss
plt.subplot(1,2,1)
sns.lineplot(x=history.epoch, y=history.history['loss'], color='red', label='Loss')
sns.lineplot(x=history.epoch, y=history.history['val_loss'], color='orange', label='Val Loss')
plt.title('Loss on train vs test')
plt.legend(loc='best')

# plot accuracy and val accuracy
plt.subplot(1,2,2)
sns.lineplot(x=history.epoch, y=history.history['accuracy'], color='blue', label='Accuracy')
sns.lineplot(x=history.epoch, y=history.history['val_accuracy'], color='green', label='Val Accuracy')
plt.title('Accuracy on train vs test')
plt.legend(loc='best')

plt.show()_____no_output_____

## Confusion matrix on test data_____no_output_____

y_test_hat = model.predict(X_test, batch_size=4)
y_test_hat = np.argmax(y_test_hat, axis=1)
y_test = np.argmax(y_test, axis=1)_____no_output_____# calculate confusion matrix & classification report
conf_m = confusion_matrix(y_test, y_test_hat)
clas_r = classification_report(y_test, y_test_hat)

print(conf_m)
# print classification report
print('Classification report on test data')
print(clas_r)_____no_output_____


# Notebook from NicoSerranoP/PySyft
Path: examples/tutorials/translations/japanese/Part 02 - Intro to Federated Learning.ipynb

# Part 2: Federated Learning入門

前回のセクションでは、PointerTensorについて学びました。PointerTensorは、プライバシーに配慮したディープラーニングを実現するための基礎となるツールで、インフラとしての役割をはたします。このセクションでは、それらのツールの使い方、特に私たちが最初に学ぶプライバシーに配慮したディープラーニングの手法である、Federated Learningの実装方法について学びます。

著者:
- Andrew Trask - Twitter: [@iamtrask](https://twitter.com/iamtrask)

### Federated Learningとは?

Federated Learningはディープラーニングのモデルを学習させるシンプル、かつパワフルな手法の一つです。

ディープラーニングの学習プロセスは、通常、データを集めるところから始まります。
データは人がパソコンやモバイルなどの端末を使って記録、生成することによって作成され、実際に学習プロセスを実行するマシンに集められます。
その後、集められたデータを使って学習処理を実行します。
これが、一般的なディープラーニングの学習プロセスです。

ですが、Federated Learningではこのプロセスをひっくり返します。

データをモデルのところへ持って行くのではなく、モデルをデータのあるところへ持って行くのです。
データの所有者はデータのコピーを誰にも渡さず、アクセス権限だけをコントロールするという考え方です。

クールでしょ?_____no_output_____# Section 2.1 - Federated Learningのトイモデル（初歩的な事例）

まずは、みなさんも良くご存知の（ごく普通の？）、データを一箇所に集めるやり方でのモデル学習を確認しましょう。
最小構成で以下の3つが必要です。

- トイ データセット
- モデル
- 基本的なトレーニングロジック

注記: もし、トイモデルを学習するPytorchのAPIについて解説が必要な方は、チュートリアルを進める前に[fast.ai](http://fast.ai)のコースを学習される事をおすすめします。_____no_output_____

import torch
from torch import nn
from torch import optim_____no_output_____# トイデータセット（初歩的なモデル学習で使うデータ）
data = torch.tensor([[0,0],[0,1],[1,0],[1,1.]], requires_grad=True)
target = torch.tensor([[0],[0],[1],[1.]], requires_grad=True)

# トイモデル（初歩的なモデル）
model = nn.Linear(2,1)

def train():
    # Training Logic
    opt = optim.SGD(params=model.parameters(),lr=0.1)
    for iter in range(20):

        # 1) Gradientsの初期化 (もし、あれば)
        opt.zero_grad()

        # 2) 予測を行う
        pred = model(data)

        # 3) ロスの計算
        loss = ((pred - target)**2).sum()

        # 4) 勾配ベクトルの計算
        loss.backward()

        # 5) 勾配ベクトルの適用
        opt.step()

        # 6) 進捗を表示
        print(loss.data)_____no_output_____train()_____no_output_____

できました。今回は基本的なモデルを（従来からある）ごくごく一般的な手法で学習させてみました。全ての学習データはわたしたちのマシン上にあります。ローカルにあるデータセットを使ってローカルにあるモデルのパラメータをアップデートしています。しかし、Federatd Learningの世界では同じようにはいきません。
では、Federated Learning"的"な方法で同じモデルを学習させてみましょう。

必要なステップ:

- 複数のワーカー（ユーザーの事です）を作成する
- 作成したワーカーが保有するPointerTensorを取得する
- Federated Learning用にトレーニングロジックを調整する

    新しいトレーニングステップ（トレーニングロジック）:
    - モデルをデータセットを保持しているワーカーへ送る
    - モデルをデータセットのある場所で学習させる
    - モデルを返してもらい、次のワーカーに対して同様の処理を行う。ワーカーの数だけ繰り返す。
    _____no_output_____

import syft as sy
hook = sy.TorchHook(torch)
_____no_output_____# データセットを保持する複数のワーカーを作成する

bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")
_____no_output_____# トイデータセット（初歩的なモデルで使うデータセット）
data = torch.tensor([[0,0],[0,1],[1,0],[1,1.]], requires_grad=True)
target = torch.tensor([[0],[0],[1],[1.]], requires_grad=True)

# 学習データを分割し、半分をBob、残り半分をAliceに送り、それぞれのポインタを取得する
data_bob = data[0:2]
target_bob = target[0:2]

data_alice = data[2:]
target_alice = target[2:]

# トイモデル（初歩的なモデル）を初期化する
model = nn.Linear(2,1)

data_bob = data_bob.send(bob)
data_alice = data_alice.send(alice)
target_bob = target_bob.send(bob)
target_alice = target_alice.send(alice)

# BobとAliceのポインタをそれぞれ纏める
datasets = [(data_bob,target_bob),(data_alice,target_alice)]

opt = optim.SGD(params=model.parameters(),lr=0.1)_____no_output_____def train():
    # トレーニングロジック
    opt = optim.SGD(params=model.parameters(),lr=0.1)
    for iter in range(10):

        # NEW) ワーカーの数だけ繰り返す
        for data,target in datasets:

            # NEW) モデルを該当ワーカーへ送る
            model.send(data.location)

            # 1) 勾配ベクトルの初期化 (もし、あれば)
            opt.zero_grad()

            # 2) 予測する
            pred = model(data)

            # 3) ロスを計算する
            loss = ((pred - target)**2).sum()

            # 4) 勾配ベクトルを計算する
            loss.backward()

            # 5) 勾配ベクトルの適用
            opt.step()

            # NEW) モデルを返してもらう (計算済み勾配ベクトル含む)
            model.get()

            # 6) 進捗を表示
            print(loss.get()) # NEW) 微調整... .get()でデータを私たちのマシンに持ってくる必要があります。

# federated learning環境下での勾配ベクトルの集計作業 ＜ 後ほど解説します_____no_output_____train()_____no_output_____

## 良く出来ました！

今回は、トイモデル（初歩的なモデル）をFederated Learningの手法を使って学習してみました。モデルをそれぞれのワーカーへ送り、そこで勾配ベクトルの計算とモデルへの適用を行い、その後、学習済みモデルを私たちのマシンへ戻しています。この一連のプロセスにおいて、私たちは学習データ自体に対するアクセス権限をBobやAliceに要求していません。つまり、BobとAliceのプライバシーは保たれています！

## この例の不完全な部分

この例はFederated Learningの入門事例としては良いものの、いくつかの点で問題があります。その中でも最もよく知られた問題は、私たちが`model.get()`によって返してもらう学習済みのモデルの勾配ベクトルを調べる事で、実際にはBobやAliceのデータについて多くを知る事が可能だと言う事です。特定の状況下では、彼らのデータを完璧に復元することも可能です。

では、どうすれば良いでしょうか？まず思いつく手法は、私たちのマシンにモデルを返してもらう前に、複数のワーカーのgradientsを平均することです。しかし、そのためにはPointerTensorオブジェクトの高度な使い方を理解する必要があります。次のセクションでは、PointerTensorの発展的な機能について学び、今回学んだシンプルな例をアップデートしていきましょう。
_____no_output_____# おめでとうございます！コミュニティに入ろう！

本チュートリアルを完了しました。おめでとうございます！もし、このチュートリアルを気に入って、プライバシーに配慮した非中央集権的なAI技術や付随する（データやモデルの）サプライチェーンにご興味があって、プロジェクトに参加したいと思われるなら、以下の方法で可能です。

### PySyftのGitHubレポジトリにスターをつける

一番簡単に貢献できる方法はこのGitHubのレポジトリにスターを付けていただくことです。スターが増えると露出が増え、より多くのデベロッパーにこのクールな技術の事を知って貰えます。

- [Star PySyft](https://github.com/OpenMined/PySyft)

### Slackに入る

最新の開発状況のトラッキングする一番良い方法はSlackに入ることです。
下記フォームから入る事ができます。
[http://slack.openmined.org](http://slack.openmined.org)

### コードプロジェクトに参加する

コミュニティに貢献する一番良い方法はソースコードのコントリビューターになることです。PySyftのGitHubへアクセスしてIssueのページを開き、"Projects"で検索してみてください。参加し得るプロジェクトの状況を把握することができます。また、"good first issue"とマークされているIssueを探す事でミニプロジェクトを探すこともできます。

- [PySyft Projects](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3AProject)
- [Good First Issue Tickets](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)

### 寄付

もし、ソースコードで貢献できるほどの時間は取れないけど、是非何かサポートしたいという場合は、寄付をしていただくことも可能です。寄附金の全ては、ハッカソンやミートアップの開催といった、コミュニティ運営経費として利用されます。

[OpenMined's Open Collective Page](https://opencollective.com/openmined)_____no_output_____


# Notebook from hms-dbmi/3d-genome-processing-tutorial
Path: notebooks/03-eigenvectors.ipynb

<code>
%matplotlib inline_____no_output_____from matplotlib.gridspec import GridSpec
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn.apionly as sns
import mirnylib.plotting
mpl.style.use('seaborn-poster')

import numpy as np
import pandas
from scipy.stats import spearmanr
from mirnylib import numutils
import cooler
import bioframe_____no_output_____"""
cis and trans eigenvector decomposition on Hi-C numpy arrays

refactored from mirnylib and hiclib

"""

def _filter_heatmap(A, transmask, perc_top, perc_bottom):
    # Truncate trans blowouts
    lim = np.percentile(A[transmask], perc_top)
    tdata = A[transmask]
    tdata[tdata > lim] = lim
    A[transmask] = tdata

    # Remove bins with poor coverage in trans
    marg = np.sum(A, axis=0)
    marg_nz = marg[np.sum(A, axis=0) > 0]
    min_cutoff = np.percentile(marg_nz, perc_bottom)
    dropmask = (marg > 0) & (marg < min_cutoff)
    idx = np.flatnonzero(dropmask)
    A[idx, :] = 0
    A[:, idx] = 0
    return A

def _fake_cis(A, cismask):
    cismask = cismask.astype(np.int64)
    s = np.abs(np.sum(A, axis=0)) <= 1e-10
    cismask[:, s] = 2
    cismask[s, :] = 2
    numutils.fakeCisImpl(A, cismask)
    return A

def _orient_eigs(lam, vecs, gc):
    # If GC is provided reorder and change signs of E1, E2, etc. by GC
    # correlation. If not, reorder by descending eigenvalue magnitude and
    # don't change signs.
    if gc is not None:
        corrs = [spearmanr(gc, vec, nan_policy='omit')[0] for vec in vecs]
        signs = np.sign(corrs)
        idx = np.argsort(-np.abs(corrs))
        lam, vecs, signs = lam[idx], vecs[idx], signs[idx]
        # change signs
        for i in range(len(vecs)):
            vecs[i] = signs[i] * vecs[i]
    else:
        idx = np.argsort(-np.abs(lam))
        lam, vecs = lam[idx], vecs[idx]
    return lam, vecs

def _eig(A, k):
    # Compute eigs
    vecs, lam = numutils.zeroEIG(A, numPCs=k)

    # eigsh returns unit vectors, but just making sure
    # then rescale by sqrt(eigval)
    for j in range(len(lam)):
        vecs[j] /= np.sqrt(np.sum(vecs[j]**2))
        vecs[j] *= np.sqrt(np.abs(lam[j]))

    return lam, np.array(vecs)

def trans_eig(A, partition, k=3, perc_top=99.95, perc_bottom=1, gc=None):
    """
    Compute compartmentalization eigenvectors on trans contact data

    Parameters
    ----------
    A : 2D array
        balanced whole genome contact matrix
    partition : sequence of int
        bin offset of each contiguous region to treat separately (e.g.,
        chromosomes or chromosome arms)
    k : int
        number of eigenvectors to compute; default = 3
    perc_top : float (percentile)
        filter - clip trans blowout contacts above this cutoff; default = 99.95
    perc_bottom : float (percentile)
        filter - remove bins with trans coverage below this cutoff; default=1
    gc : 1D array, optional
        GC content per bin for reordering and orienting the primary compartment
        eigenvector; not performed if no array is provided

    Returns
    -------
    eigenvalues, eigenvectors

    """
    if A.shape[0] != A.shape[1]:
        raise ValueError("A is not symmetric")

    A = np.array(A)
    A[np.isnan(A)] = 0
    n_bins = A.shape[0]
    if not (partition[0] == 0 and
            partition[-1] == n_bins and
            np.all(np.diff(partition) > 0)):
        raise ValueError("Not a valid partition. Must be a monotonic sequence "
                         "from 0 to {}.".format(n_bins))

    # Delete cis data and create trans mask
    extents = zip(partition[:-1], partition[1:])
    part_ids = []
    for n, (i0, i1) in enumerate(extents):
        A[i0:i1, i0:i1] = 0
        part_ids.extend([n] * (i1 - i0))
    part_ids = np.array(part_ids)
    transmask = (part_ids[:, None] != part_ids[None, :])

    # Filter heatmap
    A = _filter_heatmap(A, transmask, perc_top, perc_bottom)

    # Fake cis and re-balance
    A = _fake_cis(A, ~transmask)
    A = numutils.iterativeCorrection(A)[0]
    A = _fake_cis(A, ~transmask)
    A = numutils.iterativeCorrection(A)[0]

    # Compute eig
    Abar = A.mean()
    O = (A - Abar) / Abar
    lam, vecs = _eig(O, k)
    lam, vecs = _orient_eigs(lam, vecs, gc)

    return lam, vecs

def cis_eig(A, k=3, robust=True, gc=None, classic=False):
    """
    Compute compartment eigenvector on a cis matrix

    Parameters
    ----------
    A : 2D array
        balanced whole genome contact matrix
    k : int
        number of eigenvectors to compute; default = 3
    robust : bool
        Clip top 0.1 percentile and smooth first two diagonals
    gc : 1D array, optional
        GC content per bin for choosing and orienting the primary compartment
        eigenvector; not performed if no array is provided
    classic : bool
        Do it old-school

    Returns
    -------
    eigenvalues, eigenvectors

    """
    A = np.array(A)
    A[~np.isfinite(A)] = 0

    mask = A.sum(axis=0) > 0

    if A.shape[0] <= 5 or mask.sum() <= 5:
        return (
            np.array([np.ones(A.shape[0]) * np.nan for i in range(k)]),
            np.array([np.nan for i in range(k)]),
        )

    if robust:
        A = np.clip(A, 0, np.percentile(A, 99.9))
        fill_value = np.mean(np.diag(A, 2) * 2)
        for d in [-1, 0, 1]:
            numutils.fillDiagonal(A, fill_value, d)
            A[~mask, :] = 0
            A[:, ~mask] = 0

    OE = numutils.observedOverExpected(A[mask, :][:, mask])

    if robust:
        OE = np.clip(OE, 0, np.percentile(OE, 99.9))

    if classic:
        OE = numutils.iterativeCorrection(OE)[0]
        if (~np.isfinite(OE)).sum() > 0:
            return (
                np.array([np.ones(A.shape[0]) * np.nan for i in range(k)]),
                np.array([np.nan for i in range(k)]),
            )
        # mean-centered (subtract mean)
        eigvecs_compressed, eigvals = numutils.EIG(OE, k)
    else:
        eigvecs_compressed, eigvals = numutils.EIG((OE - 1.0), k,
            subtractMean=False, divideByMean=False)

    # Restore full eigs
    eigvecs = []
    for i in range(k):
        v = np.ones(mask.shape[0]) * np.nan
        v[mask] = eigvecs_compressed[i]
        eigvecs.append(v)
    eigvecs = np.array(eigvecs)

    # Orient and reorder
    eigvals, eigvecs = _orient_eigs(eigvals, eigvecs, gc)

    return eigvals, eigvecs

def cooler_cis_eigs(clr, bins, gc_col='GC', **kwargs):
    bins_grouped = bins.groupby('chrom')
    def _each(chrom):
        A = clr.matrix(balance=True).fetch(chrom)
        gc = None
        if gc_col in bins:
            gc = bins_grouped.get_group(chrom)[gc_col].values

        lam, vec = cis_eig(A, robust=True, gc=gc, **kwargs)

        eig = bins_grouped.get_group(chrom).copy()
        for i in range(len(vec)):
            eig['E{}'.format(i+1)] = vec[i]

        return lam, eig

    bins_chroms = list(bins_grouped.groups.keys())
    map_chroms = [chrom for chrom in clr.chromnames if chrom in bins_chroms]

    lams, vecs = zip(*map(_each, map_chroms))
    lams = pandas.DataFrame(
        index=map_chroms,
        data=np.vstack(lams),
        columns=['lam1', 'lam2', 'lam3'],
    )
    lams.index.name = 'chrom'
    vecs = pandas.concat(vecs, axis=0, ignore_index=True)
    return lams, vecs

def cooler_trans_eig(clr, bins, partition=None, gc_col='GC', **kwargs):

    if partition is None:
        partition = np.r_[
            [clr.offset(chrom) for chrom in clr.chromnames], len(clr.bins())]

    lo = partition[0]
    hi = partition[-1]
    A = clr.matrix(balance=True)[lo:hi, lo:hi]

    gc = None
    if gc_col in bins:
        gc = bins[gc_col].values
    lam, PCs = trans_eig(A, partition, gc=gc, **kwargs)

    vecs = bins.copy()
    for i in range(len(PCs)):
        vecs['E{}'.format(i+1)] = PCs[i]

    return lam, vecs
_____no_output_____fasta_records = bioframe.read_fasta('data/mm9.fa.gz')
chromsizes = cooler.util.fetch_chromsizes('mm9')[:'chr19']
chromosomes = list(chromsizes.index)
binsize = 100000
bins = cooler.binnify(chromsizes, binsize)
bins.head()_____no_output_____bins['GC'] = bioframe.tools.frac_gc(bins, fasta_records)
bins.head()_____no_output_____pal = sns.color_palette('muted')
colors = {
    'U': pal[1],
    'T': pal[0],
    'N': pal[2]
}

clrs = {
    'U': cooler.Cooler(f'data/UNTR.{binsize}.cool'),#f'/net/levsha/share/nezar/TCCSpitz2015_distiller/output/coolers/library_group/UNTR.{binsize}.cool'),
    'T': cooler.Cooler(f'data/TAM.{binsize}.cool'),
    'N': cooler.Cooler(f'data/NIPBL.{binsize}.cool'),
}_____no_output_____lam = {}
vec = {}
for cond in ['U', 'T', 'N']:
    lam[cond], vec[cond] = cooler_cis_eigs(clrs[cond], bins, gc_col='GC')eigenvalues are: [ 330.34817976  339.64614528  861.33220653]
eigenvalues are: [ 355.50484234  409.6986118   908.45224054]
eigenvalues are: [-235.57713831  315.68023411  608.47319351]
eigenvalues are: [-311.46913899  420.52139626  795.04473462]
eigenvalues are: [-356.43734207  441.50140299  769.78904077]
eigenvalues are: [ 241.28011534  299.557626    702.46528745]
eigenvalues are: [ 220.66064478  274.00342121  591.53559134]
eigenvalues are: [-204.73406117  232.9533683   591.07596555]
eigenvalues are: [-249.29002234  254.12651031  586.42898621]
eigenvalues are: [ 190.95496546  215.82917896  568.31190817]
eigenvalues are: [-269.40438287  320.06544787  634.37065826]
eigenvalues are: [-209.51931013  254.58591596  590.61344237]
eigenvalues are: [ 197.74007935  230.82594172  527.63388981]
eigenvalues are: [-220.7863243   263.58052087  525.57291197]
eigenvalues are: [-208.19029176  251.80481052  524.86260499]
eigenvalues are: [-185.69824281  205.40515402  423.72068803]
eigenvalues are: [-141.9939455   229.86657132  478.23236233]
eigenvalues are: [-128.44801691  155.24771458  415.8194942 ]
eigenvalues are: [ -72.68600688  106.87189334  262.45475375]
eigenvalues are: [ 267.45254343  325.81367397  785.44356728]
eigenvalues are: [ 323.8214429   363.85564115  828.37968599]
eigenvalues are: [-217.39353746  284.6574589   565.59626211]
eigenvalues are: [-289.79310365  366.5325151   719.68772478]
eigenvalues are: [-309.55504643  373.92655767  700.26984298]
eigenvalues are: [ 207.37463313  252.21723661  659.23608502]
eigenvalues are: [ 214.7169213   245.47920774  598.51563846]
eigenvalues are: [-188.4186217   210.52429353  546.50975824]
eigenvalues are: [-211.84926813  227.95207215  546.39103202]
eigenvalues are: [ 171.27111945  189.82353367  535.66747109]
eigenvalues are: [-229.21840235  288.98250627  583.69178264]
eigenvalues are: [-191.42497069  219.98798878  539.00508609]
eigenvalues are: [-192.23123132  212.02990714  503.29578312]
eigenvalues are: [-196.61462798  229.85936316  479.40648935]
eigenvalues are: [-197.33573903  222.2305696   485.93226657]
eigenvalues are: [-174.60124808  176.30994468  400.20742402]
eigenvalues are: [-137.25551292  213.44838727  464.40366164]
eigenvalues are: [-123.12390104  140.67376886  396.75593946]
eigenvalues are: [ -72.62387724  101.71709991  252.48773821]
eigenvalues are: [ 328.81553166  414.60230885  909.85514104]
eigenvalues are: [ 376.22370649  468.92269432  959.64834146]
eigenvalues are: [ 231.57929449  316.86291668  721.07889592]
eigenvalues are: [ 299.34209706  444.59730843  828.26229989]
eigenvalues are: [ 305.61898351  521.65796427  791.87077571]
eigenvalues are: [ 233.01864307  292.71343644  773.14285467]
eigenvalues are: [ 227.11968061  346.07535206  685.56965976]
eigenvalues are: [ 180.40971464  306.48957567  654.20332387]
eigenvalues are: [ 234.11214105  300.73620945  619.00636079]
eigenvalues are: [ 213.92935365  265.53321616  625.1498342 ]
eigenvalues are: [-262.75297083  392.22347682  706.07151228]
eigenvalues are: [ 196.53090086  221.48577015  623.3554627 ]
eigenvalues are: [ 179.14258754  270.05900992  536.28114627]
eigenvalues are: [-210.01904412  313.58042199  555.20727484]
eigenvalues are: [-201.50606035  254.74580617  532.35787434]
eigenvalues are: [-181.29315843  211.35267994  465.25423598]
eigenvalues are: [ 141.62156576  242.15316126  532.3234559 ]
eigenvalues are: [ 114.77170272  157.62243012  422.24648917]
eigenvalues are: [  74.79940205  141.98044777  306.45194345]
g = vec['U'].groupby('chrom').get_group('chr19')
plt.figure(figsize=(15, 5))
plt.step(np.arange(len(g)), g['E1'], where='post')
plt.axhline(0, c='k')
plt.figure(figsize=(15, 5))
plt.step(np.arange(len(g)), g['GC'], where='post')
plt.axhline(0.5, c='k')_____no_output_____plt.scatter(vecs['E1'], vecs['GC'], marker='.', edgecolor='k', alpha=0.1)_____no_output_____"""
Saddle plot code.

Authors
~~~~~~~
* Anton Goloborodko
* Nezar Abdennur

"""
from matplotlib.gridspec import GridSpec
import matplotlib.pyplot as plt
import seaborn as sns

from scipy.linalg import toeplitz
from cytoolz import merge
import numpy as np
import pandas as pd

pal = sns.color_palette('muted')

def digitize_track(
        bins,
        get_track,
        get_mask,
        chromosomes,
        prange=None,
        by_percentile=False):
    """
    Digitize per-chromosome genomic tracks.

    Parameters
    ----------
    get_track : function
        A function returning a genomic track given a chromosomal name/index.
    get_mask : function
        A function returning a binary mask of valid genomic bins given a
        chromosomal name/index.
    chromosomes : list or iterator
        A list of names/indices of all chromosomes.
    bins : int or list or numpy.ndarray
        If list or numpy.ndarray, `bins` must contain the bin edges for
        digitization.
        If int, then the specified number of bins will be generated
        automatically from the genome-wide range of the track values.
    prange : pair of floats
        The percentile of the genome-wide range of the track values used to
        generate bins. E.g., if `prange`=(2. 98) the lower bin would
        start at the 2-nd percentile and the upper bin would end at the 98-th
        percentile of the genome-wide signal.
        Use to prevent the extreme track values from exploding the bin range.
        Is ignored if `bins` is a list or a numpy.ndarray.
    by_percentile : bool
        If true then the automatically generated bins will contain an equal
        number of genomic bins genome-wide (i.e. track values are binned
        according to their percentile). Otherwise, bins edges are spaced equally.
        Is ignored if `bins` is a list or a numpy.ndarray.

    Returns
    -------
    digitized : dict
        A dictionary of the digitized track, split by chromosome.
        The value of -1 corresponds to the masked genomic bins, the values of 0
        and the number of bins correspond to the values lying below and above
        the bin range limits.
    binedges : numpy.ndarray
        The edges of bins used to digitize the track.
    """

    if not hasattr(bins, '__len__'):
        if prange is None:
            prange = (0, 100)

        fulltrack = np.concatenate([
            get_track(chrom)[get_mask(chrom)]
                for chrom in chromosomes
        ])

        if by_percentile:
            perc_edges = np.linspace(prange[0], prange[1], bins + 1)
            binedges = np.percentile(fulltrack, perc_edges)
        else:
            lo = np.percentile(fulltrack, prange[0])
            hi = np.percentile(fulltrack, prange[1])
            binedges = np.linspace(lo, hi, bins + 1)
    else:
        binedges = bins

    digitized = {}
    for chrom in chromosomes:
        x = np.digitize(get_track(chrom), binedges, right=False)
        x[~get_mask(chrom)] = -1
        digitized[chrom] = x

    return digitized, binedges

def fill_diagonal(A, values, k=0, wrap=False, inplace=False):
    """
    Based on numpy.fill_diagonal, but allows for kth diagonals as well.
    Only works on 2D arrays.
    """
    if not inplace:
        A = np.array(A)
    else:
        A = np.asarray(A)
    start = k
    end = None
    step = A.shape[1] + 1
    #This is needed so a tall matrix doesn't have the diagonal wrap around.
    if not wrap:
        end = start + A.shape[1] * A.shape[1]
    A.flat[start:end:step] = values
    return A

def make_saddle(
        get_matrix,
        get_digitized,
        chromosomes,
        contact_type,
        verbose=False):
    """
    Make a matrix of average interaction probabilities between genomic bin pairs
    as a function of a specified genomic track. The provided genomic track must
    be pre-binned (i.e. digitized).

    Parameters
    ----------
    get_matrix : function
        A function returning an matrix of interaction between two chromosomes
        given their names/indicies.
    get_digitized : function
        A function returning a track of the digitized target genomic track given
        a chromosomal name/index.
    chromosomes : list or iterator
        A list of names/indices of all chromosomes.
    contact_type : str
        If 'cis' then only cis interactions are used to build the matrix.
        If 'trans', only trans interactions are used.
    verbose : bool
        If True then reports progress.

    Returns
    -------
    interaction_sum : 2D array
        The matrix of summed interaction probability between two genomic bins
        given their values of the provided genomic track.
    interaction_count : 2D array
        The matrix of the number of genomic bin pairs that contributed to the
        corresponding pixel of ``interaction_sum``.

    """
    if contact_type not in ['cis', 'trans']:
        raise ValueError("The allowed values for the contact_type "
                         "argument are 'cis' or 'trans'.")

    n_bins = max([
        get_digitized(chrom).max()
            for chrom in chromosomes
    ])

    interaction_sum   = np.zeros((n_bins, n_bins))
    interaction_count = np.zeros((n_bins, n_bins))

    for k, chrom1 in enumerate(chromosomes):
        for chrom2 in chromosomes[k:]:
            if (((contact_type == 'trans') and (chrom1 == chrom2)) or
                ((contact_type == 'cis') and (chrom1 != chrom2))):
                continue

            matrix = get_matrix(chrom1, chrom2)
            for d in [-2, -1, 0, 1, 2]:
                fill_diagonal(matrix, np.nan, d)

            if verbose:
                print('chromosomes {} vs {}'.format(chrom1, chrom2))

            for i in range(n_bins):
                row_mask = (get_digitized(chrom1) == i)
                for j in range(n_bins):
                    col_mask = (get_digitized(chrom2) == j)
                    data = matrix[row_mask, :][:, col_mask]
                    data = data[np.isfinite(data)]
                    interaction_sum[i, j]   += np.sum(data)
                    interaction_count[i, j] += float(len(data))

    interaction_sum   += interaction_sum.T
    interaction_count += interaction_count.T

    return interaction_sum, interaction_count

def saddleplot(binedges,
                digitized,
                saddledata,
                color,
                cbar_label=None,
                fig_kws=None,
                heatmap_kws=None,
                margin_kws=None):
    """
    Plot saddle data and signal histograms in the margins.

    Parameters
    ----------
    binedges: 1D array
    digitized: dict of chrom to 1D array
    saddledata: 2D array
    color: str
    cbar_label: str
    fig_kws: dict, optional
        Extra keywords to pass to ``figure``.
    heatmap_kws : dict, optional
        Extra keywords to pass to ``imshow`` for saddle heatmap.
    margin_kws : dict, optional
        Extra keywords to pass to ``hist`` for left and top margins.

    """
    n_bins = len(binedges) - 1
    lo, hi = 0, n_bins  #-0.5, n_bins - 1.5

    # Populate kwargs
    fig_kws = merge(
        dict(figsize=(5, 5)),
        fig_kws if fig_kws is not None else {}

    )

    heatmap_kws = merge(
        dict(aspect='auto',
             cmap='coolwarm',
             interpolation='none',
             vmin=-1,
             vmax=1),
        heatmap_kws if heatmap_kws is not None else {},
    )
    vmin = heatmap_kws['vmin']
    vmax = heatmap_kws['vmax']

    margin_kws = merge(
        dict(bins=n_bins,
             range=(0, len(binedges)),
             histtype='stepfilled',
             edgecolor='k',
             facecolor=color,
             linewidth=1),
        margin_kws if margin_kws is not None else {},
    )

    # layout
    gs = GridSpec(
        nrows=3,
        ncols=3,
        width_ratios=[0.2, 1, 0.1],
        height_ratios=[0.2, 1, 0.1],
        wspace=0.05,
        hspace=0.05,
    )
    fig = plt.figure(**fig_kws)

    # heatmap
    ax = ax1 = plt.subplot(gs[4])
    img = ax.imshow(np.log10(saddledata), **heatmap_kws)
    plt.xticks(
        np.arange(0, n_bins)[::5],
        ['{:0.4f}'.format(t)
            for t in ((binedges[1:] + binedges[:-1])/2)[::5]],
        rotation=90,
    )
    plt.yticks([])
    plt.xlim(lo, hi)
    plt.ylim(hi, lo)

    # left margin
    plt.subplot(gs[3])
    plt.hist(np.concatenate(list(digitized.values())),
             **merge(margin_kws, {'orientation': 'horizontal'}))
    plt.xticks([])
    plt.yticks(
        np.arange(0, n_bins)[::5],
        ['{:0.4f}'.format(t)
            for t in ((binedges[1:] + binedges[:-1])/2)[::5]],
    )
    plt.xlim(plt.xlim()[1], plt.xlim()[0])  # fliplr
    plt.ylim(hi, lo)
    plt.gca().spines['top'].set_visible(False)
    plt.gca().spines['bottom'].set_visible(False)
    plt.gca().spines['left'].set_visible(False)

    # top margin
    plt.subplot(gs[1])
    plt.hist(np.concatenate(list(digitized.values())), **margin_kws)
    plt.xticks([])
    plt.yticks([])
    plt.xlim(lo, hi)
    plt.ylim(plt.ylim()[0], plt.ylim()[1])  # correct
    plt.gca().spines['top'].set_visible(False)
    plt.gca().spines['right'].set_visible(False)
    plt.gca().spines['left'].set_visible(False)

    # colorbar
    plt.subplot(gs[5])
    cb = plt.colorbar(
        img,
        fraction=0.8,
        label=cbar_label)
    if vmin is not None and vmax is not None:
        cb.set_ticks(np.arange(vmin, vmax + 0.001, 0.5))
    plt.grid(False)
    plt.axis('off')

    return fig_____no_output_____def fillna(x, value=0):
    x = x.copy()
    x[~np.isfinite(x)] = value
    return x_____no_output_____signal_bins = 50

for cond in ['T', 'N']:
    # Create the data "getters"
    get_obsexp = lambda chrom, _: (
        numutils.observedOverExpected(
            clrs[cond].matrix().fetch(chrom))
    )
    get_track = lambda chrom: (
        vec[cond].groupby('chrom').get_group(chrom)['E1']
    )
    get_track_mask = lambda chrom: (
        (vec[cond].groupby('chrom')
              .get_group(chrom)['E1']
              .notnull()) &
        (vec[cond].groupby('chrom')
              .get_group(chrom)['E1'] != 0)
    ).values

    # Bin the track
    digitized, binedges = digitize_track(
        signal_bins,
        get_track,
        get_track_mask,
        chromosomes,
        prange=(0.25, 99.75),
        by_percentile=False
    )

    # Aggregate contacts
    sum_, count = make_saddle(
        get_obsexp,
        lambda chrom: digitized[chrom],
        chromosomes,
        contact_type='cis',
        verbose=False)
    saddledata = sum_ / count

    # Plotting
    fig = saddleplot(
        binedges,
        digitized,
        saddledata,
        colors[cond],
        cbar_label='log10 (contact frequency / mean) in cis',
        fig_kws=None,
        heatmap_kws=None,
        margin_kws=None)
    plt.suptitle({
        'U': 'WT',
        'T': 'TAM',
        'N': r'$\Delta$Nipbl'
    }[cond])_____no_output_____signal_bins = 50

for cond in ['T', 'N']:
    # Create the data "getters"
    get_obsexp = lambda chrom, _: (
        numutils.observedOverExpected(
            clrs[cond].matrix().fetch(chrom))
    )
    get_track = lambda chrom: (
        vec[cond].groupby('chrom').get_group(chrom)['E1']
    )
    get_track_mask = lambda chrom: (
        (vec[cond].groupby('chrom')
              .get_group(chrom)['E1']
              .notnull()) &
        (vec[cond].groupby('chrom')
              .get_group(chrom)['E1'] != 0)
    ).values

    # Bin the track
    digitized, binedges = digitize_track(
        signal_bins,
        get_track,
        get_track_mask,
        chromosomes,
        prange=(0.25, 99.75),
        by_percentile=True
    )

    # Aggregate contacts
    sum_, count = make_saddle(
        get_obsexp,
        lambda chrom: digitized[chrom],
        chromosomes,
        contact_type='cis',
        verbose=False)
    saddledata = sum_ / count

    # Plotting
    fig = saddleplot(
        binedges,
        digitized,
        saddledata,
        colors[cond],
        cbar_label='log10 (contact frequency / mean) in cis',
        fig_kws=None,
        heatmap_kws=dict(vmin=-1, vmax=1),
        margin_kws=None)
    plt.suptitle({
        'U': 'WT',
        'T': 'TAM',
        'N': r'$\Delta$Nipbl'
    }[cond])_____no_output_____
</code>