# Notebook from wikibook/transfer-learning
Path: Chapter07/HandsOn-05_Text_Summarization.ipynb

# Chapter 7. 텍스트 문서의 범주화 - (5) CNN 모델로 문서 요약하기

- 리뷰는 많은 문장으로 구성돼 있으며, 일부를 통해 전체 문서의 성향을 결정할 수 있다.
- 첫 번째 단계는 각 문장에 중요도 점수를 할당해 문서의 돌출맵(saliency map)을 만드는 것이다.
- 문서 요약 과정
    1. 네트워크의 전방 전달을 통해 문서에 대한 클래스 예측을 생성한다.
    2. 네트워크의 예측을 뒤집어서 의사 레이블(pseudo label)을 만든다.
    3. 의사 레이블을 실제 레이블처럼 손실 함수에 제공하여 강한 긍정 문장일 수록 높은 기울기 노름을 보이도록 한다.
    4. 문장 임베딩층에 손실 함수의 미분을 계산한다.
    5. 가장 중요한 문장이 위에 가도록 내림차순 기울기 노름으로 문장을 정렬한다.
    _____no_output_____

import config
import numpy as np
import pandas as pd
from nltk.tokenize import sent_tokenize
import keras.backend as K
from model.cnn_document_model import DocumentModel
from preprocessing.utils import Preprocess, remove_empty_docs
from dataloader.loader import LoaderUsing TensorFlow backend.

## 학습된 모델 로드하기_____no_output_____

# imdb 전체 데이터셋으로 미리 학습한 CNN 모델 로드
imdb_model = DocumentModel.load_model(config.MODEL_DIR+ '/imdb/full_model_10.json')
imdb_model.load_model_weights(config.MODEL_DIR+ '/imdb/full_model_10.hdf5')

model = imdb_model.get_classification_model()
model.compile(loss="binary_crossentropy", optimizer='rmsprop', metrics=["accuracy"])
Vocab Size = 28656  and the index of vocabulary words passed has 28654 words
WARNING:tensorflow:From /Users/dhkdn9192/venv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /Users/dhkdn9192/venv/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.

## 데이터셋 로드하기_____no_output_____

# imdb 학습셋 로드
train_df = Loader.load_imdb_data(directory='train')
print(f'train_df.shape : {train_df.shape}')

corpus = train_df['review'].tolist()
target = train_df['sentiment'].tolist()
corpus, target = remove_empty_docs(corpus, target)
print(f'corpus size : {len(corpus)}')
print(f'target size : {len(target)}')train_df.shape : (25000, 2)
corpus size : 25000
target size : 25000

## 데이터셋을 인덱스 시퀀스로 변환_____no_output_____

Preprocess.NUM_SENTENCES = 20
preprocessor = Preprocess(corpus=corpus)
corpus_to_seq = preprocessor.fit()

x_train = np.array(corpus_to_seq)
y_train = np.array(target)
print(f'x_train.shape : {x_train.shape}')
print(f'y_train.shape : {y_train.shape}')Found 28654 unique tokens.
All documents processed.x_train.shape : (25000, 600)
y_train.shape : (25000,)

## 의사 레이블 생성
모델이 학습셋에 대해 예측한 결과들을 각각 1에서 뺀 값을 구한다. 이것을 의사 레이블이라 한다_____no_output_____

# 학습셋을 입력하여 예측결과들을 얻는다
preds = model.predict(x_train)
print(f'preds size : {len(preds)}')

# 예측결과를 1에서 뺀 값(의사 레이블)들을 얻는다
pseudo_label = np.subtract(1,preds)
print(f'pseudo_label size : {len(pseudo_label)}')preds size : 25000
pseudo_label size : 25000
print(preds[:5])[[0.56517434]
 [0.1569992 ]
 [0.80936414]
 [0.90274227]
 [0.0303922 ]]
print(pseudo_label[:5])[[0.43482566]
 [0.84300077]
 [0.19063586]
 [0.09725773]
 [0.96960783]]

## 그래디언트 계산 함수 정의
keras의 model.optimizer.get_gradients() 함수를 이용하여 그래디언트(기울기)를 계산하는 함수를 정의한다_____no_output_____

# # 학습된 문장 임배딩을 가져온다.(학습셋이 입력될 때 임베딩 레이어에서의 출력 벡터값)
sentence_ebd = imdb_model.get_sentence_model().predict(x_train)

input_tensors = [model.inputs[0],          # 입력 데이터
                 model.sample_weights[0],  # 샘플당 가중치
                 model.targets[0],         # 레이블
                ]

# 문장 임베딩층의 텐서 변수
weights = imdb_model.get_sentence_model().outputs

# 전체 모델의 손실에 대하여 기울기 계산 (문장 임베딩층의 변수)
gradients = model.optimizer.get_gradients(model.total_loss, weights)
get_gradients = K.function(inputs=input_tensors, outputs=gradients)   # 그래디언트 계산 함수WARNING:tensorflow:From /Users/dhkdn9192/venv/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
# 그래디언트 계산식의 입력 데이터
input_tensors_____no_output_____# 그래디언트 계산식 입력의 샘플당 가중치
weights_____no_output_____# 그래디언트 계산식의 입력 레이블
gradients_____no_output_____

## 그래디언트 계산
위에서 정의한 계산함수를 이용하여 특정 문서의 그래디언트를 계산해보자. 입력 데이터가 들어오면 실제 예측값이 아닌, 의사레이블을 출력하도록 그래디언트를 계산한다. 아래와 같이 학습셋의 11번째 텍스트에 대하여 수행한다_____no_output_____

document_number = 10                             # 학습셋 문서 중 n번째 문서에 대해
K.set_learning_phase(0)
inputs = [[x_train[document_number]],            # X
          [1],                                   # 샘플당 가중치
          [[pseudo_label[document_number][0]]],  # y (실제 레이블이 아닌 의사 레이블)
         ]

grad = get_gradients(inputs)
print(f'grad[0].shape : {grad[0].shape}')grad[0].shape : (1, 20, 90)
# n번째 문서 입력에 대한 그래디언트 출력값
grad_____no_output_____

## 그래디언트 노름 값에 따른 문장 정렬
nltk.sent_tokenize로 문장을 구분한 뒤, 각 문장의 그래디언트 노름 값을 구하여 노름 값 기준 정렬을 수행한다_____no_output_____

# 그래디언트 노름 값 추출하기
sent_score = []
sentences = sent_tokenize(corpus[document_number])  # 원본 문서를 문장 단위 split

# 각 문장의 노름 계산 (zero padding된 문장 빼고)
for i in range(min(len(sentences), Preprocess.NUM_SENTENCES)):
    sent_score.append((i, -np.linalg.norm(grad[0][0][i])))
print(f'sent_score size : {len(sent_score)}')

# 그래디언트 노름 값 기준으로 문장 인덱스들 정렬
sent_score.sort(key=lambda tup: tup[1])
summary_sentences = [ i for i, s in sent_score[:4]]
print('summary_sentences :', summary_sentences)sent_score size : 17
summary_sentences : [13, 1, 12, 0]
doc = corpus[document_number]
label = y_train[document_number]
prediction = preds[document_number]
print(doc, label , prediction)So it starts with a beautiful old house in the country. You have a group of people who get asked to come to this house and (not surprisingly) the caretakers always lock the gates at night for no apparent reason. Anywhoo, the people laugh, joke etc. This Dr tells them a spooky story of this woman and some kids. They get scared, they start to feel stuff. Oh no, a girl see's s ghost. Some more talking then this huge ghost comes and etc etc. This girl finds out that this ghost killed little kids and that she must free their souls, yeah yeah, blah blah. She does but, oh no, she dies as she does. And goes to heaven whilst this evil ghost goes to hell. Two people survive and escape the house. The script is terrible because a guy gets his head chopped off and Elanor (the one who dies saving the kids) says "oh no". The acting is wooden, the effects are crap and the set is a couple off rooms used over and over again. Basically if you like laughing at badly made films watch it, but if your looking for a scare then definitely give this film a miss. I was extremely disappointed when I watched this. A very big let down. My sister (who gets sacred very easily) got bored in this film it is appalling. 0 [0.06841457]
# 요약한 문장들
for i in summary_sentences:
    print(i, sentences[i])13 Basically if you like laughing at badly made films watch it, but if your looking for a scare then definitely give this film a miss.
1 You have a group of people who get asked to come to this house and (not surprisingly) the caretakers always lock the gates at night for no apparent reason.
12 The acting is wooden, the effects are crap and the set is a couple off rooms used over and over again.
0 So it starts with a beautiful old house in the country.

## 부정적인 리뷰 요약_____no_output_____

document_number = [(i, a) for i, a in enumerate(corpus)
                   if 'Wow, what a great cast! Julia' in a][0][0]
document_number_____no_output_____K.set_learning_phase(0)
inputs = [[x_train[document_number]],            # X
          [1],                                   # 샘플당 가중치
          [[pseudo_label[document_number][0]]],  # y (실제 레이블이 아닌 의사 레이블)
         ]
grad = get_gradients(inputs)

# 그래디언트 노름 값 추출하기
sent_score = []
sentences = sent_tokenize(corpus[document_number])  # 원본 문서를 문장 단위 split

# 각 문장의 노름 계산 (zero padding된 문장 빼고)
for i in range(min(len(sentences), Preprocess.NUM_SENTENCES)):
    sent_score.append((i, -np.linalg.norm(grad[0][0][i])))

# 그래디언트 노름 값 기준으로 문장 인덱스들 정렬
sent_score.sort(key=lambda tup: tup[1])
print(f'sent_score size : {len(sent_score)}')sent_score size : 13
# 원본 문서
corpus[document_number]_____no_output_____# 요약 문장들
for idx, score in sent_score[:6]:
    print(f'{idx} > {sentences[idx]}')8 > I've been working pretty hard lately, and this is guaranteed to be a hit with all these big names, right?
12 > Even Julia's fat suit is lame.
5 > You'd think so, but no.
1 > Julia Roberts, John Cusack, Christopher Walken, Catherine Zeta-Jones, Hank Azaria...what's that?
7 > It's like everyone involved woke up before each day's writing/shooting/editing and though "You know what?
9 > I'm just gonna cruise along and let somebody else carry the can."

## 긍정적인 리뷰 요약_____no_output_____

document_number = [(i, a) for i, a in enumerate(corpus)
                   if 'This is what I was expecting when star trek DS9 premiered.' in a][0][0]
document_number_____no_output_____K.set_learning_phase(0)
inputs = [[x_train[document_number]],            # X
          [1],                                   # 샘플당 가중치
          [[pseudo_label[document_number][0]]],  # y (실제 레이블이 아닌 의사 레이블)
         ]
grad = get_gradients(inputs)

# 그래디언트 노름 값 추출하기
sent_score = []
sentences = sent_tokenize(corpus[document_number])  # 원본 문서를 문장 단위 split

# 각 문장의 노름 계산 (zero padding된 문장 빼고)
for i in range(min(len(sentences), Preprocess.NUM_SENTENCES)):
    sent_score.append((i, -np.linalg.norm(grad[0][0][i])))

# 그래디언트 노름 값 기준으로 문장 인덱스들 정렬
sent_score.sort(key=lambda tup: tup[1])
print(f'sent_score size : {len(sent_score)}')sent_score size : 20
# 원본 문서
corpus[document_number]_____no_output_____# 요약 문장들
for idx, score in sent_score[-5:]:
    print(f'{idx} > {sentences[idx]}')3 > Enterprise is that show.
2 > That was a wonderful show in it's own right, however it never really gave the fans more of what they wanted.
6 > And that was one of the primary ingredients that made the original so loved.
19 > Doesn't have to be orchestral.
0 > This is what I was expecting when star trek DS9 premiered.


# Notebook from WormBase/gsoc-2021-bioinformatics-notebooks
Path: Tutorial-11-analysis-enrichment.ipynb

# **Bioinformatics with Jupyter Notebooks for WormBase:**
## **Analyses 7 - Enrichment Analyses**
Welcome to the seventh jupyter notebook in the WormBase tutorial series. Over this series of tutorials, we will write code in Python that allows us to retrieve and perform simple analyses with data available on the WormBase sites.

This tutorial will deal with performing 3 kinds of enrichment analyses - tissue, gene ontology, and phenotype for the given gene list input.
Let's get started!_____no_output_____For this tutorial we use the WormBase Tissue Enrichment Analysis (TEA) pip package which we need to install and then import. We also need to import other relevant python packages._____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tissue_enrichment_analysis as tea
%matplotlib inline
%config InlineBackend.figure_formats = {'png', 'retina'}_____no_output_____

Read a csv file with gene names into a dataframe for performing the analyses further.

The csv file needs to have one gene in each line with the first line being a header._____no_output_____

genes = pd.read_csv('data/tea.csv')_____no_output_____

For analysing the enrichment, we need to use the dictionaries which are maintained by WormBase regularly. There are 3 dictionaries, each for the three types of enrichment analyses._____no_output_____

Tissue_Enrichment = tea.fetch_dictionary('tissue')            #Tissue Enrichment
Phenotype_Enrichment = tea.fetch_dictionary('phenotype')      #Phenotype Enrichment
GO_Enrichment = tea.fetch_dictionary('go')                    #Gene Ontology Enrichment_____no_output_____

Now we analyse the gene list against the WormBase TEA dictionaries.

We create a python dictionary to store the TEA dictionaries._____no_output_____

dictionaries = {'Tissue_Enrichment': Tissue_Enrichment,
                'Phenotype_Enrichment': Phenotype_Enrichment,
                'GO_Enrichment': GO_Enrichment}_____no_output_____

We test the list of genes and store the results. For this, we need to set an alpha value to extract only the statistically significant results._____no_output_____

cutoff = 0.01

enrichments = {}
for analysis, dictionary in dictionaries.items():
    enrichments[analysis] = tea.enrichment_analysis(genes.gene_name, dictionary, show=False, alpha=cutoff)_____no_output_____

We have performed the enrichment analysis! Now we can obtain the results in the form of dataframes!_____no_output_____

for enrichment in enrichments:
  print('This is the ', enrichment, ' result dataframe, it has', str(len(enrichments[enrichment])), ' entries')
  display(enrichments[enrichment])_____no_output_____

It is also possible to visualize these enrichments in the form of plots that show the top n hits based on -log10 q values._____no_output_____

for enrichment in ['GO_Enrichment', 'Phenotype_Enrichment', 'Tissue_Enrichment']:
    data_for_graph = enrichments[enrichment]
    data_for_graph['minus_log10_Qvalue'] = -np.log10(data_for_graph['Q value'])

    n = 10 #Change based on how many top hits you want!

    data_for_graph = data_for_graph.head(n).iloc[::-1]
    data_for_graph.plot(x='Term', y='minus_log10_Qvalue', kind="barh", legend=False, width=0.8, color = 'red',
                        figsize = (10,5))
    for i, (Term, qval) in enumerate(zip(data_for_graph['Term'],
                                         data_for_graph["minus_log10_Qvalue"].round(1).astype(str))):
        plt.text(s = qval + '    ' + Term, x = 0, y = i, color = "k", horizontalalignment = 'left',
                 verticalalignment = "center", size = 10)
        plt.axis("off")
        plt.tight_layout()
        plt.title(f'{analysis} enrichment analysis showing -log10 Q-values top '+str(n)+' hits', fontsize = 14)
plt.show()_____no_output_____

This is the end of the seventh tutorial for WormBase data analysis! This tutorial dealt with performing enrichment analyses on WormBase data for a user provided gene list along with generating figures for easy understanding.

In the next tutorial, we will perform Literature based analyses on WormBase data!

Acknowledgements:
- Tissue Enrichment Analysis GitHub Repository (https://github.com/dangeles/TissueEnrichmentAnalysis)
- TEA Publication - 'Tissue enrichment analysis for C. elegans genomics.'  Angeles-Albores, D., N. Lee, R.Y., Chan, J., Sternberg, P.W. BMC Bioinformatics 17, 366 (2016). https://doi.org/10.1186/s12859-016-1229-9
- TEA Tutorial (https://colab.research.google.com/github/Munfred/worm-tutorials/blob/main/tissue_enrichment_analysis.ipynb)_____no_output_____