# Notebook from dcpatton/ragged_tensors
Path: ragged_imdb_test.ipynb

_____no_output_____

import tensorflow as tf
import random
import numpy as np

seed=52
tf.random.set_seed(seed)
random.seed(seed)
np.random.seed(seed)

vocab_size=10000

tf.__version_______no_output_____

# Padded to longest review length_____no_output_____

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.imdb.load_data(num_words=vocab_size, seed=seed)Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz
17465344/17464789 [==============================] - 1s 0us/step
# A couple of examples
index = tf.keras.datasets.imdb.get_word_index()
reverse_index = dict([(value, key) for (key, value) in index.items()])
decoded = " ".join( [reverse_index.get(i - 3, "#") for i in x_train[0]] )
print(decoded)
print(y_train[0])

decoded = " ".join( [reverse_index.get(i - 3, "#") for i in x_train[2]] )
print(decoded)
print(y_train[2])Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json
1646592/1641221 [==============================] - 0s 0us/step
# a must see i saw whipped at a press screening and it was hilarious we're talking # laughs it makes something about mary seem like a meandering drama amanda # screams star quality with her winning combination of beauty brains and serious acting ability peter cohen the director has made a cutting edge film that shows the raw inside of men's # in the urban dating world for all of it's comedy whipped succeeds with it's intelligence which is so rare for a first time director especially with a romantic comedy he is a major talent # # brian van holt jonathan # and # # round out the cast with depth and very strong performances as the would be slick # men you've got to see these guys go to work and get caught in # web check out the trailer on whipped com it's worth the 3 minute #
1
# phantom of the mall is a film that fits best in the easily forgotten category it's a pretty lousy # on the famous story by # # the phantom of the opera not a bad idea to itself but the plot and production of this movie are way to weak to bring a decent homage to that story on the bright side # # doesn't has to turn over in his grave just yet it could have been a lot worse br br phantom of the mall has too many useless flashbacks in it and way too many boring sequences to make it memorable also the # wanted to give too much # to the story than necessary and even though there's a lot of mystery getting build up about the character of eric the basic plot is ordinary and # # spoilers it's about a young couple that brutally gets torn apart because the boy gets killed in a fire that fire was set to his house because he and his parents refused to sell their home in order to make room for a huge mall to be build the boy survived the fire and he has hidden himself in the mall to avenge himself meanwhile he guards his girl who now works in the mall and tries to forget her loss end spoilers this pretty simple but rather effective plot gets # by lots of pointless elements and annoying conspiracy theories while it should just be an entertaining horror movie it desperately tries to be an intelligent thriller and that's not what the fans look for there are a few innovative killings but they're not satisfying enough for people who want to see a relaxing horror movie and besides phantom of the mall could have used at least a bit of humor this entire production the cast included takes itself way too serious br br i'll try to finish with a few positive aspects like for example it stars ken # die hard horror fans will certainly recognize him as peter for dawn of the dead that's like the horror # that yet has to find an equal even though his role in this movie is limited and even completely unnecessary it was good to see him again tv movie fans will also recognize morgan # as the mayor she's a fine actress and an elegant lady # shore is also in this but i can't stand him so my opinion about him may be a bit biased and finally a bit of praise for the leading actress named # # she's an extremely beautiful girl and she does have a bit of talent too bad she never made it to the top actually this movie is her biggest achievement and that says enough about her career
0
maxlen = 0
for i in range(x_train.shape[0]):
  if len(x_train[i]) > maxlen:
    maxlen = len(x_train[i])

maxlen_____no_output_____x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen, padding='post')
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen, padding='post')_____no_output_____tf.keras.backend.clear_session()
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras import Input, Model

input1 = Input(shape=(maxlen,))
x = Embedding(vocab_size, 64)(input1)
x = LSTM(64)(x)
out = Dense(1, activation='sigmoid')(x)

model = Model(inputs=[input1], outputs=[out])
model.summary()Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         [(None, 2494)]            0
_________________________________________________________________
embedding (Embedding)        (None, 2494, 64)          640000
_________________________________________________________________
lstm (LSTM)                  (None, 64)                33024
_________________________________________________________________
dense (Dense)                (None, 1)                 65
=================================================================
Total params: 673,089
Trainable params: 673,089
Non-trainable params: 0
_________________________________________________________________
model.compile(optimizer='adam', loss='bce', metrics=['acc'])

filepath = 'model.h5'

mc = tf.keras.callbacks.ModelCheckpoint(filepath, verbose=0, save_weights_only=True,
                                        monitor='val_loss', mode='auto', save_best_only=True)

es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto')
_____no_output_____%%time
history = model.fit(x_train, y_train, epochs=100, batch_size=16,
                    validation_data=(x_test, y_test), callbacks=[mc, es])Epoch 1/100
1563/1563 [==============================] - 174s 111ms/step - loss: 0.6934 - acc: 0.4994 - val_loss: 0.6933 - val_acc: 0.5000
Epoch 2/100
1563/1563 [==============================] - 173s 111ms/step - loss: 0.6932 - acc: 0.4990 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 3/100
1563/1563 [==============================] - 174s 111ms/step - loss: 0.6933 - acc: 0.4946 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 4/100
1563/1563 [==============================] - 175s 112ms/step - loss: 0.6932 - acc: 0.4993 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 5/100
1563/1563 [==============================] - 174s 112ms/step - loss: 0.6934 - acc: 0.4982 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 6/100
1563/1563 [==============================] - 175s 112ms/step - loss: 0.6932 - acc: 0.4987 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 7/100
1563/1563 [==============================] - 175s 112ms/step - loss: 0.6932 - acc: 0.4982 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 8/100
1563/1563 [==============================] - 175s 112ms/step - loss: 0.6932 - acc: 0.4984 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 9/100
1563/1563 [==============================] - 176s 112ms/step - loss: 0.6932 - acc: 0.4974 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 10/100
1563/1563 [==============================] - 176s 112ms/step - loss: 0.6932 - acc: 0.4992 - val_loss: 0.6931 - val_acc: 0.5000
Epoch 11/100
1563/1563 [==============================] - 176s 113ms/step - loss: 0.6932 - acc: 0.4964 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 12/100
1563/1563 [==============================] - 176s 113ms/step - loss: 0.6932 - acc: 0.5002 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 13/100
1563/1563 [==============================] - 177s 114ms/step - loss: 0.6932 - acc: 0.4994 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 14/100
1563/1563 [==============================] - 180s 115ms/step - loss: 0.6932 - acc: 0.4945 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 15/100
1563/1563 [==============================] - 181s 116ms/step - loss: 0.6932 - acc: 0.4988 - val_loss: 0.6937 - val_acc: 0.5000
Epoch 16/100
1563/1563 [==============================] - 178s 114ms/step - loss: 0.6932 - acc: 0.4986 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 17/100
1563/1563 [==============================] - 180s 115ms/step - loss: 0.6932 - acc: 0.4907 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 18/100
1563/1563 [==============================] - 180s 115ms/step - loss: 0.6932 - acc: 0.4985 - val_loss: 0.6931 - val_acc: 0.5000
Epoch 19/100
1563/1563 [==============================] - 180s 115ms/step - loss: 0.6932 - acc: 0.4948 - val_loss: 0.6931 - val_acc: 0.5000
Epoch 20/100
1563/1563 [==============================] - 182s 117ms/step - loss: 0.6932 - acc: 0.4960 - val_loss: 0.6932 - val_acc: 0.5000
Epoch 00020: early stopping
CPU times: user 1h 11s, sys: 1min 11s, total: 1h 1min 22s
Wall time: 59min 5s
import matplotlib.pyplot as plt
import seaborn as sns

history = history.history

fig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))

ax1.plot(history['loss'], label='Train loss')
ax1.plot(history['val_loss'], label='Validation loss')
ax1.legend(loc='best')
ax1.set_title('Loss')

ax2.plot(history['acc'], label='Train')
ax2.plot(history['val_acc'], label='Validation')
ax2.legend(loc='best')
ax2.set_title('Accuracy')

plt.xlabel('Epochs')
sns.despine()
plt.show()_____no_output_____model.load_weights('model.h5')
model.evaluate(x_test, y_test)782/782 [==============================] - 34s 44ms/step - loss: 0.6931 - acc: 0.5000

# Padded to the average review length_____no_output_____

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.imdb.load_data(num_words=vocab_size, seed=seed)_____no_output_____total = 0
for i in range(x_train.shape[0]):
  total += len(x_train[i])

maxlen = int(total/x_train.shape[0])
maxlen_____no_output_____x_train = tf.keras.preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen, padding='post')
x_test = tf.keras.preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen, padding='post')_____no_output_____tf.keras.backend.clear_session()
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras import Input, Model

input1 = Input(shape=(maxlen,))
x = Embedding(vocab_size, 64)(input1)
x = LSTM(64)(x)
out = Dense(1, activation='sigmoid')(x)

model = Model(inputs=[input1], outputs=[out])
model.summary()Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         [(None, 238)]             0
_________________________________________________________________
embedding (Embedding)        (None, 238, 64)           640000
_________________________________________________________________
lstm (LSTM)                  (None, 64)                33024
_________________________________________________________________
dense (Dense)                (None, 1)                 65
=================================================================
Total params: 673,089
Trainable params: 673,089
Non-trainable params: 0
_________________________________________________________________
model.compile(optimizer='adam', loss='bce', metrics=['acc'])

filepath = 'model.h5'

mc = tf.keras.callbacks.ModelCheckpoint(filepath, verbose=0, save_weights_only=True,
                                        monitor='val_loss', mode='auto', save_best_only=True)

es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto')_____no_output_____%%time
history = model.fit(x_train, y_train, epochs=100, batch_size=16,
                    validation_data=(x_test, y_test), callbacks=[mc, es])Epoch 1/100
1563/1563 [==============================] - 36s 23ms/step - loss: 0.6641 - acc: 0.5685 - val_loss: 0.6510 - val_acc: 0.5698
Epoch 2/100
1563/1563 [==============================] - 35s 23ms/step - loss: 0.6218 - acc: 0.6146 - val_loss: 0.6455 - val_acc: 0.5770
Epoch 3/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.5386 - acc: 0.7238 - val_loss: 0.5786 - val_acc: 0.7422
Epoch 4/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.5190 - acc: 0.7364 - val_loss: 0.5419 - val_acc: 0.7665
Epoch 5/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.2952 - acc: 0.8830 - val_loss: 0.3174 - val_acc: 0.8716
Epoch 6/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.1950 - acc: 0.9286 - val_loss: 0.3149 - val_acc: 0.8789
Epoch 7/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.1401 - acc: 0.9525 - val_loss: 0.3728 - val_acc: 0.8739
Epoch 8/100
1563/1563 [==============================] - 34s 21ms/step - loss: 0.0994 - acc: 0.9702 - val_loss: 0.4054 - val_acc: 0.8722
Epoch 9/100
1563/1563 [==============================] - 35s 22ms/step - loss: 0.0685 - acc: 0.9814 - val_loss: 0.4727 - val_acc: 0.8676
Epoch 10/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.0556 - acc: 0.9857 - val_loss: 0.5151 - val_acc: 0.8554
Epoch 11/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.0436 - acc: 0.9893 - val_loss: 0.5286 - val_acc: 0.8612
Epoch 12/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.0363 - acc: 0.9916 - val_loss: 0.5708 - val_acc: 0.8599
Epoch 13/100
1563/1563 [==============================] - 35s 22ms/step - loss: 0.0299 - acc: 0.9932 - val_loss: 0.6458 - val_acc: 0.8590
Epoch 14/100
1563/1563 [==============================] - 34s 21ms/step - loss: 0.0264 - acc: 0.9942 - val_loss: 0.5812 - val_acc: 0.8591
Epoch 15/100
1563/1563 [==============================] - 34s 21ms/step - loss: 0.0218 - acc: 0.9952 - val_loss: 0.6721 - val_acc: 0.8579
Epoch 16/100
1563/1563 [==============================] - 34s 22ms/step - loss: 0.0208 - acc: 0.9958 - val_loss: 0.6476 - val_acc: 0.8558
Epoch 00016: early stopping
CPU times: user 11min 22s, sys: 37.4 s, total: 11min 59s
Wall time: 9min 10s
history = history.history

fig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))

ax1.plot(history['loss'], label='Train loss')
ax1.plot(history['val_loss'], label='Validation loss')
ax1.legend(loc='best')
ax1.set_title('Loss')

ax2.plot(history['acc'], label='Train')
ax2.plot(history['val_acc'], label='Validation')
ax2.legend(loc='best')
ax2.set_title('Accuracy')

plt.xlabel('Epochs')
sns.despine()
plt.show()_____no_output_____model.load_weights('model.h5')
model.evaluate(x_test, y_test)782/782 [==============================] - 5s 6ms/step - loss: 0.3149 - acc: 0.8789

# Ragged_____no_output_____

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.imdb.load_data(num_words=vocab_size, seed=seed)_____no_output_____r_train_x = tf.ragged.constant(x_train)
r_test_x = tf.ragged.constant(x_test)_____no_output_____tf.keras.backend.clear_session()
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras import Input, Model

input1 = Input(shape=(None,), ragged=True)
x = Embedding(vocab_size, 64)(input1)
x = LSTM(64)(x)
out = Dense(1, activation='sigmoid')(x)

model = Model(inputs=[input1], outputs=[out])
model.summary()Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         [(None, None)]            0
_________________________________________________________________
embedding (Embedding)        (None, None, 64)          640000
_________________________________________________________________
lstm (LSTM)                  (None, 64)                33024
_________________________________________________________________
dense (Dense)                (None, 1)                 65
=================================================================
Total params: 673,089
Trainable params: 673,089
Non-trainable params: 0
_________________________________________________________________
model.compile(optimizer='adam', loss='bce', metrics=['acc'])

filepath = 'model.h5'

mc = tf.keras.callbacks.ModelCheckpoint(filepath, verbose=0, save_weights_only=True,
                                        monitor='val_loss', mode='auto', save_best_only=True)

es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto')
_____no_output_____%%time
history = model.fit(r_train_x, y_train, epochs=100, batch_size=16,
                    validation_data=(r_test_x, y_test), callbacks=[mc, es])Epoch 1/100
history = history.history

fig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))

ax1.plot(history['loss'], label='Train loss')
ax1.plot(history['val_loss'], label='Validation loss')
ax1.legend(loc='best')
ax1.set_title('Loss')

ax2.plot(history['acc'], label='Train')
ax2.plot(history['val_acc'], label='Validation')
ax2.legend(loc='best')
ax2.set_title('Accuracy')

plt.xlabel('Epochs')
sns.despine()
plt.show()_____no_output_____model.load_weights('model.h5')
model.evaluate(r_test_x, y_test)782/782 [==============================] - 167s 213ms/step - loss: 0.2931 - acc: 0.8842


# Notebook from SalvatoreRa/ALL-FastAI-2020
Path: Projects/ALL_FastAI_Densenet121.ipynb

# Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research Project_____no_output_____## ALL FastAI Densenet 121 Classifier
**Using The ALL Image Database for Image Processing & The Leukemia Blood Cell Image Classification Using Convolutional Neural Network Research Paper**

The ALL FastAI Densenet 121 Classifier was created by [Salvatore Raieli](https://github.com/salvatorera) based on his [Resnet50](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/Projects/ALL_FastAI_Resnet_50.ipynb) project. The classifier provides a Google Colab notebook that uses FastAI with Resnet18 and ALL_IDB2 from the [Acute Lymphoblastic Leukemia Image Database for Image Processing dataset](https://homes.di.unimi.it/scotti/all/)._____no_output_____## FastAI Densenet 121 Classifier Project Contributors

- [Salvatore Raieli](https://github.com/salvatorera "Salvatore Raieli") - PhD Immunolgy / Bioinformatics, Bologna, Italy

## DISCLAIMER

These projects should be used for research purposes only. The purpose of the projects is to show the potential of Artificial Intelligence for medical support systems such as diagnosis systems.

Although the classifiers are accurate and show good results both on paper and in real world testing, they are not meant to be an alternative to professional medical diagnosis.

Salvatore Raieli is a bioinformatician researcher and PhD in Immunology, but does not work in medical diagnosis. Please use these systems responsibly.

Please use this system responsibly.

  _____no_output_____# ALL Image Database for Image Processing by Fabio Scotti

The [Acute Lymphoblastic Leukemia Image Database for Image Processing](https://homes.di.unimi.it/scotti/all/) dataset created by [Fabio Scotti, Associate Professor Dipartimento di Informatica, Università degli Studi di Milano](https://homes.di.unimi.it/scotti/) is used in this notebook.

Although in the [Leukemia Blood Cell Image Classification Using Convolutional Neural Network](http://www.ijcte.org/vol10/1198-H0012.pdf "Leukemia Blood Cell Image Classification Using Convolutional Neural Network") paper the ALL_IDB1 dataset is used, in this notebook you will use the ALL_IDB2 dataset. After removing 10 images per class for further testing and demonstrations, the dataset will be split into 80% and 20% for training and testing respectively._____no_output_____# Clone the ALL FastAI 2020 repository

First of all you should clone the [ALL-FastAI-2020](https://github.com/AMLResearchProject/ALL-FastAI-2020 "ALL-FastAI-2020") repository from the [Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research Project](https://github.com/AMLResearchProject "Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research Project") Github Organization.

To do this, make sure you have Git installed, navigate to the location you want to clone the repository to on your device using terminal/commandline, and then use the following command:

```
  $ git clone https://github.com/AMLResearchProject/ALL-FastAI-2020.git
```

Once you have used the command above you will see a directory called **ALL-FastAI-2020** in the location you chose to clone to. In terminal, navigate to the **ALL-FastAI-2020** directory, this is your project root directory._____no_output_____# Google Drive / Colab
This tutorial assumes you have access to [Google Drive](https://www.google.com/drive/) with enough space to save the dataset and related files. It is also assumed that you have access to [Google Colab](https://colab.research.google.com)._____no_output_____## Import data to Google Drive
You need to import **ALL_IDB2** from the [Acute Lymphoblastic Leukemia Image Database for Image Processing dataset](https://homes.di.unimi.it/scotti/all/) dataset, to do this you need to request permission from Fabio Scotti, the creator of the dataset. You can request permission by following the steps provided on [this page](https://homes.di.unimi.it/scotti/all/#download).

Once you have permission you need to upload the negative and positive examples provided in **ALL_IDB2** to your Google Drive. In this tutorial we assume you have uploaded your copy of the dataset to a folder located on your Google drive with the location: *AML-ALL-Classifiers/Python/_FastAI*. Once you have uploaded the dataset you can continue with this tutorial._____no_output_____## Google Colab

**You should now be running this tutorial on Google Colab, if not please read this tutorial from the beginning. **

First we need import the Google Colab Drive library, mount our dataset drive from Google Drive, and set the path to the ALL_IDB2 folder on your drive. Run the following code block to do this. You will be asked to click a link that will authorize the application with the permissions it needs to mount your drive etc. Follow the steps and then past the authorization key into this application._____no_output_____

%reload_ext autoreload
%autoreload 2
%matplotlib inline

from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
dataset_dir = "/content/gdrive/My Drive/fastai-v3/ALL_IDB2"Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/gdrive

# Import required libraries
We need to import the relevant FastAI libraries, running the following code block with do this and get the paths to the dataset files._____no_output_____

from fastai.vision import *
from fastai.metrics import error_rate
fileNames = get_image_files(dataset_dir)
fileNames[:5]_____no_output_____

# Import dataset
Now we need to import the dataset into this notebook. run the following code blocks to import the ALL_IDB2 dataset as a FastAI DataBunch. In the ImageDataBunch.from_name_re function we can see that we pass the *dataset_dir* we created earlier in the tutorial, fileNames that we created earlier,  pattern for the files, some augmentation, the size of the images we need to replicate SqueezeNet input sizes and the number of batches.

For more information about getting datasets ready with FastAI you can check out [this article](https://docs.fast.ai/vision.data.html#Quickly-get-your-data-ready-for-training)._____no_output_____

np.random.seed(2)
pattern = r'/\w+_(\d)\.tif$'_____no_output_____data = ImageDataBunch.from_name_re(dataset_dir, fileNames, pattern, ds_tfms=get_transforms(),
                                   size=224, bs=32).normalize(imagenet_stats)
data_____no_output_____

# data.show_batch()
Now we use the data.show_batch() function to show a batch of our data. Run the following code block to do this and view the results._____no_output_____

data.show_batch(rows=3, figsize=(7,6))_____no_output_____

## View classes info
Now we can run the following code block which will print out the classes list and lengths._____no_output_____## View classes info
Now we can run the following code block which will print out the classes list and lengths._____no_output_____

print(data.classes)
len(data.classes),data.c['0', '1']

# The DenseNet model

## What and why to use transfer learning?

Transfer learning is meaning use a pre-trained model to build our classifier. A pre-trained model is a model that has been previously trained on a dataset. The model comprehends the updated weights and bias.  Using pre-trained model you are saving time and computational resources. Another vantage is that pre-trained models often perform better that architecture designed from scratch. To better understand this point, suppose you want to build a classifier able to sort different sailboat types. A model pre-trained on ships would have already capture in its first layers some boat features, learning faster and with better accuracy among the different sailboat types.

## The DenseNet architecture

**DenseNet** was originally released in the 2016. Densely Connected Convolutional Networks (DenseNets) are another approach to increasing the depth of convolutional neural networks. The motivation at the basis is to solve the vanishing gradient problem (which arises with increasing the depth of a model).
DenseNet introduces Highway Networks, Residual Networks,Fractal Networks in order to simplify the connectivity pathway. The idea is that is not necessary to learn redundant feature maps. Indeed, the Densenet has a small set of feature maps.

[Original research article](https://arxiv.org/pdf/1608.06993.pdf)_____no_output_____To test the Densenet 121 architecture with our dataset
Now we are going to test how the FastaAI implementation of Densenet 121 works with the ALL_IDB2 dataset.

Create the convolutional neural network
First we will create the convolutional neural network based on Densenet 121, to do this we can use the following code block which uses FastAI ( cnn_learner previously create_cnn) function. We pass the loaded data, specify the Densenet 121 model, pass error_rate & accuracy as a list for the metrics parameter specifying we want to see both error_rate and accuracy, and finally specify a weight decay of 1e-1 (1.0)._____no_output_____

learn = cnn_learner(data, models.densenet121, metrics=[error_rate,accuracy], wd=1e-1)Downloading: "https://download.pytorch.org/models/densenet121-a639ec97.pth" to /root/.cache/torch/checkpoints/densenet121-a639ec97.pth

# learn.lr_find() & learn.recorder.plot()
Now we will use the learn.lr_find() function to run LR Finder. LR Finder help to find the best learning rate to use with our network. For more information the [original paper](https://arxiv.org/pdf/1506.01186.pdf).
As shown from the output of above,  learn.recorder.plot() function plot the loss over learning rate. Run the following code block to view the graph. The best learning rate should be chosen as the learning rate value where the curve is the steepest. You may try different learning rate values in order to pick up the best._____no_output_____

learn.lr_find()
learn.recorder.plot()_____no_output_____

# learn.fit_one_cycle() & learn.recorder.plot_losses()
The learn.fit_one_cycle() function can be used to fit the model. Fit one cycle reach a comparable accuracy faster than th *fit* function in training of complex models. Fit one cycle instead of maintain fix the learning rate during all the iterations is linearly increasing the learning rate and then it is decreasing again (this process is what is called one cycle). Moreover, this learning rate variation is helping in preventing overfitting. We use 5 for the parameter *cyc_len* to specify the number of cycles to run (on cycle can be considered equivalent to an epoch), and *max_lr* to specify the maximum learning rate to use which we set as *0.001*. Fit one cycle varies the learning rate from 10 fold less the maximum learning rate selected. For more information about fit one cycle: [article](https://arxiv.org/pdf/1803.09820.pdf). We then use learn.fit_one_cycle() to plot the losses from *fit_one_cycle* as a graph._____no_output_____

lr = 1e-3
learn.fit_one_cycle(cyc_len=5, max_lr=lr)
learn.recorder.plot_losses()_____no_output_____

# Save the model

We can save the model once it has been trained._____no_output_____

learn.save('Densenet_121_model')_____no_output_____

# learn.recorder.plot_lr()
We use  learn.recorder.plot_lr() to plot the learning rate._____no_output_____

learn.recorder.plot_lr()_____no_output_____

# ClassificationInterpretation()
We use  [ClassificationInterpretation()](https://docs.fast.ai/vision.learner.html#ClassificationInterpretation) to visualize interpretations of our model._____no_output_____

preds, y, losses = learn.get_preds(with_loss=True)
interp = ClassificationInterpretation(learn, preds, y, losses)_____no_output_____

# interp.plot_top_losses()
We can use [interp.plot_top_losses()](https://docs.fast.ai/vision.learner.html#plot_top_losses) to view our top losses and their details._____no_output_____

interp.plot_top_losses(9, figsize=(7,7))_____no_output_____

# interp.plot_confusion_matrix()
Now we will use  [interp.plot_confusion_matrix()](https://docs.fast.ai/vision.learner.html#ClassificationInterpretation.plot_confusion_matrix) to display a [confusion matrix](https://en.wikipedia.org/wiki/Confusion_matrix). Below,  the top left square represents  true negatives, while the top right square represents false positives,  the bottom left square represents  false negatives, and in the bottom right represents  true positives._____no_output_____

interp.plot_confusion_matrix()
_____no_output_____

# learn.unfreeze()
Next we use learn.unfreeze() to unfreeze the model. AlexNet model was trained on ImageNet to classify images among 1000 categories. None of these categories is a leukemia cell, for these reason when fast.ai *cnn_learner* function is behind line substituting the last layer with 2 other layers. The last layer is a matrix that has the same size of our data class (*data.c*). Before, we just trained these two layers while the other model's layers were still keeping the downloaded weight. Unfreezing our model allow us to train also these other layers and updates their weights.
_____no_output_____

learn.unfreeze()_____no_output_____

# Train the entire (unfrozen) model
Now that we have unfrozen our model, we will use the following code blocks to train the whole model._____no_output_____

learn.lr_find()
learn.recorder.plot()_____no_output_____

#Slice parameter

Initial layers are activated by simple patterns (like edge, lines, circles etc...)  while the following layers are acquiring the ability to recognize more sophisticated patterns. Update too much the weight of these layers would probably decrease our accuracy. The scope of transfer learning is to exploit this ability of a pre-trained model in recognizing particular patterns and to adapt to our dataset. The parameter *slice* allows to apply ** discriminative learning rate**. In other words, we apply a smaller learning rate (in this case, 1e-5) to the earlier layer and a higher learning rate to the last layer._____no_output_____

nlr = slice(1e-4, 5e-3)
learn.fit_one_cycle(5, nlr)_____no_output_____learn.recorder.plot_losses()_____no_output_____preds,y,losses = learn.get_preds(with_loss=True)
interp = ClassificationInterpretation(learn, preds, y, losses)
interp.plot_top_losses(9, figsize=(7,7))_____no_output_____interp.plot_confusion_matrix()_____no_output_____

# Save the model

We save our model after the un-freezing_____no_output_____

learn.save('Densenet_121_unfreeze')_____no_output_____

# Contributing

The Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research project encourages and welcomes code contributions, bug fixes and enhancements from the Github.

Please read the [CONTRIBUTING](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/CONTRIBUTING.md "CONTRIBUTING") document for a full guide to forking our repositories and submitting your pull requests. You will also find information about our code of conduct on this page.

## Contributors

- [Salvatore Raieli](https://github.com/salvatorera "Salvatore Raieli") - [Peter Moss Leukemia AI Research](https://www.leukemiaresearchassociation.ai "Peter Moss Leukemia AI Research") & Biogenera, Bologna, Italy

# Versioning

We use SemVer for versioning. For the versions available, see [Releases](https://github.com/AMLResearchProject/ALL-FastAI-2019/releases "Releases").

# License

This project is licensed under the **MIT License** - see the [LICENSE](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/LICENSE "LICENSE") file for details.

# Bugs/Issues

We use the [repo issues](https://github.com/AMLResearchProject/ALL-FastAI-2019/issues "repo issues") to track bugs and general requests related to using this project. See [CONTRIBUTING](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/CONTRIBUTING.md "CONTRIBUTING") for more info on how to submit bugs, feature requests and proposals._____no_output_____


# Notebook from amichuda/Python_Boot_Camp
Path: Notes/Week_1_Intro_Python_Bret.ipynb

# Hi. _____no_output_____# Thanks for coming._____no_output_____So, this is probably a  bit more normal of a session (I hope). Someone will give a small presentation reviewing the reading and talking about some new concept. That's all subject to change given group feedbcak though. We'll see how it goes today. _____no_output_____Some of you may notice that this is a Jupyter Notebook. What I like about Jupyter Notebooks is that I can annotate my code in a clear and concise way. I'll first show you how to do what I am doing now. First, you want to start with a blank cell, like the one below. Notice the "In [ ]:", this means that it is a code cell. Thus, if you type code and run the cell, it will perform the tasks you ask it to. In order to change it to a markdown cell you can click on the cell, then go Cell -> Cell Type -> Markdown. You can also click into the cell, then use the dropdown menu at the top to change the cell to markdown. Alternatively you can go into "command mode" by clicking on the cell, then pressing Esc. You will know that you are in command mode because the cell wall will be blue. You can then press "m" to turn it into a markdown cell. You can change it back by pressing "y".  _____no_output_____There are all sorts of short cuts for Jupyter. To look at all of them you can enter command mode and press "h". If you'd prefer a bookmarked website, for Windows users I'd recommend:

https://www.cheatography.com/weidadeyue/cheat-sheets/jupyter-notebook/

For Mac Users:

https://treehouse-code-samples.s3.amazonaws.com/Python/jupyter-shortcuts.pdf_____no_output_____If you're not familiar with Markdown, its like a simpler version of Latex. Here is a good resource for some common commands:

https://www.markdownguide.org/cheat-sheet/_____no_output_____You can also use symbols in both markdown and in your code, which is kind of cool. To type theta in markdown you will want to surround \theta by \$ signs_____no_output_____$\theta$_____no_output_____To use a greek letter in your code, you can just type \theta than press tab._____no_output_____

θ = 3
θ +2 _____no_output_____

Lastly, who's bad at spelling? I am the worst. Jupyter does not have a built in spell check. To install one, open Anaconda prompt. Then enter these commands:

pip install jupyter_contrib_nbextensions
jupyter contrib nbextension install --user
jupyter nbextension enable spellchecker/main

Then open Jupyter Notebooks and you should see the little abc button. This is an extension for spell check!_____no_output_____## Questions?_____no_output_____Radical. Let's move on to the reading review. Who did the reading? Nice. I figured a brief review of the major points would be useful. Don't feel pressured to read if you're busy. I encourage the presenter to always give a timely overview of the reading. _____no_output_____# Chapter 1 - About Python_____no_output_____Python is important. People use it for a lot of stuff.

We're going to use a lot of packages. Personally, I'm not sure the best way to deal with them. I'm primarily a Stata user, so package management is never something I've had to deal with. R users, how do you normally deal with the packages you always use?_____no_output_____Responses:

_____no_output_____The fundamental packages we will use are the following (plus some extras):

- Pandas: Data management
    - https://pandas.pydata.org/Pandas_Cheat_Sheet.pdf
- Numpy: Matrix and array programing
    - https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Numpy_Python_Cheat_Sheet.pdf
- Scipy: More math stuff
    - https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Python_SciPy_Cheat_Sheet_Linear_Algebra.pdf
- Matplotlib: Graphics
    - https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Python_Matplotlib_Cheat_Sheet.pdf
- Sympy: Symbolic math (Similar to Maple of Mathmatica)
    - http://daabzlatex.s3.amazonaws.com/9065616cce623384fe5394eddfea4c52.pdf
- statsmodels: Regression stuff
_____no_output_____# Chapter 2 - Setting up Your Python Environment_____no_output_____Mostly about setting up Anaconda and using Jupyter (covered above)._____no_output_____They also talk about how to install packages. One way is using pip._____no_output_____

!pip install quanteconRequirement already satisfied: quantecon in c:\users\bretd\anaconda3\lib\site-packages (0.4.5)
Requirement already satisfied: requests in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (2.21.0)
Requirement already satisfied: scipy>=1.0.0 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.2.1)
Requirement already satisfied: numba>=0.38 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (0.43.1)
Requirement already satisfied: sympy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.3)
Requirement already satisfied: numpy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.16.2)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (3.0.4)
Requirement already satisfied: urllib3<1.25,>=1.21.1 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (1.24.1)
Requirement already satisfied: certifi>=2017.4.17 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2019.3.9)
Requirement already satisfied: idna<2.9,>=2.5 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2.8)
Requirement already satisfied: llvmlite>=0.28.0dev0 in c:\users\bretd\anaconda3\lib\site-packages (from numba>=0.38->quantecon) (0.28.0)

Some of you may be thinking, "Why is there a little exclamation point at the beginning of that command?". In truth, I don't really know. I looked it up and it seems like it has something to do with pip being a shell command. I don't really know what that means. Does anyone else? The command seems to also work without the exclamation as well?_____no_output_____

pip install quanteconRequirement already satisfied: quantecon in c:\users\bretd\anaconda3\lib\site-packages (0.4.5)
Requirement already satisfied: numpy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.16.2)
Requirement already satisfied: scipy>=1.0.0 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.2.1)
Requirement already satisfied: requests in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (2.21.0)
Requirement already satisfied: sympy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.3)
Requirement already satisfied: numba>=0.38 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (0.43.1)
Requirement already satisfied: certifi>=2017.4.17 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2019.3.9)
Requirement already satisfied: urllib3<1.25,>=1.21.1 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (1.24.1)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (3.0.4)
Requirement already satisfied: idna<2.9,>=2.5 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2.8)
Requirement already satisfied: llvmlite>=0.28.0dev0 in c:\users\bretd\anaconda3\lib\site-packages (from numba>=0.38->quantecon) (0.28.0)
Note: you may need to restart the kernel to use updated packages.

So ¯\\\_(ツ)_/¯_____no_output_____You can also use the upgrade subcommand for pip:_____no_output_____

pip install --upgrade quanteconRequirement already up-to-date: quantecon in c:\users\bretd\anaconda3\lib\site-packages (0.4.5)
Requirement already satisfied, skipping upgrade: scipy>=1.0.0 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.2.1)
Requirement already satisfied, skipping upgrade: numba>=0.38 in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (0.43.1)
Requirement already satisfied, skipping upgrade: sympy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.3)
Requirement already satisfied, skipping upgrade: requests in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (2.21.0)
Requirement already satisfied, skipping upgrade: numpy in c:\users\bretd\anaconda3\lib\site-packages (from quantecon) (1.16.2)
Requirement already satisfied, skipping upgrade: llvmlite>=0.28.0dev0 in c:\users\bretd\anaconda3\lib\site-packages (from numba>=0.38->quantecon) (0.28.0)
Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2019.3.9)
Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (2.8)
Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (1.24.1)
Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in c:\users\bretd\anaconda3\lib\site-packages (from requests->quantecon) (3.0.4)
Note: you may need to restart the kernel to use updated packages.

They give you some basic commands:_____no_output_____

pwd_____no_output_____ls Volume in drive C is Windows
 Volume Serial Number is 4AC5-C707

 Directory of C:\Users\bretd\Box\PhD\Python

12/06/2019  11:23 AM    <DIR>          .
12/06/2019  11:23 AM    <DIR>          ..
12/06/2019  01:26 AM    <DIR>          .ipynb_checkpoints
07/11/2019  04:17 PM         (617,970) Cheat_sheet.pdf
07/19/2019  01:43 PM    <DIR>          Learning
12/06/2019  11:23 AM            26,670 Python_Group_Reading_Review.ipynb
07/11/2019  04:16 PM      (20,391,584) Quantitative Economics with Python.pdf
12/05/2019  10:11 PM             (684) test.ipynb
12/05/2019  10:16 PM              (14) test2.py
               5 File(s)     21,045,114 bytes
               4 Dir(s)  403,475,337,216 bytes free
run test.ipynbNice
cd C:\Users\bretd\DesktopC:\Users\bretd\Desktop
pwd_____no_output_____cd C:\\Users\\bretd\\Box\\PhD\\PythonC:\Users\bretd\Box\PhD\Python

You can also save just the code from one cell as a file:_____no_output_____

%%file test2.py
print(Niice)Overwriting test2.py
ls Volume in drive C is Windows
 Volume Serial Number is 4AC5-C707

 Directory of C:\Users\bretd\Box\PhD\Python

12/06/2019  11:25 AM    <DIR>          .
12/06/2019  11:25 AM    <DIR>          ..
12/06/2019  01:26 AM    <DIR>          .ipynb_checkpoints
07/11/2019  04:17 PM         (617,970) Cheat_sheet.pdf
07/19/2019  01:43 PM    <DIR>          Learning
12/06/2019  11:27 AM            30,363 Python_Group_Reading_Review.ipynb
07/11/2019  04:16 PM      (20,391,584) Quantitative Economics with Python.pdf
12/05/2019  10:11 PM               684 test.ipynb
12/06/2019  11:26 AM                14 test2.py
               5 File(s)     21,048,807 bytes
               4 Dir(s)  403,475,091,456 bytes free

# Chapter 3 - An Introductory Example_____no_output_____This chapter uses an example to show you some more commands. I'm just going to show you those commands._____no_output_____

a = numpy.array([1,2,3])
a_____no_output_____import numpy_____no_output_____a = numpy.array([1,2,3])
a_____no_output_____

We can name numpy anything we want. That way we don't have to type it out every time:_____no_output_____

import numpy as np
b = np.array([7,7,7])
b_____no_output_____

Arrays are cool things in Numpy that we will talk about later. They are basically vectors. We'll go more into Numpy later. _____no_output_____There are packages, like numpy, but there are also subpackages:_____no_output_____

x = np.random.randn(10)
x_____no_output_____

This is how you do a loop in Python:_____no_output_____

for i in range(5):
    print(i)0
1
2
3
4

A couple things to note:
- The list starts at 0. This is because Python is indexed to 0. Any automatically made list in Python will start at 0.
- You have to start your loop with ":". This is like "{" in Stata loops
- You HAVE to indent the first thing in your loop. Ending the indentation ends the loop._____no_output_____We can also use *while* loops:_____no_output_____

i = 0
while i < 5:
    print(i)
    i = i + 1
0
1
2
3
4

Can also say:_____no_output_____

i = 0
while i < 5:
    print(i)
    i += 10
1
2
3
4

You can also loop over things like lists:_____no_output_____

loop_lis = [2,3,4]
for i in loop_lis:
    print(i)2
3
4

Have you had two lists of things and needed to map them? I have. Stata generally sucks at it. Python has a cool command called zip(). Check it out. _____no_output_____

names = ("Bret", "Oliver", "Gryffin")
nouns = ("human", "cat", "dog")

for name, noun in zip(names, nouns):
    print(f"{name} is a {noun}")
Bret is a human
Oliver is a cat
Gryffin is a dog

Where the "f" in the print function means format. It allows us to put variables in strings. _____no_output_____There is a similar syntax to functions:_____no_output_____

def cheez(x):
    print(x)

cheez("whiz")whiz

Lets talk about lists. A list is just a collection of stuff. Any stuffs. This can be numbers, strings, or Boolean operators. They are convenient, albeit limited in some ways. If you want to use them like a vector, you're better off using numpy arrays because the operators work better. That said, here's a list: _____no_output_____

x = [1, "fish", 2, "fish", "red", "fish", True, "fish"]
x_____no_output_____

You can add things to a list by using the append *method*:_____no_output_____

x.append("by Dr. Seuss")
x_____no_output_____

You can also get rid of the last thing in a list by using the pop method:_____no_output_____

y = [1,2,3,4,5,6,7]
y.pop(-3)_____no_output_____x_____no_output_____

To look at an individual element in a list:_____no_output_____

x[0]_____no_output_____x[1]_____no_output_____

*List comprehensions* are also a cool part of Python. With a list comprehension, you can create list, like you would using a loop, but without the loop. The basic syntax of a list comprehension is:

[ expression for item in list if conditional ]

This is the same as:

for item in list:

    if conditional:

        expression_____no_output_____

comp = [2 * i for i in range(10) if i != 6]
comp_____no_output_____

We can also use larger conditional statements in Python:_____no_output_____

def seperator(x):
    """
    This function seprtates a list into 3 lists that identify their data types
    """
    string = []
    inter = []
    boo = []
    list_len = len(x)
    for i in range(list_len):
        if type(x[i]) == str:
            string.append(x[i])
        elif type(x[i]) == int:
            inter.append(x[i])
        else:
            boo.append(x[i])
    print(string)
    print(inter)
    print(boo)

seperator(x)['fish', 'red', 'fish', 'fish']
[1, 2]
[True]

Where "len()" returns the length of list x, type() returns the type of variable the element is, and elif is shorthand for "else if". _____no_output_____

seperator?_____no_output_____seperator??_____no_output_____

We can also combine if statements with "and":_____no_output_____

def color(x):
    color = []
    for i in x:
        if type(i) == str and i != 'fish':
            color.append(i)
        else:
            pass
    return color

color(x)_____no_output_____

We can also use "in" as an operator:_____no_output_____

if 'fish' in x and 2 in comp:
    print('Nice')Nice

# Chapter 4 - Python Essentials _____no_output_____We've already touched on some of these topics, so I'll skip over some things. _____no_output_____There are a few types of primitive data:
- String: Words
- Integer: Numbers with no decimals
- Float: Numbers with decimals
- Boolean: T/F
_____no_output_____With Boolean variables you can do Boolean algebra, if you'd like. Some basic Boolean things:_____no_output_____

x = 1
y = 2

x < y_____no_output_____x > y_____no_output_____x == 2_____no_output_____x != 3_____no_output_____

Along with lists, there are containers called *tuples*. Tuples are just lists that are "immutable", that is, you cannot change them:_____no_output_____

lis = [1,2]
tup = ('1','2')
tup2 = (1,2)

lis[0] = 5
lis
type(tup2)_____no_output_____tup2[0] = 5
tup2_____no_output_____

You can use *slice* notation to finish lists:_____no_output_____

print(x)
print(x[3:5])
print(x[2:])
print(x[:4])
print(x[-2:])[1, 'fish', 2, 'fish', 'red', 'fish', True, 'fish']
['fish', 'red']
[2, 'fish', 'red', 'fish', True, 'fish']
[1, 'fish', 2, 'fish']
[True, 'fish']

Another type of container is a dictionary. Dictionaries are like a list, but instead of indexing by a number, you index by a string. That string is called a *key*._____no_output_____

best_film_franchise = {'Name': 'Star Wars', 'Time': '3pm', 'Location': 'DeLoach'}
best_film_franchise['Time']_____no_output_____

We also have a container called a *set*. These are collections of objects that do not repeat. You can do set operations with them. _____no_output_____

sett = {'a', 'b', 'b'}
sett_____no_output_____

Lastly, some helpful functions:_____no_output_____

max(1,2,3,4,100,5,6)_____no_output_____list(range(7))_____no_output_____

The command all sees if a list of Boolean statements are all True. The command any() sees if any statement out of a list is true._____no_output_____

x=1
y=2
True_1 = x<y
True_2 = x != y
False_1 = x > y
statements = (True_1, True_2, False_1)
all(statements)_____no_output_____any(statements)_____no_output_____

In-line functions are a kind of confusing topic. Before, we named functions, like "separator". What if we didn't want to name the function? We just wanted to do it in-line? We could then use an anonymous function:_____no_output_____

x**2_____no_output_____import scipy.integrate as sp
sp.quad?_____no_output_____sp.quad(x**3, 0,2)_____no_output_____sp.quad(lambda x: x**3,0,2)_____no_output_____

# Next week?_____no_output_____Probably be good to read the chapters on:
- Numpy
- Matplotlib
- Scipy
- Pandas

Maybe do the exercises for the Pandas section as its probably the most relevant package? What do we think?  _____no_output_____


# Notebook from daviddhc20120601/ModelArts-Lab
Path: notebook/DL_nlp_qa/qa.ipynb

# 自然语言处理实战 —— 问答系统

在NLP（自然语言处理）领域，与我们生活息息相关的就是问答系统（QA），它是机器与人交互最常见的方法。

针对实际生活中的各个领域，可以依靠固定领域的数据库来进行特定问题的回答。例如电商智能客服、银行智能回答等，其实质都是问答系统的流程：

- 首先对问题内容识别分析

- 再基于先验知识，找到合适的回答

在前面三期的自然语言处理的实战中，分别体验了命名实体识别、文本分类、文本相似度分析三个 NLP 领域的基础任务。在本期中，将结合命名实体识别和文本相似度这两个任务，即
[【华为云 ModelArts-Lab AI实战营】第七期：自然语言处理（I）命名实体识别](https://github.com/huaweicloud/ModelArts-Lab/issues/931)和[【华为云 ModelArts-Lab AI实战营】第九期：自然语言处理（III）文本相似度分析](https://github.com/huaweicloud/ModelArts-Lab/issues/1087)，来完成 NLP 领域的一个上层任务——问答系统（QA）。_____no_output_____### 进入ModelArts

点击如下链接：https://www.huaweicloud.com/product/modelarts.html ， 进入ModelArts主页。点击“立即使用”按钮，输入用户名和密码登录，进入ModelArts使用页面。

### 创建ModelArts notebook

下面，我们在ModelArts中创建一个notebook开发环境，ModelArts notebook提供网页版的Python开发环境，可以方便的编写、运行代码，并查看运行结果。

第一步：在ModelArts服务主界面依次点击“开发环境”、“创建”

![create_nb_create_button](./img/create_nb_create_button.png)

第二步：填写notebook所需的参数：

| 参数 | 说明 |
| - - - - - | - - - - - |
| 计费方式 | 按需计费  |
| 名称 | Notebook实例名称，如 text_sentiment_analysis |
| 工作环境 | Python3 |
| 资源池 | 选择"公共资源池"即可 |
| 类型 | 本案例使用较为复杂的深度神经网络模型，需要较高算力，选择"GPU" |
| 规格 | 选择"8核 | 64GiB | 1*p100" |
| 存储配置 | 选择EVS，磁盘规格5GB |

第三步：配置好notebook参数后，点击下一步，进入notebook信息预览。确认无误后，点击“立即创建”

![create_nb_creation_summary](./img/create_nb_creation_summary.png)

第四步：创建完成后，返回开发环境主界面，等待Notebook创建完毕后，打开Notebook，进行下一步操作。
![modelarts_notebook_index](./img/modelarts_notebook_index.png)

### 在ModelArts中创建开发环境

接下来，我们创建一个实际的开发环境，用于后续的实验步骤。

第一步：点击下图所示的“打开”按钮，进入刚刚创建的Notebook
![inter_dev_env](img/enter_dev_env.png)

第二步：创建一个Python3环境的的Notebook。点击右上角的"New"，然后创建TensorFlow 1.13.1开发环境。

第三步：点击左上方的文件名"Untitled"，并输入一个与本实验相关的名称
![notebook_untitled_filename](./img/notebook_untitled_filename.png)
![notebook_name_the_ipynb](./img/notebook_name_the_ipynb.png)

### 在Notebook中编写并执行代码

在Notebook中，我们输入一个简单的打印语句，然后点击上方的运行按钮，可以查看语句执行的结果：
![run_helloworld](./img/run_helloworld.png)
_____no_output_____## 知识库

“第74届奥斯卡金像奖的最佳影片是《美丽心灵》”

“《鲁迅全集》这本书是1981年出版的”

“乔丹出生在美国纽约布鲁克林”。

以上就是三条**知识**，而把大量的知识汇聚起来就成为了知识库。我们可以在wiki百科，百度百科等查阅到大量的知识。然而，这些百科的知识组建形式是非结构化的自然语言，这样的组织方式很适合人们阅读但并不适合计算机去处理。为了方便计算机的处理和理解，我们需要更加形式化、简洁化的方式去表示知识——即**三元组（triple）**。知识库通常由大量的三元组组成。

“第74届奥斯卡金像奖的最佳影片是《美丽心灵》” 可以用三元组表示为（第74届奥斯卡金像奖, 最佳影片, 美丽心灵）。

这里通常把三元组理解为（实体 entity，关系属性 attribute，属性值 value），即“第74届奥斯卡金像奖”为实体, “最佳影片”为关系属性, “美丽心灵”为属性值。

进一步的，如果我们把实体看作是结点，把实体关系（包括属性，类别等等）看作是一条边，那么包含了大量三元组的知识库就成为了一个庞大的知识图。如下即为一个小型知识图。

![knowledge_graph](./img/knowledge_graph.png)

知识图谱又是一个很复杂庞大的领域，近年来也越来越受到人们的重视，在此不进行具体的介绍，感兴趣的同学可以自行学习，并分享到我们的拓展案例或者体验文章中，分享位置是[contrib](https://github.com/huaweicloud/ModelArts-Lab/tree/master/contrib)。

## 数据集

本实战使用的中文数据集来自NLPCC（自然语言处理与中文计算会议，The Conference on Natural Language Processing and Chinese Computing）2016中的[***Task5：Open Domain Question Answering***](http://tcci.ccf.org.cn/conference/2017/taskdata.php)数据集。数据集包含 14,609 个问答对的训练集和包含 9870 个问答对的测试集。并提供一个知识库，包含 6,502,738 个实体、587,875 个属性以及 43,063,796 个三元组。

- 知识库：nlpcc-iccpol-2016.kbqa.kb
- 训练集：nlpcc-iccpol-2016.kbqa.traing-data
- 测试集：nlpcc-iccpol-2016.kbqa.testing-data

知识库文件中每行存储一个事实（fact），即三元组 ( 实体、属性、属性值) 。知识库样例如下所示：

```
"希望之星"英语风采大赛|||中文名|||“希望之星”英语风采大赛
"希望之星"英语风采大赛|||主办方|||中央电视台科教节目中心
"希望之星"英语风采大赛|||别名|||"希望之星"英语风采大赛
"希望之星"英语风采大赛|||外文名|||Star of Outlook English Talent Competition
"希望之星"英语风采大赛|||开始时间|||1998
"希望之星"英语风采大赛|||比赛形式|||全国选拔
"希望之星"英语风采大赛|||节目类型|||英语比赛
计算机应用基础 ||| 别名 ||| 计算机应用基础
计算机应用基础 ||| 中文名 ||| 计算机应用基础
计算机应用基础 ||| 作者 ||| 刘晓斌、魏智荣、刘庆生
计算机应用基础 ||| 类别 ||| 计算机/网络 > 计算机理论
计算机应用基础 ||| 字数 ||| 445000
计算机应用基础 ||| ISBN ||| 9787122177513
计算机应用基础 ||| 出版社 ||| 化学工业出版社
计算机应用基础 ||| 页数 ||| 255
计算机应用基础 ||| 开本 ||| 16开
计算机应用基础 ||| 出版时间 ||| 2013年
计算机应用基础 ||| 装帧 ||| 平装
计算机应用基础 ||| 原作者 ||| 刘升贵，黄敏，庄强兵
计算机应用基础 ||| 定价 ||| 29.00元
```

由于原数据集中知识库数据量庞大（2.3G），在本实战中提供的知识库文件仅为节选示例。

训练集和测试集为经过[预处理](https://github.com/huangxiangzhou/NLPCC2016KBQA)包含三元组的数据集。_____no_output_____### 准备源代码和数据

准备案例所需的源代码和数据，相关资源已经保存在OBS中，我们通过ModelArts SDK将资源下载到本地。_____no_output_____

from modelarts.session import Session
session = Session()

if session.region_name == 'cn-north-1':
    bucket_path = 'modelarts-labs/notebook/DL_nlp_qa/qa.tar.gz'

elif session.region_name == 'cn-north-4':
    bucket_path = 'modelarts-labs-bj4/notebook/DL_nlp_qa/qa.tar.gz'
else:
    print("请更换地区到北京一或北京四")

session.download_data(bucket_path=bucket_path, path='./qa.tar.gz')

!ls -laSuccessfully download file modelarts-labs/notebook/DL_nlp_qa/qa.tar.gz from OBS to local ./qa.tar.gz
total 375464
drwxrwxrwx  3 ma-user ma-group      4096 Sep 30 17:31 .
drwsrwsr-x 20 ma-user ma-group      4096 Sep 30 17:17 ..
drwxr-x---  2 ma-user ma-group      4096 Sep 30 17:17 .ipynb_checkpoints
-rw-r-----  1 ma-user ma-group     91607 Sep 30 17:30 qa.ipynb
-rw-r-----  1 ma-user ma-group 384366229 Sep 30 17:31 qa.tar.gz

解压从obs下载的压缩包，解压后删除压缩包。_____no_output_____

!tar xf ./qa.tar.gz

!rm ./qa.tar.gz

!ls -la   total 108
drwxrwxrwx  4 ma-user ma-group  4096 Sep 30 17:31 .
drwsrwsr-x 20 ma-user ma-group  4096 Sep 30 17:17 ..
drwxr-x---  2 ma-user ma-group  4096 Sep 30 17:17 .ipynb_checkpoints
drwxr-x---  8 ma-user ma-group  4096 Sep 30 17:15 qa
-rw-r-----  1 ma-user ma-group 91607 Sep 30 17:30 qa.ipynb

基于知识库的问答（KBQA）拆分为两个主要步骤: **命名实体识别**步骤和**属性映射**步骤。

其中，实体识别步骤的目的是找到问句中询问的实体名称，而属性映射步骤的目的在于找到问句中询问的相关属性。

下面分别进行这两个步骤。_____no_output_____
## 命名实体识别

[【华为云 ModelArts-Lab AI实战营】第七期：自然语言处理（I）命名实体识别](https://github.com/huaweicloud/ModelArts-Lab/issues/931)中，进行了中文命名实体识别的任务。

在这里，我们同样使用该方法来识别出问句中的实体内容，即知识库的实体，来找到实体对应的知识。两者的区别是，在[实战营的第七期](https://github.com/huaweicloud/ModelArts-Lab/issues/931)中，选择识别的命名实体包括人名、地名、组织机构名，而在本实战中，只需要识别包含在知识库的一个实体。

构造NER的数据集，需要根据三元组-Enitity 反向标注问题，给数据集中的Question 打标签。在这里采用BIO的标注方式，将每个元素标注为“B-X”、“I-X”或者“O”：

- “B-X”（Begin）表示此元素所在的片段属于X类型并且此元素在此片段的开头；

- “I-X”（Inside）表示此元素所在的片段属于X类型并且此元素在此片段的中间位置；

- “O”（Outside）表示不属于任何类型。

在第七期中，把“X”为“LOC”代表地名，“PER”代表人名，“ORG”代表组织机构名。由于本任务无需区分地点名、人名和组织名，只需要识别出实体，因此用B-LOC, I-LOC代替其他的标注类型，即B-LOC表示实体首字，I-LOC表示实体非首字。

使用本实践的数据集进行标注，标注前示例：

```
《机械设计基础》这本书的作者是谁？     机械设计基础

《高等数学》是哪个出版社出版的？        高等数学
 ```

标注后为：

![ner_tagging](./img/ner_tagging.png)

本实战已为开发者做好命名实体的数据标注工作，，储存在`data/data_ner`文件夹下的`train.txt`中.

执行下面程序，查看前50行标注。_____no_output_____

with open("./qa/data/data_ner/train.txt", "r") as f:
    d = f.readlines()
    for i in range(50):
        print(d[i])《 O

机 B-LOC

械 I-LOC

设 I-LOC

计 I-LOC

基 I-LOC

础 I-LOC

》 O

这 O

本 O

书 O

的 O

作 O

者 O

是 O

谁 O

？ O

《 O

高 B-LOC

等 I-LOC

数 I-LOC

学 I-LOC

》 O

是 O

哪 O

个 O

出 O

版 O

社 O

出 O

版 O

的 O

？ O

《 O

线 B-LOC

性 I-LOC

代 I-LOC

数 I-LOC

》 O

这 O

本 O

书 O

的 O

出 O

版 O

时 O

间 O

是 O

本实战中的命名实体识别任务与[实战营的第七期](https://github.com/huaweicloud/ModelArts-Lab/issues/931)步骤基本相同，使用**BERT**模型完成，故不在此列出，封装到`bert_ner.py`文件中。

参数设置如下：

```
max_seq_length = 128          #序列最大长度
train_batch_size = 64         #训练批尺寸
predict_batch_size = 64        #测试批尺寸
learning_rate = 5e-5          #学习率
num_train_epochs = 5.0         #训练轮数
dropout_rate = 1             #随机失活率
warmup_proportion = 0.1        #预热训练
```

执行下面程序块运行，完成命名实体识别的训练和测试。_____no_output_____

#进行命名实体识别的训练、验证和测试
from qa import bert_ner

bert_ner.main(_)WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7f8dc18b8598>) includes params argument, but params are not passed to Estimator.
INFO:tensorflow:Using config: {'_model_dir': './qa/output_ner', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 1, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f8dc15c8940>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=1000, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}
INFO:tensorflow:_TPUContext: eval_on_tpu True
WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.
INFO:tensorflow:Writing example 0 of 13637
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-0
INFO:tensorflow:tokens: 《 机 械 设 计 基 础 》 这 本 书 的 作 者 是 谁 ？
INFO:tensorflow:input_ids: 101 517 3322 3462 6392 6369 1825 4794 518 6821 3315 741 4638 868 5442 3221 6443 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label_ids: 9 1 6 7 7 7 7 7 1 1 1 1 1 1 1 1 1 1 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-1
INFO:tensorflow:tokens: 《 高 等 数 学 》 是 哪 个 出 版 社 出 版 的 ？
INFO:tensorflow:input_ids: 101 517 7770 5023 3144 2110 518 3221 1525 702 1139 4276 4852 1139 4276 4638 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label_ids: 9 1 6 7 7 7 1 1 1 1 1 1 1 1 1 1 1 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-2
INFO:tensorflow:tokens: 《 线 性 代 数 》 这 本 书 的 出 版 时 间 是 什 么 ？
INFO:tensorflow:input_ids: 101 517 5296 2595 807 3144 518 6821 3315 741 4638 1139 4276 3198 7313 3221 784 720 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label_ids: 9 1 6 7 7 7 1 1 1 1 1 1 1 1 1 1 1 1 1 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-3
INFO:tensorflow:tokens: 安 德 烈 是 哪 个 国 家 的 人 呢 ？
INFO:tensorflow:input_ids: 101 2128 2548 4164 3221 1525 702 1744 2157 4638 782 1450 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label_ids: 9 6 7 7 1 1 1 1 1 1 1 1 1 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-4
INFO:tensorflow:tokens: 《 线 性 代 数 》 的 i s b n 码 是 什 么 ？
INFO:tensorflow:input_ids: 101 517 5296 2595 807 3144 518 4638 151 161 144 156 4772 3221 784 720 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label_ids: 9 1 6 7 7 7 1 1 1 1 1 1 1 1 1 1 1 10 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

至此，问答系统构建的前半部分已经完成，接下来进行后半部分的工作——属性映射。_____no_output_____## 属性映射

属性映射目的在于找到问句中询问的相关属性，转换成文本相似度问题，即[【华为云 ModelArts-Lab AI实战营】第九期：自然语言处理（III）文本相似度分析](https://github.com/huaweicloud/ModelArts-Lab/issues/1087)。

构造用于文本相似度分析的训练集和测试集，构造测试集的整体关系集合，通过提取和去重，获得若干关系集合；每个sample由“问题句+关系属性+label”构成，原始数据中的关系属性的label为 1；从关系集合中随机采样五个属性作为 Negative Samples，label为0。

本实战已为开发者做好属性映射的数据处理，储存在`data/data_sim`文件夹下的`train.txt`中。

打印训练集前50行的问题、三元组、答案和属性，示例如下：_____no_output_____

with open("./qa/data/data_sim/train.txt", "r") as f:
    d = f.readlines()
    for i in range(50):
        print(d[i])12005	请问有没有其他出版社出版了东京暗鸦？	信噪比	0

12006	请问有没有其他出版社出版了东京暗鸦？	职位	0

12007	请问有没有其他出版社出版了东京暗鸦？	生产年份	0

12008	请问草根金融论坛的联系方式是什么？	联络方式	1

12009	请问草根金融论坛的联系方式是什么？	设计	0

12010	请问草根金融论坛的联系方式是什么？	建置时间	0

12011	请问草根金融论坛的联系方式是什么？	原料	0

12012	请问草根金融论坛的联系方式是什么？	人口总数	0

12013	请问草根金融论坛的联系方式是什么？	产生	0

12014	请问足跟骨刺是否被纳入了医保体系?	是否进入医保	1

12015	请问足跟骨刺是否被纳入了医保体系?	外观设计	0

12016	请问足跟骨刺是否被纳入了医保体系?	初次登场	0

12017	请问足跟骨刺是否被纳入了医保体系?	定名人	0

12018	请问足跟骨刺是否被纳入了医保体系?	t(厚度)	0

12019	请问足跟骨刺是否被纳入了医保体系?	友好城市	0

12020	请问西北大学的研究生有多少人？	研究生	1

12021	请问西北大学的研究生有多少人？	cpu主频	0

12022	请问西北大学的研究生有多少人？	最大输出功率	0

12023	请问西北大学的研究生有多少人？	定价	0

12024	请问西北大学的研究生有多少人？	注释	0

12025	请问西北大学的研究生有多少人？	放大倍率	0

12026	请问生地饴糖鸡属于什么食物？	所属类型	1

12027	请问生地饴糖鸡属于什么食物？	副职	0

12028	请问生地饴糖鸡属于什么食物？	cpu主频	0

12029	请问生地饴糖鸡属于什么食物？	青年队	0

12030	请问生地饴糖鸡属于什么食物？	主板架构	0

12031	请问生地饴糖鸡属于什么食物？	配 音	0

12032	请问哪个公司出版了《忘忧草》啊？	音乐公司	1

12033	请问哪个公司出版了《忘忧草》啊？	转职业年	0

12034	请问哪个公司出版了《忘忧草》啊？	网站性质	0

12035	请问哪个公司出版了《忘忧草》啊？	专长	0

12036	请问哪个公司出版了《忘忧草》啊？	领导者	0

12037	请问哪个公司出版了《忘忧草》啊？	使用平台	0

12038	请问多花秋海棠属于什么亚组？	亚组	1

12039	请问多花秋海棠属于什么亚组？	产品天线	0

12040	请问多花秋海棠属于什么亚组？	人口（2005）	0

12041	请问多花秋海棠属于什么亚组？	项目占地	0

12042	请问多花秋海棠属于什么亚组？	功能主治	0

12043	请问多花秋海棠属于什么亚组？	应用范围	0

12044	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	总经理	1

12045	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	制作人	0

12046	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	属	0

12047	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	注册时间	0

12048	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	性状	0

12049	请问广州恒大淘宝足球俱乐部的总经理是谁啊？	瞄准具型式	0

12050	请问spark乐驰的驱动方式是什么？	驱动方式	1

12051	请问spark乐驰的驱动方式是什么？	科 室	0

12052	请问spark乐驰的驱动方式是什么？	剧本	0

12053	请问spark乐驰的驱动方式是什么？	技术人员	0

12054	请问spark乐驰的驱动方式是什么？	执行标准	0

下面进行属性映射的训练和测试，与[【华为云 ModelArts-Lab AI实战营】第九期：自然语言处理（III）文本相似度分析](https://github.com/huaweicloud/ModelArts-Lab/issues/1087)步骤基本相同，使用**BERT**模型完成，故不在此列出，封装到`bert_similarity.py`文件中。

参数设置如下：

```
num_train_epochs = 5        #训练轮数
batch_size = 64           #批尺寸
learning_rate = 0.00005      #学习率
gpu_memory_fraction = 0.9     #gpu使用率
layer_indexes = [-2]        #句向量选取
max_seq_len = 64          #序列最大长度
```

执行下面程序块运行，完成属性映射（文本相似度分析）的训练和测试。_____no_output_____

from qa import bert_similarity
import tensorflow as tf

#BertSim为BERT相似度检验类
sim = bert_similarity.BertSim()

#在训练集上进行相似度模型训练
sim.set_mode(tf.estimator.ModeKeys.TRAIN)
sim.train()

#在验证集上进行相似度验证
sim.set_mode(tf.estimator.ModeKeys.EVAL)
sim.eval()INFO:tensorflow:Using config: {'_model_dir': './qa/output_sim/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': gpu_options {
  per_process_gpu_memory_fraction: 0.9
  allow_growth: true
}
, '_keep_checkpoint_max': 1, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f8dc19d62e8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensorflow:Using config: {'_model_dir': './qa/output_sim/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': gpu_options {
  per_process_gpu_memory_fraction: 0.9
  allow_growth: true
}
, '_keep_checkpoint_max': 1, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f8e5f289b70>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
INFO:tensorflow:Writing example 0 of 75621
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-0
INFO:tensorflow:tokens: [CLS] 请 问 有 没 有 其 他 出 版 社 出 版 了 东 京 暗 鸦 ？ [SEP] 信 噪 比 [SEP]
INFO:tensorflow:input_ids: 101 6435 7309 3300 3766 3300 1071 800 1139 4276 4852 1139 4276 749 691 776 3266 7887 8043 102 928 1692 3683 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label: 0 (id = 0)
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-1
INFO:tensorflow:tokens: [CLS] 请 问 有 没 有 其 他 出 版 社 出 版 了 东 京 暗 鸦 ？ [SEP] 职 位 [SEP]
INFO:tensorflow:input_ids: 101 6435 7309 3300 3766 3300 1071 800 1139 4276 4852 1139 4276 749 691 776 3266 7887 8043 102 5466 855 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label: 0 (id = 0)
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-2
INFO:tensorflow:tokens: [CLS] 请 问 有 没 有 其 他 出 版 社 出 版 了 东 京 暗 鸦 ？ [SEP] 生 产 年 份 [SEP]
INFO:tensorflow:input_ids: 101 6435 7309 3300 3766 3300 1071 800 1139 4276 4852 1139 4276 749 691 776 3266 7887 8043 102 4495 772 2399 819 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label: 0 (id = 0)
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-3
INFO:tensorflow:tokens: [CLS] 请 问 草 根 金 融 论 坛 的 联 系 方 式 是 什 么 ？ [SEP] 联 络 方 式 [SEP]
INFO:tensorflow:input_ids: 101 6435 7309 5770 3418 7032 6084 6389 1781 4638 5468 5143 3175 2466 3221 784 720 8043 102 5468 5317 3175 2466 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label: 1 (id = 1)
INFO:tensorflow:*** Example ***
INFO:tensorflow:guid: train-4
INFO:tensorflow:tokens: [CLS] 请 问 草 根 金 融 论 坛 的 联 系 方 式 是 什 么 ？ [SEP] 设 计 [SEP]
INFO:tensorflow:input_ids: 101 6435 7309 5770 3418 7032 6084 6389 1781 4638 5468 5143 3175 2466 3221 784 720 8043 102 6392 6369 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
INFO:tensorflow:label: 0 (id = 0)
INFO:tensorflow:Writing example 10000 of 75621
INFO:tensorflow:Writing example 20000 of 75621
INFO:tensorflow:Writing example 30000 of 75621
INFO:tensorflow:Writing example 40000 of 75621
INFO:tensorflow:Writing example 50000 of 75621
INFO:tensorflow:Writing example 60000 of 75621
INFO:tensorflow:Writing example 70000 of 75621
INFO:tensorflow:***** Running training *****
INFO:tensorflow:  Num examples = 75621
INFO:tensorflow:  Batch size = 64
INFO:tensorflow:  Num steps = 5907
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ./qa/output_sim/model.ckpt.
INFO:tensorflow:loss = 0.80406123, step = 0
INFO:tensorflow:global_step/sec: 1.56336
INFO:tensorflow:loss = 0.12109105, step = 100 (63.966 sec)
INFO:tensorflow:global_step/sec: 2.17991
INFO:tensorflow:loss = 0.014379987, step = 200 (45.873 sec)
INFO:tensorflow:global_step/sec: 2.18049
INFO:tensorflow:loss = 0.037250243, step = 300 (45.861 sec)
INFO:tensorflow:global_step/sec: 2.17971
INFO:tensorflow:loss = 0.004695637, step = 400 (45.878 sec)
INFO:tensorflow:global_step/sec: 2.17942
INFO:tensorflow:loss = 0.0020469145, step = 500 (45.884 sec)
INFO:tensorflow:global_step/sec: 2.17993
INFO:tensorflow:loss = 0.0147804385, step = 600 (45.873 sec)
INFO:tensorflow:global_step/sec: 2.17918
INFO:tensorflow:loss = 0.00692276, step = 700 (45.889 sec)
INFO:tensorflow:global_step/sec: 2.18035
INFO:tensorflow:loss = 0.04618354, step = 800 (45.864 sec)
INFO:tensorflow:global_step/sec: 2.18008
INFO:tensorflow:loss = 0.019808413, step = 900 (45.870 sec)
INFO:tensorflow:global_step/sec: 2.1802
INFO:tensorflow:loss = 0.0062913853, step = 1000 (45.867 sec)
INFO:tensorflow:global_step/sec: 2.18033
INFO:tensorflow:loss = 0.0133275, step = 1100 (45.864 sec)
INFO:tensorflow:global_step/sec: 2.17894
INFO:tensorflow:loss = 0.037112594, step = 1200 (45.894 sec)
INFO:tensorflow:Saving checkpoints for 1253 into ./qa/output_sim/model.ckpt.
INFO:tensorflow:global_step/sec: 1.72351
INFO:tensorflow:loss = 0.12576269, step = 1300 (58.021 sec)
INFO:tensorflow:global_step/sec: 2.17812
INFO:tensorflow:loss = 0.07432252, step = 1400 (45.911 sec)
INFO:tensorflow:global_step/sec: 2.17993
INFO:tensorflow:loss = 0.18995932, step = 1500 (45.873 sec)
INFO:tensorflow:global_step/sec: 2.17927
INFO:tensorflow:loss = 0.0027036057, step = 1600 (45.887 sec)
INFO:tensorflow:global_step/sec: 2.18044
INFO:tensorflow:loss = 0.0038441897, step = 1700 (45.862 sec)
INFO:tensorflow:global_step/sec: 2.181
INFO:tensorflow:loss = 0.014303254, step = 1800 (45.851 sec)
INFO:tensorflow:global_step/sec: 2.1825
INFO:tensorflow:loss = 0.00049416925, step = 1900 (45.819 sec)
INFO:tensorflow:global_step/sec: 2.18204
INFO:tensorflow:loss = 0.056986697, step = 2000 (45.829 sec)
INFO:tensorflow:global_step/sec: 2.1825
INFO:tensorflow:loss = 0.11168129, step = 2100 (45.819 sec)

接下来，整合前面两个步骤，进行完整的基于知识库的问答系统构建。_____no_output_____## 问答系统（QA）

整合以上两个步骤，就可以完成一个简单的基于知识库的问答系统。下面为具体说明：

####  1. 命名实体识别：输入问题，使用BERT模型得到问题中的实体，在知识库中检索出包含该实体的所有知识组合。

#### 2. 属性映射：在包含实体的知识组合中，进行文本相似度分析寻找答案，又可分为非语义匹配和语义匹配。

  - **非语义匹配**：如果一个知识三元组的关系属性是输入问题的子集（相当于字符串匹配），则该三元组对应的答案匹配为正确答案。非语义匹配步骤可以大大加速匹配。

  - **语义匹配**：即可转化为分类问题，利用BERT模型计算输入问题与知识三元组的相似度，将最相近的三元组对应的答案匹配为正确答案。

原数据集中知识库数据量庞大（2.3G），共43,063,796个三元组，在本实战中选择使用训练集的14620个问答对对应的三元组生成知识库，用于完成简易问答任务。若需要使用完整知识库，请自行下载[Task 5: Open Domain Question Answering](http://tcci.ccf.org.cn/conference/2017/taskdata.php)使用。（由于原知识库较大，建议使用数据库进行存储、查找等操作。）

本实战已为开发者做好三元组知识库的数据处理，储存在`data/data_qa`文件夹下的`triple.txt`中。

执行下面程序，读取三元组知识库，储存在`triple_data`中，并打印前50条知识。_____no_output_____

import pandas as pd

triple_data = pd.read_csv('./qa/data/data_qa/triple.txt', encoding='utf-8', sep='\t', header=None)
print(triple_data[:50])          0     1                                   2
0    机械设计基础    作者                         杨可桢，程光蕴，李仲生
1      高等数学   出版社                             武汉大学出版社
2      线性代数  出版时间                          2013/12/30
3       安德烈    国籍                                 摩纳哥
4      线性代数  isbn                   978-7-111-36843-4
5      高等数学    书名                          高等数学一（微积分）
6      万达广场   外文名                    amoy wanda plaza
7        李明  出生日期                              1963.1
8     韩娱守护力  小说进度                                  连载
9        夏想  连载网站                                潇湘书院
10  大学计算机基础    页数                                 272
11      城关镇   中文名                            城关镇[临澧县]
12       李明   出生地                                青海湟源
13  大学计算机基础    定价                                25 元
14       李军    职业                      河南省林业厅党组成员、副厅长
15       文庙  中文名称                                  文庙
16       杨明    民族                                  苗族
17      毛泽东    装帧                              平装(无盘)
18      西游记    类型                                  奇幻
19       王娟    性别                                   女
20  计算机应用基础    开本                                  16
21       周迅  公司名称                      安徽龙润农业科技开发有限公司
22     博士来拜    类别                               油画，壁画
23       李军  毕业院校                           天津卫生职工医学院
24       陈平    国籍                                  中国
25     万达广场  总部地点  重庆市南岸区江南大道8号（重庆万达）/重庆市万州区北滨路（万州万达）
26       张宇    性别                                   女
27      城关镇    面积                          134.27平方公里
28      城关镇  所属地区                                中国西南
29      和平村  地理位置                           山东省微山县南阳镇
30     西城社区  成立时间                             2006年6月
31       王辉    职业                        广东省广播电影电视局官员
32     东创建国  经营范围                      整车销售、售后服务、配件供应
33       茴香  中文学名                                  茴香
34       王军  出生年月                             1962年7月
35      丹顶鹤     界                                 动物界
36       王平    别名                         原名王惟允，曾用名王明
37       李军  主要成就                   中国美术学会会员 中国陶瓷协会会员
38      tcl  公司性质                                  私企
39      城关镇    人口                                  5万
40    圆锥花序组     科                  石竹科caryophyllaceae
41     匆匆那年   英文名                        back in time
42     《兄弟》    类别                                 图书
43       刘勇  代表作品                     《中国现代作家的宗教文化情结》
44    多花秋海棠     门                  被子植物门 angiospermae
45    类胡萝卜素    性质                                  色素
46       王军    籍贯                                甘肃高台
47    命运石之门  游戏类型                             假想科学adv
48    圆锥花序组     目                  中央种子目centrospermae
49      凤凰山    地点                              山西省定襄县

建立好知识库后，即可进行基于知识库的问答系统搭建，问答步骤依次为：

1. 请提问：输入与知识库相关问题

- 打印问题token，实体标注结果，并输出识别实体

- 找到结果的可能来源知识集合并打印

- 属性映射：分别进行语义匹配（查找属性是否在问题中）和非语义匹配（相似度匹配）

- 相似度最高即为匹配答案，并打印答案来源知识及用时

- 可进行多轮问答，输入回车结束在线KBQA

#### 首先导入依赖包_____no_output_____

import numpy as np
from qa.bert import tokenization
from qa.bert_similarity import BertSim
from qa.bert_qa import *INFO:tensorflow:Using config: {'_model_dir': './qa/output_sim/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': gpu_options {
  per_process_gpu_memory_fraction: 0.9
  allow_growth: true
}
, '_keep_checkpoint_max': 1, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f8da77a8b38>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /home/ma-user/anaconda3/envs/TensorFlow-1.13.1/lib/python3.6/site-packages/tensorflow/python/data/ops/dataset_ops.py:429: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, use
    tf.py_function, which takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.

INFO:tensorflow:Calling model_fn.

#### 读取BERT预训练模型中文字典_____no_output_____

tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=True)

tokenizer.tokenize("今天的天气真好！")INFO:tensorflow:Done calling model_fn.

#### 分别获得每个字的字向量、位置向量、文本向量和标签_____no_output_____

def convert(line):
    feature = convert_single_example(0, line, label_list, max_seq_length, tokenizer, 'p')
    input_ids = np.reshape([feature.input_ids],(batch_size, max_seq_length))
    input_mask = np.reshape([feature.input_mask],(batch_size, max_seq_length))
    segment_ids = np.reshape([feature.segment_ids],(batch_size, max_seq_length))
    label_ids =np.reshape([feature.label_ids],(batch_size, max_seq_length))
    return input_ids, input_mask, segment_ids, label_ids_____no_output_____

### 构建问答系统_____no_output_____

def kbqa():
    while True:
        print('\n\033[1;31m请提问:\033[0m\n')
        sentence = str(input())
        if len(sentence) == 0:
            print("再见啦！")
            return

        sentence_ = tokenizer.tokenize(sentence)
        print('\n你的问题是:{}'.format(sentence_))
        input_ids, input_mask, segment_ids, label_ids = convert(sentence_)

        feed_dict = {input_ids_p: input_ids,
                     input_mask_p: input_mask,
                     segment_ids_p:segment_ids,
                     label_ids_p:label_ids}

        pred_ids_result = sess.run([pred_ids], feed_dict)
        pred_label_result = convert_id_to_label(pred_ids_result, id2label)
        print("\n实体标注结果为：",pred_label_result)
        result = strage_combined_link_org_loc(sentence_, pred_label_result[0])
        print('\n识别的实体是：{}'.format(''.join(result)),'\n')

        ans_range = []
        for j in range(len(triple_data)):
            if triple_data[0][j] == result[0]:
                triple_range = [triple_data[0][j],triple_data[1][j],triple_data[2][j]]
                print("结果可能来自：",triple_range)
                ans_range.append(triple_range)

        ans = None
        ans_base = None
        score = 0

        for k in range(len(ans_range)):
            print("\n知识三元组%d："%(k+1),ans_range[k][0],ans_range[k][1],ans_range[k][2])

            #非语义匹配
            if ans_range[k][1] in sentence:
                print("属性“",ans_range[k][1],"”在问题中")
                ans_ = 1

            #语义匹配
            else:
                ans_ = max(bs.predict(ans_range[k][0]+ans_range[k][1]+ans_range[k][2],sentence)[0])
            print("相似度为：",ans_)

            if score < ans_:
                score = ans_
                ans = ans_range[k][2]
                ans_base = ans_range[k]

        if score < 0.8:
            print("\n\033[1;31m答案不确定\033[0m")

        else:
            print("\n答案来自三元组：",ans_base[0],ans_base[1],ans_base[2])
            print("相似度为：", score)
            print("\n\033[1;31m答案是：{}\033[0m".format(ans))_____no_output_____

#### 下面就可以运行`kbqa()`来进行基于知识库的多轮在线问答。

### 注意，由于使用的知识库规格限制，请先大致浏览`./qa/data/data_qa/`文件夹下`triple.txt`文件中的三元组知识，以免无关问题造成问答效果不佳。_____no_output_____

kbqa()
请提问:

西游记每集多长时间？

至此，使用BERT进行基于知识库的问答系统搭建完毕。

本实战营系列的自然语言处理（NLP）领域的四期案例到此也告一段落。

NLP是深度学习、人工智能非常重要且复杂的领域，本实战营只是带领大家浅显地了解，有待大家持续探索！_____no_output_____