# Notebook from Haydara-Mahmoud/application-project-2021
Path: LearningVolunteerCompetencies.ipynb

# importing libraries
from ipywidgets import *
from IPython.display import clear_output, display, Javascript
import pandas as pd
import ipywidgets as widgets
import plotly.express as px
import random
import datetime
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, hamming_loss, classification_report, f1_score
from sklearn.ensemble import RandomForestClassifier
from skmultilearn.adapt import BRkNNaClassifier
from sklearn.model_selection import train_test_split
from skmultilearn.problem_transform import BinaryRelevance
from skmultilearn.problem_transform import ClassifierChain
from skmultilearn.problem_transform import LabelPowerset
from skmultilearn.adapt import MLkNN
import seaborn as sns
import numpy as np
import json_____no_output_____def loginLogout():
    return True_____no_output_____out11 = widgets.Output()

def on_select_Column_Change(event):
    if (columnNullList.value !=0):
        dic = getNullDict()
        key_list = list(dic.keys())
        value_list = list(dic.values())
        position = value_list.index(columnNullList.value)
        s = key_list[position]
        getNullValuesfor(getGeneralDataset(),s)
    else:
        out11.clear_output()

columnNullList = Dropdown(
            options={'',0},
            value=0,
            description='Choose attribute:',
            style= {'description_width':'auto'}
            )
columnNullList.observe(on_select_Column_Change, names=['value'])_____no_output_____# Functions

# 1 - Showing file chooser dialog -> Returns file selected
def getFile():
    fileName = str(fileUploadbutton.metadata[0]['name'])
    if '.json' in fileName:
        fileContent = fileUploadbutton.value[fileUploadbutton.metadata[0]['name']]['content']
        a_json = json.loads(fileContent)
        dataframe = pd.DataFrame.from_dict(a_json, orient="columns")
        return fileName, dataframe
    else:
        return fileName, None
# -------------------------------------------------------------------------------------------------------------------------

# 2 - Read dataset -> This function is so specific for "PersonTasks" dataset -> Returns dataset
def read_file(fileSelected,file):
    # Reading dataset:
    if withConsole==True:
        outputText.append('>> Reading dataset file : '+str(file))
    dataset = fileSelected
    if checkIfPersonTasksDataset(dataset) == True:
        # Flattening dataset
        if withConsole==True:
            outputText.append('>> Flattening dataset file : '+str(file))
        dataset = showJSONDataset(fileSelected)
        # Check how many people in the dataset
        numberOfPersons = dataset.personID.unique();
        if withConsole==True:
            outputText.append('>> Found '+str(len(numberOfPersons))+" volunteer(s) in dataset")
        # Takes the first person
        selectedPersonID = numberOfPersons[0]
        if withConsole==True:
            if (len(numberOfPersons)>1):
                outputText.append('>> Filtering dataset to one volunteer which personID = '+str(selectedPersonID))
        # Filter dataset
        dataset = dataset[dataset.personID==selectedPersonID]
        # Replacing personID by 1 and adding personName with a random number from namesList list
        # dataset = dataset.drop(["personID"],axis=1)
        dataset.personID = dataset.personID.replace(selectedPersonID,1)
        outputText.append('>> Changing personID from '+str(selectedPersonID)+' to 1')
        namesList = ['Jack','Jordan','Axel','Ian','Luis','Nicolas',
            'Sophia','Charlotte','Scarlett','Ivy','Lucy','Ariana']
        name = random.choice(namesList)
        dataset['personName'] = name
        if withConsole==True:
            outputText.append('>> Adding personName attribute with name : '+str(name))

        # Changing taskDuration into numbers
        dataset.taskDuration = pd.to_numeric(dataset.taskDuration, downcast="float")

        # Changing taskID into numbers
        dataset.taskID = pd.to_numeric(dataset.taskID)

        # Changing iVolunteerUUID into numbers
        dataset.iVolunteerUUID = pd.to_numeric(dataset.iVolunteerUUID)

        # Visualizing personID, taskType1, taskType2 and taskDuration
        if withConsole==True:
            outputText.append('>> Making a Sunburst Visualization for taskType1, taskType2 and taskDuration for : '+str(name))
        datasetVisualization = dataset[['personName','taskType1','taskType2','taskDuration']]
        fig = px.sunburst(datasetVisualization, path=[datasetVisualization["personName"].tolist(),
                                                      datasetVisualization["taskType1"].tolist(),
                                                      datasetVisualization["taskType2"].tolist()],
                          values=datasetVisualization["taskDuration"].tolist())
        fig.update_layout(showlegend=True,
                          title_text="Sunburst chart for: "+ name)
        # Statistics for all attributes
        if withConsole==True:
            outputText.append('>> Making some statistics for: '+str(name))
        datasetNumericStatistics = dataset.describe()
        datasetStatistics = dataset.describe(include='all')

        # Getting all Null/NaN columns which null/nan values != 0 and show them in columnNullList drop down list
        allNaNNullValues = showNullNaNValues(dataset)
        dropDownValue = 1;
        allNaNNullValuesNotZero = allNaNNullValues[allNaNNullValues["# of null/nan values"]!=0].reset_index(drop=True)
        global nullDict
        nullDict = {'':0}
        for i in range(len(allNaNNullValuesNotZero)):
            nullDict[str(allNaNNullValuesNotZero["Columns"][i])] = dropDownValue;
            dropDownValue = dropDownValue + 1
        columnNullList.options = nullDict
        columnNullList.value = 0

        with output:
            out1 = widgets.Output()
            out2 = widgets.Output()
            out3 = widgets.Output()
            out4 = widgets.Output()
            out5 = widgets.Output()
            out7 = widgets.Output()
            out10 = widgets.Output()
            if withConsole==True:
                out6 = widgets.Output()
                finalResult = Accordion(children=[out6,out1,out2,out3], selected_index=None)
                finalResult.set_title(0,"Console:")
                finalResult.set_title(1,"Dataset View:")
                finalResult.set_title(2,"Visulaization:")
                finalResult.set_title(3,"Statistics:")
                # Tab inside Statistics accordion
                tabChildren = [out4,out5,out7,out10]
                tab = Tab(children=tabChildren)
                tab.set_title(0,'Numeric data:')
                tab.set_title(1,'All data:')
                tab.set_title(2,'Null/NaN values:')
                tab.set_title(3,'Null/NaN rows:')

                consoleText = Textarea(value="Console:", rows=10, disabled=True, layout=Layout(width="auto"))
                for text in outputText:
                    consoleText.value = consoleText.value + "\r" + text
            else:
                finalResult = Accordion(children=[out1,out2,out3], selected_index=None)
                finalResult.set_title(0,"Dataset View:")
                finalResult.set_title(1,"Visulaization:")
                finalResult.set_title(2,"Statistics:")
                # Tab inside Statistics accordion
                tabChildren = [out4,out5,out7,out10]
                tab = widgets.Tab(children=tabChildren)
                tab.set_title(0,'Numeric data:')
                tab.set_title(1,'All data:')
                tab.set_title(2,'Null/NaN values:')
                tab.set_title(3,'Null/NaN rows:')

            display(finalResult)
            with out1:
                display(dataset)
            with out2:
                display(fig)
            with out3:
                display(tab)
            with out4:
                display(datasetNumericStatistics)
            with out5:
                display(datasetStatistics)
            with out7:
                display(allNaNNullValues)
            with out10:
                # out11 is global
                display(VBox([columnNullList,out11]))
            if withConsole==True:
                with out6:
                    display(consoleText)
            '''
            display(title1)
            display(dataset)
            display(title2)
            display(fig)
            display(title3)
            display(datasetStatistics)
            '''
        enableDisableButton(preprocessingButton,1)
        return dataset
    else:
        errorMessage = getMessage('error','You select a wrong dataset! Please select a suitable one!')
        enableDisableButton(preprocessingButton,0)
        with output:
            display(errorMessage)
        return False

def getNullDict():
    return nullDict

def getNullValuesfor(dataset,columnName):
    out11.clear_output()
    nullValues = dataset[dataset[columnName].isna()]
    with out11:
        display(nullValues)
# -------------------------------------------------------------------------------------------------------------------------
# 3 - After reading a dataset -> ordering events
def order_events(b):
    fileselect1.icon = "spinner"
    global outputText;
    outputText = [];
    # Checking if we want it with console or not
    global withConsole
    withConsole = False
    file,content = getFile()
    if '.json' in file:
        output.clear_output()
        # defining global variable in order accessing it in the next cells.
        global finalDataset
        finalDataset = read_file(content,file)
        fileselect1.icon = "eye"
        return finalDataset
    else:
        output.clear_output()
        fileselect1.icon = "eye"
        with output:
            informationMessage = getMessage('information','You have to choose a dataset with a JSON format!!')
            enableDisableButton(preprocessingButton,0)
            display(informationMessage)

# ------------------------------------------------------------------------------------------------------------------------
def order_eventsWithOutput(b):
    fileselect2.icon = "spinner"
    global outputText;
    outputText = [];
    # Checking if we want it with console or not
    global withConsole
    withConsole = True
    file,content = getFile()
    if '.json' in file:
        output.clear_output()
        # defining global variable in order accessing it in the next cells.
        global finalDataset
        finalDataset = read_file(content,file)
        fileselect2.icon = "info-circle"
        return finalDataset
    else:
        fileselect2.icon = "info-circle"
        output.clear_output()
        with output:
            informationMessage = getMessage('information','You have to choose a dataset with a JSON format!!')
            enableDisableButton(preprocessingButton,0)
            display(informationMessage)

# ------------------------------------------------------------------------------------------------------------------------
# 4 - Check if this is the target dataset "PersonTasks" dataset
def checkIfPersonTasksDataset(dataset):
    columns = dataset.columns
    if len(columns) == 13:
        if ('taskType1' in columns) & ('taskType2' in columns) & ('taskDuration' in columns):
            return True
        else:
            return False
    else:
        return False

# ------------------------------------------------------------------------------------------------------------------------
# 5 - Messages
def getMessage(messsageType,messageText):
    finalMessage = ''
    if messsageType == 'error':
        finalMessage = Button(description=str(messageText),icon="remove",layout=Layout(width='auto'),button_style="danger",disabled=True)
        # widgets.HTML(value='&nbsp
Error:
&nbsp '+  +'')
    else:
        finalMessage = Button(description=str(messageText),icon="info-circle",layout=Layout(width='auto'),button_style="info",disabled=True)
        # widgets.HTML(value='&nbsp
Information:
&nbsp '+ str(messageText) +'')
    return finalMessage

# ------------------------------------------------------------------------------------------------------------------------
# 6 - These three functions to flatten dataset if it has nested JSON code:
# Knowing nested JSON columns in dataset
def extractNestedColumns(dataframe):
    nestedColumns = [];
    col = dataframe.columns;
    for column in col:
        value = dataframe[column][0];
        if "{" in str(value):
              nestedColumns.append(column);
    return nestedColumns

def splittingNestedColumns(dataframe):
    # Taking nested json columns in dataframe
    nestedColumns = extractNestedColumns(dataframe);
    if len(nestedColumns)!=0:
        for i in nestedColumns:
            # taking every column in nestedColumns attributes and adding it to the original dataframe
            p = pd.DataFrame(dataframe[i]);
            f = [];
            for j in p[i]:
                f.append(j);
            s = pd.DataFrame(f);
            scol = s.columns;
            for k in scol:
                s = s.rename(columns={k:"{}{}{}".format(i,"_",k)})
            scol = s.columns;
            for n in scol:
                dataframe[n] = s[n];
            for j in scol:
                dataframe[j] = s[j];
            newDataFrame = dataframe.drop(nestedColumns,axis=1)
    return newDataFrame

# A function to show [flattened] dataset
def showJSONDataset(fileSelected):
    # Reading the JSON File and put it in a variable
    datasetVariable = fileSelected
    datasetVariable = splittingNestedColumns(datasetVariable)
    return datasetVariable

# ------------------------------------------------------------------------------------------------------------------------
# 7 - Function to enable/disable a button 0-> disable , 1->enable
def enableDisableButton(buttonName,value):
    if value == 0:
        buttonName.layout = Layout(width="0",height="0")
        output2.clear_output()
        buttonName.disabled=False
        '''
        buttonName.button_style="danger"
        buttonName.disabled=True
        '''
    else:
        buttonName.layout = Layout(width="auto",height="auto")
        output2.clear_output()
        buttonName.disabled=False
        '''
        buttonName.button_style="success"
        buttonName.disabled=False
        '''

# -----------------------------------------------------------------------------------------------------------------------
# 8 - Function to show categorical/numeric attributes with null/nan values
def showNullNaNValues(dataset):
    dataset.convert_dtypes().dtypes;
    col = dataset.columns;
    nullValues = list(dataset.isnull().sum());
    colType = [];
    for i in col:
        if (dataset[i].dtype == "O") | (dataset[i].dtype == "bool"):
            colType.append("Categorical");
        else:
            colType.append("Numeric");

    # Making dataframe
    columnsDict={"Columns":list(col),"Types":list(colType),"# of null/nan values":nullValues};
    df = pd.DataFrame(columnsDict);
    return df

# -----------------------------------------------------------------------------------------------------------------------
# 9 - Function to get general dataset
def getGeneralDataset():
    return finalDataset.copy();_____no_output_____

# Learning Volunteer Competencies
***
##   Assessment Mode:_____no_output_____

assessmentMode = Dropdown(
            options={'Learning Method': 1},
            value=1,
            description='Assessment Mode : ',
            style= {'description_width':'auto'}
            )

display(assessmentMode)_____no_output_____

***

##   Dataset:_____no_output_____

# For printing widgets
output = widgets.Output()

fileUploadbutton = widgets.FileUpload(
    description = "Select Dataset", icon="file",
    accept='.json',  # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'
    multiple=False,  # True to accept multiple files upload else False
    layout = Layout(width="auto")
)
state = fileUploadbutton.get_state()

def checkIfSelect(event):
    if fileUploadbutton.value != {}:
        fileselect1.disabled=False
        fileselect2.disabled=False
    else:
        fileselect1.disabled=True
        fileselect2.disabled=True

def clickedSelect(event):
    clickedButton.click()

fileUploadbutton.observe(clickedSelect)

# Buttons
layout = widgets.Layout(width='auto')
fileselect1 = Button(description="View Dataset", icon="eye", layout=layout, tooltip="View dataset and show the result",disabled=True)
fileselect1.on_click(order_events)

fileselect2 = Button(description="View Dataset with console",
                     icon="info-circle", layout=layout, tooltip="View dataset and show the result step by step", disabled=True)
fileselect2.on_click(order_eventsWithOutput)

clickedButton = Button(description="View Data")
clickedButton.on_click(checkIfSelect)

buttons = HBox([fileUploadbutton,fileselect1,fileselect2])

display(buttons)
display(output)_____no_output_____

***
##   Pre-processing:_____no_output_____

# functions for preprocessing
def extractingDateTimeFeatures(dataset,column1,column2):
    dataset[column1] = pd.to_datetime(dataset[column1])
    dataset[column2] = pd.to_datetime(dataset[column2])

    # Extracting date and time
    dateFrom = [d.date() for d in dataset[column1]]
    dateTo = [d.date() for d in dataset[column2]]
    timeFrom = [d.time() for d in dataset[column1]]
    timeTo = [d.time() for d in dataset[column2]]

    # Insert them in dataset in specific locations
    dataset.insert(7,"dateFrom",dateFrom)
    dataset.insert(8,"dateTo",dateTo)
    dataset.insert(9,"timeFrom",timeFrom)
    dataset.insert(10,"timeTo",timeTo)

    # Return dataset
    return dataset

def deleteAttributes(dataset,listOfAttributes):
    dataset = dataset.drop(listOfAttributes,axis=1)
    return dataset

def getTaskDuration(t1,t2):
    duration = t2-t1
    duration_in_s = duration.total_seconds()
    duration_in_h = divmod(duration_in_s, 3600)
    duration_in_m = round(duration_in_h[1] / 3600,2)
    finalDuration = duration_in_h[0] + duration_in_m
    return finalDuration

# This function is specific for PersonTasks dataset
def handlingTaskDuration(dataset):
    numberOfRows = len(dataset);
    numberOfAffectedRows = 0;
    for i in range(numberOfRows):
        taskD = dataset.taskDuration[i];
        if (pd.isna(taskD)):
            t1 = pd.to_datetime(dataset.taskDateFrom[i]);
            t2 = pd.to_datetime(dataset.taskDateTo[i]);
            taskDuration = getTaskDuration(t1,t2)
            dataset.loc[i, "taskDuration"] = taskDuration;
            numberOfAffectedRows = numberOfAffectedRows + 1;
    return dataset, numberOfAffectedRows

# A function to return number of tasks that are accomplished between two specific hours
# Parameters hour1, hour2 from string datatype
def getNumberOfTasksBetweenTwoHours(dataset,column,personID,hour1,hour2):
    resultedRows = dataset[dataset.personID==personID];

    list1=[]; # A list of all hours between the two defined hours
    hoursRange = pd.date_range(hour1,hour2,freq="1min")
    for data in hoursRange:
        list1.append(data.time())

    filteredColumn = resultedRows[column];
    list2 = []; # list of all values in column which its value is in list1
    for i in filteredColumn:
        if (i in list1):
            list2.append(i)

    return len(list2)

def getGeneratedDataset(dataset):
    # Column of generated dataset
    personName = [];
    NumberOfAccomplishedTasks = [];
    OrganizationManagementProportion = [];
    TrainingLearningProportion = [];
    CoorporationProportion = [];
    VarietyWorkProportion = [];
    StartWorkingMorning = [];
    WorkingLongTime = [];
    taskLocation = [];

    numberOfVolunteers = list(dataset.personID.unique());

    for pId in numberOfVolunteers:
        # Taking tasks for each volunteer
        volunteerTasks = dataset[dataset.personID==pId]

        # Begin extracting new attributes
        # 1 - personName
        volunteerName = volunteerTasks.personName.unique()
        personName.append(volunteerName[0]);

        # 2- Knowing the number of tasks:
        totalNumberOfTasks = len(volunteerTasks.taskType1);
        NumberOfAccomplishedTasks.append(totalNumberOfTasks)

        # 3 - OrganizationManagementProportion -> Number of tasks that from
        # taskType1='Organisation | Verwaltung', 'Bewerb | Leistungsprüfung (inkl. Vorbereitung)' and 'Einsatz'
        # Number between 0-1.
        OrganizationManagementProportion.append((round(len(volunteerTasks[((volunteerTasks.taskType1=='Organisation | Verwaltung') |
                                                         (volunteerTasks.taskType1=='Einsatz'))]) / totalNumberOfTasks,2)))

        # 4 - TrainingLearningProportion -> Number of tasks that from
        # taskType1='Schulung', 'Übung' and 'Kurs | Weiterbildung'
        # Number between 0-1.
        TrainingLearningProportion.append((round(len(volunteerTasks[((volunteerTasks.taskType1=='Schulung') |
                                                         (volunteerTasks.taskType1=='Übung') |
                                                         (volunteerTasks.taskType1=='Kurs | Weiterbildung'))]) / totalNumberOfTasks,2)))

        # 5 - CoorporationProportion -> Number of tasks that from
        # taskType1='Jugend'
        # Number between 0-1.
        CoorporationProportion.append(round(len(volunteerTasks[(volunteerTasks.taskType1=='Jugend')])/ totalNumberOfTasks,2))

        # 6 - VarietyWorkProportion -> Number of tasks that from
        # taskType1='Sonstiges'
        # Number between 0-1.
        VarietyWorkProportion.append(round(len(volunteerTasks[(volunteerTasks.taskType1=='Sonstiges')|
                                                         (volunteerTasks.taskType1=='Bewerb | Leistungsprüfung (inkl. Vorbereitung)') |
                                                             (volunteerTasks.taskType1=='Arbeiten in der FW')])/ totalNumberOfTasks,2))

        # 7 - StartWorkingMorning -> Number of tasks that from
        # timeFrom and using getNumberOfTasksBetweenTwoHours function "08:00:00" and "12:00:00"
        # Number between 0-1.
        StartWorkingMorning.append(round(getNumberOfTasksBetweenTwoHours(volunteerTasks,'timeFrom',pId,'08:00:00','12:00:00')/totalNumberOfTasks,2))

        # 8 - WorkingLongTime -> Number of tasks in which
        # Taking the mean of all volunteer's taskDuration and compare each task to it and take all them that >= mean
        # Number between 0-1 and Expresses about telorate work prassure.
        taskDurationMean = volunteerTasks.taskDuration.mean();
        WorkingLongTime.append(round(len(volunteerTasks[volunteerTasks.taskDuration >= taskDurationMean])/totalNumberOfTasks,2))

        # 9 - taskLocation -> Number of unique locations
        # Number between 0-1.
        # -> Mobility.
        taskLocation.append(round(len(volunteerTasks.taskLocation.unique())/totalNumberOfTasks,2))

    # ---------------------------------------------------------------------------------------------------------
    # -------------------------------------------GETTING DICTIONARY--------------------------------------------
    # ---------------------------------------------------------------------------------------------------------
    generatedDatasetDict = {
        'personName':personName,
        'NumberOfAccomplishedTasks':NumberOfAccomplishedTasks,
        'OrganizationManagementProportion':OrganizationManagementProportion,
        'TrainingLearningProportion':TrainingLearningProportion,
        'CoorporationProportion':CoorporationProportion,
        'VarietyWorkProportion': VarietyWorkProportion,
        'StartWorkingMorning': StartWorkingMorning,
        'WorkingLongTime': WorkingLongTime,
        'taskLocation':taskLocation
    }

    # ---------------------------------------------------------------------------------------------------------
    # -------------------------------------------GETTING DATAFRAME---------------------------------------------
    # ---------------------------------------------------------------------------------------------------------
    generatedDatasetDataFrame = pd.DataFrame(generatedDatasetDict)
    return generatedDatasetDataFrame

def getFinalPreprocessedDataset():
    return finalPreprocessedDataset.copy()

# ------------------------------------------------------------------------------------------------------------------------------
# activating prediction button
def activatePredictionButton():
    if (runModelButton.disabled==False) & (preprocessingButton.disabled==True):
        predictionButton.disabled=False
    else:
        predictionButton.disabled=True

def preprocessingOnClickEvent(event):
    preprocessedDataset = getGeneralDataset()
    preprocessingOutput = [];
    preprocessingOutput.append(">> Extracting time and date from taskDateFrom and taskDateTo attributes.")
    preprocessedDataset = extractingDateTimeFeatures(preprocessedDataset,'taskDateFrom','taskDateTo')
    listOfDeletedAttributes = ['taskName', 'iVolunteerSource', 'iVolunteerUUID', 'taskGeoInformation_latitude', 'taskGeoInformation_longitude','taskGeoInformation_gridID']
    preprocessedDataset = deleteAttributes(preprocessedDataset,listOfDeletedAttributes)
    preprocessingOutput.append(">> Deleting unnecessary "+str(len(listOfDeletedAttributes))+" attributes:")
    for attribute in listOfDeletedAttributes:
        preprocessingOutput.append(">> Deleting "+str(attribute)+".")
    nullValuesList = showNullNaNValues(preprocessedDataset)
    nullValuesList = nullValuesList[nullValuesList["# of null/nan values"] != 0].reset_index(drop=True)
    for i in range(len(nullValuesList)):
        if (nullValuesList["# of null/nan values"][i] > 1):
            preprocessingOutput.append('>> Found '+str(nullValuesList["# of null/nan values"][i])+' Null/NaN values in '+str(nullValuesList['Columns'][i])+' attribute')
        else:
            preprocessingOutput.append('>> Found '+str(nullValuesList["# of null/nan values"][i])+' Null/NaN value in '+str(nullValuesList['Columns'][i])+' attribute')
    preprocessingOutput.append('>> Handling Null/NaN values.')
    preprocessedDataset, numberOfAffectedRows = handlingTaskDuration(preprocessedDataset)
    preprocessingOutput.append('>> '+str(numberOfAffectedRows)+' rows has affected and handled!!')
    preprocessingOutput.append('>> Making dataset fit the learning model......')
    global finalPreprocessedDataset
    finalPreprocessedDataset = getGeneratedDataset(preprocessedDataset)
    with output2:
        out9 = widgets.Output()
        out12 = widgets.Output()
        preporcessingResult = Accordion(children=[out9,out12], selected_index=None)
        preporcessingResult.set_title(0,"Console for pre-processing step:")
        preporcessingResult.set_title(1,"Pre-processed Dataset:")
        preprocessingConsoleText = Textarea(value="Preprocessing Console:", rows=10, disabled=True, layout=Layout(width="auto"))
        for text in preprocessingOutput:
            preprocessingConsoleText.value = preprocessingConsoleText.value + "\r" + text
        display(preporcessingResult)
        with out9:
            display(preprocessingConsoleText)
        with out12:
            display(finalPreprocessedDataset)
    preprocessingButton.disabled = True
    activatePredictionButton()_____no_output_____output2 = widgets.Output()
preprocessingButton = Button(description="Make preprocessing", layout=Layout(width="0", height="0"), icon='gears')
preprocessingButton.on_click(preprocessingOnClickEvent)

display(preprocessingButton)
display(output2)_____no_output_____

***
##   Learning Model:_____no_output_____

# Functions for learning model

# When clicking EACH Classifier button
def check_event(clickEvent):
    tempButton = clickEvent
    if (tempButton.icon == 'plus-square'):
        if tempButton.description not in choosenClassifier:
            choosenClassifier.append(tempButton.description)
        tempButton.icon = 'minus-square'
        tempButton.button_style='primary'
    else:
        tempButton.icon = 'plus-square'
        for i in choosenClassifier:
            if i == tempButton.description:
                choosenClassifier.remove(i)
                tempButton.button_style=''
    checkForStepper()
    checkIfOneSelected(listOfButtons)

def checkForStepper():
    if (MLkNNButton.button_style == '') & (BRMLkNNButton.button_style == ''):
        k.disabled = True
    else:
        k.disabled = False
# ------------------------------------------------------------------------------------------------------------------------------
# When selecting "select all unselected" and "unselect all selected" buttons
def selectAllButtonOnClick(eventClick):
    tempButton = eventClick
    if tempButton.description=='Select All Unselected':
        for button in listOfButtons:
            if button.button_style == '':
                button.click()
    else:
        for button in listOfButtons:
            if button.button_style == 'primary':
                button.click()
    checkIfOneSelected(listOfButtons)

# ------------------------------------------------------------------------------------------------------------------------------
# Activating "Run Model" button
def checkIfOneSelected(listOfButtons):
    nb = 0
    for button in listOfButtons:
        if button.button_style=='primary':
            nb = nb + 1
    if nb>=1:
        runModelButton.disabled=False
    else:
        runModelButton.disabled=True

# ------------------------------------------------------------------------------------------------------------------------------
# Filling dropListClassifier drop down list according to choosen classifiers
def fillDropListClassifier(dropList,choosenClassifier):
    try:
        dropList.options = {}
    except:
        dropList.options = {}
    index = 0;
    classifierDict = {}
    for classifier in choosenClassifier:
        classifierDict[str(classifier)] = index
        index = index + 1
    dropList.options = classifierDict
# ------------------------------------------------------------------------------------------------------------------------------
# -----------------------------------------------LEARNING MODEL FUNCTIONS-------------------------------------------------------
# ------------------------------------------------------------------------------------------------------------------------------
# For determining quality for each classifier in order to give an advice
def determineQuality(accuracyValue):
    if accuracyValue < 0.5:
        Quality.append('BAD')
    elif 0.5 <= accuracyValue <= 0.7:
        Quality.append('So-So')
    elif 0.71 <= accuracyValue <= 0.88:
        Quality.append('GOOD')
    else:
        Quality.append('VERY GOOD')

# ------------------------------------------------------------------------------------------------------------------------------
# PT_build_Model for Problem Transformation methods
def PT_build_Model(model,mlb_estimater,classifierName,algorithmName,Xtrain,ytrain,Xtest,ytest):
    Methods.append('Problem Transformation')
    clf = mlb_estimater(model,require_dense=[False,True])
    clf.fit(Xtrain,ytrain)
    # Prediction
    clf_predictions = clf.predict(Xtest)
    # Check the accuracy score and hamming loss
    acc = accuracy_score(ytest,clf_predictions)
    hmm = hamming_loss(ytest,clf_predictions)
    ClassifierName.append(classifierName)
    MethodsAlgorithms.append(algorithmName)
    accuracyArray.append(acc)
    hammingArray.append(hmm)
    determineQuality(acc)
    return clf

# ------------------------------------------------------------------------------------------------------------------------------
# AA_build_Model for Alogrithm Adaptation methods
def AA_build_Model(algorithm,kN,classifierName,algorithmName,X_train,y_train,X_test,y_test):
    Methods.append('Algorithm Adaptation')
    if algorithm==MLkNN:
        classifier = algorithm(k=kN,s=0.5)
    else:
        classifier = algorithm(k=kN)
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    acc = accuracy_score(y_test,y_pred)
    hmm = hamming_loss(y_test,y_pred)
    ClassifierName.append(classifierName)
    MethodsAlgorithms.append(algorithmName)
    accuracyArray.append(acc)
    hammingArray.append(hmm)
    determineQuality(acc)
    return classifier

# ------------------------------------------------------------------------------------------------------------------------------
# Reading training dataset -> Specific function
def readTrainingDataset():
    dataset = pd.read_json('./dataset/finalDataset1.json',orient='columns')
    return dataset

# ------------------------------------------------------------------------------------------------------------------------------
# Mapping classifiers
def MapClassifierToModelsWithFinalResult(choosenClassifier,numberOfKNN,X_train, X_test, y_train, y_test):
    listOfRunningClassifiers = []

    # For giving final result
    global Methods
    Methods = []
    global ClassifierName
    ClassifierName = []
    global MethodsAlgorithms
    MethodsAlgorithms = []
    global accuracyArray
    accuracyArray = []
    global hammingArray
    hammingArray = []
    global Quality
    Quality = []

    for classifier in choosenClassifier:
        if classifier == 'Binary Relevance with NomialNB':
            # BinaryRelevance
            BR1 = PT_build_Model(MultinomialNB(),BinaryRelevance,'BinaryRelevance','MultinominalNB',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(BR1)
        elif classifier == 'Binary Relevance with RandomForestClassifier':
            # BinaryRelevance
            BR2 = PT_build_Model(RandomForestClassifier(),BinaryRelevance,'BinaryRelevance','RandomForestClassifier',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(BR2)
        elif classifier == 'ClassifierChain with NomialNB':
            # ClassifierChain
            CC1 = PT_build_Model(MultinomialNB(),ClassifierChain,'ClassifierChain','MultinominalNB',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(CC1)
        elif classifier == 'Classifier Chain with RandomForestClassifier':
            # ClassifierChain
            CC2 = PT_build_Model(RandomForestClassifier(),ClassifierChain,'ClassifierChain','RandomForestClassifier',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(CC2)
        elif classifier == 'LabelPowerset with NomialNB':
            # LabelPowerset
            LP1 = PT_build_Model(MultinomialNB(),LabelPowerset,'LabelPowerset','MultinominalNB',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(LP1)
        elif classifier == 'LabelPowerset with RandomForestClassifier':
            # LabelPowerset
            LP2 = PT_build_Model(RandomForestClassifier(),LabelPowerset,'LabelPowerset','RandomForestClassifier',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(LP2)
        elif classifier == 'MLkNN':
            # ML-KNN
            MK1 = AA_build_Model(MLkNN,numberOfKNN,'MLkNN','MLkNN',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(MK1)
        elif classifier == 'BRMLkNN':
            # ML-KNN
            MK2 = AA_build_Model(BRkNNaClassifier,numberOfKNN,'BRkNNaClassifier','BRkNNaClassifier',X_train,y_train,X_test,y_test)
            listOfRunningClassifiers.append(MK2)

    # Making Final DataFrame
    dataDict = {'Method': Methods,
                'Classifier':ClassifierName,
                'Algorithm': MethodsAlgorithms,
                'Accuracy': accuracyArray,
                'Hamming Loss': hammingArray,
                'Quality': Quality}
    # Making a dataframe
    resultDf = pd.DataFrame(dataDict)

    # Returning final result and list of running classifiers
    return listOfRunningClassifiers, resultDf

# ------------------------------------------------------------------------------------------------------------------------------
def appendTextToConsole(textAreaPane,Text):
    textAreaPane.value = textAreaPane.value + "\r" + Text

# ------------------------------------------------------------------------------------------------------------------------------
# Train model
def runModelClickEvent(event):
    runModelButton.icon = "spinner"
    output3.clear_output()
    fillDropListClassifier(dropListClassifier,choosenClassifier)
    # For console
    out13 = widgets.Output()
    trainingResult = Accordion(children=[out13], selected_index=0)
    trainingConsoleText = Textarea(value="Training Console:", rows=10, disabled=True, layout=Layout(width="auto"))
    trainingResult.set_title(0,"Training Console:")

    with output3:
        display(trainingResult)
        with out13:
            display(trainingConsoleText)

    # Reading training dataset
    trainingDataset = readTrainingDataset()
    datasetRows = trainingDataset.shape[0]
    datasetColumns = trainingDataset.shape[1]
    appendTextToConsole(trainingConsoleText,"▶ Reading training dataset with size: "+str(datasetRows)+" rows * "+str(datasetColumns)+" columns")

    # Spliting training dataset
    X = trainingDataset[["NumberOfAccomplishedTasks", "OrganizationManagementProportion", "TrainingLearningProportion", "CoorporationProportion", "VarietyWorkProportion", "StartWorkingMorning", "WorkingLongTime", "taskLocation"]].values
    y = trainingDataset[["SelfManagement","WillingnessToLearn","Energy","Persistence","Mobility"]].values

    appendTextToConsole(trainingConsoleText,"▶ Droping personID column..")

    appendTextToConsole(trainingConsoleText,"▶ Spliting dataset into training set (80%) and test set (20%):")
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)
    trainingsetRows = X_train.shape[0]
    trainingsetColumns = X_train.shape[1]
    testsetRows = X_test.shape[0]
    testsetColumns = X_test.shape[1]
    appendTextToConsole(trainingConsoleText,"  ▶▶ Training set size: "+str(trainingsetRows)+" rows * "+str(trainingsetColumns)+" columns")
    appendTextToConsole(trainingConsoleText,"  ▶▶ Test set size: "+str(testsetRows)+" rows * "+str(testsetColumns)+" columns")

    global listOfTrainedRunningClassifiers

    numberOfKNN = k.value
    listOfTrainedRunningClassifiers, adviceResult = MapClassifierToModelsWithFinalResult(choosenClassifier,numberOfKNN,X_train, X_test, y_train, y_test)

    for item in choosenClassifier:
        appendTextToConsole(trainingConsoleText,"▶ Training model with "+str(item))

    adviceResultVeryGood = adviceResult[adviceResult.Quality=='VERY GOOD'].reset_index(drop=True)
    adviceResultGood = adviceResult[adviceResult.Quality=='GOOD'].reset_index(drop=True)
    adviceResultSoSo = adviceResult[adviceResult.Quality=='So-So'].reset_index(drop=True)
    adviceResultBAD = adviceResult[adviceResult.Quality=='BAD'].reset_index(drop=True)
    listOfPrettyGoodResult = []
    for vgr in range(len(adviceResultVeryGood)):
        text = str(adviceResultVeryGood.Classifier[vgr]) + ' with ' + str(adviceResultVeryGood.Algorithm[vgr])
        listOfPrettyGoodResult.append(Button(description=text,layout=Layout(width="auto"),disabled=True,icon='star', button_style='success'))
    for gr in range(len(adviceResultGood)):
        text = str(adviceResultGood.Classifier[gr]) + ' with ' + str(adviceResultGood.Algorithm[gr])
        listOfPrettyGoodResult.append(Button(description=text,layout=Layout(width="auto"),disabled=True,icon='star-half-o', button_style='primary'))
    for ssr in range(len(adviceResultSoSo)):
        text = str(adviceResultSoSo.Classifier[ssr]) + ' with ' + str(adviceResultSoSo.Algorithm[ssr])
        listOfPrettyGoodResult.append(Button(description=text,layout=Layout(width="auto"),disabled=True,icon='star-o', button_style='warning'))
    for br in range(len(adviceResultBAD)):
        text = str(adviceResultBAD.Classifier[br]) + ' with ' + str(adviceResultBAD.Algorithm[br])
        listOfPrettyGoodResult.append(Button(description=text,layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))

    out14 = widgets.Output()
    out15 = widgets.Output()
    trainingResult.children = [out13,out14,out15]
    trainingResult.set_title(1,"Result:")
    trainingResult.set_title(2,"Advice which classifier you should use:")

    if len(listOfTrainedRunningClassifiers)!=0:
        predictionButton.disabled=False
    else:
        predictionButton.disabled=True

    activatePredictionButton()

    with out14:
        display(adviceResult)
    with out15:
        display(HBox(listOfPrettyGoodResult,layout=Layout(width='100%',display='inline-flex',flex_flow='row wrap')))

    runModelButton.icon = "gear"_____no_output_____output3 = widgets.Output()

layout = Layout(display='flex',
                align_items='center',
                width='50%')

runModelButton = Button(description="Run Model", tooltip="Learn Model on different Algorithms to give you an advice",
                       layout = Layout(width="auto"), icon="gear", disabled=True)
runModelButton.on_click(runModelClickEvent)

myModelAlgorithms = Label(value='▶ Choose Classifiers you want to train model on : ', layout= Layout(width="auto"))
mymodelLabel = Label(value='▶ Now run learning model : ', layout= Layout(width="auto"))

BRNBButton = Button(description="Binary Relevance with NomialNB", disabled=False, layout = layout, icon="plus-square")
BRRFButton = Button(description="Binary Relevance with RandomForestClassifier", disabled=False, layout = layout, icon="plus-square")

CCNBButton = Button(description="ClassifierChain with NomialNB", disabled=False, layout = layout, icon="plus-square")
CCRFButton = Button(description="Classifier Chain with RandomForestClassifier", disabled=False, layout = layout, icon="plus-square")

LPNBButton = Button(description="LabelPowerset with NomialNB", disabled=False, layout = layout, icon="plus-square")
LPRFButton = Button(description="LabelPowerset with RandomForestClassifier", disabled=False, layout = layout, icon="plus-square")

MLkNNButton = Button(description="MLkNN", disabled=False, layout = layout, icon="plus-square")
BRMLkNNButton = Button(description="BRMLkNN", disabled=False, layout = layout, icon="plus-square")

k = IntSlider(description= "k =", min=2, max=10, step=1, readout=True, value=3, disabled = True)

listOfButtons = [BRNBButton, BRRFButton, CCNBButton, CCRFButton, LPNBButton, LPRFButton, MLkNNButton, BRMLkNNButton]

global choosenClassifier
choosenClassifier = []

global allSelected

BRNBButton.on_click(check_event)
BRRFButton.on_click(check_event)
CCNBButton.on_click(check_event)
CCRFButton.on_click(check_event)
LPNBButton.on_click(check_event)
LPRFButton.on_click(check_event)
MLkNNButton.on_click(check_event)
BRMLkNNButton.on_click(check_event)

BR = HBox([BRNBButton, BRRFButton])
CC = HBox([CCNBButton, CCRFButton])
LP = HBox([LPNBButton, LPRFButton])
KN = HBox([MLkNNButton, BRMLkNNButton])

selectAllButton = Button(description="Select All Unselected", layout=Layout(width="auto"), icon='check-square')
selectAllButton.on_click(selectAllButtonOnClick)
UnselectAllButton = Button(description="UnSelect All Selected", layout=Layout(width="auto"), icon='square-o')
UnselectAllButton.on_click(selectAllButtonOnClick)

display(HBox([myModelAlgorithms,selectAllButton,UnselectAllButton,k]))
display(VBox([BR,CC,LP,KN]))
display(HBox([mymodelLabel,runModelButton]))

display(output3)_____no_output_____

***
##   Predict volunteer competency:_____no_output_____

# Functions for prediction

# Getting the selected value from dropListClassifier
def changeEvent(change):
    global selection
    allOptions = dropListClassifier.options
    allKeys = list(allOptions.keys())
    allValues = list(allOptions.values())
    for i in allValues:
        if dropListClassifier.value == i:
            selection = i
    return selection

# -----------------------------------------------------------------------------------------------------------------------------
# Prediction Button Click
def predictionClickEvent(event):
    # clearing output4
    output4.clear_output()

    # Get pre-processed dataset
    generatedDataset = getFinalPreprocessedDataset()

    # taking the same columns of training dataset
    columnsWithoutOutput = generatedDataset[["NumberOfAccomplishedTasks", "OrganizationManagementProportion", "TrainingLearningProportion", "CoorporationProportion", "VarietyWorkProportion", "StartWorkingMorning", "WorkingLongTime", "taskLocation"]].values
    predictionResult = np.array(columnsWithoutOutput).reshape((1, -1))

    # Prediction process using the choosed classifier
    predictionClassifier = listOfTrainedRunningClassifiers[selection]
    predictionClassifier = predictionClassifier.predict(predictionResult).toarray()

    # Making result in a pretty output
    for item in predictionClassifier:
        finalPredictionResult = item

    volumnteerName = generatedDataset.personName[0]

    predictionLabel = Label(value="Prediction "+str(volumnteerName)+"'s competencies using "+str(listOfTrainedRunningClassifiers[selection]),
                           layout = Layout(width='auto'))

    listOfPredictedResult = [];
    for counter in range(len(finalPredictionResult)):
        if counter==0: # Self Management Competency
            if finalPredictionResult[counter] == 0:
                listOfPredictedResult.append(Button(description="Self Management",layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))
            else:
                listOfPredictedResult.append(Button(description="Self Management",layout=Layout(width="auto"),disabled=True,icon='check', button_style='success'))
        elif counter==1: # Willingness To Learn Competency
            if finalPredictionResult[counter] == 0:
                listOfPredictedResult.append(Button(description="Willingness To Learn",layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))
            else:
                listOfPredictedResult.append(Button(description="Willingness To Learn",layout=Layout(width="auto"),disabled=True,icon='check', button_style='success'))
        elif counter==2: # Energy Competency
            if finalPredictionResult[counter] == 0:
                listOfPredictedResult.append(Button(description="Energy",layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))
            else:
                listOfPredictedResult.append(Button(description="Energy",layout=Layout(width="auto"),disabled=True,icon='check', button_style='success'))
        elif counter==3: # Persistence Competency
            if finalPredictionResult[counter] == 0:
                listOfPredictedResult.append(Button(description="Persistence",layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))
            else:
                listOfPredictedResult.append(Button(description="Persistence",layout=Layout(width="auto"),disabled=True,icon='check', button_style='success'))
        elif counter==4: # Mobility Competency
            if finalPredictionResult[counter] == 0:
                listOfPredictedResult.append(Button(description="Mobility",layout=Layout(width="auto"),disabled=True,icon='close', button_style='danger'))
            else:
                listOfPredictedResult.append(Button(description="Mobility",layout=Layout(width="auto"),disabled=True,icon='check', button_style='success'))

    out16 = widgets.Output();
    out17 = widgets.Output();
    with output4:
        display(VBox([out16,out17]))
        with out16:
            display(predictionLabel)
        with out17:
            display(HBox(listOfPredictedResult,layout=Layout(width='100%',display='inline-flex',flex_flow='row wrap')))_____no_output_____output4 = widgets.Output()
layout = Layout(width="auto")

dropListClassifier = Dropdown(
            options={'':0},
            value=0,
            description='Choose Algorithm/Estimator:',
            layout = layout,
            style= {'description_width':'auto', 'description_color':'red'}
            )
dropListClassifier.observe(changeEvent, names=['value'])

predictionButton = Button(description="Predict volunteer competencies", layout =Layout(width="auto"), icon='question-circle', disabled=True)
predictionButton.on_click(predictionClickEvent)

items = HBox([dropListClassifier])

display(items)

display(predictionButton)
display(output4)_____no_output_____


# Notebook from sidraju/pycon-tutorial
Path: All-sid coding with the video .ipynb

# creates everything like graphs and diagrams right in the notebook
%matplotlib inline
import pandas as pd
idx = pd.IndexSlice_____no_output_____# this formats the dataframes into a html style sheet
from IPython.core.display import HTML
css = open('style-table.css').read() + open('style-notebook.css').read()
HTML(''.format(css))_____no_output_____%%time
cast = pd.DataFrame.from_csv('data/cast.csv', index_col=None)CPU times: user 5.09 s, sys: 658 ms, total: 5.75 s
Wall time: 6.1 s
cast.head()_____no_output_____%%time
release_dates = pd.read_csv('data/release_dates.csv', index_col=None,
                            parse_dates=['date'], infer_datetime_format=True)CPU times: user 553 ms, sys: 59.7 ms, total: 613 ms
Wall time: 649 ms
release_dates.head()_____no_output_____titles = cast[['title', 'year']].drop_duplicates().reset_index(drop=True)
titles.head()_____no_output_____

### Years_____no_output_____

titles = pd.dataframe.from_csv('data/titles.csv', index_col=None,encoding='itf-8')
_____no_output_____# 1. What is the name and year of the very first movie ever made?

titles.sort('year').head(1)_____no_output_____# 1. How many years into the future does the IMDB database list movie titles?

titles.sort('year').tail(3)#.year - 2015_____no_output_____# 1. How many movies listed in `titles` came out in 1950?

len(titles[titles.year == 1950])

# or: (titles.year == 1950).sum()_____no_output_____# 1. How many movies came out in 1960?

len(titles[titles.year == 1960])_____no_output_____# 1. How many movies came out in each year of the 1970s?
#    (Hint: try a Python "for" loop.)

for y in range(1970, 1980):
    print(y, (titles.year == y).sum())1970 1876
1971 1774
1972 1840
1973 1813
1974 1795
1975 1704
1976 1716
1977 1686
1978 1679
1979 1757
# 1. How many movies came out during your own lifetime,
#    from the year of your birth through 2014?

len(titles[(titles.year >= 1974) & (titles.year <= 2014)])_____no_output_____# 2. Use "value_counts" to determine how many movies came out
#    in each year of the 1970s.

titles[titles.year // 10 == 197].year.value_counts().sort_index()_____no_output_____# 3. Use "groupby" to determine how many movies came out in each year of the 1970s.

titles.groupby('year').size().loc[1970:1979]_____no_output_____

### Titles_____no_output_____

# 1. What are the names of the movies made through 1906?

titles[titles.year <= 1906][['title']]_____no_output_____# 1. What movies have titles that fall between Star Trek and Star Wars in the alphabet?

titles[(titles.title >= 'Star Trek') & (titles.title <= 'Star Wars')]_____no_output_____# 2. Use an index and .loc[] to find the movies whose titles fall between Star Trek
#    and Star Wars in the alphabet.

t = titles.copy()
t = t.set_index('title').sort_index()
t.loc['Star Trek':'Star Wars']_____no_output_____# 2. Use an index and .loc[] to retrieve the names of the movies made through 1906.

titles.set_index('year').sort_index().loc[1800:1906]_____no_output_____# 2. What are the 15 most common movie titles in film history?

titles.title.value_counts().head(15)_____no_output_____# Use this for session 3?

i = cast.set_index('name').sort_index()_____no_output_____a = i.loc['Cary Grant',['year','n']].groupby('year').agg(['min', 'mean', 'max'])
a.loc[:1942].plot(kind='area', stacked=False)
a_____no_output_____# 5. What are the 5 longest movie titles ever?

pd.set_option('max_colwidth', 300)

t = titles.copy()
t['len'] = t.title.str.len()
t = t.sort('len', ascending=False)
t.head()_____no_output_____# 5. What are the 15 most popular movie titles, if you strip off the suffixes like
#    (II) and (III) that the IMDB adds to distinguish movies shown in the same year?

titles.title.str.extract('^([^(]*)').value_counts().head(15)_____no_output_____

### How many movies actors have been in_____no_output_____

# 1. How many movies has Judi Dench acted in?

len(cast[cast.name == 'Judi Dench'])_____no_output_____# 1. How many movies did Sidney Poitier appear in?

c = cast
c = c[c.name == 'Sidney Poitier']
len(c)_____no_output_____# 1. In how many of his movies was Sidney Poitier the lead (`n==1`)?

c = cast
c = c[c.name == 'Sidney Poitier']
c = c[c.n == 1]
len(c)_____no_output_____

### Pulling and displaying movie credits_____no_output_____

# 1. List the movies, sorted by year, in which Judi Dench starred as lead actor.

c = cast
c = c[c.name == 'Judi Dench']
c = c[c.n == 1]
c.sort('year')_____no_output_____# 1. Who was credited in the 1972 version of Sleuth, in order by `n` rank?

c = cast
c = c[c.title == 'Sleuth']
c = c[c.year == 1972]
c.sort('n')_____no_output_____

### Common character names_____no_output_____

# 2. What are the 11 most common character names in movie history?

cast.character.value_counts().head(11)_____no_output_____# 3. Which actors have played the role “Zombie” the most times?

c = cast
c = c[c.character == 'Zombie']
c = c.groupby('name').size().order()
c.tail(5)_____no_output_____# 3. Which ten people have appeared most often as “Herself” over the history of film?

c = cast
c = c[c.character == 'Herself']
c = c.groupby('name').size().order()
c.tail(10)_____no_output_____# 3. Which ten people have appeared most often as “Himself” over the history of film?

c = cast
c = c[c.character == 'Himself']
c = c.groupby('name').size().order()
c.tail(10)_____no_output_____# 4. Take the 50 most common character names in film.
#    Which are most often played by men?

c = cast
clist = c.character.value_counts().head(50)
clist.head()_____no_output_____clist.tail()_____no_output_____cast_by_character = cast.sort('character').set_index('character')_____no_output_____c = cast_by_character.loc[clist.index][['type']]
c = c.reset_index()
c = c.groupby(['character', 'type']).size()
c = c.unstack()
c['ratio'] = c.actress / (c.actor + c.actress)
c = c.sort('ratio')
c.head()_____no_output_____# 4. …which of those 50 characters are most often played by women?

c.tail()_____no_output_____# 4. …which of those 50 characters have a ratio closest to 0.5?

c[(c.ratio > 0.4) & (c.ratio < 0.6)]_____no_output_____

### Who has been in the most movies_____no_output_____

# 2. Which actors or actresses appeared in the most movies in the year 1945?

cast[cast.year == 1945].name.value_counts().head(10)_____no_output_____# 2. Which actors or actresses appeared in the most movies in the year 1985?

cast[cast.year == 1985].name.value_counts().head(10)_____no_output_____%%time
# 2. Create a `cast_by_title_year` dataframe indexed by title and year
#    to use in the next few questions.

cast_by_title_year = cast.set_index(['title', 'year']).sort_index()
cast_by_title_year.head()CPU times: user 15.8 s, sys: 2.68 s, total: 18.5 s
Wall time: 18.9 s
%%time
# 2. Use `cast_by_title_year` to find the stars of the film Inception
#    and order them by `n` before displaying the top 10.

cast_by_title_year.loc['Inception'].sort('n').head(10)CPU times: user 183 ms, sys: 4.19 ms, total: 187 ms
Wall time: 186 ms
# 2. Use `cast_by_title_year` to find the first 10 stars in the 1996 film Hamlet,
#    and order them by `n`.

cast_by_title_year.loc['Hamlet',1996].sort('n').head(10)_____no_output_____%%time
# 2. Write a `for` loop that, for the top 9 actors in the 1977 movie Star Wars,
#    determines how many movies they starred in after 1977.

names = cast_by_title_year.loc['Star Wars',1977].sort('n').head(9).name
for name in names:
    print(name, len(cast[(cast.name == name) & (cast.year > 1977)]))Mark Hamill 42
Harrison Ford 44
Carrie Fisher 40
Peter Cushing 10
Alec Guinness 9
Anthony Daniels 11
Kenny Baker 18
Peter Mayhew (II) 7
David Prowse 5
CPU times: user 8.7 s, sys: 11.8 ms, total: 8.71 s
Wall time: 8.73 s
# 2. Create an indexed version of `cast` that, once built, lets you answer
#    the previous question with a `for` loop that finishes in under a second.

i = cast.set_index('name').sort_index()_____no_output_____%%time
for name in names:
    c = i.loc[name]
    c = c[c.year > 1977]
    #c = c[(c.character != 'Himself') & (c.character != 'Herself')]
    print(name, len(c))Mark Hamill 42
Harrison Ford 44
Carrie Fisher 40
Peter Cushing 10
Alec Guinness 9
Anthony Daniels 11
Kenny Baker 18
Peter Mayhew (II) 7
David Prowse 5
CPU times: user 2.8 s, sys: 64.4 ms, total: 2.86 s
Wall time: 2.89 s
# 3. How many people were cast in each of the movies named "Hamlet”?

c = cast
c = c[c.title == 'Hamlet']
c = c.groupby('year').size()
c_____no_output_____# 5. How many actors are in the cast of each version of Hamlet,
#    including Hamlets with IMDB name collisions like "Hamlet (II)"
#    and "Hamlet (III)"? [BAD]

c = cast_by_title_year
# c.loc['Hamlet':'Hamlet (Z'].index.value_counts()    - Drat
# c.loc['Hamlet':'Hamlet (Z'].groupby(level=0).size() - Drat
# c.loc['Hamlet':'Hamlet (Z'].groupby(level=1).size() - Drat
c.loc['Hamlet':'Hamlet (Z'].groupby(level=[0,1]).size()

# Or:
#c = cast[(cast.title >= 'Hamlet') & (cast.title < 'Hamlet (Z')]
#c.groupby(['title', 'year']).size()_____no_output_____

### Actors and Actresses_____no_output_____

# 4. Build a dataframe with a row for each year with two columns:
#    the number of roles for actors in that year's films,
#    and the number of roles for actresses.

aa = cast[['year', 'type']].groupby(['year', 'type']).size()
aa = aa.loc[:2014].unstack()
aa.head()_____no_output_____# 4. Use that dataframe to make a kind='area' plot showing the total
#    number of roles available over the history of film.

aa.plot(kind='area')_____no_output_____f = aa.actor / (aa.actor + aa.actress)
f.plot(ylim=[0,1], kind='area')_____no_output_____c = cast
#c = c[c.year // 10 == 198]
c = c[c.n <= 3]
c = c.groupby(['year', 'type', 'n']).size()
c = c.unstack(1)
c.swaplevel(0,1).loc[1].plot(ylim=0, kind='area')
#f = c.actor / (c.actor + c.actress)
#f = f.unstack()
#f.plot(ylim=[0,1])_____no_output_____

### Rank over time_____no_output_____

# 2. Define “leading actor” as an actor or actress whose `n==1`
#    and “supporting actor” as `n==2` — what is the average year
#    of all the supporting roles Judi Dench has had?

c = cast
c = c[c.name == 'Judi Dench']
print(c[c.n == 2].year.mean())1986.83333333
# 2. What is the average year of Judi Dench’s leading roles —
#    is her career moving forwards toward leading roles
#    or backwards towards supporting ones?

print(c[c.n == 1].year.mean())2006.0
# 2. Did Sidney Poitier move forward or back over his career?

c = cast
c = c[c.name == 'Sidney Poitier']
print(c[c.n == 2].year.mean())
print(c[c.n == 1].year.mean())1961.75
1970.47619048
# 2. What about Michael Caine?

c = cast
c = c[c.name == 'Michael Caine']
print(c[c.n == 2].year.mean())
print(c[c.n == 1].year.mean())1988.38095238
1983.51923077
c = cast
#c = c[c.year // 10 == 195]
c = c[c.n.notnull()].groupby('name').n.agg(['size', 'mean'])
c.head()_____no_output_____c = c[c['size'] >= 10]
c = c.sort('mean')
c.head(60)_____no_output_____

### Release dates_____no_output_____

release_dates.head()_____no_output_____# 5. In which month is a movie whose name starts with the text
#    "The Lord of the Rings" most likely to be released?

r = release_dates
r = r[r.title.str.startswith('The Lord of the Rings')]
r = r[r.country == 'USA']
r.date.dt.month.value_counts()_____no_output_____# 5. In which months is a movie whose name ends in the word "Christmas"
#    most likely to be released?

r = release_dates
r = r[r.title.str.endswith('Christmas')]
r = r[r.country == 'USA']
r.date.dt.month.value_counts()_____no_output_____rd = release_dates.set_index(['title', 'year']).sort_index()
rd.head()_____no_output_____rd.loc[[('#Beings', 2015), ('#Horror', 2015)]]_____no_output_____c = cast
c = c[c.name == 'Tom Cruise'][['title', 'year']].drop_duplicates()
#c = c.join(rd, ['title', 'year'])
#c = c[c.country == 'USA']
#c.date.dt.month.value_counts().sort_index().plot(kind='bar')
c.values_____no_output_____# ASK
# rd.loc[c]
# rd.loc[c.values]
# rd.loc[list(c.values)]_____no_output_____# 5. In what months of the year have Helen Mirren movies been most often released?

c = cast
c = c[c.name == 'Helen Mirren'][['title', 'year']].drop_duplicates()
c = c.join(rd, ['title', 'year'])
c = c[c.country == 'USA']
c.date.dt.month.value_counts().sort_index().plot(kind='bar')_____no_output_____# 5. …Jeff Bridges movies?

c = cast
c = c[c.name == 'Jeff Bridges'][['title', 'year']].drop_duplicates()
c = c.join(rd, ['title', 'year'])
c = c[c.country == 'USA']
c.date.dt.month.value_counts().sort_index().plot(kind='bar')_____no_output_____# 5. …Tom Cruise movies?

c = cast
c = c[c.name == 'Tom Cruise'][['title', 'year']].drop_duplicates()
c = c.join(rd, ['title', 'year'])
c = c[c.country == 'USA']
c.date.dt.month.value_counts().sort_index().plot(kind='bar')_____no_output_____%%time
# 5. Use join() to build a table of release dates indexed by actor,
#    and use it to re-run the previous three questions efficiently.

c = cast
c = c[['name', 'title', 'year']]
c = c.join(rd, ['title', 'year'])
c = c[c.country == 'USA']
c = c.set_index('name').sort_index()
releases = c
releases.head()CPU times: user 19.8 s, sys: 3.83 s, total: 23.6 s
Wall time: 23.7 s
releases.loc['Tom Cruise'].date.dt.month.value_counts().sort_index().plot(kind='bar')_____no_output_____# pivot(self, index=None, columns=None, values=None)_____no_output_____cast.head()_____no_output_____c = cast
c = c[c.year >= 1990]
c = c[c.year <= 1993]
c = c[c.name == 'George Clooney']
#c = c[c.title == 'Inception']
#c = c[c.n.notnull()]
#c = c.pivot('name', 'year', 'title')
c.fillna('')_____no_output_____release_dates.head()_____no_output_____r = release_dates
r = r[r.title.str.startswith('Star Wars: Episode')]
r = r[r.country.str.startswith('U')]
r.pivot('title', 'country', 'date')
#r.pivot('country', 'title', 'date')_____no_output_____r = release_dates
r = r[r.title.str.startswith('Star Wars: Episode')]
r = r[r.country.str.startswith('U')]
r.set_index(['title', 'country'])[['date']].unstack()_____no_output_____cast.head()_____no_output_____t = titles

t.head()_____no_output_____c = cast
c = c[c.title == 'Hamlet']
c = c.set_index(['year', 'character'])#.unstack('type')
c_____no_output_____c = cast
c = c[c.title == 'Hamlet']
c = c.set_index(['year', 'type'])#.unstack('type')
c_____no_output_____


# Notebook from gvvynplaine/lessons
Path: notebooks/02_Basics/04_Data_and_Models/04_TF_Data_and_Models.ipynb

# Data and Models
In the subsequent lessons, we will continue to learn deep learning. But we've ignored a fundamental concept about data and modeling: quality and quantity._____no_output_____

_____no_output_____In a nutshell, a machine learning model consumes input data and produces predictions. The quality of the predictions directly corresponds to the quality and quantity of data you train the model with; **garbage in, garbage out**. Check out this [article](https://venturebeat.com/2018/06/30/understanding-the-practical-applications-of-business-ai/) on where it makes sense to use AI and how to properly apply it._____no_output_____We're going to go through all the concepts with concrete code examples and some synthesized data to train our models on. The task is to determine whether a tumor will be benign (harmless) or malignant (harmful) based on leukocyte (white blood cells) count and blood pressure. This is a synethic dataset that we created and has no clinical relevance._____no_output_____# Full dataset_____no_output_____We'll first train a model with the entire dataset. Later we'll remove a subset of the dataset and see the effect it has on our model._____no_output_____## Data_____no_output_____### Load data_____no_output_____

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from pandas.plotting import scatter_matrix
import urllib_____no_output_____SEED = 1234
DATA_FILE = 'tumors.csv'_____no_output_____# Set seed for reproducibility
np.random.seed(SEED)_____no_output_____# Load data from GitHub to this notebook's local drive
url = "https://raw.githubusercontent.com/madewithml/lessons/master/data/tumors.csv"
response = urllib.request.urlopen(url)
html = response.read()
with open(DATA_FILE, 'wb') as fp:
    fp.write(html)_____no_output_____# Raw data
df = pd.read_csv(DATA_FILE, header=0)
df.head()_____no_output_____# Define X and y
X = df[['leukocyte_count', 'blood_pressure']].values
y = df['tumor_class'].values_____no_output_____# Plot data
colors = {'benign': 'red', 'malignant': 'blue'}
plt.scatter(X[:, 0], X[:, 1], c=[colors[_y] for _y in y], s=25, edgecolors='k')
plt.xlabel('leukocyte count')
plt.ylabel('blood pressure')
plt.legend(['malignant ', 'benign'], loc="upper right")
plt.show()_____no_output_____

We want to choose features that have strong predictive signal for our task. If you want to improve performance, you need to continuously do feature engineering by collecting and adding new signals. So you may run into a new feature that has high correlation (orthogonal signal) with your existing features but it may still possess some unique signal to boost your predictive performance. _____no_output_____

# Correlation matrix
scatter_matrix(df, figsize=(5, 5));
df.corr()_____no_output_____

### Split data_____no_output_____

import collections
from sklearn.model_selection import train_test_split_____no_output_____TRAIN_SIZE = 0.70
VAL_SIZE = 0.15
TEST_SIZE = 0.15
SHUFFLE = True_____no_output_____def train_val_test_split(X, y, val_size, test_size, shuffle):
    """Split data into train/val/test datasets.
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, stratify=y, shuffle=shuffle)
    X_train, X_val, y_train, y_val = train_test_split(
        X_train, y_train, test_size=val_size, stratify=y_train, shuffle=shuffle)
    return X_train, X_val, X_test, y_train, y_val, y_test_____no_output_____# Create data splits
X_train, X_val, X_test, y_train, y_val, y_test = train_val_test_split(
    X=X, y=y, val_size=VAL_SIZE, test_size=TEST_SIZE, shuffle=SHUFFLE)
class_counts = dict(collections.Counter(y))
print (f"X_train: {X_train.shape}, y_train: {y_train.shape}")
print (f"X_val: {X_val.shape}, y_val: {y_val.shape}")
print (f"X_test: {X_test.shape}, y_test: {y_test.shape}")
print (f"Sample point: {X_train[0]} → {y_train[0]}")
print (f"Classes: {class_counts}")X_train: (722, 2), y_train: (722,)
X_val: (128, 2), y_val: (128,)
X_test: (150, 2), y_test: (150,)
Sample point: [18.01865938 15.48133647] → benign
Classes: {'malignant': 611, 'benign': 389}

### Label encoder_____no_output_____

from sklearn.preprocessing import LabelEncoder_____no_output_____# Output vectorizer
y_tokenizer = LabelEncoder()_____no_output_____# Fit on train data
y_tokenizer = y_tokenizer.fit(y_train)
classes = list(y_tokenizer.classes_)
print (f"classes: {classes}")classes: ['benign', 'malignant']
# Convert labels to tokens
print (f"y_train[0]: {y_train[0]}")
y_train = y_tokenizer.transform(y_train)
y_val = y_tokenizer.transform(y_val)
y_test = y_tokenizer.transform(y_test)
print (f"y_train[0]: {y_train[0]}")y_train[0]: benign
y_train[0]: 0
# Class weights
counts = collections.Counter(y_train)
class_weights = {_class: 1.0/count for _class, count in counts.items()}
print (f"class counts: {counts},\nclass weights: {class_weights}")class counts: Counter({1: 441, 0: 281}),
class weights: {0: 0.0035587188612099642, 1: 0.0022675736961451248}

### Standardize data_____no_output_____

from sklearn.preprocessing import StandardScaler_____no_output_____# Standardize the data (mean=0, std=1) using training data
X_scaler = StandardScaler().fit(X_train)_____no_output_____# Apply scaler on training and test data (don't standardize outputs for classification)
X_train = X_scaler.transform(X_train)
X_val = X_scaler.transform(X_val)
X_test = X_scaler.transform(X_test)_____no_output_____# Check (means should be ~0 and std should be ~1)
print (f"X_train[0]: mean: {np.mean(X_train[:, 0], axis=0):.1f}, std: {np.std(X_train[:, 0], axis=0):.1f}")
print (f"X_train[1]: mean: {np.mean(X_train[:, 1], axis=0):.1f}, std: {np.std(X_train[:, 1], axis=0):.1f}")
print (f"X_val[0]: mean: {np.mean(X_val[:, 0], axis=0):.1f}, std: {np.std(X_val[:, 0], axis=0):.1f}")
print (f"X_val[1]: mean: {np.mean(X_val[:, 1], axis=0):.1f}, std: {np.std(X_val[:, 1], axis=0):.1f}")
print (f"X_test[0]: mean: {np.mean(X_test[:, 0], axis=0):.1f}, std: {np.std(X_test[:, 0], axis=0):.1f}")
print (f"X_test[1]: mean: {np.mean(X_test[:, 1], axis=0):.1f}, std: {np.std(X_test[:, 1], axis=0):.1f}")X_train[0]: mean: 0.0, std: 1.0
X_train[1]: mean: -0.0, std: 1.0
X_val[0]: mean: 0.1, std: 1.0
X_val[1]: mean: 0.1, std: 1.0
X_test[0]: mean: -0.1, std: 1.0
X_test[1]: mean: -0.1, std: 1.0

## Modeling_____no_output_____### Model_____no_output_____

# Use TensorFlow 2.x
%tensorflow_version 2.x
import tensorflow as tfTensorFlow 2.x selected.
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Input
from tensorflow.keras.models import Model
from tensorflow.keras.utils import plot_model_____no_output_____# Set seed for reproducability
tf.random.set_seed(SEED)_____no_output_____INPUT_DIM = 2 # X is 2-dimensional
HIDDEN_DIM = 100
NUM_CLASSES = 2_____no_output_____class MLP(Model):
    def __init__(self, hidden_dim, num_classes):
        super(MLP, self).__init__(name='mlp')
        self.fc1 = Dense(units=hidden_dim, activation='relu', name='W1')
        self.fc2 = Dense(units=num_classes, activation='softmax', name='W2')

    def call(self, x_in, training=False):
        z = self.fc1(x_in)
        y_pred = self.fc2(z)
        return y_pred

    def summary(self, input_shape):
        x_in = Input(shape=input_shape, name='X')
        summary = Model(inputs=x_in, outputs=self.call(x_in), name=self.name)
        return plot_model(summary, show_shapes=True) # forward pass_____no_output_____# Initialize model
model = MLP(hidden_dim=HIDDEN_DIM, num_classes=NUM_CLASSES)
model.summary(input_shape=(INPUT_DIM,))_____no_output_____

### Training_____no_output_____

from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.metrics import SparseCategoricalAccuracy
from tensorflow.keras.optimizers import Adam_____no_output_____LEARNING_RATE = 1e-3
NUM_EPOCHS = 5
BATCH_SIZE = 32_____no_output_____# Compile
model.compile(optimizer=Adam(lr=LEARNING_RATE),
              loss=SparseCategoricalCrossentropy(),
              metrics=[SparseCategoricalAccuracy()])_____no_output_____# Training
model.fit(x=X_train,
          y=y_train,
          validation_data=(X_val, y_val),
          epochs=NUM_EPOCHS,
          batch_size=BATCH_SIZE,
          class_weight=class_weights,
          shuffle=False,
          verbose=1)WARNING:tensorflow:sample_weight modes were coerced from
  ...
    to
  ['...']
WARNING:tensorflow:sample_weight modes were coerced from
  ...
    to
  ['...']
Train on 722 samples, validate on 128 samples
Epoch 1/5
722/722 [==============================] - 2s 3ms/sample - loss: 0.0016 - sparse_categorical_accuracy: 0.7008 - val_loss: 0.0014 - val_sparse_categorical_accuracy: 0.7812
Epoch 2/5
722/722 [==============================] - 0s 132us/sample - loss: 0.0012 - sparse_categorical_accuracy: 0.8864 - val_loss: 0.0011 - val_sparse_categorical_accuracy: 0.8359
Epoch 3/5
722/722 [==============================] - 0s 128us/sample - loss: 8.6852e-04 - sparse_categorical_accuracy: 0.9072 - val_loss: 8.7229e-04 - val_sparse_categorical_accuracy: 0.8672
Epoch 4/5
722/722 [==============================] - 0s 206us/sample - loss: 6.7633e-04 - sparse_categorical_accuracy: 0.9321 - val_loss: 7.1961e-04 - val_sparse_categorical_accuracy: 0.8828
Epoch 5/5
722/722 [==============================] - 0s 141us/sample - loss: 5.4061e-04 - sparse_categorical_accuracy: 0.9571 - val_loss: 6.0240e-04 - val_sparse_categorical_accuracy: 0.9219

### Evaluation_____no_output_____

import itertools
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix_____no_output_____def plot_confusion_matrix(y_true, y_pred, classes, cmap=plt.cm.Blues):
    """Plot a confusion matrix using ground truth and predictions."""
    # Confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    #  Figure
    fig = plt.figure()
    ax = fig.add_subplot(111)
    cax = ax.matshow(cm, cmap=plt.cm.Blues)
    fig.colorbar(cax)

    # Axis
    plt.title("Confusion matrix")
    plt.ylabel("True label")
    plt.xlabel("Predicted label")
    ax.set_xticklabels([''] + classes)
    ax.set_yticklabels([''] + classes)
    ax.xaxis.set_label_position('bottom')
    ax.xaxis.tick_bottom()

    # Values
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, f"{cm[i, j]:d} ({cm_norm[i, j]*100:.1f}%)",
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    # Display
    plt.show()_____no_output_____def plot_multiclass_decision_boundary(model, X, y, savefig_fp=None):
    """Plot the multiclass decision boundary for a model that accepts 2D inputs.

    Arguments:
        model {function} -- trained model with function model.predict(x_in).
        X {numpy.ndarray} -- 2D inputs with shape (N, 2).
        y {numpy.ndarray} -- 1D outputs with shape (N,).
    """
    # Axis boundaries
    x_min, x_max = X[:, 0].min() - 0.1, X[:, 0].max() + 0.1
    y_min, y_max = X[:, 1].min() - 0.1, X[:, 1].max() + 0.1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 101),
                         np.linspace(y_min, y_max, 101))

    # Create predictions
    x_in = np.c_[xx.ravel(), yy.ravel()]
    y_pred = model.predict(x_in)
    y_pred = np.argmax(y_pred, axis=1).reshape(xx.shape)

    # Plot decision boundary
    plt.contourf(xx, yy, y_pred, cmap=plt.cm.Spectral, alpha=0.8)
    plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.RdYlBu)
    plt.xlim(xx.min(), xx.max())
    plt.ylim(yy.min(), yy.max())

    # Plot
    if savefig_fp:
        plt.savefig(savefig_fp, format='png')_____no_output_____# Predictions
pred_train = model.predict(X_train)
pred_test = model.predict(X_test)
print (f"sample probability: {pred_test[0]}")
pred_train = np.argmax(pred_train, axis=1)
pred_test = np.argmax(pred_test, axis=1)
print (f"sample class: {pred_test[0]}")sample probability: [0.21060377 0.7893962 ]
sample class: 1
# Accuracy
train_acc = accuracy_score(y_train, pred_train)
test_acc = accuracy_score(y_test, pred_test)
print (f"train acc: {train_acc:.2f}, test acc: {test_acc:.2f}")train acc: 0.96, test acc: 0.93

We're going to plot a white point, which we know belongs to the malignant tumor class. Our well trained model here would accurately predict that it is indeed a malignant tumor!_____no_output_____

# Visualize the decision boundary
plt.figure(figsize=(8,5))
plt.title("Test")
plot_multiclass_decision_boundary(model=model, X=X_test, y=y_test)

# Sample point near the decision boundary
mean_leukocyte_count, mean_blood_pressure = X_scaler.transform(
    [[np.mean(df.leukocyte_count), np.mean(df.blood_pressure)]])[0]
plt.scatter(mean_leukocyte_count+0.05, mean_blood_pressure-0.05, s=200,
            c='b', edgecolor='w', linewidth=2)

# Annotate
plt.annotate('true: malignant,\npred: malignant',
             color='white',
             xy=(mean_leukocyte_count, mean_blood_pressure),
             xytext=(0.4, 0.65),
             textcoords='figure fraction',
             fontsize=16,
             arrowprops=dict(facecolor='white', shrink=0.1))
plt.show()_____no_output_____

Great! We received great performances on both our train and test data splits. We're going to use this dataset to show the importance of data quality and quantity._____no_output_____# Reduced dataset_____no_output_____Let's remove some training data near the decision boundary and see how robust the model is now._____no_output_____## Data_____no_output_____### Load data_____no_output_____

REDUCED_DATA_FILE = 'tumors_reduced.csv'_____no_output_____# Load data from GitHub to this notebook's local drive
url = "https://raw.githubusercontent.com/madewithml/lessons/master/data/tumors_reduced.csv"
response = urllib.request.urlopen(url)
html = response.read()
with open(REDUCED_DATA_FILE, 'wb') as fp:
    fp.write(html)_____no_output_____# Raw reduced data
df_reduced = pd.read_csv(REDUCED_DATA_FILE, header=0)
df_reduced.head()_____no_output_____# Define X and y
X = df_reduced[['leukocyte_count', 'blood_pressure']].values
y = df_reduced['tumor_class'].values_____no_output_____# Plot data
colors = {'benign': 'red', 'malignant': 'blue'}
plt.scatter(X[:, 0], X[:, 1], c=[colors[_y] for _y in y], s=25, edgecolors='k')
plt.xlabel('leukocyte count')
plt.ylabel('blood pressure')
plt.legend(['malignant ', 'benign'], loc="upper right")
plt.show()_____no_output_____

### Split data_____no_output_____

# Create data splits
X_train, X_val, X_test, y_train, y_val, y_test = train_val_test_split(
    X, y, val_size=VAL_SIZE, test_size=TEST_SIZE, shuffle=SHUFFLE)
class_counts = dict(collections.Counter(y_train))
print (f"X_train: {X_train.shape}, y_train: {y_train.shape}")
print (f"X_val: {X_val.shape}, y_val: {y_val.shape}")
print (f"X_test: {X_test.shape}, y_test: {y_test.shape}")
print (f"Sample point: {X_train[0]} → {y_train[0]}")
print (f"Classes: {class_counts}")X_train: (520, 2), y_train: (520,)
X_val: (92, 2), y_val: (92,)
X_test: (108, 2), y_test: (108,)
Sample point: [14.4110029  13.14842457] → benign
Classes: {'benign': 281, 'malignant': 239}

### Label encoder_____no_output_____

# Encode class labels
y_tokenizer = LabelEncoder()
y_tokenizer = y_tokenizer.fit(y_train)
num_classes = len(y_tokenizer.classes_)
y_train = y_tokenizer.transform(y_train)
y_val = y_tokenizer.transform(y_val)
y_test = y_tokenizer.transform(y_test)_____no_output_____# Class weights
counts = collections.Counter(y_train)
class_weights = {_class: 1.0/count for _class, count in counts.items()}
print (f"class counts: {counts},\nclass weights: {class_weights}")class counts: Counter({0: 281, 1: 239}),
class weights: {0: 0.0035587188612099642, 1: 0.0041841004184100415}

### Standardize data_____no_output_____

# Standardize inputs using training data
X_scaler = StandardScaler().fit(X_train)
X_train = X_scaler.transform(X_train)
X_val = X_scaler.transform(X_val)
X_test = X_scaler.transform(X_test)_____no_output_____

## Modeling_____no_output_____### Model_____no_output_____

# Initialize model
model = MLP(hidden_dim=HIDDEN_DIM, num_classes=NUM_CLASSES)_____no_output_____

### Training_____no_output_____

# Compile
model.compile(optimizer=Adam(lr=LEARNING_RATE),
              loss=SparseCategoricalCrossentropy(),
              metrics=[SparseCategoricalAccuracy()])_____no_output_____# Training
model.fit(x=X_train,
          y=y_train,
          validation_data=(X_val, y_val),
          epochs=NUM_EPOCHS,
          batch_size=BATCH_SIZE,
          class_weight=class_weights,
          shuffle=False,
          verbose=1)WARNING:tensorflow:sample_weight modes were coerced from
  ...
    to
  ['...']
WARNING:tensorflow:sample_weight modes were coerced from
  ...
    to
  ['...']
Train on 520 samples, validate on 92 samples
Epoch 1/5
520/520 [==============================] - 1s 1ms/sample - loss: 0.0024 - sparse_categorical_accuracy: 0.8615 - val_loss: 0.0020 - val_sparse_categorical_accuracy: 0.9239
Epoch 2/5
520/520 [==============================] - 0s 164us/sample - loss: 0.0017 - sparse_categorical_accuracy: 0.9942 - val_loss: 0.0016 - val_sparse_categorical_accuracy: 0.9457
Epoch 3/5
520/520 [==============================] - 0s 154us/sample - loss: 0.0012 - sparse_categorical_accuracy: 0.9981 - val_loss: 0.0012 - val_sparse_categorical_accuracy: 0.9674
Epoch 4/5
520/520 [==============================] - 0s 156us/sample - loss: 8.8105e-04 - sparse_categorical_accuracy: 0.9981 - val_loss: 9.2301e-04 - val_sparse_categorical_accuracy: 0.9674
Epoch 5/5
520/520 [==============================] - 0s 155us/sample - loss: 6.4092e-04 - sparse_categorical_accuracy: 0.9981 - val_loss: 7.2682e-04 - val_sparse_categorical_accuracy: 0.9674

### Evaluation_____no_output_____

# Predictions
pred_train = model.predict(X_train)
pred_test = model.predict(X_test)
print (f"sample probability: {pred_test[0]}")
pred_train = np.argmax(pred_train, axis=1)
pred_test = np.argmax(pred_test, axis=1)
print (f"sample class: {pred_test[0]}")sample probability: [0.9758981  0.02410194]
sample class: 0
# Accuracy
train_acc = accuracy_score(y_train, pred_train)
test_acc = accuracy_score(y_test, pred_test)
print (f"train acc: {train_acc:.2f}, test acc: {test_acc:.2f}")train acc: 1.00, test acc: 1.00
# Visualize the decision boundary
plt.figure(figsize=(8,5))
plt.title("Test")
plot_multiclass_decision_boundary(model=model, X=X_test, y=y_test)

# Sample point near the decision boundary (same point as before)
plt.scatter(mean_leukocyte_count+0.05, mean_blood_pressure-0.05, s=200,
            c='b', edgecolor='w', linewidth=2)

# Annotate
plt.annotate('true: malignant,\npred: benign',
             color='white',
             xy=(mean_leukocyte_count, mean_blood_pressure),
             xytext=(0.45, 0.60),
             textcoords='figure fraction',
             fontsize=16,
             arrowprops=dict(facecolor='white', shrink=0.1))
plt.show()_____no_output_____

This is a very scary but highly realistic scenario. Based on our reduced synthetic dataset, we have achieved a model that generalized really well on the test data. But when we ask for the prediction for the same point tested earlier (which we known is malignant), the prediction is now a benign tumor. We would have completely missed the tumor. To mitigate this, we can:
1. get more data around the space we are concerned about
2. consume predictions with caution when they are close to the decision boundary_____no_output_____# Takeaway_____no_output_____Models are not crystal balls. So it's important that before any machine learning, we really look at our data and ask ourselves if it is truly representative for the task we want to solve. The model itself may fit really well and generalize well on your data but if the data is of poor quality to begin with, the model cannot be trusted._____no_output_____Once you are confident that your data is of good quality and quantity, you can finally start thinking about modeling. The type of model you choose depends on many factors, including the task, type of data, complexity required, etc.

So once you figure out what type of model your task needs, start with simple models and then slowly add complexity. You don’t want to start with neural networks right away because that may not be right model for your data and task. Striking this balance in model complexity is one of the key tasks of your data scientists. **simple models → complex models**

_____no_output_____---
Share and discover ML projects at Made With ML.

             _____no_output_____


# Notebook from foerstner-lab/2021-06-21-Supervised_Machine_Learning_as_part_of_an_EBI_Systems_Biology_course
Path: jupyter_notebooks/04-Classification_incl_feature_selection_cancer_gene_expresison.ipynb

# Cancer classification based on gene expression

- reference paper: https://www.nature.com/articles/nmeth.2810#Sec15
- further reading: https://machinelearningmastery.com/feature-selection-with-real-and-categorical-data/_____no_output_____#### pandas
fast, powerful, flexible and easy to use open source data analysis and manipulation tool for python.
#### numpy
essential component in the burgeoning Python visualization landscape.
#### seaborn
**Seaborn** is a library for making statistical graphics in Python. It builds on top of matplotlib and integrates closely with pandas data structures.
#### scikit-learn
Simple and efficient tools for predictive data analysis.
##### OneHotEncoder
Encode categorical integer features using a one-hot aka one-of-K scheme. The input to this transformer should be a matrix of integers, denoting the values taken on by categorical (discrete) features. The output will be a sparse matrix where each column corresponds to one possible value of one feature.
##### StandardScaler
standardize the distribution making the values lie in the same range.
##### LabelEncoder
encode the DataFrame of string labels. The data frame has columns above 50 and avoids creating LabelEncoder object for each column. The column LabelEnconder will create the below error and deal with the columns.
##### SelectKBest
selects the features according to the k highest score.
##### f_classif
uses the ANOVA f-test for the features, and take into consideration only linear dependency unlike mutual information based feature selection which can capture any kind of statistical dependency.
##### train_test_split
Using the function train_test_split() from scikit-learn, you can split your dataset into subsets that minimize the potential for bias in your evaluation and validation process.
##### StandardScaler
standardize the distribution making the values lie in the same range.
##### DummyClassifier
a classifier that makes predictions using simple rules. This classifier is useful as a simple baseline to compare with other (real) classifiers.
##### MLPClassifier
trains iteratively since at each time step the partial derivatives of the loss function with respect to the model parameters are computed to update the parameters. It can also have a regularization term added to the loss function that shrinks model parameters to prevent overfitting.
##### KNeighborsClassifier
implements learning based on the k nearest neighbors of each query point, where k is an integer value specified by the user.
##### SVC
C-support vector classification whose implementation is based on libsvm. The module used by scikit-learn is sklearn.svm.SVC. This class handles the multiclass support according to one-vs-one scheme.
##### GaussianProcessClassifier
supports multi-class classification by performing either one-versus-rest or one-versus-one based training and prediction. In one-versus-rest, one binary Gaussian process classifier is fitted for each class, which is trained to separate this class from the rest.
##### RBF
a stationary kernel. It is also known as the "squared exponential" kernel. It is parameterized by a length scale parameter l > 0, which can either be a scalar (isotropic variant of the kernel) or a vector with the same number of dimensions as the inputs X (anisotropic variant of the kernel).
##### DecisionTreeClassifier
a non-parametric supervised learning method used for classification and regression. The goal is to create a model that predicts the value of a target variable by learning simple decision rules inferred from the data features.
##### RandomForestClassifier
A random forest is a meta estimator that fits a number of decision tree classifiers on various sub-samples of the dataset and uses averaging to improve the predictive accuracy and control over-fitting.
##### AdaBoostClassifier
a meta-estimator that begins by fitting a classifier on the original dataset and then fits additional copies of the classifier on the same dataset but where the weights of incorrectly classified instances are adjusted such that subsequent classifiers focus more on difficult cases.
##### GaussianNB
contains the params theta and sigma which is the variance and mean of each feature per class.
##### QuadraticDiscriminantAnalysis
classifier with a quadratic decision boundary, generated by fitting class conditional densities to the data and using Bayes' rule. The model fits a Gaussian density to each class.
##### classification_report
is about key metrics in a classification problem. You'll have precision, recall, f1-score and support for each class you're trying to find. The recall means "how many of this class you find over the whole number of element of this class".
##### confusion_matrix
evaluate the quality of the output of a classifier on the iris data set. The diagonal elements represent the number of points for which the predicted label is equal to the true label, while off-diagonal elements are those that are mislabeled by the classifier. The higher the diagonal values of the confusion matrix the better, indicating many correct predictions.
##### accuracy_score
computes the accuracy, either the fraction (default) or the count (normalize=False) of correct predictions. _____no_output_____

import pandas as pd
import numpy as np
import requests, zipfile, io, os
from sklearn.preprocessing import OneHotEncoder, StandardScaler, LabelEncoder
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import seaborn as sns

# classifiers
from sklearn.dummy import DummyClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

# evaluation
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score_____no_output_____

## Getting the data - Option 1: start from scratch_____no_output_____

# download Dataset with Python

data_directory = "cancer_datasets"

headers = {'Connection': 'keep-alive',
           'Host': 'static-content.springer.com',
           'Referer': 'https://www.nature.com/',
           'Upgrade-Insecure-Requests': '1',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
           'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0',
           'Accept-Language': 'en-US,en;q=0.5'}

url = 'https://static-content.springer.com/esm/art%3A10.1038%2Fnmeth.2810/MediaObjects/41592_2014_BFnmeth2810_MOESM206_ESM.zip'
request = requests.get(url, headers= headers, stream=True)
zip_reader = zipfile.ZipFile(io.BytesIO(request.content))
zip_reader.extractall(data_directory)
_____no_output_____# Create dataframes
data_directory = "cancer_datasets"

cancer_types = ["Breast", "Colon", "GBM", "Lung"]

breast_gene_exp_df = pd.read_csv(
    "{}/Breast/BREAST_Gene_Expression.txt".format(data_directory),
    delimiter=r"\s+", index_col=0)
colon_gene_exp_df = pd.read_csv(
    "{}/Colon/COLON_Gene_Expression.txt".format(data_directory),
    delimiter=r"\s+", index_col=0)
gbm_gene_exp_df = pd.read_csv(
    "{}/GBM/GLIO_Gene_Expression.txt".format(data_directory),
    delimiter=r"\s+", index_col=0)
lung_gene_exp_df = pd.read_csv(
    "{}/Lung/LUNG_Gene_Expression.txt".format(data_directory),
    delimiter=r"\s+", index_col=0)

breast_gene_exp_df_____no_output_____# Not all genes are found in all expression sets - only overlap can be used

genes_breast = set(breast_gene_exp_df.index)
genes_colon = set(colon_gene_exp_df.index)
genes_gdm = set(gbm_gene_exp_df.index)
genes_lung = set(lung_gene_exp_df.index)
genes_found_in_all_sets = genes_breast.intersection(
    genes_colon).intersection(
    genes_gdm).intersection(
    genes_lung)
len(genes_found_in_all_sets)_____no_output_____# Create universal dataframe with all cancer types

exp_df = pd.DataFrame()
for df, cancer_type in zip([breast_gene_exp_df, colon_gene_exp_df,
                            gbm_gene_exp_df,lung_gene_exp_df],
                            cancer_types):
    transposed_df = pd.DataFrame(df.reindex(genes_found_in_all_sets).transpose())
    transposed_df["cancer_type"] = cancer_type
    exp_df = exp_df.append(transposed_df)

# export to csv
exp_df.to_csv("cancer_cells_expression.csv.bz2")_____no_output_____

## Getting the data - Option 2: read the processed file_____no_output_____

# The file can be found in the git repo
exp_df = pd.read_csv("cancer_cells_expression.csv.bz2", index_col=0)_____no_output_____

## Preparing the data_____no_output_____

# create a new dataframe by excluding the target column

X = exp_df.drop(["cancer_type"], axis=1).values
targets = exp_df["cancer_type"].values_____no_output_____X.shape_____no_output_____targets.shape_____no_output_____# fitting the transformer and then return the transformed training instances
targets = LabelEncoder().fit_transform(targets)_____no_output_____targets_____no_output_____X_scaled = StandardScaler().fit_transform(X)_____no_output_____# train test split of the data set

X_train, X_test, y_train, y_test = train_test_split(X_scaled, targets, random_state=42)_____no_output_____# Classification with a random forest classifier

model = RandomForestClassifier()
model.fit(X_train, y_train).score(X_test, y_test)_____no_output_____# Explaining feature selection with SVC. The optimal model is feature size independent

model =  SVC(gamma='auto')
data_dict = {}

for K in range(1000, len(overlap_genes), 100):
    selector = SelectKBest(score_func=f_classif, k=K)
    selected_input = selector.fit_transform(X_scaled, targets.ravel())
    best_genes_index = selector.get_support(indices=True)
    best_genes =[list(overlap_genes)[i] for i in best_genes_index]
    X_train, X_test, y_train, y_test = train_test_split(selected_input,
                                                    target_array.ravel(),
                                                    test_size=0.2,
                                                    random_state=42)
    model.fit(X_train, y_train)
    mean_accuracy = model.score(X_test, y_test)
    data_dict[K] = mean_accuracy

sns.lineplot(list(data_dict.keys()), list(data_dict.values()))
    _____no_output_____# benchmark with dummy classifier

dummy_clf = DummyClassifier(strategy="stratified")
dummy_clf.fit(X_train, y_train)
dummy_clf.score(X_test, y_test)_____no_output_____# set up different classifiers

names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
         "Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
         "Naive Bayes", "QDA"]

classifiers = [
    KNeighborsClassifier(),
    SVC(kernel="linear", C=0.025),
    SVC(gamma="scale", C=1),
    GaussianProcessClassifier(1.0 * RBF(1.0)),
    DecisionTreeClassifier(max_depth=5),
    RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
    MLPClassifier(alpha=1, max_iter=40),
    AdaBoostClassifier(),
    GaussianNB(),
    QuadraticDiscriminantAnalysis()]_____no_output_____# compare different classifiers

for name, clf in zip(names, classifiers):
    model = clf
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    accuracy = accuracy_score(y_test, y_pred)
    report = classification_report(y_test, y_pred)
    print(name)
    print("accuracy: {}".format(accuracy))
    print("report:")
    print(report)Nearest Neighbors
accuracy: 0.4307692307692308
report:
              precision    recall  f1-score   support

           0       1.00      0.09      0.16        34
           1       0.67      0.11      0.19        18
           2       0.41      1.00      0.59        51
           3       0.00      0.00      0.00        27

    accuracy                           0.43       130
   macro avg       0.52      0.30      0.23       130
weighted avg       0.52      0.43      0.30       130

Linear SVM
accuracy: 0.046153846153846156
report:
              precision    recall  f1-score   support

           0       0.00      0.00      0.00        34
           1       0.00      0.00      0.00        18
           2       0.11      0.12      0.11        51
           3       0.00      0.00      0.00        27

    accuracy                           0.05       130
   macro avg       0.03      0.03      0.03       130
weighted avg       0.04      0.05      0.04       130

RBF SVM
accuracy: 0.6307692307692307
report:
              precision    recall  f1-score   support

           0       0.88      0.41      0.56        34
           1       0.62      0.28      0.38        18
           2       0.71      1.00      0.83        51
           3       0.35      0.44      0.39        27

    accuracy                           0.63       130
   macro avg       0.64      0.53      0.54       130
weighted avg       0.67      0.63      0.61       130