# Notebook from enzo-ca/multilabel-url-classification
Path: notebooks/data_exploration_iscx.ipynb

import glob
import os
import re
import urllib.parse

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

import warnings
warnings.filterwarnings("ignore")
from sklearn import metrics
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB_____no_output_____# Change this to where your datasets are downloaded
datasets_path = r'D:\\dev\\CyberML\\Datasets'_____no_output_____

# ISCX2016 Dataset
https://www.unb.ca/cic/datasets/url-2016.html_____no_output_____

# Dataset paths
dataset_base_path = os.path.join(datasets_path, r"ISCXURL2016\\URL")
raw_url_paths = []
for fn in glob.glob(os.path.join(dataset_base_path, "*.csv")):
    raw_url_paths.append(os.path.normpath(fn))

# Make sure files get added in specific
raw_url_paths.sort(reverse=False)
mapping_label = {0:"benign", 1:"malware", 2:"phishing", 3:"spam"}
mapping_label_r = dict((v, k) for k, v in mapping_label.items())

print(raw_url_paths)
print(mapping_label)['D:\\dev\\CyberML\\Datasets\\ISCXURL2016\\URL\\Benign_list_big_final.csv', 'D:\\dev\\CyberML\\Datasets\\ISCXURL2016\\URL\\Malware_dataset.csv', 'D:\\dev\\CyberML\\Datasets\\ISCXURL2016\\URL\\phishing_dataset.csv', 'D:\\dev\\CyberML\\Datasets\\ISCXURL2016\\URL\\spam_dataset.csv']
{0: 'benign', 1: 'malware', 2: 'phishing', 3: 'spam'}
def ends(df, n=5):
    '''Get the head and tail of a dataframe'''
    return df.head(n).append(df.tail(n))

# Load and parse the dataset
raw_urls = []
for idx, fn in enumerate(raw_url_paths):
    with open(fn,'r', encoding='utf-8') as f:
        data = f.readlines()
        # unquote unescapes any char sequence in the URL
        data = [(str(urllib.parse.unquote(d)), idx) for d in data]
        # data = [(str(d), idx) for d in data]
        raw_urls.extend(data)

# Create dataframe with columns for URL and label
urldata = pd.DataFrame(raw_urls, columns=['url', 'label'])
urldata['url'] = urldata['url'].apply(lambda x: x.strip("\n"))
# Categorize into good or bad just like the FWAF dataset
urldata['label_good_bad'] = urldata['label'].apply(lambda x: 0 if x == 0 else 1)
# Convert ground_truth column according to mapping
urldata['label_multi'] = urldata['label']
urldata['label'] = urldata['label'].apply(lambda x: mapping_label[x])
urldata['label'] = urldata['label'].astype('category')

print(urldata.info())
display(urldata.groupby('label').count())
display(ends(urldata))
# urldata['label'].cat.codes
# urldata['label'].cat.codesurldata['label'].cat.codesurldata['label'].cat.codes<class 'pandas.core.frame.DataFrame'>
RangeIndex: 68909 entries, 0 to 68908
Data columns (total 4 columns):
url               68909 non-null object
label             68909 non-null category
label_good_bad    68909 non-null int64
label_multi       68909 non-null int64
dtypes: category(1), int64(2), object(1)
memory usage: 1.6+ MB
None
if 'url_length' not in urldata.columns: urldata['url_length'] = urldata['url'].apply(lambda i: len(str(i)))
if 'count-' not in urldata.columns: urldata['count-'] = urldata['url'].apply(lambda i: i.count('-'))
if 'count@' not in urldata.columns: urldata['count@'] = urldata['url'].apply(lambda i: i.count('@'))
if 'count?' not in urldata.columns: urldata['count?'] = urldata['url'].apply(lambda i: i.count('?'))
if 'count%' not in urldata.columns: urldata['count%'] = urldata['url'].apply(lambda i: i.count('%'))
if 'count.' not in urldata.columns: urldata['count.'] = urldata['url'].apply(lambda i: i.count('.'))
if 'count=' not in urldata.columns: urldata['count='] = urldata['url'].apply(lambda i: i.count('='))
if 'count<' not in urldata.columns: urldata['count<'] = urldata['url'].apply(lambda i: i.count('<'))
if 'count>' not in urldata.columns: urldata['count>'] = urldata['url'].apply(lambda i: i.count('>'))
if 'count/' not in urldata.columns: urldata['count/'] = urldata['url'].apply(lambda i: i.count('/'))
if 'count\\' not in urldata.columns: urldata['count\\'] = urldata['url'].apply(lambda i: i.count('\\'))

# Count number of non alphanumeric characters in url
if 'count_non_alphanum' not in urldata.columns: urldata["count_non_alphanum"] = urldata.apply(lambda p: sum( not q.isalnum() for q in p["url"] ), axis=1)

display(ends(urldata, 10))

# Display total sum of each column by class (benign, malware, phishing, spam)
grouped_data = urldata.groupby(['label',]).agg('sum')
display("Sum", grouped_data)
# Display percentages of total
display("Percentage", grouped_data.apply(lambda x:  100*x / x.sum()))
# Display average number
display("Mean", urldata.groupby(['label',]).agg('mean'))
# display(urldata.groupby(['label',]).agg(['mean', 'sum']))

# Display total sum of each column by class (good/bad)
grouped_data = urldata.groupby(['label_good_bad',]).agg('sum')
display("Sum", grouped_data)
# Display percentages of total
display("Percentage", grouped_data.apply(lambda x:  100*x / x.sum()))
# Display average number
display("Mean", urldata.groupby(['label_good_bad',]).agg('mean'))_____no_output_____# Correlation heatmap
urldata2 = urldata.copy()
urldata2['label'] = urldata2['label'].cat.codes

corrmat = urldata2[urldata2.columns[~urldata2.columns.isin(['label_good_bad', 'label_multi'])]].corr()
# corrmat = urldata2[urldata2.columns[~urldata2.columns.isin(['label'])]].corr()
sns.set_style("darkgrid")
# f, ax = plt.subplots(figsize=(25,19))
f, ax = plt.subplots(figsize=(13,13))
sns.heatmap(corrmat, square=True, annot = True, annot_kws={'size':10}, cmap='Blues')_____no_output_____urldata.describe(include='all')_____no_output_____

# Char-level Features_____no_output_____

def custom_tokenizer(s):
    return re.split(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./<>?]', s)
    # return s

def no_number_preprocessor(s):
    return re.sub(r'\b[0-9][0-9.,-]*\b', 'NUMBER-SPECIAL-TOKEN', s)

def custom_preprocessor(s):
    return re.sub(r'[_]', ' ', s)

# TF-IDF Vectoritzer Experiments
vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="char", sublinear_tf=True, strip_accents='unicode', max_features=79) #converting data to vectors
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", tokenizer=None, sublinear_tf=True, token_pattern=r"(?u)\b\w\w+\b")
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", tokenizer=custom_tokenizer, sublinear_tf=True, token_pattern=r"\S+")
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", preprocessor=custom_preprocessor, sublinear_tf=True, token_pattern=r"(?u)\b\w\w+\b")

# Splitting data: binary vs multilabel
# X_train, X_test, y_train, y_test = train_test_split(urldata['url'], urldata['label_good_bad'], test_size=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(urldata["url"][urldata['label'] != "spam"], urldata['label_good_bad'][urldata['label'] != "spam"], test_size=0.3, random_state=42)

X_train = vectorizer.fit_transform(X_train)
X_test = vectorizer.transform(X_test)_____no_output_____features = vectorizer.get_feature_names()
print(features)
# print(features[:200] + features[-150:-50])
# features = pd.DataFrame(vectorizer.get_feature_names())
# display(features.sample(n=8))[' ', '!', '#', '%', '&', ')', '+', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '=', '?', '[', ']', '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'а', 'е', 'и', 'о', 'ا', 'ب', 'ة', 'ت', 'د', 'ر', 'س', 'ع', 'ف', 'ل', 'م', 'ن', 'و', 'ي', '儿', '剪', '图', '画', '粘', '纸', '贴', '�']
goodCount = len(urldata[urldata['label_good_bad'] == 0])
badCount = len(urldata[urldata['label_good_bad'] > 0])
print(goodCount)
print(badCount)35378
33531

## Logistic Regression_____no_output_____

# lgs = LogisticRegression(class_weight={1: 2 * goodCount / badCount, 0: 1.0}, max_iter=500) # class_weight='balanced')
lgs = LogisticRegression(max_iter=500)
lgs.fit(X_train, y_train) #training our model_____no_output_____predicted = lgs.predict(X_test)

fpr, tpr, _ = metrics.roc_curve(y_test, (lgs.predict_proba(X_test)[:, 1]))
auc = metrics.auc(fpr, tpr)

# print("Bad samples: %d" % badCount)
# print("Good samples: %d" % goodCount)
print("------------")
print("Accuracy: %f" % lgs.score(X_test, y_test))
print("Precision: %f" % metrics.precision_score(y_test, predicted))
print("Recall: %f" % metrics.recall_score(y_test, predicted))
print("F1-Score: %f" % metrics.f1_score(y_test, predicted))
print("AUC: %f" % auc)

cm = metrics.confusion_matrix(y_test, predicted)
print(cm)

sns.set_style("darkgrid")
plt.figure(figsize=(9,9))
sns.heatmap(cm, annot=True, fmt=".3f", linewidths=.5, square = True, cmap = 'Blues_r');
plt.ylabel('Actual label');
plt.xlabel('Predicted label');
all_sample_title = 'Accuracy Score: {0}'.format(lgs.score(X_test, y_test))
plt.title(all_sample_title, size = 15);------------
Accuracy: 0.934692
Precision: 0.941186
Recall: 0.882006
F1-Score: 0.910636
AUC: 0.965843
[[10277   355]
 [  760  5681]]
# sample1 = ["http://google.com"]
sample = urldata[urldata['label'] == "spam"]
sample1 = sample["url"]
sample1 = vectorizer.transform(sample1)
predicted = lgs.predict(sample1)
print(predicted)

fpr, tpr, _ = metrics.roc_curve(sample['label_good_bad'], (lgs.predict_proba(sample1)[:, 1]))
auc = metrics.auc(fpr, tpr)

goodCount = len(sample[sample['label_good_bad'] == 0])
badCount = len(sample[sample['label_good_bad'] > 0])

print("Bad samples: %d" % badCount)
print("Good samples: %d" % goodCount)
print("------------")
print("Accuracy: %f" % lgs.score(sample1, sample['label_good_bad']))
print("Precision: %f" % metrics.precision_score(sample['label_good_bad'], predicted))
print("Recall: %f" % metrics.recall_score(sample['label_good_bad'], predicted))
print("F1-Score: %f" % metrics.f1_score(sample['label_good_bad'], predicted))
print("AUC: %f" % auc)

cm = metrics.confusion_matrix(sample['label_good_bad'], predicted)
print(cm)[0 1 1 ... 1 0 0]
Bad samples: 12000
Good samples: 0
------------
Accuracy: 0.597500
Precision: 1.000000
Recall: 0.597500
F1-Score: 0.748044
AUC: nan
[[   0    0]
 [4830 7170]]

# Word Level Features
Here we use TF-IDF to extract features (word) from our URLs_____no_output_____

from sklearn.feature_extraction import text

def custom_tokenizer(s):
    s = s.split("NUMBERSPECIALTOKEN")
    return re.split(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./<>?]', s)
    # return s

def no_number_preprocessor(s):
    '''Since different sequences of numbers should have no special meaning as
    features, we can replace all of them with a special token so that it is
    interpreted as a single generic feature'''

    return re.sub(r'\b[0-9][0-9.,-]*\b', 'NUMBERSPECIALTOKEN', s)

# my_additional_stop_words = ['amp', 'app', 'archive', 'asp', 'aspx', 'com', 'content', 'cpath', 'csp', 'css', 'd3', 'en', 'htm', 'html', 'http', 'https', 'id', 'index', 'info', 'js', 'kickass', 'login', 'net', 'news', 'olx', 'org', 'page', 'php', 'pl', 'ref', 'ru', 'search', 'title', 'torcache', 'torrent', 'uk', 'video', 'view', 'vn', 'wp', 'www', 'cc', 'cgi', 'cn']
# stop_words = text.ENGLISH_STOP_WORDS.union(my_additional_stop_words)
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="char", sublinear_tf=True, strip_accents='unicode', max_features=83, stop_words=stop_words) #converting data to vectors
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", tokenizer=None, sublinear_tf=True, token_pattern=r"(?u)\b\w\w+\b", max_features=79)
# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", tokenizer=custom_tokenizer, sublinear_tf=True, token_pattern=r"\S+")

# vectorizer = TfidfVectorizer(min_df = 0.05, analyzer="word", preprocessor=custom_preprocessor, sublinear_tf=True, token_pattern=r"\w")

# vectorizer = TfidfVectorizer(min_df = 0.0, analyzer="word", preprocessor=no_number_preprocessor, tokenizer=custom_tokenizer, sublinear_tf=True, token_pattern=r"(?u)\b\w\w+\b", max_features=79, strip_accents='unicode')

vectorizer = TfidfVectorizer(min_df = 0.001, analyzer="word", preprocessor=no_number_preprocessor, sublinear_tf=True, token_pattern = '\w*')
# vectorizer = TfidfVectorizer(min_df = 0.001, analyzer="word", preprocessor=no_number_preprocessor, tokenizer=custom_tokenizer, sublinear_tf=True, strip_accents='unicode')

X_train, X_test, y_train, y_test = train_test_split(urldata['url'], urldata['label_multi'], test_size=0.3, random_state=42) #splitting data
# X_train, X_test, y_train, y_test = train_test_split(urldata["url"][urldata['label'] != "spam"], urldata['label_good_bad'][urldata['label'] != "spam"], test_size=0.2, random_state=42) #splitting data

X_train = vectorizer.fit_transform(X_train)
X_test = vectorizer.transform(X_test)

features = vectorizer.get_feature_names()
print(features)['', '0a530e17ea959c39afe6ce5fbae147ec', '1080p', '1337x', '15a6848b9976441f73230911ec8eac09', '2a', '39e0fc26c1e8462a296039a38f87fdc9', '3cf', '3d', '500px', '6848d1f0', '720p', '8dd59c825746c7ea64549f74628a4afe', 'A', 'A3P5ROKL5A1OLE', 'A77C20A90C6790855CB2882AD06183ED', 'AB', 'AK', 'AP', 'Apple', 'Approved', 'Area', 'B', 'B7', 'Banking', 'Bedroom', 'C224', 'CCCCCC', 'CSPCHD', 'Content', 'Detail', 'E072', 'E119', 'EW4Hou_aN610rPsocimkJ_IuLBofatBy4b0MLqoh7Lf', 'FS', 'FTOOTH', 'FreeMarket', 'GameType', 'HALLID', 'HDTV', 'Health', 'Home', 'Homewares', 'How', 'In', 'Itemid', 'JOcK7gzdoyrcIT2176iLzhCz8UoPOGcT6Y1ax8FNTPf', 'Kitchen', 'Kitchenware', 'LgrpCategory', 'License', 'LiveTop', 'Login', 'MAdvertis', 'MallDisplay', 'May', 'NOT', 'NUMBERSPECIALTOKEN', 'NUMBERSPECIALTOKEN720p', 'NUMBERSPECIALTOKENchn', 'NUMBERSPECIALTOKENcom', 'NUMBERSPECIALTOKENconfigurazione', 'NUMBERSPECIALTOKENcustom', 'NUMBERSPECIALTOKENdetail', 'NUMBERSPECIALTOKENe', 'NUMBERSPECIALTOKENhtm', 'NUMBERSPECIALTOKENhtml', 'NUMBERSPECIALTOKENinfo', 'NUMBERSPECIALTOKENjpg', 'NUMBERSPECIALTOKENmillion', 'NUMBERSPECIALTOKENphotos', 'NUMBERSPECIALTOKENtek', 'NUMBERSPECIALTOKENtorrent', 'Name', 'News', 'OSI', 'Open', 'Operating', 'Para', 'Params', 'Personal', 'Press', 'Qty', 'Radio', 'Recipe', 'Release', 'S', 'STNO', 'ShowItem', 'Source', 'SportOption', 'State', 'StationLeader', 'System', 'Systems', 'The', 'Thread', 'TuneInSearch', 'Type', 'UI', 'UTF8', 'UploadFiles', 'Web_nMngr', 'X', '_', '_login', '_method', '_talk', '_vti_bin', 'a', 'aac', 'abc', 'abdn', 'about', 'abyssinica', 'ac', 'ac3', 'acaidx', 'acard4u', 'account', 'accounts', 'action', 'active', 'ad', 'add', 'add_facet_filter', 'admin', 'administrator', 'ads', 'adservers', 'aerade', 'after', 'against', 'age', 'agesage', 'ahds', 'aig', 'akhbarelyom', 'album', 'aljazeera', 'all', 'all2', 'all3', 'all4', 'alldblist', 'allegro', 'allezvinsfrenchr', 'allgirltogaparty', 'allianceleague', 'allrecipes', 'amazon', 'amber', 'america', 'american', 'amluMjAxNQ', 'amp', 'an', 'ancestry', 'and', 'android', 'angel', 'anh', 'answering', 'antonjulianov777', 'any', 'aol', 'aolpictures', 'aolshop', 'ap', 'api', 'app', 'app7', 'apple', 'appleid', 'applerugs', 'apps', 'archive', 'archives', 'are', 'area1', 'area2', 'args', 'art', 'article', 'articles', 'as', 'asia', 'askubuntu', 'asp', 'aspx', 'assets', 'astore', 'at', 'atwiki', 'au', 'audilinks', 'audio', 'auth', 'author', 'auto', 'avengers', 'awesome', 'b', 'babal', 'baby', 'back', 'ball', 'bam', 'ban', 'bang', 'banner', 'bao', 'barcelona', 'bat', 'battle', 'bb35bdc9f245a1182ce52675c91e306f', 'bbs', 'bclassid', 'bdnews24', 'be', 'beautiful', 'beauty', 'before', 'behind', 'best', 'bestblackhatforum', 'better', 'bi', 'bien', 'big', 'bigcinema', 'bigtruckstopseek', 'bimabn', 'bin', 'bjpgqsc', 'black', 'bleacherreport', 'blog', 'blog_post_id', 'blogspot', 'bm', 'bm1', 'bm2', 'bmatch', 'bmw', 'bo', 'bo_table', 'board', 'board_idx', 'book', 'books', 'bottom', 'box', 'boys', 'br', 'brigade', 'brrip', 'business', 'but', 'buy_now', 'buzzfil', 'by', 'c', 'cPath', 'c_id', 'ca', 'cac', 'cache', 'caixa', 'call', 'cam', 'camera', 'can', 'cant', 'car', 'cars', 'case', 'cat', 'category', 'cau', 'cc', 'cdsb', 'censor', 'center', 'central', 'cfm', 'cgi', 'ch', 'chCatId', 'champions', 'change', 'channelId', 'chat', 'check', 'cheezburger', 'chi', 'chinacxyy', 'chinesevie', 'chl', 'cho', 'chuyen', 'cid', 'citicard', 'city', 'cl', 'class', 'classid', 'classified', 'click', 'clk', 'closure', 'cmd', 'cn', 'cntKey', 'cntTypeId', 'co', 'code', 'codecanyon', 'college', 'com', 'com_content', 'comcast', 'comicbook', 'comments', 'commonFile', 'communications', 'community', 'complete', 'component', 'components', 'con', 'config', 'confirm', 'cong', 'constraint', 'contact', 'contatta', 'content', 'cool', 'core', 'correios', 'could', 'country', 'cpage', 'cranfield', 'create', 'credit', 'cross', 'cs', 'csp', 'css', 'ctype', 'cua', 'cung', 'cuoi', 'custom', 'customer', 'd', 'd0b6dd74a8394345128e20d296cb1c5e', 'd3', 'da', 'dai', 'daily', 'dam', 'dan', 'dang', 'dantri', 'dao', 'dap', 'data', 'date', 'dau', 'day', 'db', 'dbid', 'dc', 'dddgaming', 'de', 'dead', 'deadspin', 'decor', 'default', 'den', 'depositphotos', 'design', 'desktop', 'detail', 'development', 'di', 'diem', 'dien', 'different', 'digg', 'digital', 'diply', 'directory', 'dispNo', 'disp_no', 'dispatch', 'displaycorner', 'distractify', 'distrimarsanitarios', 'div', 'divulgacao', 'djcatalog2', 'dl', 'dll', 'do', 'doc', 'docs', 'does', 'doi', 'dokuwiki', 'domain', 'dong', 'dont', 'dot', 'douga', 'down', 'download', 'downloads_details', 'drive', 'dropbox', 'dstk', 'du', 'duc', 'dung', 'duoc', 'duong', 'e', 'ead', 'ec', 'ecnavi', 'edu', 'eksisozluk', 'elcomercio', 'electrocasnice', 'electronice', 'elektronik', 'elitedaily', 'em', 'email', 'emgn', 'empires', 'empresa', 'en', 'english', 'enterprise', 'entertainment', 'eor', 'episode', 'ero', 'es', 'espn', 'ettv', 'eu', 'ever', 'every', 'evt19', 'exe', 'extratorrent', 'eztv', 'f', 'f1f16e8c6142fab8553f2b9cfe045961', 'f554ab9b50d41d8cd98fra3', 'facebook', 'facet', 'family', 'fan', 'fanpage', 'fashion', 'fazenda', 'fb', 'fbn', 'fbtrack', 'feature', 'features', 'fh', 'file', 'files', 'film', 'finda', 'first', 'firstrec', 'fishki', 'five', 'flash', 'folder', 'font', 'food', 'football', 'for', 'form', 'formkey', 'fortune', 'forum', 'fr', 'free', 'free1', 'free2', 'free3', 'from', 'ft', 'fucked', 'full', 'full_screen_preview', 'g', 'g2_GALLERYSID', 'g2_highlightId', 'g2_imageViewsIndex', 'g2_itemId', 'g2_return', 'g2_returnName', 'g2_view', 'gadget', 'galaxy', 'galleries', 'gallery', 'gallery2', 'game', 'game_cn', 'games', 'gawker', 'gay', 'gb', 'general', 'genius', 'get', 'getpocket', 'gets', 'gia', 'gid', 'gioi', 'girl', 'girlfriend', 'girls', 'gives', 'gizmodo', 'go', 'golfsource', 'good', 'google', 'gov', 'government', 'governo', 'gp', 'gpt', 'gr', 'grand', 'grantland', 'grasslandhotel', 'gtm', 'guardian', 'guest', 'guide', 'h', 'ha', 'haber', 'hai', 'haishundl', 'hang', 'has', 'have', 'hc360', 'hdfcbank', 'hdtv', 'he', 'header', 'header_text', 'help', 'her', 'here', 'heritage', 'hero', 'hi', 'highlight', 'hinh', 'his', 'hit', 'hitposition', 'ho', 'hoa', 'hobi', 'hoc', 'hoi', 'hollywood', 'hollywoodlife', 'home', 'home_feed', 'hon', 'hook', 'hot', 'hotlinegsm', 'house', 'how', 'href', 'htm', 'html', 'http', 'https', 'hub', 'hubpages', 'human', 'i', 'i2', 'ic', 'icicibank', 'icon', 'ictguides', 'id', 'id3', 'id8', 'identify', 'ie', 'if', 'il', 'ilovespeedbumps', 'im', 'image', 'images', 'img', 'in', 'includes', 'index', 'india', 'indian', 'indianexpress', 'info', 'information', 'inner_website', 'insider', 'instagram', 'international', 'internet', 'interpark', 'interval', 'interview', 'into', 'io9', 'iphone', 'ipl', 'ir', 'irecommend', 'is', 'isikpandizot', 'islam', 'isqm', 'it', 'ita', 'item', 'itemAreaList', 'itemCateList', 'itementry3', 'items', 'its', 'j', 'jalopnik', 'jay', 'jezebel', 'jin0044', 'john', 'jokes', 'jp', 'jpg', 'jquery', 'js', 'js182', 'jsessionid', 'jsp', 'just', 'k', 'kakaku', 'katproxy', 'kenh14', 'key', 'keyword', 'khi', 'khong', 'kickass', 'kids', 'kienthuc', 'kim', 'know', 'kotaku', 'ky', 'l', 'l7', 'l_code', 'la', 'lai', 'lam', 'lan', 'lang', 'language', 'langx', 'last', 'latest', 'launches', 'layer', 'le', 'league', 'left', 'levsign', 'li', 'lib', 'license', 'life', 'lifehacker', 'lifestyle', 'like', 'limit', 'line', 'list', 'listing', 'live', 'livehall', 'lm2', 'lmname', 'lo', 'local', 'location', 'login', 'loginform', 'lol', 'look', 'love', 'lovettz', 'ly', 'm', 'm2u', 'ma', 'mac', 'mad', 'made', 'magazine', 'mail', 'main', 'make', 'man', 'mang', 'manufacturers_id', 'mark', 'marketing', 'marvel', 'mashable', 'mashhuda', 'masini', 'mat', 'may', 'maybank2u', 'mbb', 'mbb_info', 'mdiv', 'me', 'media', 'mediaset', 'medium', 'member', 'memberoffers', 'men', 'method', 'methodId', 'methods', 'metro', 'mflag', 'mic', 'microsoft', 'mid', 'middle', 'min', 'mirtesen', 'misc', 'mkv', 'mo', 'mobile', 'mod', 'moda', 'mode', 'module', 'modules', 'moi', 'momoshop', 'montecarlo', 'month', 'more', 'most', 'moto', 'motthegioi', 'movie', 'moviepilot', 'movies', 'mp3', 'mp4', 'mpp', 'music', 'musik', 'mx', 'my', 'myappleid', 'mylust', 'mysite', 'myspace', 'n', 'na', 'nam', 'name', 'name1st', 'nao', 'nav', 'navsection', 'naylorantiques', 'need', 'nepal', 'nesn', 'net', 'network', 'never', 'new', 'newdetails', 'news', 'newscodejs', 'newstudy', 'next', 'ng', 'ngay', 'nguoi', 'nguyentandung', 'nha', 'nhan', 'nhat', 'nhung', 'nics', 'night', 'nl', 'no', 'node', 'noi', 'not', 'noticia', 'noticias', 'now', 'nu', 'numreq', 'nuoc', 'nymag', 'nypost', 'o', 'oembed', 'of', 'oferta', 'off', 'offer', 'offers', 'ok', 'olahraga', 'old', 'olx', 'on', 'one', 'onedio', 'online', 'only', 'op', 'open', 'operation', 'option', 'or', 'order', 'org', 'os', 'osCsid', 'osobowe', 'other', 'others', 'otomoto', 'our', 'out', 'over', 'p', 'p_id', 'p_sed', 'p_sem', 'p_sey', 'p_ssd', 'p_ssm', 'p_ssy', 'page', 'page_site', 'pages', 'para', 'pastehtml', 'patch', 'path', 'paypal', 'paytm', 'pc', 'pe', 'people', 'perezhilton', 'period', 'pers', 'persianblog', 'personal', 'personalDetail04', 'pf_rd_i', 'pf_rd_m', 'pf_rd_p', 'pf_rd_r', 'pf_rd_s', 'pf_rd_t', 'phan', 'phone', 'photo', 'photo_news', 'photos', 'php', 'phpBB2', 'phu', 'pic', 'pid', 'pikabu', 'pitchfork', 'pjhku', 'pk', 'pl', 'plarium', 'platform', 'play', 'player', 'pls', 'plugin', 'plugins', 'plus', 'pn', 'pocket', 'police', 'pornsharing', 'portal', 'post', 'posting', 'posts', 'pp', 'prdNo', 'press', 'prettyPhoto', 'prev', 'price', 'pricedown', 'printphoto', 'privatbank', 'pro', 'pro_control', 'product', 'product_info', 'product_reviews', 'products', 'products_id', 'program', 'programId', 'project', 'projects', 'promo', 'proper', 'prospectus', 'proxinfo', 'pub', 'public', 'pubyun', 'pussy', 'putlocker', 'q', 'qinpengqipaiguanfangxiazai', 'qua', 'quan', 'que', 'questions', 'qui', 'quoc', 'quote', 'qz', 'r', 'radio', 'rand', 'read', 'readmore_perezhilton', 'real', 'really', 'recom', 'recommended', 'rectype', 'red', 'redir', 'redirect', 'ref', 'register', 'related', 'relevant', 'remax', 'report', 'responsive', 'results', 'rh', 'richmedia', 'right', 'ringring', 'ro', 'road', 'route', 'rs', 'rsid', 'rt', 'ru', 'rugs', 'rules', 'run', 'runescape', 's', 'sa', 'sales', 'salisburyjournal', 'sample', 'samsung', 'san', 'sao', 'says', 'sc', 'schertzauto', 'science', 'scr', 'scripts', 'se', 'search', 'search_organic', 'searchengineland', 'searchuser', 'season', 'secret', 'section', 'secure', 'security', 'see', 'seekingalpha', 'seng', 'seria', 'series', 'server', 'serverfault', 'service', 'services', 'set', 'set_id', 'sex', 'sexy', 'sfglobe', 'sflag', 'sh_dwh_token', 'share', 'she', 'shop', 'shopNo', 'should', 'show', 'show_item', 'showhot', 'showphotos', 'showphotos2', 'shows', 'showtj', 'shtml', 'si', 'sid', 'sign', 'signin', 'signup', 'simple_click', 'sinh', 'site', 'siteId', 'siteconfig', 'sites', 'skin', 'skip', 'skyrock', 'slashdot', 'sm', 'smp', 'so', 'sobre', 'social', 'solucoes', 'solutions', 'some', 'son', 'song', 'sort', 'sortby', 'source', 'sourceLink', 'sourceforge', 'soydg', 'space', 'spacedesc', 'sparta', 'sport', 'sports', 'spreadsheet', 'spreadsheets', 'spring', 'squarespace', 'src', 'srv', 'ssl', 'stackoverflow', 'star', 'stars', 'start', 'startup', 'state', 'static', 'status', 'steamcommunity', 'still', 'stock', 'store', 'stories', 'stormfall', 'story', 'strategy', 'string', 'style', 'su', 'sua', 'subcats2', 'submission', 'submit', 'subs', 'summary', 'super', 'superuser', 'support', 'supporto', 'syosetu', 'system', 't', 'tablet', 'tag', 'tags', 'tai', 'take', 'takes', 'tam', 'task', 'tcolor', 'te', 'team', 'tech', 'techcrunch', 'teespring', 'template', 'templates', 'term', 'text', 'than', 'thanh', 'that', 'the', 'theargus', 'thechive', 'their', 'them', 'theme', 'themeforest', 'themes', 'thenextweb', 'thenorthernecho', 'there', 'thereefrestaurant', 'these', 'they', 'thi', 'this', 'thisischeshire', 'thisisoxfordshire', 'thisisyork', 'thong', 'thrones', 'thu', 'thuc', 'tien', 'time', 'tin', 'tinh', 'tinnhanh360', 'title', 'tj', 'to', 'tobogo', 'tom', 'too', 'tool', 'tools', 'tooth', 'top', 'topics', 'torcache', 'torrent', 'torrent_download', 'torrentdn', 'tours', 'tp', 'tpl', 'trailer', 'training', 'tran', 'trang', 'travel', 'tren', 'triangle', 'trong', 'true', 'trung', 'truong', 'tseed', 'tu', 'tuc', 'tune', 'tunein', 'tuong', 'tuyen', 'tv', 'tw', 'twitter', 'two', 'type', 'u', 'ua', 'ubuntu', 'udn', 'ui', 'uid', 'uk', 'ukvisit', 'ultimas', 'un', 'unterhaltung', 'uol', 'up', 'update', 'updated2', 'upgrade', 'upload', 'uploads', 'uproxx', 'url', 'us', 'use', 'user', 'user_id', 'userid', 'users', 'using', 'utm_campaign', 'utm_medium', 'utm_source', 'v', 'v10', 'va', 'van', 'vao', 'variety', 've', 'vector', 'venturebeat', 'ver', 'ver1', 'ver16', 'ver31', 'verify8', 'verify9', 'version', 'vi', 'video', 'video_news', 'videos', 'vien', 'viet', 'view', 'vilagnomad', 'vision', 'vn', 'vo', 'voce', 'voi', 'vong', 'vs', 'vu', 'w', 'wall', 'wam', 'wangqiao365', 'want', 'wants', 'wap', 'war', 'wars', 'was', 'watch', 'way', 'we', 'web', 'webcam', 'weblogin', 'webobjects', 'webscr', 'week', 'what', 'when', 'where', 'white', 'who', 'why', 'wife', 'wiki', 'wikiwiki', 'will', 'win', 'windows', 'with', 'woa', 'woa1', 'woman', 'women', 'wordpress', 'work', 'working', 'world', 'wp', 'wr_id', 'ws', 'wu8188', 'www', 'x', 'x264', 'xa', 'xe', 'xhamster', 'xj9599', 'xpj5988', 'xvid', 'xvideo', 'y', 'yahoo', 'year', 'yes', 'yify', 'yiyuanxiangguan', 'york', 'yorkpress', 'you', 'young', 'your', 'yourlust', 'z', 'za', 'zh', 'zozo', 'zzly', 'عن', 'في', 'كتاب', 'معلومات', 'アニメ', 'ヌける', '巨乳', '树叶粘贴画', '树叶贴画作品', '素人', '衍纸贴画图片欣赏']
# Print the most important features based on idf since there is no way to access them based on tf-idf
word2tfidf = list(zip(vectorizer.get_feature_names(), vectorizer.idf_))
word2tfidf = sorted(word2tfidf, key=lambda l:l[1], reverse=True)
print(len(word2tfidf))
df = pd.DataFrame(word2tfidf, columns=['Term', 'IDF'])
# df = df.sort_values(by=['IDF', 'Term'], ascending=False)
display(ends(df, n=20))1378

## Multinomial Bayes Classifier
We are going to train this classifier once, on the big ISCX2016 dataset, and later test its performance on predicting for other distinct datasets, without retraining or finetuning to see how well it can generalize._____no_output_____

mnb = MultinomialNB()
mnb.fit(X_train, y_train) #training our model

predicted = mnb.predict(X_test)

# print("Bad samples: %d" % badCount)
# print("Good samples: %d" % goodCount)
print("------------")
print("Accuracy: %f" % mnb.score(X_test, y_test))
print('Naive Bayes Train Accuracy = ',metrics.accuracy_score(y_train,mnb.predict(X_train)))
print('Naive Bayes Test Accuracy = ',metrics.accuracy_score(y_test,mnb.predict(X_test)))
print("Precision: %f" % metrics.precision_score(y_test, predicted, average='weighted'))
print("Recall: %f" % metrics.recall_score(y_test, predicted, average='weighted'))
print("F1-Score: %f" % metrics.f1_score(y_test, predicted, average='weighted'))
print("")
print(metrics.precision_recall_fscore_support(y_test, predicted, average='weighted'))

print('Report : ')
print(metrics.classification_report(y_test, predicted, target_names=list(sorted(mapping_label.values()))))

cm = metrics.confusion_matrix(y_test, predicted)
print(cm)

sns.set_style("darkgrid")

x_axis_labels = list(sorted(mapping_label.values()))
y_axis_labels = list(sorted(mapping_label.values()))
plt.figure(figsize=(9,9))
p = sns.heatmap(cm, annot=True, fmt="g", linewidths=.5, square = True, cmap = 'Blues_r',
                xticklabels=x_axis_labels, yticklabels=y_axis_labels, annot_kws={"size":15},
                    cbar=True,cbar_kws={"shrink": .80});
plt.ylabel('Actual label', size=15);
plt.xlabel('Predicted label', size=15);
p.set_xticklabels(x_axis_labels, size=13)
p.set_yticklabels(y_axis_labels, size=13)
all_sample_title = 'Accuracy Score: {0:.2}'.format(mnb.score(X_test, y_test))
plt.title(all_sample_title, size = 15);------------
Accuracy: 0.961979
Naive Bayes Train Accuracy =  0.9656480636868728
Naive Bayes Test Accuracy =  0.9619793934116965
Precision: 0.961748
Recall: 0.961979
F1-Score: 0.961626

(0.9617480045682395, 0.9619793934116965, 0.9616258487604782, None)
Report :
              precision    recall  f1-score   support

      benign       0.96      0.99      0.98     10593
     malware       0.95      0.92      0.94      3535
    phishing       0.93      0.89      0.91      2914
        spam       0.99      0.98      0.99      3631

    accuracy                           0.96     20673
   macro avg       0.96      0.94      0.95     20673
weighted avg       0.96      0.96      0.96     20673

[[10488    56    37    12]
 [  158  3245   125     7]
 [  230    97  2580     7]
 [   20     4    33  3574]]

# Individual Sample Testing_____no_output_____

# Sources:
# https://github.com/uppusaikiran/Malicious-URL-Detector/blob/master/malicious_url_detector/data/final_training_set.csv
# https://www.phishtank.com/phish_archive.php?__cf_chl_captcha_tk__=877785b9893dcc1f35feea1297996cc31c9da583-1616481681-0-AbOm2eDPPQpB1N6_kBXGIfhKm2qwcTZdPU-rYsbh6Cu2pisvGHNtsZdDEm3QxREHRmhTRJDS0WOTdX7weeK06svvwclD41QM2-mrWOL4-gH6dag-ulqWvYIrqnCbvz8fTqc_5gyzNh7AjDqIa54P3TDmXFTDz1IS4T4ERpUqcwXqEPg1tIHiSYbGrocOC-LLK-tp9GfCR2J9Hnh50bDvqjTTmrKVy_uC201inv7LHzxEw58ZgIotLbdM3UB-kI7sYdOIfGf7gqNi6-AL0z3psDqOiT6lRtcbqHo0Ly7G_28JlMpoGGvhdkKvHAGNpJU5dWdJ_o-g-unQ9SjHPNdU6pd7kmMtjLY0yR4T9JfXq0FKHLfoHvWXxLGzhBoEVWc7_fe3Ii56XV7kcwTW0njfsYYR1Xk6wL2ZIkQ_guKD-WXxA9_ZDgh9eZIktyy9tcYF3HDYGkDCvBmDGs83O7TH2-ydUA1peqpEmssQpbMFHWSJ5dxh1UIxfpmlRE-zGD5IWkMSoC0tYoZCZC_l5X3ufX_unNn_pItHC7ku-CgctCFWxyiDEygrhayno0ouPeY_AkXZfWP9f3PJHvTmmgKAOWL6svnHK6fF8_SHCsZe_jp5pwMorhEe-lwGvypaWZEu1w

test_sample = [
    [0, 'https://google.com'],
    [0, 'https://rit.edu'],
    [0, 'https://pythonexamples.org/pandas-dataframe-sort-by-index/'],
    [0, 'https://amazon.com/Python-Data-Science-Handbook-Essential/dp/1491912057'],
    [0, 'https://amazon.com/Practical-Malware-Analysis-Hands-Dissecting/dp/1593272901'],
    [0, 'https://stackoverflow.com/questions/56717542/how-to-make-prediction-with-single-sample-in-sklearn-model-predict'],
    [2, 'https://anzhuoc.ddns.net/index.html'],
    [2, 'https://retuken.retukenssw.buzz/'],
    [2, 'https://www.ifh.org.ar/tr/MARKET/'],
    [2, 'http://www.ulsanbiz.co.kr/wp-content/plugins/updraftplus/languages/confirm.php'],
    [2, 'https://blueebadge.cf/'],
    [1, 'http://03.pcsoft.xiazai47.com/TZGeiPaiShui2016GuoQiBuDing.rar'],
    [1, 'http://03za.ru/russradio.exe'],
    [1, r'http://10021.url.246546.com/down/%25E7%2588%25B1%25E5%25A5%2587%25E8%2589%25BAvip%25E7%25A0%25B4%25E8%25A7%25A3%25E7%2589%2588%2540292_236801.exe'],
    [9, r'/examples/jsp/cal/dcforum.cgi?az=list&forum=../../../../../../../etc/passwd\x00'],
    [9, r'/cgi-bin/error.php?selected_theme='],
    [0, r'/121006_dakotacwpressconf/'],
    [0, r'/103886/'],
    [9, r"/top.php?stuff='uname >q36497765 #"]
    ]

test_sample_list = [i[1] for i in test_sample]
# print(test_sample_list)
# print(len(test_sample_list))

test_sample_list = vectorizer.transform(test_sample_list)
predicted = mnb.predict(test_sample_list)
all_results = []
for idx, pr in enumerate(predicted):
    if test_sample[idx][0] == 9:
        yt = 'malicious'
    else:
        yt = mapping_label[test_sample[idx][0]]

    yp = mapping_label[pr]

    # if yt == yp:
    #     print("y_true:{}, y_pred:{}".format(yt, yp))
    # else:
    #     print("--> y_true:{}, y_pred:{}".format(yt, yp))

    if test_sample[idx][0] == 9:
        # as long as it is not classified as benign, it's fine
        all_results.append([test_sample[idx][1], yt, yp, yt == 'benign'])
    else:
        all_results.append([test_sample[idx][1], yt, yp, yt != yp])

results_df = pd.DataFrame(all_results, columns=['URL', 'y_true', 'y_pred', 'misclassified']).reset_index()
results_df = results_df.set_index(['misclassified','index']).sort_index()
display(results_df)_____no_output_____

## Phishing Dataset_____no_output_____

# Load the dataset
all_phishing = []
with open('Datasets\PhishingDatabase\phishing-links-ACTIVE-today.txt', encoding='utf-8') as f:
    for line in f:
        all_phishing.append([2, line])
with open('Datasets\PhishingDatabase\phishing-links-ACTIVE-large.txt', encoding='utf-8') as f:
    for line in f:
        all_phishing.append([2, line])

print("Done")Done
from tqdm import tqdm

all_phishing_list = [i[1] for i in all_phishing]
all_phishing_list = vectorizer.transform(all_phishing_list)
predicted = mnb.predict(all_phishing_list)
all_results = []
for idx, pr in tqdm(enumerate(predicted)):
    if all_phishing[idx][0] == 9:
        yt = 'malicious'
    else:
        yt = mapping_label[all_phishing[idx][0]]

    yp = mapping_label[pr]

    if all_phishing[idx][0] == 9:
        # as long as it is not classified as benign, it's fine
        all_results.append([all_phishing[idx][1], yt, yp, yt == 'benign'])
    else:
        all_results.append([all_phishing[idx][1], yt, yp, yt != yp])

results_df = pd.DataFrame(all_results, columns=['URL', 'y_true', 'y_pred', 'misclassified']).reset_index()
results_df = results_df.set_index(['misclassified','index']).sort_index()
results_df['label'] = results_df['y_true'].map(mapping_label_r)
display(results_df)38758it [00:00, 637089.08it/s]
# Get percentage of what the classifier got right and wrong
display(results_df.groupby(level=[0]).size())
display(results_df.index.get_level_values(0).value_counts(normalize=True).sort_values(ascending=False) * 100)_____no_output_____df_test = results_df
test_sample_list = vectorizer.transform(results_df['URL'])
predicted_multi = mnb.predict(test_sample_list)
display(predicted_multi)

print("------------")
print(metrics.precision_recall_fscore_support(df_test['label'], predicted_multi, average='weighted'))

print('Report : ')
print(metrics.classification_report(df_test['label'], predicted_multi, target_names=list(sorted(mapping_label.values()))))

cm = metrics.confusion_matrix(df_test['label'], predicted_multi)
print(cm)

sns.set_style("darkgrid")

x_axis_labels = list(sorted(mapping_label.values()))
y_axis_labels = list(sorted(mapping_label.values()))
plt.figure(figsize=(9,9))
p = sns.heatmap(cm, annot=True, fmt="g", linewidths=.5, square = True, cmap = 'Blues_r',
                xticklabels=x_axis_labels, yticklabels=y_axis_labels, annot_kws={"size":15},
                    cbar=True,cbar_kws={"shrink": .80});
plt.ylabel('Actual label', size=15);
plt.xlabel('Predicted label', size=15);
p.set_xticklabels(x_axis_labels, size=13)
p.set_yticklabels(y_axis_labels, size=13)
all_sample_title = 'Accuracy Score: {0:.2}'.format(metrics.accuracy_score(df_test['label'], predicted_multi))
plt.title(all_sample_title, size = 15);_____no_output_____

## FWAF Malicious Queries Dataset
https://github.com/faizann24/Fwaf-Machine-Learning-driven-Web-Application-Firewall_____no_output_____

# Dataset paths
dataset_base_path = os.path.join(datasets_path, "FWAF")

queries_raw = []
for fn in glob.glob(os.path.join(dataset_base_path, "*.txt")):
    queries_raw.append(os.path.normpath(fn))

# Make sure files get added in specific order
queries_raw.sort(reverse=True)

mapping_label_binary = {0:"benign", 9:"malicious"}
mlbl = list(mapping_label_binary.keys())

print(queries_raw)
print(mapping_label_binary)

raw_urls = []
for idx, fn in enumerate(queries_raw):
    with open(fn,'r', encoding='utf-8') as f:
        data = f.readlines()
        data = [(str(urllib.parse.unquote(d)), mlbl[idx]) for d in data]
        raw_urls.extend(data)

# Create dataframe with columns for URL and label
urldata = pd.DataFrame(raw_urls, columns=['url', 'label'])
urldata['url'] = urldata['url'].apply(lambda x: x.strip("\n"))
# Convert gt column according to mapping
# urldata['label'] = urldata['label'].apply(lambda x: mapping_label_binary[x])

print(urldata.info())
display(urldata.groupby('label').count())
display(ends(urldata))['D:\\dev\\CyberML\\Datasets\\FWAF\\goodqueries.txt', 'D:\\dev\\CyberML\\Datasets\\FWAF\\badqueries.txt']
{0: 'benign', 9: 'malicious'}
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1342657 entries, 0 to 1342656
Data columns (total 2 columns):
url      1342657 non-null object
label    1342657 non-null int64
dtypes: int64(1), object(1)
memory usage: 20.5+ MB
None

### Experiment: Rebalance classes and measure classifier performance_____no_output_____

from sklearn.utils import resample

urldata.groupby('label').count()['url'].values.tolist()

# Separate majority and minority classes
df_majority = urldata[urldata['label'] == 0]
df_minority = urldata[urldata['label'] == 9]

min_samples = min(urldata.groupby('label').count()['url'].values.tolist())

# Downsample majority class
df_majority_downsampled = resample(df_majority,
                                 replace=False,    # sample without replacement
                                 n_samples=min_samples, # to match minority class
                                 random_state=123) # reproducible results

# Combine minority class with downsampled majority class
df_downsampled = pd.concat([df_majority_downsampled, df_minority])

# Display new class counts
display(urldata.groupby('label').count())
display(df_downsampled.groupby('label').count())_____no_output_____# df_test = urldata
df_test = df_downsampled
test_sample_list = vectorizer.transform(df_test['url'])
predicted_multi = mnb.predict(test_sample_list)
predicted = [0 if x == 0 else 9 for x in predicted_multi]

print("------------")
print(metrics.precision_recall_fscore_support(df_test['label'], predicted, average='weighted'))

print('Report : ')
print(metrics.classification_report(df_test['label'], predicted, target_names=list(sorted(mapping_label_binary.values()))))

cm = metrics.confusion_matrix(df_test['label'], predicted)
# cm = metrics.confusion_matrix(urldata['label'], predicted, normalize='all')
print(cm)

sns.set_style("darkgrid")

x_axis_labels = list(sorted(mapping_label_binary.values()))
y_axis_labels = list(sorted(mapping_label_binary.values()))
plt.figure(figsize=(9,9))
p = sns.heatmap(cm, annot=True, fmt="g", linewidths=.5, square = True, cmap = 'Blues_r',
                xticklabels=x_axis_labels, yticklabels=y_axis_labels, annot_kws={"size":15},
                    cbar=True,cbar_kws={"shrink": .80});
plt.ylabel('Actual label', size=15);
plt.xlabel('Predicted label', size=15);
p.set_xticklabels(x_axis_labels, size=13)
p.set_yticklabels(y_axis_labels, size=13)
all_sample_title = 'Accuracy Score: {0:.2}'.format(metrics.accuracy_score(df_test['label'], predicted))
plt.title(all_sample_title, size = 15);------------
(0.7516480373545104, 0.7082658022690438, 0.695126317967531, None)
Report :
              precision    recall  f1-score   support

      benign       0.65      0.92      0.76     48126
   malicious       0.86      0.50      0.63     48126

    accuracy                           0.71     96252
   macro avg       0.75      0.71      0.70     96252
weighted avg       0.75      0.71      0.70     96252

[[44077  4049]
 [24031 24095]]

## Phishing Dataset
https://github.com/mitchellkrogza/Phishing.Database_____no_output_____

# Dataset paths
dataset_base_path = os.path.join(datasets_path, "PhishingDatabase")

queries_raw = []
for fn in glob.glob(os.path.join(dataset_base_path, "*.csv")):
    queries_raw.append(os.path.normpath(fn))

# Make sure files get added in specific order
queries_raw.sort(reverse=True)

mapping_label_binary = {0:"benign", 2:"phishing"}
mlbl = list(mapping_label_binary.keys())

print(queries_raw)
print(mapping_label_binary)

h = 'phish_id,url,phish_detail_url,submission_time,verified,verification_time,online,target'.split(",")
urldata = pd.read_csv(queries_raw[0], names=h)
# Remove headers in last line
urldata = urldata.iloc[:-1]
urldata.dropna(subset=['url'], inplace=True)
# Remove rows that don't start with http
urldata = urldata[urldata['url'].astype(str).str.startswith('http')]
urldata = urldata.filter(['url'])
urldata['label'] = 2

# Create dataframe with columns for URL and label
# urldata = pd.DataFrame(urldata, columns=['url', 'label'])
# urldata['url'] = urldata['url'].apply(lambda x: x.strip("\n"))
# Convert gt column according to mapping
# urldata['label'] = urldata['label'].apply(lambda x: mapping_label_binary[x])

print(urldata.info())
display(urldata.groupby('label').count())
display(ends(urldata))['D:\\dev\\CyberML\\Datasets\\PhishingDatabase\\ALL-phishing-links.csv']
{0: 'benign', 2: 'phishing'}
<class 'pandas.core.frame.DataFrame'>
Int64Index: 10286 entries, 0 to 10285
Data columns (total 2 columns):
url      10286 non-null object
label    10286 non-null int64
dtypes: int64(1), object(1)
memory usage: 241.1+ KB
None
mapping_label_tri = {0:"benign", 2: 'phishing', 9:"malicious"}

mlbl = [value for key,value in sorted(mapping_label_tri.items(), reverse=False)]

df_test = urldata
test_sample_list = vectorizer.transform(df_test['url'])
predicted_multi = mnb.predict(test_sample_list)

(unique, counts) = np.unique(predicted_multi, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies)
print("="*10)

mask = (predicted_multi != 0) & (predicted_multi != 2)
predicted_multi[mask] = 9
predicted = [x for x in predicted_multi]

(unique, counts) = np.unique(predicted, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies)

print("------------")
display(urldata.groupby('label').count())
print(metrics.precision_recall_fscore_support(df_test['label'], predicted, average='weighted'))

print('Report : ')
print(metrics.classification_report(df_test['label'], predicted, target_names=mlbl))

cm = metrics.confusion_matrix(df_test['label'], predicted)
# cm = metrics.confusion_matrix(urldata['label'], predicted, normalize='all')
print(cm)

sns.set_style("darkgrid")

x_axis_labels = mlbl
y_axis_labels = mlbl
plt.figure(figsize=(9,9))
p = sns.heatmap(cm, annot=True, fmt="g", linewidths=.5, square = True, cmap = 'Blues_r',
                xticklabels=x_axis_labels, yticklabels=y_axis_labels, annot_kws={"size":15},
                    cbar=True,cbar_kws={"shrink": .80});
plt.ylabel('Actual label', size=15);
plt.xlabel('Predicted label', size=15);
p.set_xticklabels(x_axis_labels, size=13)
p.set_yticklabels(y_axis_labels, size=13)
all_sample_title = 'Accuracy Score: {0:.2}'.format(metrics.accuracy_score(df_test['label'], predicted))
plt.title(all_sample_title, size = 15);[[   0 4820]
 [   1  427]
 [   2 4929]
 [   3  110]]
==========
[[   0 4820]
 [   2 4929]
 [   9  537]]
------------
# https://github.com/mitchellkrogza/Phishing.Database
# Dataset paths 2
dataset_base_path = os.path.join(datasets_path, "PhishingDatabase")

queries_raw = []
for fn in glob.glob(os.path.join(dataset_base_path, "*.txt")):
# for fn in glob.glob(os.path.join(dataset_base_path, "*today.txt")):
    queries_raw.append(os.path.normpath(fn))

# Make sure files get added in specific order
queries_raw.sort(reverse=True)

print(queries_raw)

raw_urls = []
for idx, fn in enumerate(queries_raw):
    with open(fn,'r', encoding='utf-8') as f:
        data = f.readlines()
        data = [(str(urllib.parse.unquote(d)), 2) for d in data]
        raw_urls.extend(data)

# Create dataframe with columns for URL and label
urldata = pd.DataFrame(raw_urls, columns=['url', 'label'])
urldata['url'] = urldata['url'].apply(lambda x: x.strip("\n"))

print(urldata.info())
display(urldata.groupby('label').count())
display(ends(urldata))['D:\\dev\\CyberML\\Datasets\\PhishingDatabase\\phishing-links-ACTIVE-today.txt', 'D:\\dev\\CyberML\\Datasets\\PhishingDatabase\\phishing-links-ACTIVE-large.txt']
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 38758 entries, 0 to 38757
Data columns (total 2 columns):
url      38758 non-null object
label    38758 non-null int64
dtypes: int64(1), object(1)
memory usage: 605.7+ KB
None
df_test = urldata
test_sample_list = vectorizer.transform(df_test['url'])
predicted_multi = mnb.predict(test_sample_list)

(unique, counts) = np.unique(predicted_multi, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies)
print("="*10)

mask = (predicted_multi != 0) & (predicted_multi != 2)

mapping_label_tri = {0:"benign", 2: 'phishing', 9:"malicious"}
predicted_multi[mask] = 9
# mapping_label_tri = {0:"benign", 2: 'malicious'}
# predicted_multi[mask] = 2

mlbl = [value for key,value in sorted(mapping_label_tri.items(), reverse=False)]

predicted = [x for x in predicted_multi]

(unique, counts) = np.unique(predicted, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies)

print("------------")
display(urldata.groupby('label').count())
print(metrics.precision_recall_fscore_support(df_test['label'], predicted, average='weighted'))

print('Report : ')
print(metrics.classification_report(df_test['label'], predicted, target_names=mlbl))

# cm = metrics.confusion_matrix(df_test['label'], predicted)
cm = metrics.confusion_matrix(urldata['label'], predicted, normalize='all')
print(cm)

sns.set_style("darkgrid")

x_axis_labels = mlbl
y_axis_labels = mlbl
plt.figure(figsize=(9,9))
p = sns.heatmap(cm, annot=True, fmt="g", linewidths=.5, square = True, cmap = 'Blues_r',
                xticklabels=x_axis_labels, yticklabels=y_axis_labels, annot_kws={"size":15},
                    cbar=True,cbar_kws={"shrink": .80});
plt.ylabel('Actual label', size=15);
plt.xlabel('Predicted label', size=15);
p.set_xticklabels(x_axis_labels, size=13)
p.set_yticklabels(y_axis_labels, size=13)
all_sample_title = 'Accuracy Score: {0:.2}'.format(metrics.accuracy_score(df_test['label'], predicted))
plt.title(all_sample_title, size = 15);[[    0 12537]
 [    1  2023]
 [    2 23812]
 [    3   386]]
==========
[[    0 12537]
 [    2 23812]
 [    9  2409]]
------------

# Visualization of ISCX2016 Dataset using t-SNE and PCA_____no_output_____

from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import matplotlib.cm as cm

dense_X_train = X_train.todense()

# PCA plot
pca = PCA(n_components=2).fit(dense_X_train)
print(pca.explained_variance_ratio_)
data2D_PCA = pca.transform(dense_X_train)

# t-SNE plot
embeddings = TSNE(
    n_components=2,
    init='pca',
    # perplexity=70,
    # perplexity=7,
    n_jobs=6, verbose=1
)
data2D_TSNE = embeddings.fit_transform(dense_X_train)
print("Final Embeddings KL Divergence: {}".format(embeddings.kl_divergence_))[0.04536378 0.02683723]
[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 48236 samples in 40.567s...
[t-SNE] Computed neighbors for 48236 samples in 974.069s...
[t-SNE] Computed conditional probabilities for sample 1000 / 48236
[t-SNE] Computed conditional probabilities for sample 2000 / 48236
[t-SNE] Computed conditional probabilities for sample 3000 / 48236
[t-SNE] Computed conditional probabilities for sample 4000 / 48236
[t-SNE] Computed conditional probabilities for sample 5000 / 48236
[t-SNE] Computed conditional probabilities for sample 6000 / 48236
[t-SNE] Computed conditional probabilities for sample 7000 / 48236
[t-SNE] Computed conditional probabilities for sample 8000 / 48236
[t-SNE] Computed conditional probabilities for sample 9000 / 48236
[t-SNE] Computed conditional probabilities for sample 10000 / 48236
[t-SNE] Computed conditional probabilities for sample 11000 / 48236
[t-SNE] Computed conditional probabilities for sample 12000 / 48236
[t-SNE] Computed conditional probabilities for sample 13000 / 48236
[t-SNE] Computed conditional probabilities for sample 14000 / 48236
[t-SNE] Computed conditional probabilities for sample 15000 / 48236
[t-SNE] Computed conditional probabilities for sample 16000 / 48236
[t-SNE] Computed conditional probabilities for sample 17000 / 48236
[t-SNE] Computed conditional probabilities for sample 18000 / 48236
[t-SNE] Computed conditional probabilities for sample 19000 / 48236
[t-SNE] Computed conditional probabilities for sample 20000 / 48236
[t-SNE] Computed conditional probabilities for sample 21000 / 48236
[t-SNE] Computed conditional probabilities for sample 22000 / 48236
[t-SNE] Computed conditional probabilities for sample 23000 / 48236
[t-SNE] Computed conditional probabilities for sample 24000 / 48236
[t-SNE] Computed conditional probabilities for sample 25000 / 48236
[t-SNE] Computed conditional probabilities for sample 26000 / 48236
[t-SNE] Computed conditional probabilities for sample 27000 / 48236
[t-SNE] Computed conditional probabilities for sample 28000 / 48236
[t-SNE] Computed conditional probabilities for sample 29000 / 48236
[t-SNE] Computed conditional probabilities for sample 30000 / 48236
[t-SNE] Computed conditional probabilities for sample 31000 / 48236
[t-SNE] Computed conditional probabilities for sample 32000 / 48236
[t-SNE] Computed conditional probabilities for sample 33000 / 48236
[t-SNE] Computed conditional probabilities for sample 34000 / 48236
[t-SNE] Computed conditional probabilities for sample 35000 / 48236
[t-SNE] Computed conditional probabilities for sample 36000 / 48236
[t-SNE] Computed conditional probabilities for sample 37000 / 48236
[t-SNE] Computed conditional probabilities for sample 38000 / 48236
[t-SNE] Computed conditional probabilities for sample 39000 / 48236
[t-SNE] Computed conditional probabilities for sample 40000 / 48236
[t-SNE] Computed conditional probabilities for sample 41000 / 48236
[t-SNE] Computed conditional probabilities for sample 42000 / 48236
[t-SNE] Computed conditional probabilities for sample 43000 / 48236
[t-SNE] Computed conditional probabilities for sample 44000 / 48236
[t-SNE] Computed conditional probabilities for sample 45000 / 48236
[t-SNE] Computed conditional probabilities for sample 46000 / 48236
[t-SNE] Computed conditional probabilities for sample 47000 / 48236
[t-SNE] Computed conditional probabilities for sample 48000 / 48236
[t-SNE] Computed conditional probabilities for sample 48236 / 48236
[t-SNE] Mean sigma: 0.000000
[t-SNE] KL divergence after 250 iterations with early exaggeration: 80.326408
[t-SNE] KL divergence after 1000 iterations: 1.248222
Final Embeddings KL Divergence: 1.2482221126556396
# data2D = data2D_PCA
data2D = data2D_TSNE

N = len(np.unique(y_train))
print(N,"classes")
labels = urldata['label'].unique().tolist()
print(labels)

# Combined plot
# https://stackoverflow.com/a/54944523/4017530
fig, ax = plt.subplots(figsize = (12, 8))
colors = cm.rainbow(np.linspace(0, 1, N))
for group, c in zip(np.unique(y_train), colors):
    ix = np.where(y_train == group)
    ax.scatter(data2D[ix,0], data2D[ix,1], color=c, label=group, alpha=0.2)
ax.legend(urldata['label'].unique())
xl = ax.get_xlim()
yl = ax.get_xlim()
plt.xlabel("t-SNE dimension 1")
plt.ylabel("t-SNE dimension 2")

plt.show()4 classes
[2]
# Separated Plot for better viewing
fig, axes = plt.subplots(N, 1, figsize = (12, 8*4))
for group, c, ax in zip(np.unique(y_train), colors, axes.flatten()):
    ix = np.where(y_train == group)
    ax.scatter(data2D[ix,0], data2D[ix,1], color=c, label=mapping_label[group], alpha=0.2)
    ax.legend()
    ax.set_ylim(yl)
    ax.set_xlim(xl)
#     plt.xlabel("dimension 1")
#     plt.ylabel("dimension 2")_____no_output_____