# Notebook from marcomussi/RecommenderSystemPolimi
Path: Jupyter notebook/Practice 9 - Collaborative Boosted FW.ipynb

# Recommender Systems 2018/19

### Practice 9 - Collaborative Boosted FW

### Underlying idea... the way to capture the feature importance from the user point of view, is to... use its point of view (e.g., ratings, collaborative similarity)_____no_output_____## A few info about feature weightign techniques

* Information retrieval methods (TF-IDF, BM25..) do not optimize a predictive model
* Embedding methods (FISM, UFSM, Factorization Machines) optimize everything at the same time, model and weights, but are difficult to train and sensitive to noise
* Wrapper methods (LFW, CFW) apply a two step approach approximating an already built collaborative model, more robust and easier to tune
_____no_output_____

from urllib.request import urlretrieve
import zipfile

# skip the download
#urlretrieve ("http://files.grouplens.org/datasets/movielens/ml-10m.zip", "data/Movielens_10M/movielens_10m.zip")
dataFile = zipfile.ZipFile("data/Movielens_10M/movielens_10m.zip")
URM_path = dataFile.extract("ml-10M100K/ratings.dat", path = "data/Movielens_10M")
URM_file = open(URM_path, 'r')

def rowSplit (rowString):

    split = rowString.split("::")
    split[3] = split[3].replace("\n","")

    split[0] = int(split[0])
    split[1] = int(split[1])
    split[2] = float(split[2])
    split[3] = int(split[3])

    result = tuple(split)

    return result

URM_file.seek(0)
URM_tuples = []

for line in URM_file:
   URM_tuples.append(rowSplit (line))

userList, itemList, ratingList, timestampList = zip(*URM_tuples)

userList = list(userList)
itemList = list(itemList)
ratingList = list(ratingList)
timestampList = list(timestampList)

import scipy.sparse as sps

URM_all = sps.coo_matrix((ratingList, (userList, itemList)))
URM_all = URM_all.tocsr()

_____no_output_____ICM_path = dataFile.extract("ml-10M100K/tags.dat", path = "data/Movielens_10M")
ICM_file = open(ICM_path, 'r')

def rowSplit (rowString):
    split = rowString.split("::")
    split[3] = split[3].replace("\n","")

    split[0] = int(split[0])
    split[1] = int(split[1])
    split[2] = str(split[2]) # tag is a string, not a float like the rating
    split[3] = int(split[3])

    result = tuple(split)

    return result

ICM_file.seek(0)
ICM_tuples = []

for line in ICM_file:
    ICM_tuples.append(rowSplit(line))

userList_icm, itemList_icm, tagList_icm, timestampList_icm = zip(*ICM_tuples)

userList_icm = list(userList_icm)
itemList_icm = list(itemList_icm)
tagList_icm = list(tagList_icm)
timestampList_icm = list(timestampList_icm)

userList_unique = list(set(userList_icm))
itemList_unique = list(set(itemList_icm))
tagList_unique = list(set(tagList_icm))

numUsers = len(userList_unique)
numItems = len(itemList_unique)
numTags = len(tagList_unique)

print ("Number of items\t {}, Number of users\t {}".format(numItems, numUsers))
print ("Number of tags\t {}, Number of item-tag tuples {}".format(numTags, len(tagList_icm)))

print("\nData example:")
print(userList_icm[0:10])
print(itemList_icm[0:10])
print(tagList_icm[0:10])

from sklearn import preprocessing
le = preprocessing.LabelEncoder()
le.fit(tagList_icm)

tagList_icm = le.transform(tagList_icm)

import numpy as np

ones = np.ones(len(tagList_icm))

ICM_all = sps.coo_matrix((ones, (itemList_icm, tagList_icm)), shape=(URM_all.shape[1], tagList_icm.max()+1))
ICM_all = ICM_all.tocsr()Number of items	 7601, Number of users	 4009
Number of tags	 16529, Number of item-tag tuples 95580

Data example:
[15, 20, 20, 20, 20, 20, 20, 20, 20, 20]
[4973, 1747, 1747, 2424, 2424, 2424, 2947, 2947, 3033, 3033]
['excellent!', 'politics', 'satire', 'chick flick 212', 'hanks', 'ryan', 'action', 'bond', 'spoof', 'star wars']

from Notebooks_utils.data_splitter import train_test_holdout

URM_train, URM_test = train_test_holdout(URM_all, train_perc = 0.8)
URM_train, URM_validation = train_test_holdout(URM_train, train_perc = 0.9)_____no_output_____

### To shorten the notebook let's use the parameters we computed the previous time (Warning: not very clean, the data split may be different, also, running BayesianSearch multiple times may result in a different solution)_____no_output_____

best_params_ItemKNNCF = {'topK': 700, 'shrink': 200, 'similarity': 'cosine', 'normalize': True}
best_params_ItemKNNCBF = {'topK': 500, 'shrink': 1000, 'similarity': 'cosine', 'normalize': True}_____no_output_____

### Step 1: Create the two models, collaborative and content based_____no_output_____

from KNN.ItemKNNCFRecommender import ItemKNNCFRecommender
from KNN.ItemKNNCBFRecommender import ItemKNNCBFRecommender

itemKNNCF = ItemKNNCFRecommender(URM_train)
itemKNNCF.fit(**best_params_ItemKNNCF)

itemKNNCBF = ItemKNNCBFRecommender(ICM_all, URM_train)
itemKNNCBF.fit(**best_params_ItemKNNCBF)Similarity column 65134 ( 100 % ), 2269.37 column/sec, elapsed time 0.48 min
Similarity column 65134 ( 100 % ), 2535.53 column/sec, elapsed time 0.43 min

### Question, how the two similarities differ?_____no_output_____

W_sparse_CF = itemKNNCF.W_sparse
W_sparse_CBF = itemKNNCBF.W_sparse_____no_output_____

## Let's plt the global similarity distribution _____no_output_____

W_sparse_CF_sorted = np.sort(W_sparse_CF.data.copy())
W_sparse_CBF_sorted = np.sort(W_sparse_CBF.data.copy())_____no_output_____import numpy as np
import matplotlib.pyplot as pyplot

pyplot.plot(W_sparse_CF_sorted, label = 'CF')
pyplot.plot(W_sparse_CBF_sorted, label = 'CBF')
pyplot.ylabel('Similarity cell ')
pyplot.xlabel('Similarity value')
pyplot.legend()
pyplot.show()_____no_output_____

## Let's now plot the delta for the common values_____no_output_____

# Get common structure
W_sparse_CF_structure = W_sparse_CF.copy()
W_sparse_CF_structure.data = np.ones_like(W_sparse_CF_structure.data)

W_sparse_CBF_structure = W_sparse_CBF.copy()
W_sparse_CBF_structure.data = np.ones_like(W_sparse_CBF_structure.data)

W_sparse_common = W_sparse_CF_structure.multiply(W_sparse_CBF_structure)

# Get values of both in common structure of CF
W_sparse_delta = W_sparse_CBF.copy().multiply(W_sparse_common)
W_sparse_delta -= W_sparse_CF.copy().multiply(W_sparse_common)_____no_output_____W_sparse_delta_sorted = np.sort(W_sparse_delta.data.copy())_____no_output_____pyplot.plot(W_sparse_delta_sorted, label = 'delta')
pyplot.ylabel('Similarity cell ')
pyplot.xlabel('Similarity value')
pyplot.legend()
pyplot.show()_____no_output_____# How many values we have in common?

print("W_sparse_CF has {:.2E} values and {:.2f} % in common with CBF".format(W_sparse_CF.nnz, W_sparse_common.nnz/W_sparse_CF.nnz*100))
print("W_sparse_CBF has {:.2E} values and {:.2f} % in common with CF".format(W_sparse_CBF.nnz, W_sparse_common.nnz/W_sparse_CBF.nnz*100))

print("W_sparse_delta has {:.2E} values".format(W_sparse_delta.nnz))
W_sparse_CF has 7.00E+06 values and 7.19 % in common with CBF
W_sparse_CBF has 2.26E+06 values and 22.24 % in common with CF
W_sparse_delta has 5.03E+05 values

## What do we see?
## The great majority of similarities are not common... the models are modeling two very different phenomena_____no_output_____# Idea:

## what if we were to weight features in such a way to approximate as well as possible the collaborative similarity?

### Basic model:  $S_{CBF} = ICM \cdot diag(feature\_weights) \cdot ICM^{T}$

### We want  $S_{CBF} = S_{CF}$

### Hence we wish to find the weights such that  $S_{CF} = ICM \cdot diag(feature\_weights) \cdot ICM^{T}$

### Now, we can rearrange the equations considering that we need only weights for features the two items have in common... this becomes a regression problem in form Ax = b. In our case A are the common features, x the weights and b the collaborative similarity
_____no_output_____### Since it is a regression problem, we can use an already available solver like linalg in scipy.sparse

### It takes as imput two data structures, one is sparse and the other is the target value_____no_output_____

W_sparse_delta = W_sparse_delta.tocoo()

item_index_1 = W_sparse_delta.row[666]
item_index_2 = W_sparse_delta.col[666]_____no_output_____print("Item indices are {} and {}".format(item_index_1, item_index_2))Item indices are 5 and 158
print("Collaborative similarity value is {}".format(W_sparse_CF[item_index_1, item_index_2]))
print("Content-based similarity value is {}".format(W_sparse_CBF[item_index_1, item_index_2]))Collaborative similarity value is 0.061169322580099106
Content-based similarity value is 0.005825620144605637

### Compute the common features_____no_output_____

common_features = ICM_all[item_index_1,:].multiply(ICM_all[item_index_2,:])
common_features_____no_output_____common_features.indices_____no_output_____

## These items have just a common feature, that is a common scenario

### Let's look for items having  a few features in common, say 5_____no_output_____

min_common_features = 5

for inner_index in range(len(W_sparse_delta.row)):

    item_index_1 = W_sparse_delta.row[inner_index]
    item_index_2 = W_sparse_delta.col[inner_index]

    common_features = ICM_all[item_index_1,:].multiply(ICM_all[item_index_2,:])

    if common_features.nnz >= min_common_features:

        # Bad programming, don't do this at home
        break

print("Item indices are {} and {}".format(item_index_1, item_index_2))

print("Collaborative similarity value is {}".format(W_sparse_CF[item_index_1, item_index_2]))
print("Content-based similarity value is {}".format(W_sparse_CBF[item_index_1, item_index_2]))

common_features.indicesItem indices are 1 and 50
Collaborative similarity value is 0.10406135022640228
Content-based similarity value is 0.003887253813445568
from scipy.sparse import linalg

loss_tolerance = 1e-6
iteration_limit = 50000
damp_coeff=0.0

W_sparse_CF_value = W_sparse_CF[item_index_1, item_index_2]_____no_output_____linalg_result = linalg.lsqr(common_features, W_sparse_CF_value, show = False, atol=loss_tolerance, btol=loss_tolerance,
                  iter_lim = iteration_limit, damp=damp_coeff)_____no_output_____feature_weights = linalg_result[0].copy()
feature_weights_____no_output_____feature_weights[common_features.indices]_____no_output_____

### Note that all weights will be zeros except for the features those items have in common, nothing can be learned if a feature is not common

### If a dataset has a very sparse ICM, this FW method will struggle_____no_output_____

new_CBF_value = common_features.dot(feature_weights)
new_CBF_value_____no_output_____print("Collaborative similarity value is {}".format(W_sparse_CF[item_index_1, item_index_2]))
print("Content-based similarity value is {}".format(W_sparse_CBF[item_index_1, item_index_2]))

print("Weighted Content-based similarity value is {}".format(new_CBF_value[0]))Collaborative similarity value is 0.10406135022640228
Content-based similarity value is 0.003887253813445568
Weighted Content-based similarity value is 0.10406135642893984

### Note that we learned the CF similarity amost exactly...

### Warning: 5 common features and 1 similarity value, more parameters than data points, overfitting!

### if we want to apply this method to the whole dataset we have to compute the common features of all items having a common similarity value. Whe already know who they are, thanks to W_sparse_delta_____no_output_____

from FW_Similarity.CFW_D_Similarity_Linalg import CFW_D_Similarity_Linalg_____no_output_____CFW_weithing = CFW_D_Similarity_Linalg(URM_train, ICM_all, W_sparse_CF)
CFW_weithing.fit()CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7078.34 column/sec, elapsed time 0.15 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 3.72E-04, nonzero cells 1576400
CFW_D_Similarity_Linalg: Content S structure has 391759 out of 1576400 ( 24.85%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 2.15E+04, average is: 5.48E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 8529.33 column/sec, elapsed time 0.13 min
from Base.Evaluation.Evaluator import SequentialEvaluator

evaluator_validation = SequentialEvaluator(URM_validation, cutoff_list=[5])
evaluator_test = SequentialEvaluator(URM_test, cutoff_list=[5])
_____no_output_____results_dict, _ = evaluator_test.evaluateRecommender(itemKNNCF)
results_dictSequentialEvaluator: Processed 26001 ( 37.21% ) in 30.60 seconds. Users per second: 850
SequentialEvaluator: Processed 52001 ( 74.42% ) in 60.66 seconds. Users per second: 857
SequentialEvaluator: Processed 69878 ( 100.00% ) in 81.15 seconds. Users per second: 861
results_dict, _ = evaluator_test.evaluateRecommender(itemKNNCBF)
results_dictSequentialEvaluator: Processed 26001 ( 37.21% ) in 30.10 seconds. Users per second: 864
SequentialEvaluator: Processed 53001 ( 75.85% ) in 60.49 seconds. Users per second: 876
SequentialEvaluator: Processed 69878 ( 100.00% ) in 78.78 seconds. Users per second: 887
results_dict, _ = evaluator_test.evaluateRecommender(CFW_weithing)
results_dictSequentialEvaluator: Processed 27001 ( 38.64% ) in 30.63 seconds. Users per second: 882
SequentialEvaluator: Processed 56174 ( 80.39% ) in 60.63 seconds. Users per second: 926
SequentialEvaluator: Processed 69878 ( 100.00% ) in 74.20 seconds. Users per second: 942

### This algorithm has parameters too:
* topK, the usual number of neighbors
* add_zeros_quota, the percentage of zero collaborative similarities to add to the train set, this helps the regressor in finding non-relevant features
* normalize_similarity, apply or not the usual cosine denominator_____no_output_____

from ParameterTuning.BayesianSearch import BayesianSearch
from ParameterTuning.AbstractClassSearch import DictionaryKeys

from ParameterTuning.AbstractClassSearch import EvaluatorWrapper

evaluator_validation_tuning = EvaluatorWrapper(evaluator_validation)
evaluator_test_tuning = EvaluatorWrapper(evaluator_test)

recommender_class = CFW_D_Similarity_Linalg

parameterSearch = BayesianSearch(recommender_class,
                                 evaluator_validation=evaluator_validation_tuning,
                                 evaluator_test=evaluator_test_tuning)

hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [5, 10, 20, 50, 100, 150, 200, 300, 400, 500, 600, 700, 800]
hyperparamethers_range_dictionary["add_zeros_quota"] = range(0,1)
hyperparamethers_range_dictionary["normalize_similarity"] = [True, False]

recommenderDictionary = {DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train, ICM_all, W_sparse_CF],
                         DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
                         DictionaryKeys.FIT_POSITIONAL_ARGS: dict(),
                         DictionaryKeys.FIT_KEYWORD_ARGS: dict(),
                         DictionaryKeys.FIT_RANGE_KEYWORD_ARGS: hyperparamethers_range_dictionary}

output_root_path = "result_experiments/"

import os

# If directory does not exist, create
if not os.path.exists(output_root_path):
    os.makedirs(output_root_path)

output_root_path += recommender_class.RECOMMENDER_NAME

n_cases = 2
metric_to_optimize = "MAP"

best_parameters_CFW_D = parameterSearch.search(recommenderDictionary,
                                                 n_cases = n_cases,
                                                 output_root_path = output_root_path,
                                                 metric=metric_to_optimize)

Initialization
--------------------------------------------------------------------------------------
 Step |   Time |      Value |   add_zeros_quota |   normalize_similarity |      topK |
BayesianSearch: Testing config: {'topK': 100, 'add_zeros_quota': 0.554339736755939, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 8015.05 column/sec, elapsed time 0.14 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 1.47E-04, nonzero cells 624111
CFW_D_Similarity_Linalg: Content S structure has 176941 out of 624111 ( 28.35%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 9.84E+03, average is: 5.56E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 8082.12 column/sec, elapsed time 0.13 min
SequentialEvaluator: Processed 30248 ( 43.30% ) in 30.00 seconds. Users per second: 1008
SequentialEvaluator: Processed 63629 ( 91.08% ) in 60.00 seconds. Users per second: 1060
SequentialEvaluator: Processed 69857 ( 100.00% ) in 65.40 seconds. Users per second: 1068
BayesianSearch: New best config found. Config: {'topK': 100, 'add_zeros_quota': 0.554339736755939, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.23251189334402225, 'PRECISION': 0.12500202795230692, 'RECALL': 0.02567822778469299, 'RECALL_TEST_LEN': 0.12500202795230692, 'MAP': 0.07402332622356764, 'MRR': 0.24433676415155742, 'NDCG': 0.04988431943163831, 'F1': 0.04260452746921793, 'HIT_RATE': 0.6223284710193682, 'ARHR': 0.3005773699605592, 'NOVELTY': 0.0007108966812643189, 'DIVERSITY_MEAN_INTER_LIST': 0.874110298247455, 'DIVERSITY_HERFINDAHL': 0.9748195570791022, 'COVERAGE_ITEM': 0.0235514477845672, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.05523756651501048, 'SHANNON_ENTROPY': 6.384996402251202}

BayesianSearch: Saving model in result_experiments/

CFW_D_Similarity_Linalg: Saving model in file 'result_experiments/_best_model'
CFW_D_Similarity_Linalg: Saving complete
CFW_D_Similarity_Linalg: Loading model from file 'result_experiments/_best_model'
CFW_D_Similarity_Linalg: Loading complete
SequentialEvaluator: Processed 32001 ( 45.80% ) in 30.48 seconds. Users per second: 1050
SequentialEvaluator: Processed 64680 ( 92.56% ) in 60.48 seconds. Users per second: 1069
SequentialEvaluator: Processed 69878 ( 100.00% ) in 65.12 seconds. Users per second: 1073
BayesianSearch: Best result evaluated on URM_test. Config: {'topK': 100, 'add_zeros_quota': 0.554339736755939, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: CUTOFF: 5 - ROC_AUC: 0.3474985, PRECISION: 0.2481146, RECALL: 0.0256617, RECALL_TEST_LEN: 0.2481146, MAP: 0.1740920, MRR: 0.4101472, NDCG: 0.0635737, F1: 0.0465128, HIT_RATE: 1.2405621, ARHR: 0.5979152, NOVELTY: 0.0007109, DIVERSITY_MEAN_INTER_LIST: 0.8741307, DIVERSITY_HERFINDAHL: 0.9748236, COVERAGE_ITEM: 0.0235514, COVERAGE_USER: 0.9763861, DIVERSITY_GINI: 0.0552469, SHANNON_ENTROPY: 6.3852649,

    1 | 03m16s |    0.07402 |            0.5543 |                 0.8860 |    4.3510 |
BayesianSearch: Testing config: {'topK': 800, 'add_zeros_quota': 0.38009173827507636, 'normalize_similarity': True}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 8196.22 column/sec, elapsed time 0.13 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 6.73E-04, nonzero cells 2853774
CFW_D_Similarity_Linalg: Generating train data. Sample 1184107 ( 57.26 %)
CFW_D_Similarity_Linalg: Content S structure has 646097 out of 2853774 ( 22.64%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 3.92E+06, average is: 6.07E+00, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 2910.95 column/sec, elapsed time 0.37 min
SequentialEvaluator: Processed 27001 ( 38.65% ) in 30.18 seconds. Users per second: 895
SequentialEvaluator: Processed 56001 ( 80.17% ) in 60.91 seconds. Users per second: 919
SequentialEvaluator: Processed 69857 ( 100.00% ) in 75.01 seconds. Users per second: 931
BayesianSearch: Config is suboptimal. Config: {'topK': 800, 'add_zeros_quota': 0.38009173827507636, 'normalize_similarity': True, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.02059445247672627, 'PRECISION': 0.00969933817178923, 'RECALL': 0.0016942935632998503, 'RECALL_TEST_LEN': 0.00969933817178923, 'MAP': 0.004405209531216943, 'MRR': 0.020701814659471286, 'NDCG': 0.0017849015533740248, 'F1': 0.002884686220306829, 'HIT_RATE': 0.048184147615843796, 'ARHR': 0.021312586187974755, 'NOVELTY': 0.001096712609973419, 'DIVERSITY_MEAN_INTER_LIST': 0.8465851094629667, 'DIVERSITY_HERFINDAHL': 0.9693145981265872, 'COVERAGE_ITEM': 0.02520956796757454, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.048351961089522846, 'SHANNON_ENTROPY': 6.157188492481505}

    2 | 04m05s |    0.00441 |            0.3801 |                 0.2698 |   11.7663 |
BayesianSearch: Testing config: {'topK': 600, 'add_zeros_quota': 0.05176644703066691, 'normalize_similarity': True}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 8228.27 column/sec, elapsed time 0.13 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 5.89E-04, nonzero cells 2499662
CFW_D_Similarity_Linalg: Generating train data. Sample 594418 ( 25.01 %)
CFW_D_Similarity_Linalg: Content S structure has 578522 out of 2499662 ( 23.14%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 3.53E+06, average is: 6.10E+00, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 3218.35 column/sec, elapsed time 0.34 min
SequentialEvaluator: Processed 29001 ( 41.51% ) in 30.23 seconds. Users per second: 959
SequentialEvaluator: Processed 58001 ( 83.03% ) in 60.41 seconds. Users per second: 960
SequentialEvaluator: Processed 69857 ( 100.00% ) in 71.93 seconds. Users per second: 971
BayesianSearch: Config is suboptimal. Config: {'topK': 600, 'add_zeros_quota': 0.05176644703066691, 'normalize_similarity': True, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.016221232899590113, 'PRECISION': 0.007397254391113004, 'RECALL': 0.0013590164218846461, 'RECALL_TEST_LEN': 0.007397254391113004, 'MAP': 0.0034095763885906055, 'MRR': 0.016252725806528836, 'NDCG': 0.0013664665433071297, 'F1': 0.0022961807392842177, 'HIT_RATE': 0.0367608113718024, 'ARHR': 0.016615132818949465, 'NOVELTY': 0.001130608325510885, 'DIVERSITY_MEAN_INTER_LIST': 0.8792430157580405, 'DIVERSITY_HERFINDAHL': 0.9758460858862925, 'COVERAGE_ITEM': 0.028188043111124758, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.05351828269070927, 'SHANNON_ENTROPY': 6.5155959298829185}

    3 | 03m02s |    0.00341 |            0.0518 |                 0.0056 |    9.8201 |
BayesianSearch: Testing config: {'topK': 300, 'add_zeros_quota': 0.32839844435750476, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7875.97 column/sec, elapsed time 0.14 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 3.72E-04, nonzero cells 1576400
CFW_D_Similarity_Linalg: Content S structure has 391759 out of 1576400 ( 24.85%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 2.15E+04, average is: 5.48E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 8241.45 column/sec, elapsed time 0.13 min
SequentialEvaluator: Processed 29001 ( 41.51% ) in 30.58 seconds. Users per second: 948
SequentialEvaluator: Processed 60001 ( 85.89% ) in 60.67 seconds. Users per second: 989
SequentialEvaluator: Processed 69857 ( 100.00% ) in 69.58 seconds. Users per second: 1004
BayesianSearch: Config is suboptimal. Config: {'topK': 300, 'add_zeros_quota': 0.32839844435750476, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.22182935616091967, 'PRECISION': 0.11966636605251318, 'RECALL': 0.024743930734398335, 'RECALL_TEST_LEN': 0.11966636605251318, 'MAP': 0.06997291689530447, 'MRR': 0.23365708995999748, 'NDCG': 0.04797324648429329, 'F1': 0.041008381517416995, 'HIT_RATE': 0.5953734056715863, 'ARHR': 0.28566619427308815, 'NOVELTY': 0.0007018729924844246, 'DIVERSITY_MEAN_INTER_LIST': 0.7770335796612085, 'DIVERSITY_HERFINDAHL': 0.9554044912916769, 'COVERAGE_ITEM': 0.019836030337458163, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.04127973684745727, 'SHANNON_ENTROPY': 5.579810643928271}

    4 | 02m43s |    0.06997 |            0.3284 |                 0.9144 |    6.5349 |
BayesianSearch: Testing config: {'topK': 50, 'add_zeros_quota': 0.8760636327595164, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data

### The best solution has a MAP of 0.1756062, slightly better than the pure CBF

### Let's try another ICM, the genres of the movie_____no_output_____

genres_path = dataFile.extract("ml-10M100K/movies.dat", path = "data/Movielens_10M")

def _loadICM_genres(genres_path, header=True, separator='::', genresSeparator="|"):

    # Genres
    from data.IncrementalSparseMatrix import IncrementalSparseMatrix

    ICM_builder = IncrementalSparseMatrix(auto_create_col_mapper = True,
                                          auto_create_row_mapper = False)

    fileHandle = open(genres_path, "r", encoding="latin1")
    numCells = 0

    if header:
        fileHandle.readline()

    for line in fileHandle:
        numCells += 1
        if (numCells % 1000000 == 0):
            print("Processed {} cells".format(numCells))

        if (len(line)) > 1:
            line = line.split(separator)

            line[-1] = line[-1].replace("\n", "")

            movie_id = int(line[0])

            title = line[1]
            # In case the title contains commas, it is enclosed in "..."
            # genre list will always be the last element
            genreList = line[-1]

            genreList = genreList.split(genresSeparator)

            # Rows movie ID
            # Cols features
            ICM_builder.add_single_row(movie_id, genreList, data = 1.0)

    fileHandle.close()

    return ICM_builder.get_SparseMatrix(), ICM_builder.get_column_token_to_id_mapper(), ICM_builder.get_row_token_to_id_mapper()

ICM_genres, _, _ = _loadICM_genres(genres_path, header=True, separator='::', genresSeparator="|")
ICM_genres = ICM_genres.tocsr()

ICM_genres_____no_output_____CFW_weithing_genre = CFW_D_Similarity_Linalg(URM_train, ICM_genres, W_sparse_CF)
CFW_weithing_genre.fit()

results_dict, _ = evaluator_test.evaluateRecommender(CFW_weithing_genre)
results_dictCFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7304.08 column/sec, elapsed time 0.15 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 7.54E-04, nonzero cells 3199152
CFW_D_Similarity_Linalg: Content S structure has 343180 out of 3199152 ( 10.73%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 1.52E+04, average is: 4.43E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 7440.28 column/sec, elapsed time 0.15 min
SequentialEvaluator: Processed 33001 ( 47.23% ) in 30.22 seconds. Users per second: 1092
SequentialEvaluator: Processed 67001 ( 95.88% ) in 60.71 seconds. Users per second: 1104
SequentialEvaluator: Processed 69878 ( 100.00% ) in 62.66 seconds. Users per second: 1115
from ParameterTuning.BayesianSearch import BayesianSearch
from ParameterTuning.AbstractClassSearch import DictionaryKeys

from ParameterTuning.AbstractClassSearch import EvaluatorWrapper

evaluator_validation_tuning = EvaluatorWrapper(evaluator_validation)
evaluator_test_tuning = EvaluatorWrapper(evaluator_test)

recommender_class = CFW_D_Similarity_Linalg

parameterSearch = BayesianSearch(recommender_class,
                                 evaluator_validation=evaluator_validation_tuning,
                                 evaluator_test=evaluator_test_tuning)

hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [5, 10, 20, 50, 100, 150, 200, 300, 400, 500, 600, 700, 800]
hyperparamethers_range_dictionary["add_zeros_quota"] = range(0,1)
hyperparamethers_range_dictionary["normalize_similarity"] = [True, False]

recommenderDictionary = {DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train, ICM_genres, W_sparse_CF],
                         DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
                         DictionaryKeys.FIT_POSITIONAL_ARGS: dict(),
                         DictionaryKeys.FIT_KEYWORD_ARGS: dict(),
                         DictionaryKeys.FIT_RANGE_KEYWORD_ARGS: hyperparamethers_range_dictionary}

output_root_path = "result_experiments/ICM_genres/"

import os

# If directory does not exist, create
if not os.path.exists(output_root_path):
    os.makedirs(output_root_path)

output_root_path += recommender_class.RECOMMENDER_NAME

n_cases = 2
metric_to_optimize = "MAP"

best_parameters_CFW_D = parameterSearch.search(recommenderDictionary,
                                                 n_cases = n_cases,
                                                 output_root_path = output_root_path,
                                                 metric=metric_to_optimize)
Initialization
--------------------------------------------------------------------------------------
 Step |   Time |      Value |   add_zeros_quota |   normalize_similarity |      topK |
BayesianSearch: Testing config: {'topK': 50, 'add_zeros_quota': 0.5460399111630392, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7442.35 column/sec, elapsed time 0.15 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 1.26E-04, nonzero cells 533928
CFW_D_Similarity_Linalg: Content S structure has 65514 out of 533928 ( 12.27%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 3.02E+03, average is: 4.60E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 7324.09 column/sec, elapsed time 0.15 min
SequentialEvaluator: Processed 38948 ( 55.75% ) in 30.00 seconds. Users per second: 1298
SequentialEvaluator: Processed 69857 ( 100.00% ) in 53.91 seconds. Users per second: 1296
BayesianSearch: New best config found. Config: {'topK': 50, 'add_zeros_quota': 0.5460399111630392, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.04233768030882133, 'PRECISION': 0.019541825920573092, 'RECALL': 0.002923853014383998, 'RECALL_TEST_LEN': 0.019541825920573092, 'MAP': 0.009481341247914655, 'MRR': 0.04089664123375849, 'NDCG': 0.004327835108402573, 'F1': 0.005086641431123445, 'HIT_RATE': 0.09744191705913509, 'ARHR': 0.04394143273639938, 'NOVELTY': 0.0010364723459785942, 'DIVERSITY_MEAN_INTER_LIST': 0.9838253076530643, 'DIVERSITY_HERFINDAHL': 0.9967622448470803, 'COVERAGE_ITEM': 0.06462062824331379, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.17520149629050483, 'SHANNON_ENTROPY': 9.674670809271444}

BayesianSearch: Saving model in result_experiments/ICM_genres/

CFW_D_Similarity_Linalg: Saving model in file 'result_experiments/ICM_genres/_best_model'
CFW_D_Similarity_Linalg: Saving complete
CFW_D_Similarity_Linalg: Loading model from file 'result_experiments/ICM_genres/_best_model'
CFW_D_Similarity_Linalg: Loading complete
SequentialEvaluator: Processed 37001 ( 52.95% ) in 30.10 seconds. Users per second: 1229
SequentialEvaluator: Processed 69878 ( 100.00% ) in 54.94 seconds. Users per second: 1272
BayesianSearch: Best result evaluated on URM_test. Config: {'topK': 50, 'add_zeros_quota': 0.5460399111630392, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: CUTOFF: 5 - ROC_AUC: 0.0758334, PRECISION: 0.0392026, RECALL: 0.0029488, RECALL_TEST_LEN: 0.0392026, MAP: 0.0204536, MRR: 0.0769627, NDCG: 0.0055672, F1: 0.0054849, HIT_RATE: 0.1960131, ARHR: 0.0884196, NOVELTY: 0.0010365, DIVERSITY_MEAN_INTER_LIST: 0.9838275, DIVERSITY_HERFINDAHL: 0.9967627, COVERAGE_ITEM: 0.0646206, COVERAGE_USER: 0.9763861, DIVERSITY_GINI: 0.1752086, SHANNON_ENTROPY: 9.6748105,

    1 | 02m14s |    0.00948 |            0.5460 |                 0.7785 |    2.9236 |
BayesianSearch: Testing config: {'topK': 300, 'add_zeros_quota': 0.9818092972417922, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7389.16 column/sec, elapsed time 0.15 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 7.54E-04, nonzero cells 3199152
CFW_D_Similarity_Linalg: Content S structure has 343180 out of 3199152 ( 10.73%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 1.52E+04, average is: 4.43E-02, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 7043.79 column/sec, elapsed time 0.15 min
SequentialEvaluator: Processed 33001 ( 47.24% ) in 30.04 seconds. Users per second: 1099
SequentialEvaluator: Processed 67001 ( 95.91% ) in 60.16 seconds. Users per second: 1114
SequentialEvaluator: Processed 69857 ( 100.00% ) in 62.06 seconds. Users per second: 1126
BayesianSearch: Config is suboptimal. Config: {'topK': 300, 'add_zeros_quota': 0.9818092972417922, 'normalize_similarity': False, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.031187521174208276, 'PRECISION': 0.015442976366006009, 'RECALL': 0.0022998228659914257, 'RECALL_TEST_LEN': 0.015442976366006009, 'MAP': 0.006873255364530581, 'MRR': 0.03019334736199182, 'NDCG': 0.003083033222764459, 'F1': 0.004003439333457135, 'HIT_RATE': 0.07704310233763259, 'ARHR': 0.03216666428084033, 'NOVELTY': 0.0010484862796817551, 'DIVERSITY_MEAN_INTER_LIST': 0.9748942082494275, 'DIVERSITY_HERFINDAHL': 0.994976050536015, 'COVERAGE_ITEM': 0.07334111216875978, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.1108203529553254, 'SHANNON_ENTROPY': 9.013769641884808}

    2 | 01m48s |    0.00687 |            0.9818 |                 0.9002 |    7.2822 |
BayesianSearch: Testing config: {'topK': 100, 'add_zeros_quota': 0.5408037722416156, 'normalize_similarity': True}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7548.72 column/sec, elapsed time 0.14 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 2.52E-04, nonzero cells 1067828
CFW_D_Similarity_Linalg: Content S structure has 127182 out of 1067828 ( 11.91%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 1.61E+04, average is: 1.26E-01, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 3064.32 column/sec, elapsed time 0.35 min
SequentialEvaluator: Processed 37001 ( 52.97% ) in 30.27 seconds. Users per second: 1222
SequentialEvaluator: Processed 69857 ( 100.00% ) in 55.67 seconds. Users per second: 1255
BayesianSearch: Config is suboptimal. Config: {'topK': 100, 'add_zeros_quota': 0.5408037722416156, 'normalize_similarity': True, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.03446445357420634, 'PRECISION': 0.01577222039308997, 'RECALL': 0.0023044971055562143, 'RECALL_TEST_LEN': 0.01577222039308997, 'MAP': 0.00769780885714169, 'MRR': 0.0333414451426962, 'NDCG': 0.0033392181536924413, 'F1': 0.0040214199560061345, 'HIT_RATE': 0.07863206264225489, 'ARHR': 0.03575470842816976, 'NOVELTY': 0.0010813311116455071, 'DIVERSITY_MEAN_INTER_LIST': 0.989816524259286, 'DIVERSITY_HERFINDAHL': 0.9979604710155222, 'COVERAGE_ITEM': 0.0855467190714527, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.16316928961188898, 'SHANNON_ENTROPY': 10.05444038952249}

    3 | 01m36s |    0.00770 |            0.5408 |                 0.2024 |    4.2721 |
BayesianSearch: Testing config: {'topK': 700, 'add_zeros_quota': 0.02591915073946116, 'normalize_similarity': True}
CFW_D_Similarity_Linalg: Generating train data
Similarity column 65134 ( 100 % ), 7334.40 column/sec, elapsed time 0.15 min
CFW_D_Similarity_Linalg: Collaborative S density: 1.65E-03, nonzero cells 6997140
CFW_D_Similarity_Linalg: Content S density: 1.73E-03, nonzero cells 7331808
CFW_D_Similarity_Linalg: Generating train data. Sample 624340 ( 8.74 %)
CFW_D_Similarity_Linalg: Content S structure has 705850 out of 7331808 ( 9.63%) nonzero collaborative cells
CFW_D_Similarity_Linalg: Nonzero collaborative cell sum is: 7.84E+04, average is: 1.11E-01, average over all collaborative data is 3.96E-02
Similarity column 65134 ( 100 % ), 3133.95 column/sec, elapsed time 0.35 min
SequentialEvaluator: Processed 30001 ( 42.95% ) in 30.35 seconds. Users per second: 989
SequentialEvaluator: Processed 60001 ( 85.89% ) in 60.38 seconds. Users per second: 994
SequentialEvaluator: Processed 69857 ( 100.00% ) in 69.39 seconds. Users per second: 1007
BayesianSearch: Config is suboptimal. Config: {'topK': 700, 'add_zeros_quota': 0.02591915073946116, 'normalize_similarity': True, 'epochs': 0, 'max_epochs': 0} - results: {'ROC_AUC': 0.03521718176656118, 'PRECISION': 0.016675971389172612, 'RECALL': 0.0023562633893919223, 'RECALL_TEST_LEN': 0.016675971389172612, 'MAP': 0.008000617929007574, 'MRR': 0.03389925132771226, 'NDCG': 0.003454565540612396, 'F1': 0.004129097956600361, 'HIT_RATE': 0.08318421919063229, 'ARHR': 0.036735283030571726, 'NOVELTY': 0.0010870922586515988, 'DIVERSITY_MEAN_INTER_LIST': 0.9921662375152077, 'DIVERSITY_HERFINDAHL': 0.9984304069394974, 'COVERAGE_ITEM': 0.13321767433291368, 'COVERAGE_USER': 0.9760926671137938, 'DIVERSITY_GINI': 0.20589119134999923, 'SHANNON_ENTROPY': 10.876395364725553}

    4 | 02m14s |    0.00800 |            0.0259 |                 0.0021 |   11.3005 |
BayesianSearch: Testing config: {'topK': 5, 'add_zeros_quota': 0.35747392814147605, 'normalize_similarity': False}
CFW_D_Similarity_Linalg: Generating train data

### Here the best solution has a MAP of 0.0204536... not great_____no_output_____### Since we are weighting features, and weights can be applied only to common features, we expect a sparse ICM to have limited quality gains, whereas a more dense one to exhibit much higher improvements.

## This however greatly depends on the ICM structure...

## Tip:
### If your ICM is very sparse, you may try to aggregate features (e.g., put them in OR, add them...) to reduce their number and get a new denser ICM on which to try feature weighting_____no_output_____