# Notebook from desaizeeshan22/ECOMMERCE_ML
Path: ECOMMERCE_BIGDATA_ML.ipynb

import numpy as np
import pandas as pd
import math

import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib import style

style.use('ggplot')

import os
_____no_output_____df=pd.read_csv('C:\\Users\\desai\\Documents\\summer-products-with-rating-and-performance_2020-08.csv')_____no_output_____df.info()<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1573 entries, 0 to 1572
Data columns (total 43 columns):
 #   Column                        Non-Null Count  Dtype
---  ------                        --------------  -----
 0   title                         1573 non-null   object
 1   title_orig                    1573 non-null   object
 2   price                         1573 non-null   float64
 3   retail_price                  1573 non-null   int64
 4   currency_buyer                1573 non-null   object
 5   units_sold                    1573 non-null   int64
 6   uses_ad_boosts                1573 non-null   int64
 7   rating                        1573 non-null   float64
 8   rating_count                  1573 non-null   int64
 9   rating_five_count             1528 non-null   float64
 10  rating_four_count             1528 non-null   float64
 11  rating_three_count            1528 non-null   float64
 12  rating_two_count              1528 non-null   float64
 13  rating_one_count              1528 non-null   float64
 14  badges_count                  1573 non-null   int64
 15  badge_local_product           1573 non-null   int64
 16  badge_product_quality         1573 non-null   int64
 17  badge_fast_shipping           1573 non-null   int64
 18  tags                          1573 non-null   object
 19  product_color                 1532 non-null   object
 20  product_variation_size_id     1559 non-null   object
 21  product_variation_inventory   1573 non-null   int64
 22  shipping_option_name          1573 non-null   object
 23  shipping_option_price         1573 non-null   int64
 24  shipping_is_express           1573 non-null   int64
 25  countries_shipped_to          1573 non-null   int64
 26  inventory_total               1573 non-null   int64
 27  has_urgency_banner            473 non-null    float64
 28  urgency_text                  473 non-null    object
 29  origin_country                1556 non-null   object
 30  merchant_title                1573 non-null   object
 31  merchant_name                 1569 non-null   object
 32  merchant_info_subtitle        1572 non-null   object
 33  merchant_rating_count         1573 non-null   int64
 34  merchant_rating               1573 non-null   float64
 35  merchant_id                   1573 non-null   object
 36  merchant_has_profile_picture  1573 non-null   int64
 37  merchant_profile_picture      226 non-null    object
 38  product_url                   1573 non-null   object
 39  product_picture               1573 non-null   object
 40  product_id                    1573 non-null   object
 41  theme                         1573 non-null   object
 42  crawl_month                   1573 non-null   object
dtypes: float64(9), int64(15), object(19)
memory usage: 528.6+ KB
df.describe()_____no_output_____cat_cols=[x for x in df.columns if df[x].dtype=='O']_____no_output_____cat_cols_____no_output_____num_cols=[x for x in df.columns if df[x].dtype!='O']_____no_output_____num_cols_____no_output_____for col in df.columns:
    print(col,"\t",df[col].isna().mean()*100)title 	 0.0
title_orig 	 0.0
price 	 0.0
retail_price 	 0.0
currency_buyer 	 0.0
units_sold 	 0.0
uses_ad_boosts 	 0.0
rating 	 0.0
rating_count 	 0.0
rating_five_count 	 2.860775588048315
rating_four_count 	 2.860775588048315
rating_three_count 	 2.860775588048315
rating_two_count 	 2.860775588048315
rating_one_count 	 2.860775588048315
badges_count 	 0.0
badge_local_product 	 0.0
badge_product_quality 	 0.0
badge_fast_shipping 	 0.0
tags 	 0.0
product_color 	 2.6064844246662426
product_variation_size_id 	 0.8900190718372537
product_variation_inventory 	 0.0
shipping_option_name 	 0.0
shipping_option_price 	 0.0
shipping_is_express 	 0.0
countries_shipped_to 	 0.0
inventory_total 	 0.0
has_urgency_banner 	 69.93006993006993
urgency_text 	 69.93006993006993
origin_country 	 1.080737444373808
merchant_title 	 0.0
merchant_name 	 0.25429116338207247
merchant_info_subtitle 	 0.06357279084551812
merchant_rating_count 	 0.0
merchant_rating 	 0.0
merchant_id 	 0.0
merchant_has_profile_picture 	 0.0
merchant_profile_picture 	 85.63254926891291
product_url 	 0.0
product_picture 	 0.0
product_id 	 0.0
theme 	 0.0
crawl_month 	 0.0
df['units_sold'].astype('O').value_counts()

_____no_output_____## Inspecting the target variable units_sold
plt.figure(figsize=(14,6))
sns.countplot(data=df,x="units_sold")
## 9 significant categories in the units_sold which is a discrete variable the remaining categories can be replaced by the
# values closest to them 1,2,3,6,7,8 are the closest in value to 10 units sold
_____no_output_____units_sold_list=list(df['units_sold'].astype('O').value_counts().index)_____no_output_____units_sold_list_____no_output_____for val in units_sold_list:
    if (val == 8 or val ==1 or val==7 or val==3 or val ==2 or val==6):
        df['units_sold']=df['units_sold'].replace(val,10)
       _____no_output_____plt.figure(figsize=(14,6))
sns.countplot(data=df,x="units_sold")_____no_output_____df['product_color'].value_counts()_____no_output_____color_list=list(df['product_color'].value_counts().index)_____no_output_____for x in color_list:
    print(x)black
white
yellow
blue
pink
red
green
grey
purple
armygreen
navyblue
winered
orange
multicolor
beige
lightblue
khaki
gray
white & green
skyblue
rosered
brown
coffee
darkblue
floral
rose
leopard
lightpink
black & green
navy
fluorescentgreen
White
lightgreen
black & white
camouflage
Black
orange-red
pink & blue
watermelonred
wine
lightyellow
burgundy
Army green
applegreen
camel
pink & grey
coolblack
mintgreen
navy blue
white & black
pink & black
pink & white
coralred
lightred
apricot
Pink
black & yellow
lakeblue
black & blue
whitefloral
silver
dustypink
army green
army
light green
brown & yellow
offwhite
gold
black & stripe
orange & camouflage
lightpurple
navyblue & white
wine red
greysnakeskinprint
blue & pink
gray & white
offblack
lightgray
lightkhaki
nude
denimblue
RED
prussianblue
white & red
lightgrey
whitestripe
claret
jasper
blackwhite
rosegold
violet
tan
darkgreen
rainbow
winered & yellow
Blue
ivory
red & blue
star
leopardprint
Rose red
for x in color_list:
    if '&' in x:
        df['product_color']=df['product_color'].replace(x,'dual')
    _____no_output_____for x in color_list:
    if 'green' in x or 'khaki' in x:
         df['product_color']=df['product_color'].replace(x,'green')_____no_output_____for x in color_list:
    if 'blue' in x or 'navy'  in x :
          df['product_color']=df['product_color'].replace(x,'blue')_____no_output_____for x in color_list:
    if 'red' in x or 'rose'  in x :
          df['product_color']=df['product_color'].replace(x,'red')_____no_output_____df['product_color'] = df['product_color'].replace('Black', 'black')
df['product_color'] = df['product_color'].replace('White', 'white')

df['product_color'] = df['product_color'].replace('lightpink', 'pink')

df['product_color'] = df['product_color'].replace('gray', 'grey')

df['product_color'] = df['product_color'].replace('coffee', 'brown')

df['product_color'] = df['product_color'].replace('multicolor', 'other')
df['product_color'] = df['product_color'].replace('floral', 'other')
df['product_color'] = df['product_color'].replace('leopard', 'other')
df['product_color'] = df['product_color'].replace('camouflage', 'other')

df['product_color'] =df['product_color'].replace(np.nan, 'other')_____no_output_____df['product_color'] = df['product_color'].replace('apricot', 'other')
df['product_color'] = df['product_color'].replace('camel', 'yellow')
df['product_color'] = df['product_color'].replace('lightyellow', 'yellow')
df['product_color'] = df['product_color'].replace('burgundy', 'red')
df['product_color'] = df['product_color'].replace('wine', 'red')
df['product_color'] = df['product_color'].replace('silver', 'other')
df['product_color'] = df['product_color'].replace('whitefloral', 'white')
_____no_output_____df['product_color'] = df['product_color'].replace('RED', 'red')_____no_output_____df['product_color'].value_counts()_____no_output_____df['shipping_is_express'].astype('O').value_counts()_____no_output_____plt.figure(figsize=(14,6))
sns.countplot(x='shipping_is_express',data=df)
##Majority items are not shipped under express shipping_____no_output_____plt.figure(figsize=(14,6))
sns.scatterplot(x='shipping_is_express',y='units_sold',data=df)

## The shipping methods influence on units sold can not be aptly inferred as only four products use express shipping but it
## can be observed that the highest units sold is for products without express shipping_____no_output_____df['origin_country'].astype('O').value_counts()_____no_output_____plt.figure(figsize=(14,6))
sns.countplot(x='origin_country',data=df)

## Majority of goods are manufactured in China,we can observe China as a manufacturing hub
## US comes in second with 31 products ,the remaining countries can all be clubbed under others as they dont contribute significantly_____no_output_____country_list=list(df['origin_country'].astype('O').value_counts().index)_____no_output_____country_list_____no_output_____for x in country_list:
    if x!='US'and x!='CN':
        df['origin_country']=df['origin_country'].replace(x,'OTHER')

df['origin_country']=df['origin_country'].replace(np.nan,'OTHER')    _____no_output_____df['origin_country'].astype('O').value_counts()_____no_output_____plt.figure(figsize=(14,6))
sns.countplot(x='origin_country',data=df)_____no_output_____def null_val_viz(df):
    cols_with_null=[col for col in df.columns if df[col].isna().mean()>0]
    pct_null_vals=df[cols_with_null].isna().mean()*100
    plt.figure(figsize=(14,6))
    sns.barplot(y=pct_null_vals.index,x=pct_null_vals,orient='h')
    plt.title("NA VALUES BY COLUMN")
    _____no_output_____null_val_viz(df)_____no_output_____plt.figure(figsize=(14,6))
sns.scatterplot(data=df,x="price",y="units_sold")

## products between 1 and 10 euros have a few products with 100000 units sold and products priced between 10 and 20 euros cap
## at about 50000 units sold
### products priced between 20 and 30 euros cap at 20000 units sold while premium prodcuts priced slightly less than 50 euros
## has one unit sold
## Highest variability in sales is between low and mid priced products_____no_output_____df["units_sold"].describe()_____no_output_____plt.figure(figsize=(14,6))
sns.scatterplot(data=df,x="rating",y="units_sold")_____no_output_____plt.figure(figsize=(14,6))
sns.distplot(df['price'], color='green', label='Price')
sns.distplot(df['retail_price'], color='yellow', label='Retail price')
plt.legend();

 _____no_output_____plt.figure(figsize=(14,6))
sns.distplot(df['units_sold'].astype('O'))_____no_output_____
pd.crosstab(df['units_sold'],df['uses_ad_boosts'])_____no_output_____df_use_boost=df[df['uses_ad_boosts']== 1]
df_not_use_boost=df[df['uses_ad_boosts']== 0]
_____no_output_____

# T test to test if using an ad boost aka advertising leads to a statistically significant difference in units sold
_____no_output_____

from scipy.stats import ttest_ind
ttest_ind(df_use_boost['units_sold'],df_not_use_boost['units_sold'])
## H0:Mean units sold for products with ad boosts - Mean units sold for products without ad boosts == 0
##HA:Mean units sold for products with ad boosts - Mean units sold for products without ad boosts  !=0
###Difference in mean units sold for populations using ad boosts and  not using ad boosts
## A very high p value suggests we cannot reject the null hypothesis aka zero differences in
## mean units_sold by presence and absence of ad boosts_____no_output_____df['badge_local_product'].astype('O').value_counts()_____no_output_____local_vec=df['badge_local_product'].astype('O')_____no_output_____plt.figure(figsize=(14,6))
sns.scatterplot(x=local_vec,y=df["units_sold"])_____no_output_____df_local_product_badge=df[df['badge_local_product']== 1]
df_no_local_product_badge=df[df['badge_local_product']== 0]_____no_output_____

# T test to test if having a quality badge leads to a statistically significant difference in units sold
_____no_output_____

ttest_ind(df_local_product_badge['units_sold'],df_no_local_product_badge['units_sold'])
## H0:Mean units sold for productis with local badge - Mean units sold for productis without local badge == 0
##HA:Mean units sold for productis with local badge - Mean units sold for productis without local badge != 0
## p value greater than 0.05 we cannot reject the null hyptothesis hence there's no statistically significant effect of the
## presence of local product badge on units sold_____no_output_____df['badge_product_quality'].astype('O').value_counts()_____no_output_____df_has_quality_badge=df[df['badge_product_quality']== 1]
df_no_quality_badge=df[df['badge_product_quality']== 0]_____no_output_____

# T test to test if having a quality badge leads to a statistically significant difference in units sold
_____no_output_____

ttest_ind(df_has_quality_badge['units_sold'],df_no_quality_badge['units_sold'])
## H0:Mean units sold for productis with quality badge - Mean units sold for productis without quality badge == 0
##HA:Mean units sold for productis with quality badge - Mean units sold for productis without quality badge != 0
## p value less than 0.05 we can reject the null hyptothesis hence there's a statistically significant effect of the
## presence of quality badge on units sold_____no_output_____plt.figure(figsize=(14,6))
sns.scatterplot(x=df['badge_product_quality'].astype('O'),y=df["units_sold"])_____no_output_____df['product_variation_size_id'].value_counts()_____no_output_____df['product_variation_size_id'].value_counts().index_____no_output_____df['product_variation_size_id'] = df['product_variation_size_id'].replace('Size -XXS', 'S')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('SIZE-XXS', 'S')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('Size S.', 'S')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('s', 'S')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('SizeL', 'L')

df['product_variation_size_id'] =df['product_variation_size_id'].replace('5XL', 'XL')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('4XL', 'XL')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('3XL', 'XL')
df['product_variation_size_id'] = df['product_variation_size_id'].replace('2XL', 'XL')_____no_output_____for x in df['product_variation_size_id'].value_counts().index:
     if x != 'XXXS' \
    and x != 'XXS' \
    and x != 'XS' \
    and x != 'S' \
    and x != 'M' \
    and x != 'L' \
    and x != 'XL' \
    and x != 'XXL' \
    and x != 'XXXXL' \
    and x != 'XXXXXL':
            df['product_variation_size_id']= df['product_variation_size_id'].replace(x,'other')
           _____no_output_____df['product_variation_size_id']= df['product_variation_size_id'].replace(np.nan,'other')_____no_output_____df['product_variation_size_id'].value_counts()_____no_output_____plt.figure(figsize=(14,6))
sns.countplot(data=df,x='product_variation_size_id',order=df['product_variation_size_id'].value_counts().index)_____no_output_____fig,ax=plt.subplots(figsize=(27,27))
fig=sns.heatmap(data=df.corr(),annot=True,cbar=True,vmin=0,vmax=1,ax=ax)
plt.xticks(rotation=45, ha='right')
plt.yticks(rotation=45, ha='right')
plt.show()_____no_output_____df['has_urgency_banner'].replace(np.nan,0,inplace=True)_____no_output_____df.drop(['title','title_orig','currency_buyer','merchant_title','crawl_month','product_id','product_picture',
         'product_url', 'merchant_profile_picture', 'merchant_id', 'currency_buyer','theme','urgency_text',
         'merchant_info_subtitle','title','title_orig','tags','shipping_option_name'],axis=1,inplace=True)_____no_output_____df.drop(['rating_count'],axis=1,inplace=True)_____no_output_____## Rating count suggests the number of ratings for every product but the same information is conveyed by the
## breakdown of the number of each type of rating rating_five_count to rating_one_count columns,hence due to redundancy
## we can drop this column._____no_output_____df.drop(['merchant_name'],axis=1,inplace=True)_____no_output_____df['rating_five_count'].describe()_____no_output_____df.isna().sum()_____no_output_____df['rating_four_count'].replace(np.nan,0,inplace=True)
df['rating_five_count'].replace(np.nan,0,inplace=True)
df['rating_three_count'].replace(np.nan,0,inplace=True)
df['rating_two_count'].replace(np.nan,0,inplace=True)
df['rating_one_count'].replace(np.nan,0,inplace=True)_____no_output_____df.isna().sum()_____no_output_____df=pd.get_dummies(df,columns=['product_color'],prefix='COLOR_',drop_first=True)

_____no_output_____df=pd.get_dummies(df,columns=['origin_country'],prefix='origin_',drop_first=True)_____no_output_____df.columns_____no_output_____df=pd.get_dummies(df,columns = ['product_variation_size_id'],prefix = 'prodsize_',drop_first = True)_____no_output_____df.head(20)_____no_output_____df.to_csv('preprocessed_ecom.csv')_____no_output_____X=df.drop(['units_sold'],axis=1)
y=df['units_sold']_____no_output_____from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.3,random_state=10)_____no_output_____from sklearn.linear_model import LogisticRegression as lr
from sklearn.model_selection import GridSearchCV
lr=lr(max_iter=10000,multi_class='multinomial',solver='saga',random_state = 10)
lr.fit(X_train,y_train)

_____no_output_____print(f"Training data accuracy : {lr.score(X_train,y_train)}")
Training data accuracy : 0.31244323342415986
from sklearn.metrics import confusion_matrix, accuracy_score
y_pred_lr = lr.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_lr)}")Test data accuracy : 0.298728813559322

from sklearn.tree import DecisionTreeClassifier

classifier_DT = DecisionTreeClassifier(random_state = 10)
classifier_DT.fit(X_train, y_train)
_____no_output_____from sklearn import tree
tree.plot_tree(classifier_DT)_____no_output_____from sklearn.model_selection import cross_val_score
print(f"Training data cross val accuracy :{np.mean(cross_val_score(classifier_DT,X_train,y_train,cv=5))}")Training data cross val accuracy :0.666659810777458

y_pred_DT = classifier_DT.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_DT)}")Test data accuracy : 0.7161016949152542
from xgboost import XGBClassifier
XGB=XGBClassifier()
XGB.fit(X_train,y_train)
print(f"Training data cross val accuracy :{np.mean(cross_val_score(XGB,X_train,y_train,cv=5))}")Training data cross val accuracy :0.7493048128342246
##XGB tends to overfit slightly
y_pred_DT = classifier_DT.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_DT)}")Test data accuracy : 0.7161016949152542
from sklearn.ensemble import AdaBoostClassifier
ADB_Classifier=AdaBoostClassifier()
ADB_Classifier.fit(X_train,y_train)
print(f"Training data cross val accuracy :{np.mean(cross_val_score(ADB_Classifier,X_train,y_train,cv=5))}")Training data cross val accuracy :0.5231715343480049
y_pred_ADB = ADB_Classifier.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_ADB)}")Test data accuracy : 0.5338983050847458
from sklearn.ensemble import RandomForestClassifier

RFC = RandomForestClassifier()
RFC.fit(X_train, y_train)

print(f"Training data cross val accuracy :{np.mean(cross_val_score(RFC,X_train,y_train,cv=5))}")Training data cross val accuracy :0.75475935828877
y_pred_RFC = RFC.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_RFC)}")Test data accuracy : 0.7754237288135594
from sklearn.ensemble import GradientBoostingClassifier

GBC = GradientBoostingClassifier()
GBC.fit(X_train, y_train)
print(f"Training data cross val accuracy :{np.mean(cross_val_score(GBC,X_train,y_train,cv=5))}")Training data cross val accuracy :0.7347758124228714
y_pred_GBC = GBC.predict(X_test)
print(f"Test data accuracy : {accuracy_score(y_test, y_pred_GBC)}")Test data accuracy : 0.7351694915254238
classifiers=[classifier_DT,GBC,RFC,ADB_Classifier,XGB]
name_keys=["Decision_Tree","Gradient_Boosting_Classifier","Random_Forest_Classifier","AdaBoostClassifier","XGBoost"]
mean_cross_val_score=[]
for clf in classifiers:
    accuracies = cross_val_score(estimator = clf,
                                 X = X_train,
                                 y = y_train,
                                 cv = 5)
    mean_cross_val_score.append(np.mean(accuracies)*100)

accuracies = pd.DataFrame({'Classifier': name_keys,
                           'mean_cross_val_score': mean_cross_val_score})
accuracies.sort_values('mean_cross_val_score',ascending=False)
    _____no_output_____from sklearn.model_selection import GridSearchCV
param_grid_RFC=[{"criterion":["gini","entropy"],"n_estimators":[100,200,300,500],"max_features":["auto","sqrt","log2"]}]
RFC_grid=GridSearchCV(estimator=RFC,param_grid=param_grid_RFC,cv=5,return_train_score=True)
RFC_grid.fit(X_train,y_train)_____no_output_____print(f"The optimum hyperparameters to be used : {RFC_grid.best_params_}")The optimum hyperparameters to be used : {'criterion': 'entropy', 'max_features': 'auto', 'n_estimators': 100}
print(f"The best cross val score :{RFC_grid.best_score_}")The best cross val score :0.7692801316330729
y_RFC_tuned=RFC_grid.predict(X_test)_____no_output_____
print(f"Test data accuracy : {accuracy_score(y_test, y_RFC_tuned)}")Test data accuracy : 0.7817796610169492

# Random Forest Classifier with tuned hyperparameters gives the best results _____no_output_____


# Notebook from Asurada2015/pymoo
Path: doc/source/algorithms/cmaes.ipynb

## CMA-ES

**Disclaimer:** We make use of the implementation available at [PyPi](https://pypi.org/project/cma/)  published by the author Nikolaus Hansen under the BSD license.

CMA-ES which was proposed in . Moreover, a comparing review can be found in .
CMA-ES stands for covariance matrix adaptation evolution strategy. Evolution strategies (ES) are stochastic, derivative-free methods for numerical optimization of non-linear or non-convex continuous optimization problems. They belong to the class of evolutionary algorithms and evolutionary computation. An evolutionary algorithm is broadly based on the principle of biological evolution, namely the repeated interplay of variation (via recombination and mutation) and selection: in each generation (iteration) new individuals (candidate solutions) are generated by variation, usually in a stochastic way, of the current parental individuals. Then, some individuals are selected to become the parents in the next generation based on their fitness or objective function value
$f(x)$. Like this, over the generation sequence, individuals with better and better $f$-values are generated.
(excerpt from [Wikipedia](https://en.wikipedia.org/wiki/CMA-ES))._____no_output_____ _____no_output_____### Example_____no_output_____

from pymoo.algorithms.so_cmaes import CMAES
from pymoo.factory import get_problem
from pymoo.optimize import minimize

problem = get_problem("sphere")

algorithm = CMAES()

res = minimize(problem,
               algorithm,
               seed=1,
               verbose=False)

print("Best solution found: \nX = %s\nF = %s" % (res.X, res.F))Best solution found:
X = [0.49999994 0.5        0.50000001 0.50000003 0.49999996 0.49999994
 0.5        0.49999999 0.49999995 0.50000004]
F = [1.36882511e-14]

CMA-ES already has several stopping criteria implemented. However, as for other algorithms, the number of iterations or function evaluations can be directly passed to `minimize`._____no_output_____

res = minimize(problem,
               algorithm,
               ('n_iter', 10),
               seed=1,
               verbose=True)

print("Best solution found: \nX = %s\nF = %s" % (res.X, res.F))==============================================================================
n_gen |  n_eval |     fopt     |    sigma     | min std  | max std  |   axis
==============================================================================
    1 |      10 |  0.693281236 |  0.500000000 |  0.50000 |  0.50002 |  1.00005
    2 |      20 |  0.510635963 |  0.467010358 |  0.44671 |  0.47745 |  1.19477
    3 |      30 |  0.510635963 |  0.436791171 |  0.40501 |  0.44470 |  1.24024
    4 |      40 |  0.510635963 |  0.422510052 |  0.38513 |  0.43316 |  1.28711
    5 |      50 |  0.510635963 |  0.414590098 |  0.37047 |  0.42855 |  1.35280
    6 |      60 |  0.499588048 |  0.399593307 |  0.35758 |  0.42099 |  1.42064
    7 |      70 |  0.499588048 |  0.377241108 |  0.32669 |  0.41240 |  1.57503
    8 |      80 |  0.499588048 |  0.360917895 |  0.30394 |  0.39292 |  1.59267
    9 |      90 |  0.452070968 |  0.352635143 |  0.29372 |  0.37350 |  1.64823
   10 |     100 |  0.435363577 |  0.361760636 |  0.29683 |  0.38860 |  1.75806
Best solution found:
X = [0.52575679 0.65999627 0.65540969 0.48303056 0.96943187 0.78243048
 0.35731446 0.42159602 0.26088296 0.52906313]
F = [0.43536358]
res = minimize(problem,
               algorithm,
               ('n_evals', 50),
               seed=1,
               verbose=True)

print("Best solution found: \nX = %s\nF = %s" % (res.X, res.F))==============================================================================
n_gen |  n_eval |     fopt     |    sigma     | min std  | max std  |   axis
==============================================================================
    1 |      10 |  0.693281236 |  0.500000000 |  0.50000 |  0.50002 |  1.00005
    2 |      20 |  0.510635963 |  0.467010358 |  0.44671 |  0.47745 |  1.19477
    3 |      30 |  0.510635963 |  0.436791171 |  0.40501 |  0.44470 |  1.24024
    4 |      40 |  0.510635963 |  0.422510052 |  0.38513 |  0.43316 |  1.28711
    5 |      50 |  0.510635963 |  0.414590098 |  0.37047 |  0.42855 |  1.35280
Best solution found:
X = [0.00518307 0.70907123 0.53830206 0.64531015 0.16284682 0.36370915
 0.4825207  0.35503139 0.59110102 0.69398715]
F = [0.51063596]

Also, easily restarts can be used which are known to work very well on multi-modal functions. For instance, `Rastrigin` can be solved rather quickly by:_____no_output_____

problem = get_problem("rastrigin")

algorithm = CMAES(restarts=10, restart_from_best=True)

res = minimize(problem,
               algorithm,
               ('n_evals', 2500),
               seed=1,
               verbose=False)

print("Best solution found: \nX = %s\nF = %s" % (res.X, res.F))Best solution found:
X = [ 5.20199958e-06 -4.04980429e-05]
F = [3.3074982e-07]

Our framework internally calls the `cma.fmin2` function. All parameters which can be used there either as a keyword argument or an option can also be passed to the `CMAES` constructor as well.
An example with a few selected `cma.fmin2` parameters is shown below:_____no_output_____

import numpy as np
from pymoo.util.normalization import denormalize

np.random.seed(1)

# define an intitial point for the search
x0 = denormalize(np.random.random(problem.n_var), problem.xl, problem.xu)

algorithm = CMAES(x0=x0,
                 sigma=0.5,
                 restarts=2,
                 maxfevals=np.inf,
                 tolfun=1e-6,
                 tolx=1e-6,
                 restart_from_best='False',
                 bipop=True)

res = minimize(problem,
               algorithm,
               seed=1,
               verbose=False)

print("Best solution found: \nX = %s\nF = %s" % (res.X, res.F))Best solution found:
X = [-9.94958818e-01 -2.66237603e-07]
F = [0.99495906]

For more details about hyperparameters we refer to the software documentation of the `fmin2` in CMA-ES which can be found [here](http://cma.gforge.inria.fr/apidocs-pycma/cma.evolution_strategy.html#fmin2).
A quick explanation of possible parameters is also provided in the API documentation below._____no_output_____### API_____no_output_____


# Notebook from FRBs/PreFRBLE
Path: notebooks/RedshiftEstimate.ipynb

import matplotlib
%matplotlib inline

import numpy as np, matplotlib as mpl
from pandas import DataFrame
import PreFRBLE
from PreFRBLE.convenience import *
from PreFRBLE.estimate_redshift import *
from PreFRBLE.plot import *
from PreFRBLE.likelihood import *
from PreFRBLE.physics import *
from PreFRBLE.file_system import file_estimated_redshifts_DM
_____no_output_____

## Estimate host redshift
Here we infer the host redhisft of unlocalized FRBs based on their DM.
We start with verifying our results with the localized Spitler burst FRB121102 and then infer redshifts of all FRBs in FRBcat observed by ASKAP, CHIME or Parkes_____no_output_____

## benchmark scenario that realistically accounts for intervening galaxies along the LoS
from PreFRBLE.Scenario import scenario_benchmark as scenario

file_tex_redshifts_DM = root_results+"table_redshifts_DM.tex"_____no_output_____

### estimate redshift of Spitler Burst
Here we show in detail the full exemplary redshift estimate for the Spitler burst FRB121102. From likelihood $L(\text{DM}_{\rm EG}|z)$, we obtain $L(\text{DM}_{\rm EG}|z)$ for different redshifts to produce the observed extragalactic component $\text{DM}_{\rm EG} = \text{DM}_{\rm obs} - \text{DM}_{\rm MW}$.
According to Bayes theorem
$$
P(M|O) \propto L(O|M)\pi(M),
$$
$L(\text{DM}_{\rm EG}|z)$ has to be multiplied by the prior of host redshift $\pi(z)$, that we obtain with FRBpoppy (see _notebooks/model/Gardneier20.ipynb_), in order to account for the history of source, telescope selection effects and, most importantly, the increase of probed volume with distance.
The expected value and shape of posterior $P(z|\rm DM_{EG}$ provides an estimate on the host redshift as well as upper and possibly lower limits, which are suitable to estimate the redshift distribution of unlocalized FRBs._____no_output_____

## Spitler burst,  Tendulkar et al. 2017
redshift_spitler = 0.19273
DM_total_spitler = 558 # pc cm^-3, total
DM_spitler = 340 # pc cm^-3, extragalactic = total-MW

sigma=1

telescope='Parkes'  #ARECIBO, actually  !!! PROVIDE prior vor ARECIBO

spitler_kwargs = {'color':'red',  'marker':'x', "markersize":20, 'markeredgewidth':4}

## L(DM|z) evolution with DM-marker
PlotLikelihoodEvolution( measure='DM', scenario=scenario, alpha=0.3 )
plt.plot( DM_spitler, 1e-2, **spitler_kwargs )
plt.title(r"Likelihood $L(DM|z)$", fontsize=16)
plt.loglog()
plt.ylim(1e-3,10)
plt.tick_params(axis='both', which='major', labelsize=14)

plt.show()

## L(z|DM) for flat prior with z-Marker
tmp = Scenario( population='flat', telescope='None', **scenario.Properties( identifier=False ) )

L = LikelihoodRedshiftMeasure( measure='DM', data=DM_spitler, scenario=tmp, prior=False, renormalize=True )
L.Plot( density=True )
print(L.Norm())
#Ps, z = LikelihoodRedshift( DMs=[DM_spitler], population='flat', telescope='None', scenario=scenario )
#PlotLikelihood(x=z, P=Ps[0], measure='z', density=False)
plt.xscale('linear')
plt.yscale('log')
plt.plot( redshift_spitler, 1e-1, **spitler_kwargs )
plt.title(r"Likelihood $L(z|DM_{\rm Spitler})$", fontsize=16)
plt.text( redshift_spitler+0.05, 7e-2, 'Spitler\nBurst', fontsize=18 )
plt.xlim(0,.8)
plt.ylim(1e-2,1e1)
plt.tick_params(axis='both', which='major', labelsize=14)
plt.show()

## pi(z) priors with z-Marker
fig, ax = plt.subplots()
for population, linestyle in zip( populations, linestyles_population ):
    Lz = GetLikelihood( 'z', Scenario( population=population, telescope=telescope) )
    Lz.Plot( linestyle=linestyle, ax=ax, linewidth=2, label=population, density=True )
#    PlotLikelihood(x=z, P=P, measure='z', linestyle=linestyle, ax=ax, linewidth=2, label=population, density=False)
ax.set_title("Prior $\pi(z)$", fontsize=16)
ax.set_xlim(0,0.8)
ax.set_xscale('linear')
ax.set_yscale('linear')
ax.legend(fontsize=16)
ax.plot( redshift_spitler, 0.1, **spitler_kwargs )
plt.tick_params(axis='both', which='major', labelsize=14)
plt.show()

##  P(z|DM) ~ L(z|DM)*pi(z) with additional estimated z-Marker
fig, ax = plt.subplots()
colors = ['green','orange','blue']
shift=list(0.8+np.arange(len(populations))/5)
for population, linestyle in zip( populations, linestyles_population ):
    tmp = Scenario( population=population, telescope=telescope, **scenario.Properties( identifier=False) )
    Ls = LikelihoodRedshiftMeasures( measure='DM', datas=[DM_spitler], scenario=tmp, prior=True, renormalize=True )
    L = Ls[0]
    L.PlotExpectation( y=1e-1*shift.pop(), ax=ax, marker='+', markersize=20, color=colors.pop() )
    L.Plot( linestyle=linestyle, ax=ax, density=True )
#    est, dev = L.Expectation( sigma=sigma )
#    Ps, z = LikelihoodRedshift( DMs=[DM_spitler], population=population, telescope=telescope, scenario=scenario )
#    PlotLikelihood(x=z, P=Ps[0], measure='z', linestyle=linestyle, ax=ax, density=False)
#    est, dev = Likelihood2Expectation( x=z, P=Ps[0], log=False, density=True, sigma=sigma )
#    ax.errorbar( est, 1e-2*shift.pop(), xerr=dev, marker='+', markersize=20, color=colors.pop() )

ax.set_title(r"Posterior $P(z|DM) \propto L(DM|z) \pi(z)$ & Estimate", fontsize=16)
ax.set_xscale('linear')
ax.set_yscale('log')
ax.plot( redshift_spitler, 7e-2, **spitler_kwargs )
#ax.set_ylabel(r"$P(z)\Delta z$")
ax.set_xlim(0,0.8)
ax.set_ylim(1e-2,6)
plt.tick_params(axis='both', which='major', labelsize=14)
plt.show()

/data/prefrble/PreFRBLE/venv_PreFRBLE/lib/python3.4/site-packages/PreFRBLE/LikelihoodFunction.py:176: RuntimeWarning: divide by zero encountered in power
  self.dev =  ( self.Probability()*N )**-0.5

All this can be done in a single line by calling_____no_output_____

est, dev = RedshiftEstimate( DM_spitler, tmp, deviation=True, sigma=3 )
print( "z = {} - {} + {}".format( est, dev[0][0], dev[1][0] ) )z = 0.3313891390506023 - 0.3313891390506023 + 0.2686108609493978

### Estimate redshifts in FRBcat
Here we estimate redshift of all FRBs listed in FRBcat following the procedure above.
Results are written to a numpy array for later use as well as latex table for presentation of results._____no_output_____

###

sigma = 3  ## indicate order of sigma deviation to be obtained for <z>, choose >= 3 for lower limits

plot=True

redshift_estimates, deviations, teles = [], [], []

with open( file_tex_redshifts_DM, 'w' ) as f:

    ## prepare header for latex tabular
    f.write( R"\begin{tabular}{l|c|c|c|c|c|c}" )
    f.write( "\n\t")
#    f.write( R"ID & $\DMobs$ / $\unitDM$  & $\DMMW$ / $\unitDM$ & $z_{\rm SMD}(\DM)$  & $z_{\rm SFR}(\DM)$ & $z_{\rm coV}(\DM)$ \\" )
    f.write( R"ID & $\DMobs$ / $\unitDM$  & $\DMMW$ / $\unitDM$" )
    f.write( R"".join( [ R" & $z_{{\rm {0}}}(\DM)$ ".format(pop) for pop in populations] ) )
    f.write( R"\\")

    ## load FRBs from FRBcat
    FRBs = GetFRBcat(telescopes=telescopes)

    for FRB in FRBs:
        ## estimate redshift of FRBs based on extragalactic DM, considering all populations and selection effects of the detecting telescope

        telescope = telescopes_FRBcat_inv[FRB['tele']]

        ests, devs = [], []
        no_limit = False
        for population in populations:
            tmp = Scenario( telescope=telescope, population=population, **scenario.Properties( identifier=False) )
            est, dev = RedshiftEstimate( DM=FRB['DM']-FRB['DM_gal'], scenario=tmp, deviation=True, sigma=sigma )
            ests.append(est)
            devs.append(dev)
            ## check whether estimate provides a lower limit, otherwise, do not consider for output
            if np.isnan(est) or est - dev[0] <= 0:
                no_limit = True
                break
        if no_limit: continue

#        ests, devs = RedshiftEstimates( plot=False, DM=FRB['DM']-FRB['DM_gal'], telescope=telescopes_FRBcat_inv[FRB['tele']], scenario=scenario, sigma=sigma )

        ## only if redshift can be estimated    some have too low DM and can hence not be located in far Universe. These need to be investigated with the constrained simulation
        ## and only if lower limit is > 0
        if not np.any(np.isnan(ests)):
            ##  collect the FRB and write entry to tabular
            redshift_estimates.append( ests )
            deviations.append( devs )
            teles.append( telescope )
#            if ests[0] > 0.15:  ## only write down FRBs with reasonable redshift estimates, lowest two bins are z=[0.1,0.2], so nothing below .15 can deliver lower limits
            f.write( "\n\t")
            f.write( R"%s & %.1f & %.1f & $%.2f _{-%.2f} ^{+%.2f}$ & $%.2f _{-%.2f} ^{+%.2f}$ & $%.2f _{-%.2f} ^{+%.2f}$ \\" % ( FRB['ID'], FRB['DM'], FRB['DM_gal'], ests[0], devs[0][0], devs[0][1], ests[1], devs[1][0], devs[1][1], ests[2], devs[2][0], devs[2][1] ) )

    ## close tabular
    f.write( "\n")
    f.write( R"\end{tabular}" )

## write redshift estimates to npy file for later use. can be read with GetEstimatedRedshifts
redshift_estimates = np.array(redshift_estimates)
deviations = np.array(deviations)
names = ['redshift/SFR','dev-/SFR', 'dev+/SFR', 'redshift/coV','dev-/coV', 'dev+/coV', 'redshift/SMD','dev-/SMD', 'dev+/SMD', 'telescope']
formats = ['f','f','f','f','f','f','f','f','f','S']
dtype = np.dtype( {'names':names, 'formats':formats})
columns = [ redshift_estimates[:,0], deviations[:,0,0], deviations[:,0,1], redshift_estimates[:,1], deviations[:,1,0], deviations[:,1,1], redshift_estimates[:,2], deviations[:,2,0], deviations[:,2,1], teles ]

a = DataFrame()
for name, data in zip( names, columns ):
    a[name] = data

a.to_csv( file_estimated_redshifts_DM, index=False )

#a = np.array( list(zip( *columns )), dtype=dtype )
#np.savetxt( file_estimated_redshifts_DM, a )

#redshift_estimates = np.array(redshift_estimates)
#redshift_estimates.tofile( file_estimated_redshifts_DM, sep=' ')

/data/prefrble/PreFRBLE/venv_PreFRBLE/lib/python3.4/site-packages/PreFRBLE/LikelihoodFunction.py:176: RuntimeWarning: divide by zero encountered in power
  self.dev =  ( self.Probability()*N )**-0.5


# Notebook from krassowski/meningitis-integration
Path: exploration/Molecular_data_analysis.ipynb

### This notebook has moved!_____no_output_____As this notebook grew, it has been split up into:
- [protein/Unsupervised_analysis](protein/Unsupervised_analysis.ipynb)
  - [Notes_on_PCA_with_prcomp_and_factoextra](Notes_on_PCA_with_prcomp_and_factoextra.ipynb)
- [rna/Unsupervised_analysis](rna/Unsupervised_analysis.ipynb)

and others notebooks.

NB: the supervised part will be placed outside of `data_exploration` directory._____no_output_____