# Notebook from Jerome-maker/ensae_teaching_cs
Path: _doc/notebooks/2a/cffi_linear_regression.ipynb

# Optimisation de code avec cffi, numba, cython

L'idée est de recoder une fonction en C. On prend comme exemple la fonction de prédiction de la régression linéaire de [scikit-learn](http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html) et de prévoir le gain de temps qu'on obtient en recodant la fonction dans un langage plus rapide._____no_output_____

from jyquickhelper import add_notebook_menu
add_notebook_menu()_____no_output_____memo_time = []
import timeit

def unit(x):
    if x >= 1: return "%1.2f s" % x
    elif x >= 1e-3: return "%1.2f ms" % (x* 1000)
    elif x >= 1e-6: return "%1.2f µs" % (x* 1000**2)
    elif x >= 1e-9: return "%1.2f ns" % (x* 1000**3)
    else:
        return "%1.2g s" % x

def timeexe(legend, code, number=100, repeat=1000):
    rep = timeit.repeat(code, number=number, repeat=repeat, globals=globals())
    ave = sum(rep) / (number * repeat)
    std = (sum((x/number - ave)**2 for x in rep) / repeat)**0.5
    fir = rep[0]/number
    fir3 = sum(rep[:3]) / (3 * number)
    las3 = sum(rep[-3:]) / (3 * number)
    rep.sort()
    mini = rep[len(rep)//20] / number
    maxi = rep[-len(rep)//20] / number
    print("Moyenne: %s Ecart-type %s (with %d runs) in [%s, %s]" % (
                unit(ave), unit(std), number, unit(mini), unit(maxi)))
    return dict(legend=legend, average=ave, deviation=std, first=fir, first3=fir3,
                last3=las3, repeat=repeat, min5=mini, max5=maxi, code=code, run=number)_____no_output_____

## Régression linéaire_____no_output_____

from sklearn.datasets import load_diabetes
diabetes = load_diabetes()
diabetes_X_train = diabetes.data[:-20]
diabetes_X_test  = diabetes.data[-20:]
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test  = diabetes.target[-20:]_____no_output_____from sklearn.linear_model import LinearRegression
clr = LinearRegression()
clr.fit(diabetes_X_train, diabetes_y_train)_____no_output_____clr.coef______no_output_____clr.intercept______no_output_____z = diabetes_X_test[0:1,:]
memo_time.append(timeexe("sklearn.predict", "clr.predict(z)"))Moyenne: 37.24 µs Ecart-type 18.35 µs (with 100 runs) in [30.90 µs, 59.07 µs]
%timeit clr.predict(z)33.4 µs ± 3.04 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)

### optimisation avec cffi

On s'inspire de l'exemple [Purely for performance (API level, out-of-line)](http://cffi.readthedocs.io/en/latest/overview.html?highlight=example#purely-for-performance-api-level-out-of-line)._____no_output_____

from cffi import FFI
ffibuilder = FFI()

ffibuilder.cdef("int linreg(int, double *, double *, double, double *);")

ffibuilder.set_source("_linear_regression",
r"""
    static int linreg(int dimension, double * x, double *coef, double intercept, double * out)
    {
        for(; dimension > 0; --dimension, ++x, ++coef)
            intercept += *x * *coef;
        *out = intercept;
        return 1;
    }
""")

ffibuilder.compile(verbose=True)generating .\_linear_regression.c
(already up-to-date)
the current directory is 'C:\\xavierdupre\\__home_\\GitHub\\ensae_teaching_cs\\_doc\\notebooks\\2a'
running build_ext
building '_linear_regression' extension
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -Ic:\python370_x64\include -Ic:\python370_x64\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\cppwinrt" /Tc_linear_regression.c /Fo.\Release\_linear_regression.obj
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\link.exe /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:c:\python370_x64\libs /LIBPATH:c:\python370_x64\PCbuild\amd64 "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\lib\x64" "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\lib\um\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\um\x64" /EXPORT:PyInit__linear_regression .\Release\_linear_regression.obj /OUT:.\_linear_regression.cp37-win_amd64.pyd /IMPLIB:.\Release\_linear_regression.cp37-win_amd64.lib

La fonction compilée est accessible comme suit._____no_output_____

from _linear_regression import ffi, lib
lib.linreg_____no_output_____

On s'inspire de l'exemple [How to pass a Numpy array into a cffi function and how to get one back out?](https://stackoverflow.com/questions/16276268/how-to-pass-a-numpy-array-into-a-cffi-function-and-how-to-get-one-back-out)._____no_output_____

import numpy
out = numpy.zeros(1)_____no_output_____ptr_coef = clr.coef_.__array_interface__['data'][0]
cptr_coef = ffi.cast ( "double*" , ptr_coef )_____no_output_____x = diabetes_X_test[0:1,:]
ptr_x = x.__array_interface__['data'][0]
cptr_x = ffi.cast ( "double*" , ptr_x )_____no_output_____ptr_out = out.__array_interface__['data'][0]
cptr_out = ffi.cast ( "double*" , ptr_out )_____no_output_____n = len(clr.coef_)
lib.linreg(n, cptr_x, cptr_coef, clr.intercept_, cptr_out)_____no_output_____out_____no_output_____

On vérifie qu'on obtient bien la même chose._____no_output_____

clr.predict(x)_____no_output_____

Et on mesure le temps d'exécution :_____no_output_____

memo_time.append(timeexe("cffi-linreg", "lib.linreg(n, cptr_x, cptr_coef, clr.intercept_, cptr_out)"))Moyenne: 781.50 ns Ecart-type 541.34 ns (with 100 runs) in [458.27 ns, 1.43 µs]

C'est beaucoup plus rapide. Pour être totalement honnête, il faut mesurer les étapes qui consiste à extraire les pointeurs._____no_output_____

def predict_clr(x, clr):
    out = numpy.zeros(1)
    ptr_coef = clr.coef_.__array_interface__['data'][0]
    cptr_coef = ffi.cast ( "double*" , ptr_coef )
    ptr_x = x.__array_interface__['data'][0]
    cptr_x = ffi.cast ( "double*" , ptr_x )
    ptr_out = out.__array_interface__['data'][0]
    cptr_out = ffi.cast ( "double*" , ptr_out )
    lib.linreg(len(x), cptr_x, cptr_coef, clr.intercept_, cptr_out)
    return out

predict_clr(x, clr)_____no_output_____memo_time.append(timeexe("cffi-linreg-wrapped", "predict_clr(x, clr)"))Moyenne: 8.21 µs Ecart-type 3.68 µs (with 100 runs) in [6.86 µs, 14.30 µs]

Cela reste plus rapide._____no_output_____### cffi - seconde version

Comme on construit la fonction en dynamique (le code est connu lors de l'exécution), on peut facilement se passer de la boucle et écrire le code sans boucle et avec les coefficients._____no_output_____

res = " + ".join("{0}*x[{1}]".format(c, i) for i, c in enumerate(clr.coef_))
res_____no_output_____code = """
    static int linreg_custom(double * x, double * out)
    {{
        out[0] = {0} + {1};
    }}
""".format(clr.intercept_, res)
print(code)
    static int linreg_custom(double * x, double * out)
    {
        out[0] = 152.76430691633442 + 0.3034995490660432*x[0] + -237.63931533353403*x[1] + 510.5306054362253*x[2] + 327.7369804093466*x[3] + -814.1317093725387*x[4] + 492.81458798373217*x[5] + 102.8484521916802*x[6] + 184.60648905984*x[7] + 743.519616750542*x[8] + 76.09517221662392*x[9];
    }

from cffi import FFI
ffibuilder = FFI()

ffibuilder.cdef("int linreg_custom(double *, double *);")
ffibuilder.set_source("_linear_regression_custom", code)
ffibuilder.compile(verbose=True)generating .\_linear_regression_custom.c
(already up-to-date)
the current directory is 'C:\\xavierdupre\\__home_\\GitHub\\ensae_teaching_cs\\_doc\\notebooks\\2a'
running build_ext
building '_linear_regression_custom' extension
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -Ic:\python370_x64\include -Ic:\python370_x64\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\cppwinrt" /Tc_linear_regression_custom.c /Fo.\Release\_linear_regression_custom.obj
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\link.exe /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:c:\python370_x64\libs /LIBPATH:c:\python370_x64\PCbuild\amd64 "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\lib\x64" "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\lib\um\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\um\x64" /EXPORT:PyInit__linear_regression_custom .\Release\_linear_regression_custom.obj /OUT:.\_linear_regression_custom.cp37-win_amd64.pyd /IMPLIB:.\Release\_linear_regression_custom.cp37-win_amd64.lib
from _linear_regression_custom.lib import linreg_custom
linreg_custom(cptr_x, cptr_out)
out_____no_output_____memo_time.append(timeexe("cffi-linreg-custom", "linreg_custom(cptr_x, cptr_out)"))Moyenne: 400.93 ns Ecart-type 176.46 ns (with 100 runs) in [320.00 ns, 655.80 ns]

On a gagné un facteur 2._____no_output_____

def predict_clr_custom(x):
    out = numpy.zeros(1)
    ptr_x = x.__array_interface__['data'][0]
    cptr_x = ffi.cast("double*", ptr_x)
    ptr_out = out.__array_interface__['data'][0]
    cptr_out = ffi.cast("double*", ptr_out)
    linreg_custom(cptr_x, cptr_out)
    return out

predict_clr_custom(x)_____no_output_____memo_time.append(timeexe("cffi-linreg-custom wrapped", "predict_clr_custom(x)"))Moyenne: 5.53 µs Ecart-type 2.17 µs (with 100 runs) in [4.85 µs, 8.51 µs]

C'est un peu plus rapide._____no_output_____### et en float?

L'ordinateur fait la distinction entre les [double](https://en.wikipedia.org/wiki/Double-precision_floating-point_format) code sur 64 bit et les [float](https://en.wikipedia.org/wiki/Single-precision_floating-point_format) codé sur 32 bits. La précision est meilleure dans le premier cas et les calculs sont plus rapides dans le second. Dans le cas du machine learning, on préfère la rapidité à une perte précision en précision qui est souvent compensée par l'optimisation inhérente à tout problème de machine learning. Ce qu'on perd sur une observation, on le retrouve sur une autre._____no_output_____

res = " + ".join("{0}f*x[{1}]".format(c, i) for i, c in enumerate(clr.coef_))
res_____no_output_____code = """
    static int linreg_custom_float(float * x, float * out)
    {{
        out[0] = {0}f + {1};
    }}
""".format(clr.intercept_, res)
print(code)
    static int linreg_custom_float(float * x, float * out)
    {
        out[0] = 152.76430691633442f + 0.3034995490660432f*x[0] + -237.63931533353403f*x[1] + 510.5306054362253f*x[2] + 327.7369804093466f*x[3] + -814.1317093725387f*x[4] + 492.81458798373217f*x[5] + 102.8484521916802f*x[6] + 184.60648905984f*x[7] + 743.519616750542f*x[8] + 76.09517221662392f*x[9];
    }

from cffi import FFI
ffibuilder = FFI()

ffibuilder.cdef("int linreg_custom_float(float *, float *);")
ffibuilder.set_source("_linear_regression_custom_float", code)
ffibuilder.compile(verbose=True)generating .\_linear_regression_custom_float.c
(already up-to-date)
the current directory is 'C:\\xavierdupre\\__home_\\GitHub\\ensae_teaching_cs\\_doc\\notebooks\\2a'
running build_ext
building '_linear_regression_custom_float' extension
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -Ic:\python370_x64\include -Ic:\python370_x64\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\cppwinrt" /Tc_linear_regression_custom_float.c /Fo.\Release\_linear_regression_custom_float.obj
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\link.exe /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:c:\python370_x64\libs /LIBPATH:c:\python370_x64\PCbuild\amd64 "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\lib\x64" "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\lib\um\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\um\x64" /EXPORT:PyInit__linear_regression_custom_float .\Release\_linear_regression_custom_float.obj /OUT:.\_linear_regression_custom_float.cp37-win_amd64.pyd /IMPLIB:.\Release\_linear_regression_custom_float.cp37-win_amd64.lib
from _linear_regression_custom_float.lib import linreg_custom_float_____no_output_____def predict_clr_custom_float(x):
    out = numpy.zeros(1, dtype=numpy.float32)
    ptr_x = x.__array_interface__['data'][0]
    cptr_x = ffi.cast ( "float*" , ptr_x )
    ptr_out = out.__array_interface__['data'][0]
    cptr_out = ffi.cast ( "float*" , ptr_out )
    linreg_custom_float(cptr_x, cptr_out)
    return out_____no_output_____

Avant d'appeler la fonction, on doit transformer le vecteur iniatial en [float32](https://docs.scipy.org/doc/numpy/user/basics.types.html)._____no_output_____

x32 = x.astype(numpy.float32)
predict_clr_custom(x32)_____no_output_____memo_time.append(timeexe("cffi-linreg-custom-float wrapped", "predict_clr_custom(x32)"))Moyenne: 7.21 µs Ecart-type 3.59 µs (with 100 runs) in [5.15 µs, 13.26 µs]

La différence n'est pas flagrante. Mesurons le code C uniquement même si la partie Python ne peut pas être complètement évitée._____no_output_____

out = numpy.zeros(1, dtype=numpy.float32)
ptr_x = x32.__array_interface__['data'][0]
cptr_x = ffi.cast ( "float*" , ptr_x )
ptr_out = out.__array_interface__['data'][0]
cptr_out = ffi.cast ( "float*" , ptr_out )

memo_time.append(timeexe("cffi-linreg-custom-float32", "linreg_custom_float(cptr_x, cptr_out)"))Moyenne: 499.05 ns Ecart-type 250.34 ns (with 100 runs) in [331.85 ns, 1.05 µs]

La différence n'est pas significative._____no_output_____### SIMD

C'est un ensemble d'instructions processeur pour faire des opérations terme à terme sur 4 float32 aussi rapidement qu'une seule. Le processeur ne peut faire des opérations que les nombres sont copiés dans ses registres. Le programme passe alors son temps à copier des nombres depuis la mémoire vers les registres du processeur puis à faire la copie dans le chemin inverse pour le résultat. Les instructions [SIMD](https://en.wikipedia.org/wiki/SIMD) font gagner du temps du niveau du calcul. Au lieu de faire 4 opérations de multiplication terme à terme, il n'en fait plus qu'une. Il suffit de savoir comment utiliser ces instructions. Avec Visual Studio, elles sont accessible via ces fonctions [Memory and Initialization Using Streaming SIMD Extensions](https://msdn.microsoft.com/en-us/library/0hey67c0%28v=vs.100%29.aspx).
Le code suivant n'est probablement pas optimal mais il n'est pas trop compliqué à suivre._____no_output_____

code = """
#include <xmmintrin.h>

static int linreg_custom_float_simd(float * x, float * out)
{
    __m128 c1 = _mm_set_ps(0.3034995490664121f, -237.63931533353392f, 510.5306054362245f, 327.7369804093466f);
    __m128 c2 = _mm_set_ps(-814.1317093725389f, 492.81458798373245f, 102.84845219168025f, 184.60648905984064f);
    __m128 r1 = _mm_set_ss(152.76430691633442f);
    r1 = _mm_add_ss(r1, _mm_mul_ps(c1, _mm_load_ps(x)));
    r1 = _mm_add_ss(r1, _mm_mul_ps(c2, _mm_load_ps(x+4)));
    float r[4];
    _mm_store_ps(r, r1);
    out[0] = r[0] + r[1] + r[2] + r[3] + 743.5196167505419f * x[8] + 76.095172216624f * x[9];
    return 1;
}
"""_____no_output_____from cffi import FFI
ffibuilder = FFI()

ffibuilder.cdef("int linreg_custom_float_simd(float *, float *);")
ffibuilder.set_source("_linear_regression_custom_float_simd", code)
ffibuilder.compile(verbose=True)generating .\_linear_regression_custom_float_simd.c
(already up-to-date)
the current directory is 'C:\\xavierdupre\\__home_\\GitHub\\ensae_teaching_cs\\_doc\\notebooks\\2a'
running build_ext
building '_linear_regression_custom_float_simd' extension
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\cl.exe /c /nologo /Ox /W3 /GL /DNDEBUG /MD -Ic:\python370_x64\include -Ic:\python370_x64\include "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include" "-IC:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\include\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\cppwinrt" /Tc_linear_regression_custom_float_simd.c /Fo.\Release\_linear_regression_custom_float_simd.obj
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX86\x64\link.exe /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:c:\python370_x64\libs /LIBPATH:c:\python370_x64\PCbuild\amd64 "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\ATLMFC\lib\x64" "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6.1\lib\um\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.17763.0\um\x64" /EXPORT:PyInit__linear_regression_custom_float_simd .\Release\_linear_regression_custom_float_simd.obj /OUT:.\_linear_regression_custom_float_simd.cp37-win_amd64.pyd /IMPLIB:.\Release\_linear_regression_custom_float_simd.cp37-win_amd64.lib
from _linear_regression_custom_float_simd.lib import linreg_custom_float_simd_____no_output_____out = numpy.zeros(1, dtype=numpy.float32)
ptr_x = x32.__array_interface__['data'][0]
cptr_x = ffi.cast ( "float*" , ptr_x )
ptr_out = out.__array_interface__['data'][0]
cptr_out = ffi.cast ( "float*" , ptr_out )

linreg_custom_float_simd(cptr_x, cptr_out)
out_____no_output_____memo_time.append(timeexe("cffi-linreg-custom-float32-simd", "linreg_custom_float_simd(cptr_x, cptr_out)"))Moyenne: 515.47 ns Ecart-type 663.25 ns (with 100 runs) in [316.05 ns, 987.65 ns]

C'est légèrement mieux, quelques références :

* [aligned_vs_unaligned_load.c](https://gist.github.com/rmcgibbo/7689820) : c'est du code mais facile à lire.
* [How to Write Fast Numerical Code](https://www.inf.ethz.ch/personal/markusp/teaching/263-2300-ETH-spring11/slides/class17.pdf)

Les processeurs évoluent au fil du temps, 4 float, 8 float, [SIMD2](https://msdn.microsoft.com/en-us/library/kcwz153a%28v=vs.100%29.aspx), [FMA4 Intrinsics Added for Visual Studio 2010 SP1](https://msdn.microsoft.com/en-us/library/gg445134%28v=vs.100%29.aspx), [AVX](https://software.intel.com/en-us/articles/introduction-to-intel-advanced-vector-extensions/)._____no_output_____### Réécriture purement Python

On continue avec uniquement du Python sans numpy._____no_output_____

coef = clr.coef_
list(coef)_____no_output_____code = str(clr.intercept_) + "+" + "+".join("x[{0}]*({1})".format(i, c) for i, c in enumerate(coef))
code_____no_output_____def predict_clr_python(x):
    return 152.764306916+x[0]*0.3034995490664121+x[1]*(-237.63931533353392)+x[2]*510.5306054362245+ \
            x[3]*327.7369804093466+ \
            x[4]*(-814.1317093725389)+x[5]*492.81458798373245+x[6]*102.84845219168025+ \
            x[7]*184.60648905984064+x[8]*743.5196167505419+x[9]*76.095172216624

predict_clr_python(x[0])_____no_output_____z = list(x[0])
memo_time.append(timeexe("python-linreg-custom", "predict_clr_python(z)"))Moyenne: 3.15 µs Ecart-type 1.81 µs (with 100 runs) in [2.00 µs, 6.32 µs]

De façon assez surprenante, c'est plutôt rapide. Et si on y mettait une boucle._____no_output_____

def predict_clr_python_loop(x, coef, intercept):
    return intercept + sum(a*b for a, b in zip(x, coef))

predict_clr_python_loop(x[0], list(clr.coef_), clr.intercept_)_____no_output_____coef = list(clr.coef_)
intercept = clr.intercept_
memo_time.append(timeexe("python-linreg", "predict_clr_python_loop(z, coef, intercept)"))Moyenne: 5.87 µs Ecart-type 3.95 µs (with 100 runs) in [3.48 µs, 10.08 µs]

A peine plus long._____no_output_____### Réécriture avec Python et numpy_____no_output_____

def predict_clr_numpy(x, coef, intercept):
    return intercept + numpy.dot(coef, x).sum()

predict_clr_numpy(x[0], clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("numpy-linreg-numpy", "predict_clr_numpy(z, coef, clr.intercept_)"))Moyenne: 9.97 µs Ecart-type 4.96 µs (with 100 runs) in [7.37 µs, 19.62 µs]

Les dimensions des tableaux sont trop petites pour que le calcul matriciel apporte une différence. On se retrouve dans le cas *cffi* où les échanges Python - C grignotent tout le temps de calcul._____no_output_____### numba

[numba](http://numba.pydata.org/) essaye de compiler à la volée des bouts de codes écrits en Python. On induque quelle fonction optimiser en faisant précéder la fonction de ``@jit``. Toutes les écritures ne fonctionnent, typiquement, certaines listes en compréhension soulèvent une exception. Il faut donc écrire son code en Python d'une façon assez proche de ce qu'il serait en C._____no_output_____

from numba import jit_____no_output_____@jit
def predict_clr_numba(x, coef, intercept):
    s = intercept
    for i in range(0, len(x)):
        s += coef[i] * x[i]
    return s

predict_clr_numba(z, clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("numba-linreg-notype", "predict_clr_numba(z, clr.coef_, clr.intercept_)"))Moyenne: 32.13 µs Ecart-type 15.74 µs (with 100 runs) in [20.77 µs, 59.50 µs]

Plutôt rapide !_____no_output_____

@jit('double(double[:], double[:], double)')
def predict_clr_numba_cast(x, coef, intercept):
    s = intercept
    for i in range(0, len(x)):
        s += coef[i] * x[i]
    return s

# La fonction ne fonctionne qu'avec un numpy.array car le langage C est fortement typé.
predict_clr_numba_cast(x[0], clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("numba-linreg-type", "predict_clr_numba_cast(x[0], clr.coef_, clr.intercept_)"))Moyenne: 1.30 µs Ecart-type 1.45 µs (with 100 runs) in [821.72 ns, 2.50 µs]

On voit que plus on donne d'information au compilateur, plus il est capable d'optimiser._____no_output_____

@jit('float32(float32[:], float32[:], float32)')
def predict_clr_numba_cast_float(x, coef, intercept):
    s = intercept
    for i in range(0, len(x)):
        s += coef[i] * x[i]
    return s

# La fonction ne fonctionne qu'avec un numpy.array car le langage C est fortement typé.
x32 = x[0].astype(numpy.float32)
c32 = clr.coef_.astype(numpy.float32)
i32 = numpy.float32(clr.intercept_)
predict_clr_numba_cast_float(x32, c32, i32)_____no_output_____memo_time.append(timeexe("numba-linreg-type-float32", "predict_clr_numba_cast_float(x32, c32, i32)"))Moyenne: 1.17 µs Ecart-type 5.36 µs (with 100 runs) in [580.73 ns, 2.03 µs]

On essaye avec les coefficients dans la fonction._____no_output_____

@jit('double(double[:])')
def predict_clr_numba_cast_custom(x):
    coef = [ 3.03499549e-01, -2.37639315e+02,  5.10530605e+02,  3.27736980e+02,
            -8.14131709e+02,  4.92814588e+02,  1.02848452e+02,  1.84606489e+02,
             7.43519617e+02,  7.60951722e+01]
    s = 152.76430691633442
    for i in range(0, len(x)):
        s += coef[i] * x[i]
    return s

predict_clr_numba_cast_custom(x[0])_____no_output_____memo_time.append(timeexe("numba-linreg-type-custom", "predict_clr_numba_cast_custom(x[0])"))Moyenne: 958.94 ns Ecart-type 492.60 ns (with 100 runs) in [687.40 ns, 1.90 µs]

On se rapproche des temps obtenus avec *cffi* sans *wrapping*, cela signifie que *numba* fait un bien meilleur travail à ce niveau que le wrapper rapidement créé._____no_output_____

@jit('double(double[:], double[:], double)')
def predict_clr_numba_numpy(x, coef, intercept):
    return intercept + numpy.dot(coef, x).sum()

predict_clr_numba_numpy(x[0], clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("numba-linreg-type-numpy", "predict_clr_numba_numpy(x[0], clr.coef_, clr.intercept_)"))Moyenne: 8.31 µs Ecart-type 4.03 µs (with 100 runs) in [5.84 µs, 15.30 µs]

*numba* est moins performant quand *numpy* est impliqué car le code de numpy n'est pas réécrit, il est appelé._____no_output_____### cython

[cython](http://cython.org/) permet de créer des extensions C de plus grande envergure que *numba*. C'est l'option choisie par [scikit-learn](http://scikit-learn.org/stable/). Il vaut mieux connaître le C pour s'en servir et là encore, l'objectif est de réduire les échanges Python / C qui coûtent cher._____no_output_____

%load_ext cython_____no_output_____%%cython
def predict_clr_cython(x, coef, intercept):
    s = intercept
    for i in range(0, len(x)):
        s += coef[i] * x[i]
    return s_____no_output_____predict_clr_cython(x[0], clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("cython-linreg", "predict_clr_cython(x[0], clr.coef_, clr.intercept_)"))Moyenne: 3.55 µs Ecart-type 2.19 µs (with 100 runs) in [2.28 µs, 7.82 µs]

Cython fait moins bien que *numba* dans notre cas et l'optimisation proposée est assez proche du temps déjà obtenue avec le langage Python seul. Cela est dû au fait que la plupart des objets tels que du code associé aux listes ou aux dictionnaires ont été réécrits en C._____no_output_____

%%cython
cimport numpy as npc

def predict_clr_cython_type(npc.ndarray[double, ndim=1, mode='c'] x,
                            npc.ndarray[double, ndim=1, mode='c'] coef,
                            double intercept):
    cdef double s = intercept
    for i in range(0, x.shape[0]):
        s += coef[i] * x[i]
    return s_____no_output_____predict_clr_cython_type(x[0], clr.coef_, clr.intercept_)_____no_output_____memo_time.append(timeexe("cython-linreg-type", "predict_clr_cython_type(x[0], clr.coef_, clr.intercept_)"))Moyenne: 1.82 µs Ecart-type 1.10 µs (with 100 runs) in [1.37 µs, 3.66 µs]

Le temps est quasi identique avec un écart type moins grand de façon significative._____no_output_____### Une dernière option : ONNX

[ONNX](https://onnx.ai/) est un format de sérialisation qui permet de décrire un modèle de modèle de machine learning ou de deep learning. Cela permet de dissocer le modèle de la librairie qui a servi à le produire (voir [ML.net and ONNX](http://www.xavierdupre.fr/app/machinelearningext/helpsphinx/aonnx.html)._____no_output_____

try:
    from skl2onnx import convert_sklearn
    from skl2onnx.common.data_types import FloatTensorType
    import onnxruntime
    import onnx
    ok_onnx = True
    print("onnx, skl2onnx, onnxruntime sont disponibles.")

    def save_model(onnx_model, filename):
        with open(filename, "wb") as f:
            f.write(onnx_model.SerializeToString())
except ImportError as e:
    print("La suite requiert onnx, skl2onnx et onnxruntime.")
    print(e)
    ok_onnx = Falseonnx, skl2onnx, onnxruntime sont disponibles.

On convertit le modèle au format [ONNX](https://onnx.ai/)._____no_output_____

if ok_onnx:
    onnx_model = convert_sklearn(clr, 'model', [('input', FloatTensorType([1, clr.coef_.shape[0]]))])
    save_model(onnx_model, 'model.onnx')

    model_onnx = onnx.load('model.onnx')
    print("Modèle sérialisé au format ONNX")
    print(model_onnx)
else:
    print("onnx, onnxmltools, onnxruntime sont disponibles.")The maximum opset needed by this model is only 1.

On calcule les prédictions. Le module {onnxruntime](https://docs.microsoft.com/en-us/python/api/overview/azure/onnx/intro?view=azure-onnx-py) optimise les calculs pour des modèles de deep learning. Cela explique pourquoi tous les calculs sont réalisés avec des réels représentés sur 4 octets [numpy.float32](https://docs.scipy.org/doc/numpy/user/basics.types.html?highlight=float32)._____no_output_____

if ok_onnx:
    sess = onnxruntime.InferenceSession("model.onnx")
    for i in sess.get_inputs():
        print('Input:', i)
    for o in sess.get_outputs():
        print('Output:', o)

    def predict_onnxrt(x):
        return sess.run(["variable"], {'input': x})

    print("Prediction:", predict_onnxrt(x.astype(numpy.float32)))Input: NodeArg(name='input', type='tensor(float)', shape=[1, 10])
Output: NodeArg(name='variable', type='tensor(float)', shape=[1, 1])
Prediction: [array([[197.61847]], dtype=float32)]
if ok_onnx:
    x32 = x.astype(numpy.float32)
    memo_time.append(timeexe("onnxruntime-float32", "predict_onnxrt(x32)"))
    memo_time.append(timeexe("onnxruntime-float64", "predict_onnxrt(x.astype(numpy.float32))"))Moyenne: 15.61 µs Ecart-type 6.97 µs (with 100 runs) in [11.03 µs, 28.25 µs]
Moyenne: 16.05 µs Ecart-type 6.55 µs (with 100 runs) in [12.40 µs, 31.08 µs]

### Récapitulatif_____no_output_____

import pandas
df = pandas.DataFrame(data=memo_time)
df = df.set_index("legend").sort_values("average")
df_____no_output_____

On enlève quelques colonnes et on rappelle :

* **cffi**: signifie optimisé avec cffi
* **custom**: pas de boucle mais la fonction ne peut prédire qu'une seule régression linéaire
* **float32**: utilise des float et non des double
* **linreg**: régression linéaire
* **numba**: optimisation avec numba
* **numpy**: optimisation avec numpy
* **python**: pas de C, que du python
* **simd**: optimisé avec les instructions SIMD
* **sklearn**: fonction sklearn.predict
* **static**: la fonction utilise des variables statiques
* **type**: la fonction est typée et ne fonctionne qu'avec un type précis en entrée.
* **wrapped**: code optimisé mais embabllé dans une fonction Python qui elle ne l'est pas (les containers sont recréés à chaque fois)_____no_output_____

cols = ["average", "deviation", "min5", "max5", "run", "code"]
df[cols]_____no_output_____%matplotlib inline
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 1, figsize=(14,6))
df[["average", "deviation"]].plot(kind="barh", logx=True, ax=ax, xerr="deviation",
                                  legend=False, fontsize=12, width=0.8)
ax.set_ylabel("")
ax.grid(b=True, which="major")
ax.grid(b=True, which="minor");_____no_output_____

Il manque à ce comparatif le [GPU](https://en.wikipedia.org/wiki/Graphics_processing_unit) mais c'est un peu plus complexe à mettre en oeuvre, il faut une carte [GPU](https://fr.wikipedia.org/wiki/Processeur_graphique) et la parallélisation n'apporterait pas énormément compte tenu de la faible dimension du problème._____no_output_____### Prédiction one-off et biais de mesure

Le graphique précédent montre que la fonction ``predict`` de *scikit-learn* est la plus lente. La première raison est que ce code est valable pour toutes les régresssions linéaires alors que toutes les autres fonctions sont spécialisées pour un seul modèle. La seconde raison est que le code de *scikit-learn* est optimisé pour le calcul de plusieurs prédictions à la fois alors que toutes les autres fonctions n'en calcule qu'une seule (scénario dit *one-off*). On compare à ce que donnerait unev version purement python et numpy._____no_output_____

def predict_clr_python_loop_multi(x, coef, intercept):
    # On s'attend à deux dimension.
    res = numpy.zeros((x.shape[0], 1))
    res[:, 0] = intercept
    for i in range(0, x.shape[0]):
        res[i, 0] += sum(a*b for a, b in zip(x[i, :], coef))
    return res

predict_clr_python_loop_multi(diabetes_X_test[:2], clr.coef_, clr.intercept_)_____no_output_____def predict_clr_numpy_loop_multi(x, coef, intercept):
    # On s'attend à deux dimension.
    res = numpy.ones((x.shape[0], 1)) * intercept
    res += x @ coef.reshape((len(coef), 1))
    return res

predict_clr_numpy_loop_multi(diabetes_X_test[:2], clr.coef_, clr.intercept_)_____no_output_____def predict_clr_numba_cast_multi(X, coef, intercept):
    return [predict_clr_numba_cast(x, coef, intercept) for x in X]

predict_clr_numba_cast_multi(diabetes_X_test[:2], clr.coef_, clr.intercept_)_____no_output_____def predict_clr_cython_type_multi(X, coef, intercept):
    return [predict_clr_cython_type(x, coef, intercept) for x in X]

predict_clr_cython_type_multi(diabetes_X_test[:2], clr.coef_, clr.intercept_)_____no_output_____memo = []
batch = [1, 10, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 10000,
         20000, 50000, 75000, 100000, 150000, 200000, 300000, 400000,
         500000, 600000]
number = 10
for i in batch:
    if i <= diabetes_X_test.shape[0]:
        mx = diabetes_X_test[:i]
    else:
        mxs = [diabetes_X_test] * (i // diabetes_X_test.shape[0] + 1)
        mx = numpy.vstack(mxs)
        mx = mx[:i]

    print("batch", "=", i)
    repeat=20 if i >= 5000 else 100

    memo.append(timeexe("sklearn.predict %d" % i, "clr.predict(mx)",
                        repeat=repeat, number=number))
    memo[-1]["batch"] = i
    memo[-1]["lib"] = "sklearn"

    if i <= 1000:
        # très lent
        memo.append(timeexe("python %d" % i, "predict_clr_python_loop_multi(mx, clr.coef_, clr.intercept_)",
                            repeat=20, number=number))
        memo[-1]["batch"] = i
        memo[-1]["lib"] = "python"

    memo.append(timeexe("numpy %d" % i, "predict_clr_numpy_loop_multi(mx, clr.coef_, clr.intercept_)",
                        repeat=repeat, number=number))
    memo[-1]["batch"] = i
    memo[-1]["lib"] = "numpy"

    if i <= 10000:
        # très lent
        memo.append(timeexe("numba %d" % i, "predict_clr_numba_cast_multi(mx, clr.coef_, clr.intercept_)",
                            repeat=repeat, number=number))
        memo[-1]["batch"] = i
        memo[-1]["lib"] = "numba"

    if i <= 1000:
        # très lent
        memo.append(timeexe("cython %d" % i, "predict_clr_cython_type_multi(mx, clr.coef_, clr.intercept_)",
                            repeat=repeat, number=number))
        memo[-1]["batch"] = i
        memo[-1]["lib"] = "cython"

    if ok_onnx:
        memo.append(timeexe("onnxruntime %d" % i, "predict_onnxrt(mx.astype(numpy.float32))",
                            repeat=repeat, number=number))
        memo[-1]["batch"] = i
        memo[-1]["lib"] = "onnxruntime"batch = 1
Moyenne: 61.72 µs Ecart-type 20.51 µs (with 10 runs) in [36.70 µs, 100.19 µs]
Moyenne: 12.42 µs Ecart-type 6.08 µs (with 10 runs) in [9.92 µs, 36.42 µs]
Moyenne: 8.15 µs Ecart-type 1.73 µs (with 10 runs) in [7.47 µs, 9.17 µs]
Moyenne: 2.38 µs Ecart-type 508.42 ns (with 10 runs) in [2.21 µs, 3.16 µs]
Moyenne: 3.34 µs Ecart-type 2.32 µs (with 10 runs) in [2.65 µs, 6.76 µs]
Moyenne: 20.95 µs Ecart-type 10.21 µs (with 10 runs) in [14.85 µs, 40.61 µs]
batch = 10
Moyenne: 51.02 µs Ecart-type 19.16 µs (with 10 runs) in [36.46 µs, 93.71 µs]
Moyenne: 131.17 µs Ecart-type 64.49 µs (with 10 runs) in [80.24 µs, 261.33 µs]
Moyenne: 7.72 µs Ecart-type 653.54 ns (with 10 runs) in [7.59 µs, 7.74 µs]
Moyenne: 10.04 µs Ecart-type 5.02 µs (with 10 runs) in [9.01 µs, 15.13 µs]
Moyenne: 19.72 µs Ecart-type 9.62 µs (with 10 runs) in [13.95 µs, 36.23 µs]
Moyenne: 36.17 µs Ecart-type 6.33 µs (with 10 runs) in [31.92 µs, 42.75 µs]
batch = 100
Moyenne: 74.33 µs Ecart-type 43.73 µs (with 10 runs) in [35.67 µs, 152.30 µs]
Moyenne: 1.21 ms Ecart-type 478.46 µs (with 10 runs) in [782.46 µs, 2.28 ms]
Moyenne: 13.86 µs Ecart-type 9.37 µs (with 10 runs) in [8.18 µs, 34.17 µs]
Moyenne: 91.47 µs Ecart-type 39.33 µs (with 10 runs) in [75.06 µs, 205.27 µs]
Moyenne: 166.69 µs Ecart-type 74.59 µs (with 10 runs) in [126.10 µs, 332.92 µs]
Moyenne: 41.07 µs Ecart-type 15.02 µs (with 10 runs) in [26.03 µs, 69.77 µs]
batch = 200
Moyenne: 67.71 µs Ecart-type 42.18 µs (with 10 runs) in [36.15 µs, 126.30 µs]
Moyenne: 2.53 ms Ecart-type 734.36 µs (with 10 runs) in [1.65 ms, 4.07 ms]
Moyenne: 16.80 µs Ecart-type 6.97 µs (with 10 runs) in [9.05 µs, 24.02 µs]
Moyenne: 182.37 µs Ecart-type 73.63 µs (with 10 runs) in [149.81 µs, 416.43 µs]
Moyenne: 448.78 µs Ecart-type 206.74 µs (with 10 runs) in [252.88 µs, 841.91 µs]
Moyenne: 47.45 µs Ecart-type 22.24 µs (with 10 runs) in [32.47 µs, 82.41 µs]
batch = 500
Moyenne: 44.61 µs Ecart-type 13.69 µs (with 10 runs) in [39.59 µs, 90.75 µs]
Moyenne: 5.09 ms Ecart-type 923.90 µs (with 10 runs) in [3.97 ms, 7.15 ms]
Moyenne: 10.60 µs Ecart-type 1.09 µs (with 10 runs) in [10.39 µs, 10.51 µs]
Moyenne: 442.09 µs Ecart-type 128.29 µs (with 10 runs) in [355.32 µs, 823.74 µs]
Moyenne: 777.43 µs Ecart-type 208.18 µs (with 10 runs) in [607.56 µs, 1.27 ms]
Moyenne: 96.23 µs Ecart-type 50.76 µs (with 10 runs) in [60.33 µs, 219.46 µs]
batch = 1000
Moyenne: 67.87 µs Ecart-type 56.17 µs (with 10 runs) in [44.05 µs, 222.46 µs]
Moyenne: 10.01 ms Ecart-type 1.75 ms (with 10 runs) in [7.98 ms, 14.46 ms]
Moyenne: 17.70 µs Ecart-type 10.98 µs (with 10 runs) in [13.59 µs, 34.69 µs]
Moyenne: 861.28 µs Ecart-type 216.40 µs (with 10 runs) in [721.14 µs, 1.42 ms]
Moyenne: 1.25 ms Ecart-type 83.58 µs (with 10 runs) in [1.21 ms, 1.46 ms]
Moyenne: 112.48 µs Ecart-type 19.49 µs (with 10 runs) in [105.96 µs, 147.99 µs]
batch = 2000
Moyenne: 66.24 µs Ecart-type 13.14 µs (with 10 runs) in [53.02 µs, 85.93 µs]
Moyenne: 16.39 µs Ecart-type 570.74 ns (with 10 runs) in [16.16 µs, 17.03 µs]
Moyenne: 1.48 ms Ecart-type 91.96 µs (with 10 runs) in [1.42 ms, 1.75 ms]
Moyenne: 204.37 µs Ecart-type 12.70 µs (with 10 runs) in [197.41 µs, 237.55 µs]
batch = 3000
Moyenne: 67.97 µs Ecart-type 19.24 µs (with 10 runs) in [58.67 µs, 101.97 µs]
Moyenne: 22.16 µs Ecart-type 5.87 µs (with 10 runs) in [19.52 µs, 29.83 µs]
Moyenne: 2.26 ms Ecart-type 157.83 µs (with 10 runs) in [2.16 ms, 2.67 ms]
Moyenne: 303.46 µs Ecart-type 38.51 µs (with 10 runs) in [288.95 µs, 395.06 µs]
batch = 4000
Moyenne: 76.48 µs Ecart-type 17.38 µs (with 10 runs) in [64.83 µs, 88.18 µs]
Moyenne: 23.36 µs Ecart-type 5.57 µs (with 10 runs) in [22.24 µs, 25.17 µs]
Moyenne: 2.92 ms Ecart-type 152.70 µs (with 10 runs) in [2.85 ms, 3.20 ms]
Moyenne: 396.96 µs Ecart-type 46.83 µs (with 10 runs) in [380.32 µs, 429.04 µs]
batch = 5000
Moyenne: 90.18 µs Ecart-type 6.40 µs (with 10 runs) in [89.05 µs, 111.25 µs]
Moyenne: 24.74 µs Ecart-type 1.37 µs (with 10 runs) in [24.38 µs, 30.70 µs]
Moyenne: 3.81 ms Ecart-type 294.09 µs (with 10 runs) in [3.62 ms, 4.88 ms]
Moyenne: 483.24 µs Ecart-type 20.95 µs (with 10 runs) in [471.78 µs, 569.48 µs]
batch = 10000
Moyenne: 103.63 µs Ecart-type 6.08 µs (with 10 runs) in [101.02 µs, 127.53 µs]
Moyenne: 39.31 µs Ecart-type 3.28 µs (with 10 runs) in [37.10 µs, 49.98 µs]
Moyenne: 7.50 ms Ecart-type 461.95 µs (with 10 runs) in [7.19 ms, 8.81 ms]
Moyenne: 1.24 ms Ecart-type 319.43 µs (with 10 runs) in [967.50 µs, 1.82 ms]
batch = 20000
Moyenne: 196.19 µs Ecart-type 42.92 µs (with 10 runs) in [170.47 µs, 354.37 µs]
Moyenne: 70.45 µs Ecart-type 11.47 µs (with 10 runs) in [62.46 µs, 103.39 µs]
Moyenne: 2.08 ms Ecart-type 244.46 µs (with 10 runs) in [1.88 ms, 2.60 ms]
batch = 50000
Moyenne: 944.13 µs Ecart-type 176.43 µs (with 10 runs) in [673.07 µs, 1.28 ms]
Moyenne: 323.42 µs Ecart-type 60.33 µs (with 10 runs) in [283.26 µs, 487.47 µs]
Moyenne: 7.10 ms Ecart-type 691.55 µs (with 10 runs) in [6.69 ms, 9.75 ms]
batch = 75000
Moyenne: 1.28 ms Ecart-type 170.95 µs (with 10 runs) in [1.10 ms, 1.88 ms]
Moyenne: 337.19 µs Ecart-type 35.55 µs (with 10 runs) in [302.46 µs, 398.42 µs]
Moyenne: 10.44 ms Ecart-type 649.35 µs (with 10 runs) in [10.10 ms, 12.47 ms]
batch = 100000
Moyenne: 1.75 ms Ecart-type 199.38 µs (with 10 runs) in [1.56 ms, 2.31 ms]
Moyenne: 487.81 µs Ecart-type 49.89 µs (with 10 runs) in [426.90 µs, 566.83 µs]
Moyenne: 13.82 ms Ecart-type 307.45 µs (with 10 runs) in [13.60 ms, 14.58 ms]
batch = 150000
Moyenne: 3.92 ms Ecart-type 285.28 µs (with 10 runs) in [3.72 ms, 4.99 ms]
Moyenne: 2.69 ms Ecart-type 106.66 µs (with 10 runs) in [2.54 ms, 2.95 ms]
Moyenne: 20.91 ms Ecart-type 750.62 µs (with 10 runs) in [20.53 ms, 23.89 ms]
batch = 200000
Moyenne: 5.33 ms Ecart-type 228.93 µs (with 10 runs) in [5.04 ms, 6.02 ms]
Moyenne: 3.58 ms Ecart-type 140.76 µs (with 10 runs) in [3.35 ms, 3.89 ms]
Moyenne: 28.24 ms Ecart-type 1.46 ms (with 10 runs) in [27.45 ms, 33.64 ms]
batch = 300000
Moyenne: 9.47 ms Ecart-type 1.21 ms (with 10 runs) in [8.04 ms, 11.99 ms]
Moyenne: 6.58 ms Ecart-type 923.96 µs (with 10 runs) in [5.45 ms, 8.43 ms]
Moyenne: 44.73 ms Ecart-type 2.43 ms (with 10 runs) in [42.35 ms, 50.52 ms]
batch = 400000
Moyenne: 11.12 ms Ecart-type 751.22 µs (with 10 runs) in [10.08 ms, 12.47 ms]
Moyenne: 7.29 ms Ecart-type 406.01 µs (with 10 runs) in [6.81 ms, 8.54 ms]
Moyenne: 58.54 ms Ecart-type 2.16 ms (with 10 runs) in [56.73 ms, 64.76 ms]
batch = 500000
Moyenne: 13.46 ms Ecart-type 799.10 µs (with 10 runs) in [12.65 ms, 15.37 ms]
Moyenne: 9.22 ms Ecart-type 720.11 µs (with 10 runs) in [8.68 ms, 11.41 ms]
Moyenne: 72.30 ms Ecart-type 1.37 ms (with 10 runs) in [70.96 ms, 74.87 ms]
batch = 600000
Moyenne: 15.86 ms Ecart-type 583.58 µs (with 10 runs) in [15.16 ms, 17.41 ms]
Moyenne: 10.88 ms Ecart-type 380.11 µs (with 10 runs) in [10.29 ms, 11.95 ms]
Moyenne: 87.84 ms Ecart-type 3.57 ms (with 10 runs) in [84.77 ms, 97.15 ms]
dfb = pandas.DataFrame(memo)[["average", "lib", "batch"]]
piv = dfb.pivot("batch", "lib", "average")
piv_____no_output_____for c in piv.columns:
    piv["ave_" + c] = piv[c] / piv.index
piv_____no_output_____libs = list(c for c in piv.columns if "ave_" in c)
ax = piv.plot(y=libs, logy=True, logx=True, figsize=(10, 5))
ax.set_title("Evolution du temps de prédiction selon la taille du batch")
ax.grid(True);_____no_output_____

Le minimum obtenu est pour $10^{-8} s$ soit 10 ns. Cela montre que la comparaisson précédente était incomplète voire biaisée. Tout dépend de l'usage qu'on fait de la fonction de prédiction même s'il sera toujours possible de d'écrire un code spécialisé plus rapide que toute autre fonction générique. En général, plus on reste du côté Python, plus le programme est lent. Le nombre de passage de l'un à l'autre, selon la façon dont il est fait ralenti aussi. En tenant compte de cela, le programme rouge sera plus lent que le vert._____no_output_____

from pyquickhelper.helpgen import NbImage
NbImage("pycpp.png")_____no_output_____

Ces résultats sont d'une façon générale assez volatile car le temps de calcul est enrobé dans plusieurs fonctions Python qui rendent une mesure précise difficile. Il reste néanmoins une bonne idée des ordres de grandeurs._____no_output_____## Random Forest

On reproduit les mêmes résultats pour une random forest mais la réécriture n'est plus aussi simple qu'une régression linéaire._____no_output_____### Une prédiction à la fois_____no_output_____

from sklearn.datasets import load_diabetes
diabetes = load_diabetes()
diabetes_X_train = diabetes.data[:-20]
diabetes_X_test  = diabetes.data[-20:]
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test  = diabetes.target[-20:]_____no_output_____from sklearn.ensemble import RandomForestRegressor
rf = RandomForestRegressor(n_estimators=10)
rf.fit(diabetes_X_train, diabetes_y_train)_____no_output_____memo_time = []
x = diabetes_X_test[:1]
memo_time.append(timeexe("sklearn-rf", "rf.predict(x)", repeat=100, number=20))Moyenne: 598.88 µs Ecart-type 55.79 µs (with 20 runs) in [546.51 µs, 736.22 µs]

C'est beaucoup plus long que la régression linéaire. On essaye avec *onnx*._____no_output_____

if ok_onnx:
    onnxrf_model = convert_sklearn(rf, 'model', [('input', FloatTensorType([1, clr.coef_.shape[0]]))])
    save_model(onnxrf_model, 'model_rf.onnx')
    model_onnx = onnx.load('model_rf.onnx')The maximum opset needed by this model is only 1.
if ok_onnx:
    sess = onnxruntime.InferenceSession("model_rf.onnx")
    for i in sess.get_inputs():
        print('Input:', i)
    for o in sess.get_outputs():
        print('Output:', o)

    def predict_onnxrt_rf(x):
        return sess.run(["variable"], {'input': x})

    print(predict_onnxrt_rf(x.astype(numpy.float32)))
    memo_time.append(timeexe("onnx-rf", "predict_onnxrt_rf(x.astype(numpy.float32))", repeat=100, number=20))Input: NodeArg(name='input', type='tensor(float)', shape=[1, 10])
Output: NodeArg(name='variable', type='tensor(float)', shape=[1, 1])
[array([[172.3]], dtype=float32)]
Moyenne: 28.42 µs Ecart-type 7.85 µs (with 20 runs) in [18.47 µs, 47.33 µs]

C'est beaucoup plus rapide._____no_output_____

import pandas
df2 = pandas.DataFrame(data=memo_time)
df2 = df2.set_index("legend").sort_values("average")
df2_____no_output_____fig, ax = plt.subplots(1, 1, figsize=(14,4))
df2[["average", "deviation"]].plot(kind="barh", logx=True, ax=ax, xerr="deviation",
                                   legend=False, fontsize=12, width=0.8)
ax.set_ylabel("")
ax.grid(b=True, which="major")
ax.grid(b=True, which="minor");_____no_output_____

### Prédiction en batch_____no_output_____

memo = []
batch = [1, 10, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 10000,
         20000, 50000, 75000, 100000, 150000, 200000, 300000, 400000,
         500000, 600000]
number = 10
repeat = 10
for i in batch[:15]:
    if i <= diabetes_X_test.shape[0]:
        mx = diabetes_X_test[:i]
    else:
        mxs = [diabetes_X_test] * (i // diabetes_X_test.shape[0] + 1)
        mx = numpy.vstack(mxs)
        mx = mx[:i]

    print("batch", "=", i)

    memo.append(timeexe("sklearn.predict %d" % i, "rf.predict(mx)",
                        repeat=repeat, number=number))
    memo[-1]["batch"] = i
    memo[-1]["lib"] = "sklearn"

    if ok_onnx:
        memo.append(timeexe("onnxruntime %d" % i, "predict_onnxrt_rf(mx.astype(numpy.float32))",
                            repeat=repeat, number=number))
        memo[-1]["batch"] = i
        memo[-1]["lib"] = "onnxruntime"batch = 1
Moyenne: 674.08 µs Ecart-type 216.17 µs (with 10 runs) in [535.19 µs, 1.25 ms]
Moyenne: 23.35 µs Ecart-type 7.29 µs (with 10 runs) in [18.57 µs, 44.44 µs]
batch = 10
Moyenne: 603.57 µs Ecart-type 46.82 µs (with 10 runs) in [554.90 µs, 720.59 µs]
Moyenne: 48.79 µs Ecart-type 10.54 µs (with 10 runs) in [38.44 µs, 72.73 µs]
batch = 100
Moyenne: 720.38 µs Ecart-type 85.23 µs (with 10 runs) in [625.93 µs, 878.73 µs]
Moyenne: 277.19 µs Ecart-type 61.05 µs (with 10 runs) in [234.94 µs, 434.57 µs]
batch = 200
Moyenne: 754.27 µs Ecart-type 89.03 µs (with 10 runs) in [666.03 µs, 1.01 ms]
Moyenne: 469.01 µs Ecart-type 24.81 µs (with 10 runs) in [428.60 µs, 506.47 µs]
batch = 500
Moyenne: 937.39 µs Ecart-type 119.09 µs (with 10 runs) in [817.82 µs, 1.14 ms]
Moyenne: 1.14 ms Ecart-type 108.41 µs (with 10 runs) in [1.03 ms, 1.39 ms]
batch = 1000
Moyenne: 1.14 ms Ecart-type 103.66 µs (with 10 runs) in [1.01 ms, 1.38 ms]
Moyenne: 1.98 ms Ecart-type 60.76 µs (with 10 runs) in [1.89 ms, 2.12 ms]
batch = 2000
Moyenne: 1.49 ms Ecart-type 156.51 µs (with 10 runs) in [1.40 ms, 1.95 ms]
Moyenne: 3.99 ms Ecart-type 207.44 µs (with 10 runs) in [3.76 ms, 4.41 ms]
batch = 3000
Moyenne: 1.81 ms Ecart-type 112.98 µs (with 10 runs) in [1.69 ms, 2.01 ms]
Moyenne: 5.92 ms Ecart-type 213.19 µs (with 10 runs) in [5.61 ms, 6.42 ms]
batch = 4000
Moyenne: 2.17 ms Ecart-type 137.53 µs (with 10 runs) in [2.01 ms, 2.46 ms]
Moyenne: 7.76 ms Ecart-type 389.39 µs (with 10 runs) in [7.36 ms, 8.73 ms]
batch = 5000
Moyenne: 2.61 ms Ecart-type 253.06 µs (with 10 runs) in [2.37 ms, 3.05 ms]
Moyenne: 9.85 ms Ecart-type 415.63 µs (with 10 runs) in [9.49 ms, 10.63 ms]
batch = 10000
Moyenne: 4.47 ms Ecart-type 337.08 µs (with 10 runs) in [4.20 ms, 5.31 ms]
Moyenne: 20.77 ms Ecart-type 1.84 ms (with 10 runs) in [19.49 ms, 26.13 ms]
batch = 20000
Moyenne: 9.72 ms Ecart-type 896.79 µs (with 10 runs) in [8.17 ms, 11.10 ms]
Moyenne: 40.78 ms Ecart-type 4.39 ms (with 10 runs) in [36.75 ms, 47.99 ms]
batch = 50000
Moyenne: 19.34 ms Ecart-type 198.30 µs (with 10 runs) in [19.15 ms, 19.83 ms]
Moyenne: 94.49 ms Ecart-type 402.90 µs (with 10 runs) in [93.97 ms, 95.18 ms]
batch = 75000
Moyenne: 29.40 ms Ecart-type 1.49 ms (with 10 runs) in [28.36 ms, 32.51 ms]
Moyenne: 148.87 ms Ecart-type 9.32 ms (with 10 runs) in [141.62 ms, 166.68 ms]
batch = 100000
Moyenne: 39.84 ms Ecart-type 2.04 ms (with 10 runs) in [37.37 ms, 42.91 ms]
Moyenne: 305.83 ms Ecart-type 175.02 ms (with 10 runs) in [191.99 ms, 709.25 ms]
dfbrf = pandas.DataFrame(memo)[["average", "lib", "batch"]]
pivrf = dfbrf.pivot("batch", "lib", "average")
for c in pivrf.columns:
    pivrf["ave_" + c] = pivrf[c] / pivrf.index
libs = list(c for c in pivrf.columns if "ave_" in c)
ax = pivrf.plot(y=libs, logy=True, logx=True, figsize=(10, 5))
ax.set_title("Evolution du temps de prédiction selon la taille du batch\nrandom forest")
ax.grid(True);_____no_output_____


# Notebook from Intelligent-Systems-Phystech/2022-Project-vc
Path: code/prep-sentence-similarity.ipynb

import pandas as pd
import numpy as np
import plotly.express as px
import copy
import datetime
import matplotlib.pyplot as plt_____no_output_____

# Классификация строк _____no_output_____

print(datetime.datetime.now())2022-03-14 12:44:13.640808

## JSON to DataFrame_____no_output_____

## Данные_____no_output_____

df_education = pd.read_csv('data/uuid_x_education_fields.csv', index_col=0)
df_education = df_education[~df_education['education_field_of_study'].isna()][['org_uuid', 'education_field_of_study']].reset_index(drop=True)_____no_output_____df_education_____no_output_____

# sentence-transformers/multi-qa-MiniLM-L6-cos-v1_____no_output_____## Education field of study_____no_output_____

# ToDo DataFrame with first, second, third, ..._____no_output_____from sentence_transformers import SentenceTransformer, util

threshold = 0.35
classes = ["Biology", "Computer Science", "Management", "Finance"]

#Load the model
model = SentenceTransformer('sentence-transformers/multi-qa-MiniLM-L6-cos-v1')

classification_vec = []
first_vec, second_vec = [], []

for i, query in enumerate(df_education['education_field_of_study']):

    print('{} / {}'.format(i+1, df_education.shape[0]), end="\r")

    #Encode query and documents
    query_emb = model.encode(query)
    classes_emb = model.encode(classes)

    #Compute dot score between query and all document embeddings
    scores = util.dot_score(query_emb, classes_emb)[0].cpu().tolist()

    first_score = np.partition(scores, -1)[-1]
    second_score = np.partition(scores, -2)[-2]

    if first_score > threshold:
        i = np.argmax(scores)
        classification_vec.append(classes[i])
    else:
        classification_vec.append("Other")
    first_vec.append(first_score)
    second_vec.append(second_score)

df_classes = pd.DataFrame(data=classification_vec, columns=["Class"])
df_first = pd.DataFrame(data=first_vec, columns=["top1"])
df_second = pd.DataFrame(data=second_vec, columns=["top2"])

df = pd.concat(
    [
        df_education,
        df_classes,
        df_first,
        df_second
    ], axis=1
)2622 / 2622
model.save('models/multi-qa-MiniLM-L6-cos-v1')_____no_output_____df.to_csv('data/features/education_field_of_study_classes.csv')_____no_output_____df.head(20)_____no_output_____df['Class'].value_counts()_____no_output_____plt.figure(figsize=(10,6))
plt.title('Top 1 and top 2 semilarity metric')
df['top1'].hist(bins=20, color='r', alpha=0.3, label='top 1')
df['top2'].hist(bins=20, color='b', alpha=0.3, label='top 2')
plt.xlim(0, 1)
plt.legend()
plt.show()_____no_output_____


# Notebook from ajlee21/generic-expression-patterns
Path: pseudomonas_analysis/archive/2_identify_generic_genes_pathways_pao1.ipynb

# Identify generic genes and pathways using subset of training compendium

Studies have found that some genes are more likely to be differentially expressed even across a wide range of experimental designs. These generic genes and subsequent pathways are not necessarily specific to the biological process being studied but instead represent a more systematic change.

We have developed an approach, outlined below, to automatically identify these generic genes and pathways. We have validated this simulation approach can identify generic genes and pathways in the analysis notebooks: [human_general_analysis](../human_general_analysis/) and [human_cancer_analysis](../human_cancer_analysis/). Here

This notebook applies this approach to identify generic genes and pathways in the pseudomonas compendium.

**Steps to identify generic genes:**
1. Simulates N gene expression experiments using [ponyo](https://github.com/ajlee21/ponyo)
2. Perform DE analysis to get association statistics for each gene

In this case the DE analysis is based on the experimental design of the template experiment, described in the previous [notebook](1_process_pseudomonas_data.ipynb).
The template experiment is [GEOD-33245](https://www.ebi.ac.uk/arrayexpress/experiments/E-GEOD-33245/?s_sortby=col_8&s_sortorder=ascending), which contains multiple different comparisons including WT vs *crc* mutants, WT vs *cbr* mutants in different conditions. So the DE analysis is comparing WT vs mutant.

3. For each gene, aggregate statistics across all simulated experiments
4. Rank genes based on this aggregated statistic

**Steps to identify generic gene sets (pathways):**
1. Using the same simulated experiments from above, perform GSEA analysis. This analysis will determine whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list of genes, where genes are ranked by log fold change, indicating a correlation with change in expression.
2. For each gene set (pathway), aggregate statistics across all simulated experiments
3. Rank gene sets based on this aggregated statistic_____no_output_____

%load_ext autoreload
%load_ext rpy2.ipython
%autoreload 2

import os
import sys
import pandas as pd
import numpy as np
import seaborn as sns
import pickle

from rpy2.robjects import pandas2ri
from ponyo import utils, simulate_expression_data
from generic_expression_patterns_modules import process, stats, ranking

# Imports for in-notebook function
import glob
import warnings
from keras.models import load_model
from sklearn import preprocessing

pandas2ri.activate()

np.random.seed(123)/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/matplotlib/__init__.py:886: MatplotlibDeprecationWarning:
examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.
  "found relative to the 'datapath' directory.".format(key))
Using TensorFlow backend.
# Read in config variables
base_dir = os.path.abspath(os.path.join(os.getcwd(), "../"))

config_filename = os.path.abspath(
    os.path.join(base_dir, "configs", "config_pseudomonas_pao1.tsv")
)
params = utils.read_config(config_filename)_____no_output_____# Load params
local_dir = params["local_dir"]
dataset_name = params["dataset_name"]
NN_architecture = params["NN_architecture"]
num_runs = params["num_simulated"]
project_id = params["project_id"]
metadata_col_id = params["metadata_colname"]
raw_template_filename = params["raw_template_filename"]
processed_template_filename = params["processed_template_filename"]
normalized_compendium_filename = params["normalized_compendium_filename"]
scaler_filename = params["scaler_filename"]
col_to_rank_genes = params["rank_genes_by"]
col_to_rank_pathways = params["rank_pathways_by"]
statistic = params["gsea_statistic"]

# Load metadata file with grouping assignments for samples
sample_id_metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_process_samples.tsv"
)

# Load metadata file with grouping assignments for samples
metadata_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", f"{project_id}_groups.tsv"
)

# Load pickled file
scaler = pickle.load(open(scaler_filename, "rb"))_____no_output_____# Output files
gene_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_gene_summary_{project_id}_pao1.tsv"
)
pathway_summary_filename = os.path.join(
    base_dir, dataset_name, f"generic_pathway_summary_{project_id}_pao1.tsv"
)_____no_output_____

### Simulate experiments using selected template experiment
Workflow:

1. Get the gene expression data for the selected template experiment
2. Encode this experiment into a latent space using the trained VAE model
3. Linearly shift the encoded template experiment in the latent space
4. Decode the samples. This results in a new experiment
5. Repeat steps 1-4 to get multiple simulated experiments_____no_output_____

# Note: We are manually changing the function to suit our nex experiment where we
# subset our training compendium. Ponyo will be updated to address this change to
# be more robust to user inputs
def get_sample_ids(experiment_id, dataset_name, sample_id_colname):
    """
    Returns sample ids (found in gene expression df) associated with
    a given list of experiment ids (found in the metadata)

    Arguments
    ----------
    experiment_ids_file: str
        File containing all cleaned experiment ids

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    sample_id_colname: str
        Column header that contains sample id that maps expression data
        and metadata

    """
    base_dir = os.path.abspath(os.path.join(os.getcwd(), "../"))

    if "pseudomonas" in dataset_name.lower():
        # metadata file
        mapping_file = os.path.join(
            base_dir, dataset_name, "data", "metadata", "sample_annotations.tsv"
        )

        # Read in metadata
        metadata = pd.read_csv(mapping_file, header=0, sep="\t", index_col=0)

        selected_metadata = metadata.loc[experiment_id]
        sample_ids = list(selected_metadata[sample_id_colname])

    else:
        # metadata file
        mapping_file = os.path.join(
            base_dir, dataset_name, "data", "metadata", "recount2_metadata.tsv"
        )

        # Read in metadata
        metadata = pd.read_csv(mapping_file, header=0, sep="\t", index_col=0)

        selected_metadata = metadata.loc[experiment_id]
        sample_ids = list(selected_metadata[sample_id_colname])

    return sample_ids

def shift_template_experiment_tmp(
    normalized_data_file,
    selected_experiment_id,
    sample_id_colname,
    NN_architecture,
    dataset_name,
    scaler,
    local_dir,
    base_dir,
    run,
):
    """
    Generate new simulated experiment using the selected_experiment_id as a template
    experiment and shifting this template experiment in the latent space to a new
    location. This shifted experiment will be the new simulated experiment.

    This will return a file with a single simulated experiment following the workflow mentioned.
    This function can be run multiple times to generate multiple simulated experiments from a
    single selected_experiment_id.

    Arguments
    ----------
    normalized_data_file: str
        File containing normalized gene expression data

        ------------------------------| PA0001 | PA0002 |...
        05_PA14000-4-2_5-10-07_S2.CEL | 0.8533 | 0.7252 |...
        54375-4-05.CEL                | 0.7789 | 0.7678 |...
        ...                           | ...    | ...    |...

    selected_experiment_id: str
        Experiment id selected as template

    sample_id_colname: str
        Column header that contains sample id that maps expression data and metadata

    NN_architecture: str
        Name of neural network architecture to use.
        Format 'NN_<intermediate layer>_<latent layer>'

    dataset_name: str
        Name for analysis directory. Either "Human" or "Pseudomonas"

    scaler: minmax model
        Model used to transform data into a different range

    local_dir: str
        Parent directory on local machine to store intermediate results

    base_dir: str
        Root directory containing analysis subdirectories

    run: int
        Simulation run

    Returns
    --------
    simulated_data_file: str
        File containing simulated gene expression data

    """

    # Files
    NN_dir = os.path.join(base_dir, dataset_name, "models", NN_architecture)
    latent_dim = NN_architecture.split("_")[
        -2
    ]  # Here we manually changed the index to work

    model_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_model.h5"))[0]

    weights_encoder_file = glob.glob(os.path.join(NN_dir, "*_encoder_weights.h5"))[0]

    model_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_model.h5"))[0]

    weights_decoder_file = glob.glob(os.path.join(NN_dir, "*_decoder_weights.h5"))[0]

    # Load saved models
    loaded_model = load_model(model_encoder_file, compile=False)
    loaded_decode_model = load_model(model_decoder_file, compile=False)

    loaded_model.load_weights(weights_encoder_file)
    loaded_decode_model.load_weights(weights_decoder_file)

    # Read data
    normalized_data = pd.read_csv(normalized_data_file, header=0, sep="\t", index_col=0)

    # Get corresponding sample ids
    sample_ids = get_sample_ids(selected_experiment_id, dataset_name, sample_id_colname)

    # Gene expression data for selected samples
    selected_data_df = normalized_data.loc[sample_ids]

    # Encode selected experiment into latent space
    data_encoded = loaded_model.predict_on_batch(selected_data_df)
    data_encoded_df = pd.DataFrame(data_encoded, index=selected_data_df.index)

    # Get centroid of original data
    centroid = data_encoded_df.mean(axis=0)

    # Add individual vectors(centroid, sample point) to new_centroid

    # Encode original gene expression data into latent space
    data_encoded_all = loaded_model.predict_on_batch(normalized_data)
    data_encoded_all_df = pd.DataFrame(data_encoded_all, index=normalized_data.index)

    data_encoded_all_df.head()

    # Find a new location in the latent space by sampling from the latent space
    encoded_means = data_encoded_all_df.mean(axis=0)
    encoded_stds = data_encoded_all_df.std(axis=0)

    latent_dim = int(latent_dim)
    new_centroid = np.zeros(latent_dim)

    for j in range(latent_dim):
        new_centroid[j] = np.random.normal(encoded_means[j], encoded_stds[j])

    shift_vec_df = new_centroid - centroid

    simulated_data_encoded_df = data_encoded_df.apply(
        lambda x: x + shift_vec_df, axis=1
    )

    # Decode simulated data into raw gene space
    simulated_data_decoded = loaded_decode_model.predict_on_batch(
        simulated_data_encoded_df
    )

    simulated_data_decoded_df = pd.DataFrame(
        simulated_data_decoded,
        index=simulated_data_encoded_df.index,
        columns=selected_data_df.columns,
    )

    # Un-normalize the data in order to run DE analysis downstream
    simulated_data_scaled = scaler.inverse_transform(simulated_data_decoded_df)

    simulated_data_scaled_df = pd.DataFrame(
        simulated_data_scaled,
        columns=simulated_data_decoded_df.columns,
        index=simulated_data_decoded_df.index,
    )

    # Save template data for visualization validation
    test_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "template_normalized_data_" + selected_experiment_id + "_test.txt",
    )

    selected_data_df.to_csv(test_file, float_format="%.3f", sep="\t")

    # Save
    out_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        "selected_simulated_data_" + selected_experiment_id + "_" + str(run) + ".txt",
    )

    simulated_data_scaled_df.to_csv(out_file, float_format="%.3f", sep="\t")

    out_encoded_file = os.path.join(
        local_dir,
        "pseudo_experiment",
        f"selected_simulated_encoded_data_{selected_experiment_id}_{run}.txt",
    )

    simulated_data_encoded_df.to_csv(out_encoded_file, float_format="%.3f", sep="\t")_____no_output_____# Simulate multiple experiments
# This step creates the following files in "<local_dir>/pseudo_experiment/" directory:
#   - selected_simulated_data_SRP012656_<n>.txt
#   - selected_simulated_encoded_data_SRP012656_<n>.txt
#   - template_normalized_data_SRP012656_test.txt
# in which "<n>" is an integer in the range of [0, num_runs-1]
os.makedirs(os.path.join(local_dir, "pseudo_experiment"), exist_ok=True)
for run_id in range(num_runs):
    shift_template_experiment_tmp(
        normalized_compendium_filename,
        project_id,
        metadata_col_id,
        NN_architecture,
        dataset_name,
        scaler,
        local_dir,
        base_dir,
        run_id,
    )_____no_output_____

### Process template and simulated data

* Remove samples not required for comparison.
* Make sure ordering of samples matches metadata for proper comparison_____no_output_____

if not os.path.exists(sample_id_metadata_filename):
    sample_id_metadata_filename = None

stats.process_samples_for_limma(
    raw_template_filename,
    metadata_filename,
    processed_template_filename,
    sample_id_metadata_filename,
)

for i in range(num_runs):
    simulated_filename = os.path.join(
        local_dir, "pseudo_experiment", f"selected_simulated_data_{project_id}_{i}.txt"
    )
    stats.process_samples_for_limma(
        simulated_filename,
        metadata_filename,
        None,
        sample_id_metadata_filename,
    )_____no_output_____# Quick check
template_data = pd.read_csv(
    processed_template_filename, header=0, index_col=0, sep="\t"
)

assert template_data.shape[0] == 4_____no_output_____

### Differential expression analysis_____no_output_____

# Create subdirectory: "<local_dir>/DE_stats/"
os.makedirs(os.path.join(local_dir, "DE_stats"), exist_ok=True)_____no_output_____%%R -i metadata_filename -i project_id -i processed_template_filename -i local_dir -i base_dir

source(paste0(base_dir, '/generic_expression_patterns_modules/DE_analysis.R'))

get_DE_stats_limma(metadata_filename,
                   project_id,
                   processed_template_filename,
                   "template",
                   local_dir,
                   "real")/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: S4Vectors

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: stats4

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: BiocGenerics

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: parallel

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning:
Attaching package: ‘BiocGenerics’

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:parallel’:

    clusterApply, clusterApplyLB, clusterCall, clusterEvalQ,
    clusterExport, clusterMap, parApply, parCapply, parLapply,
    parLapplyLB, parRapply, parSapply, parSapplyLB

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following object is masked from ‘package:limma’:

    plotMA

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:stats’:

    IQR, mad, sd, var, xtabs

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:base’:

    anyDuplicated, append, as.data.frame, basename, cbind, colnames,
    dirname, do.call, duplicated, eval, evalq, Filter, Find, get, grep,
    grepl, intersect, is.unsorted, lapply, Map, mapply, match, mget,
    order, paste, pmax, pmax.int, pmin, pmin.int, Position, rank,
    rbind, Reduce, rownames, sapply, setdiff, sort, table, tapply,
    union, unique, unsplit, which, which.max, which.min

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning:
Attaching package: ‘S4Vectors’

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following object is masked from ‘package:base’:

    expand.grid

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: IRanges

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: GenomicRanges

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: GenomeInfoDb

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: SummarizedExperiment

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: Biobase

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Welcome to Bioconductor

    Vignettes contain introductory material; view with
    'browseVignettes()'. To cite Bioconductor, see
    'citation("Biobase")', and for packages 'citation("pkgname")'.

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: DelayedArray

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: matrixStats

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning:
Attaching package: ‘matrixStats’

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:Biobase’:

    anyMissing, rowMedians

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: BiocParallel

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning:
Attaching package: ‘DelayedArray’

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:matrixStats’:

    colMaxs, colMins, colRanges, rowMaxs, rowMins, rowRanges

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: The following objects are masked from ‘package:base’:

    aperm, apply, rowsum

  warnings.warn(x, RRuntimeWarning)
%%R -i metadata_filename -i project_id -i base_dir -i local_dir -i num_runs -o num_sign_DEGs_simulated

source(paste0(base_dir,'/generic_expression_patterns_modules/DE_analysis.R'))

num_sign_DEGs_simulated <- c()

for (i in 0:(num_runs-1)){
    simulated_data_filename <- paste(
        local_dir,
        "pseudo_experiment/selected_simulated_data_",
        project_id,
        "_",
        i,
        ".txt",
        sep=""
    )

    run_output <- get_DE_stats_limma(
        metadata_filename,
        project_id,
        simulated_data_filename,
        "simulated",
        local_dir,
        i
    )
    num_sign_DEGs_simulated <- c(num_sign_DEGs_simulated, run_output)
}_____no_output_____

### Rank genes_____no_output_____

analysis_type = "DE"
template_DE_stats_filename = os.path.join(
    local_dir, "DE_stats", f"DE_stats_template_data_{project_id}_real.txt"
)
template_DE_stats, simulated_DE_summary_stats = ranking.process_and_rank_genes_pathways(
    template_DE_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_genes,
)_____no_output_____

### Gene summary table_____no_output_____

summary_gene_ranks = ranking.generate_summary_table(
    template_DE_stats_filename,
    template_DE_stats,
    simulated_DE_summary_stats,
    col_to_rank_genes,
    local_dir,
    "gene",
    params,
)

summary_gene_ranks.sort_values(by="Z score", ascending=False).head()/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/pandas/core/reshape/merge.py:544: UserWarning: merging between different levels can give an unintended result (1 levels on the left, 2 on the right)
  warnings.warn(msg, UserWarning)
# Add gene name as column to summary dataframe
summary_gene_ranks = ranking.add_pseudomonas_gene_name_col(summary_gene_ranks, base_dir)
summary_gene_ranks.sort_values(by="Z score", ascending=False).head()_____no_output_____# Check if there is an NaN values, there should not be
summary_gene_ranks.isna().any()_____no_output_____# Create `gene_summary_filename`
summary_gene_ranks.to_csv(gene_summary_filename, sep="\t")_____no_output_____

## Compare gene ranking_____no_output_____

# Get generic genes identified by Crow et. al.
GAPE_filename = params["reference_gene_filename"]
ref_gene_col = params["reference_gene_name_col"]
ref_rank_col = params["reference_rank_col"]

figure_filename = f"gene_ranking_{col_to_rank_genes}_pao1.svg"

corr, shared_ranking = ranking.compare_gene_ranking(
    summary_gene_ranks, GAPE_filename, ref_gene_col, ref_rank_col, figure_filename
)r = 0.3743837960745028
p = 3.621704632637787e-184
ci_low = 0.3509873530823741
ci_high = 0.3988725360726553
# Get genes that are highly generic in both
generic_both = shared_ranking[
    (shared_ranking["Rank (simulated)"] > 4500) & (shared_ranking["prop DEGs"] > 3500)
]
generic_both.to_csv(os.path.join(local_dir, "SOPHIE_GAPE_generic.tsv"), sep="\t")_____no_output_____# Get genes that are highly generic by SOPHIE but not by GAPE
generic_SOPHIE_only = shared_ranking[
    (shared_ranking["Rank (simulated)"] > 4500) & (shared_ranking["prop DEGs"] < 2000)
]
generic_SOPHIE_only.to_csv(os.path.join(local_dir, "SOPHIE_generic_only.tsv"), sep="\t")_____no_output_____

**Takeaway:**

* X-axis: gene ranking using SOPHIE (trained on Pseudomonas compendium containing ~1K experiments)
* Y-axis: gene ranking using GAPE (curated set of 73 experiments)

* Compared to the correlation plot comparing SOPHIE trained on the full pseudomonas compendium vs GAPE experiments [here](2_identify_generic_genes_pathways.ipynb), it looks like there is more noise in the above correlation plot. Looking up the experiment ids associated with GAPE [here](https://github.com/DartmouthStantonLab/GAPE/blob/main/Pa_GSEnum_unzip.rds) it looks like GAPE does contain a mix of PAO1, PA14 and clinical strains. So by subsetting out training compendium to only represent PAO1 patterns, which is adding more noise to the bottom left of the plot as expected (genes that GAPE finds generic are not found by SOPHIE due to our limited representation in our training compendium)_____no_output_____### GSEA
**Goal:** To detect modest but coordinated changes in prespecified sets of related genes (i.e. those genes in the same pathway or share the same GO term).

1. Ranks all genes based using DE association statistics.
2. An enrichment score (ES) is defined as the maximum distance from the middle of the ranked list. Thus, the enrichment score indicates whether the genes contained in a gene set are clustered towards the beginning or the end of the ranked list (indicating a correlation with change in expression).
3. Estimate the statistical significance of the ES by a phenotypic-based permutation test in order to produce a null distribution for the ES( i.e. scores based on permuted phenotype)_____no_output_____

# Create "<local_dir>/GSEA_stats/" subdirectory
os.makedirs(os.path.join(local_dir, "GSA_stats"), exist_ok=True)_____no_output_____# Load pathway data
adage_kegg_DB_filename = params["pathway_DB_filename"]_____no_output_____# Need to format data into tab-delimited matrix
# with columns= KEGG pathway name, description, gene ids
# Each gene ids is tab separated
adage_kegg_DB_processed_filename = os.path.join(
    base_dir, dataset_name, "data", "metadata", "adage_kegg_DB_process_filename.gmt"
)
stats.format_pseudomonas_pathway_DB(
    adage_kegg_DB_filename, local_dir, adage_kegg_DB_processed_filename
)_____no_output_____%%R -i base_dir -i template_DE_stats_filename -i adage_kegg_DB_processed_filename -i statistic -o template_enriched_pathways

source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))

out_filename <- paste(local_dir,
                     "GSA_stats/GSEA_stats_template_data_",
                     project_id,
                     "_real.txt",
                     sep = "")

template_enriched_pathways <- find_enriched_pathways(template_DE_stats_filename, adage_kegg_DB_processed_filename, statistic)
write.table(as.data.frame(template_enriched_pathways[1:7]), file = out_filename, row.names = F, sep = "\t")/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/rinterface/__init__.py:146: RRuntimeWarning: Loading required package: Rcpp

  warnings.warn(x, RRuntimeWarning)
/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/rpy2/robjects/pandas2ri.py:191: FutureWarning: from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict(OrderedDict(items)) may be used to preserve the key order.
  res = PandasDataFrame.from_items(items)
print(template_enriched_pathways.shape)
template_enriched_pathways[template_enriched_pathways["padj"] < 0.05].sort_values(
    by="padj"
).head()(169, 8)
%%R -i project_id -i local_dir -i adage_kegg_DB_processed_filename -i num_runs -i statistic

source(paste0(base_dir, '/generic_expression_patterns_modules/GSEA_analysis.R'))

# New files created: "<local_dir>/GSEA_stats/GSEA_stats_simulated_data_<project_id>_<n>.txt"
for (i in 0:(num_runs-1)) {
    simulated_DE_stats_filename <- paste(local_dir,
                                     "DE_stats/DE_stats_simulated_data_",
                                     project_id,
                                     "_",
                                     i,
                                     ".txt",
                                     sep = "")

    out_filename <- paste(local_dir,
                     "GSA_stats/GSEA_stats_simulated_data_",
                     project_id,
                     "_",
                     i,
                     ".txt",
                     sep = "")

    enriched_pathways <- find_enriched_pathways(simulated_DE_stats_filename, adage_kegg_DB_processed_filename, statistic)

    # Remove column with leading edge since its causing parsing issues
    write.table(as.data.frame(enriched_pathways[1:7]), file = out_filename, row.names = F, sep = "\t")
}_____no_output_____

### Rank pathways_____no_output_____

analysis_type = "GSA"
template_GSEA_stats_filename = os.path.join(
    local_dir, "GSA_stats", f"GSEA_stats_template_data_{project_id}_real.txt"
)
(
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
) = ranking.process_and_rank_genes_pathways(
    template_GSEA_stats_filename,
    local_dir,
    num_runs,
    project_id,
    analysis_type,
    col_to_rank_pathways,
    "GSEA",
)_____no_output_____

### Pathway summary table_____no_output_____

# Create intermediate file: "<local_dir>/gene_summary_table_<col_to_rank_pathways>.tsv"
summary_pathway_ranks = ranking.generate_summary_table(
    template_GSEA_stats_filename,
    template_GSEA_stats,
    simulated_GSEA_summary_stats,
    col_to_rank_pathways,
    local_dir,
    "pathway",
    params,
)

summary_pathway_ranks.sort_values(by="Rank (simulated)", ascending=False).head()/home/alexandra/anaconda3/envs/generic_expression/lib/python3.7/site-packages/pandas/core/reshape/merge.py:544: UserWarning: merging between different levels can give an unintended result (1 levels on the left, 2 on the right)
  warnings.warn(msg, UserWarning)
# Create `pathway_summary_filename`
summary_pathway_ranks.to_csv(pathway_summary_filename, sep="\t")_____no_output_____

**Quick check:**

Looks like Ribosomes are found to be significantly differentiated in the template and across 25 simulated experiments. So this pathway looks to be generic, which is consistent with findings from [Crow. et. al.](https://www.pnas.org/content/116/13/6491.abstract), using Human data. This is also consistent with [paper](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2679180/) which observed that Ribosome proteins have many different functions in *S. cerevisiae*, which might explain their variability across a wide range of contexts.

* Need to think about why it might make sense that Ribosomes being generic in Humans would also imply that they are generic in P. aeruginosa._____no_output_____