# Notebook from uah-cao1/CEO
Path: simul_scripts_test/Phasing_control_in_AO_mode.ipynb

import sys
import numpy as np
import math
import ceo
import matplotlib.pyplot as plt
import IPython
%matplotlib inline
import scipy.io as sio
import os.path
import datetime
import csv
import commands_____no_output_____sys.stdout = open('Phasing_Ctrl.log','wb')_____no_output_____from Phasing_in_AO_mode_params import *pupil sampling: 481 pixel
number of SH SAs across pupil: 60
number of SH pixels per SA: 8
## GENERIC STUFF:
tid = ceo.StopWatch()
CEO_git_commit = commands.getoutput("git rev-parse --verify --short HEAD")
VISU=True_____no_output_____

# Initialize Telescope and Turbulence_____no_output_____

# Telescope
if simul_onaxis_AO==True:
    if onaxis_AO_modes==u"Karhunen-Loeve":
        gmt = ceo.GMT_MX(D, nPx, M2_mirror_modes=onaxis_AO_modes, M2_N_MODE=M2_n_modes)
    elif onaxis_AO_modes==u"zernike":
        gmt = ceo.GMT_MX(D, nPx, M2_mirror_modes=onaxis_AO_modes,
                         M2_radial_order=M2_radial_order)

    # Telescope to be used for CL IM calibrations
    #gmt0 = ceo.GMT_MX(D, nPx, M2_mirror_modes=u"zernike", M2_radial_order=M2_radial_order)
    gmt0 = ceo.GMT_MX(D, nPx, M2_mirror_modes=u"Karhunen-Loeve", M2_N_MODE=91)

else:
    gmt = ceo.GMT_MX(D, nPx)

p = D/(nPx-1)_____no_output_____# Turbulence parameters:
if simul_turb == True:
    atm = ceo.JGmtAtmosphere(jsonfile=atm_fname)
    r0 = atm.r0
    L0 = atm.L0
    print r0, L0_____no_output_____

# Initialize Simplified NGAO/LTAO system
The simplified LTAO system comprises:
1. **on-axis geometrical SH sensor** with 60x60 sub-apertures controlling 300 KL modes / M2 segment

The simplified NGAO system comprises:
1. **on-axis geometrical SH sensor** with 60x60 sub-apertures controlling 300 KL modes / M2 segment
2. **on-axis idealized SPP sensor** controlling segment piston._____no_output_____

#---- ON-AXIS AO SH WFS:
if simul_onaxis_AO == True:
    d = D/nLenslet
    if SHtype == 'diff':
        wfs = ceo.ShackHartmann(nLenslet, n, d, N_PX_IMAGE=2*(n+1), N_GS=1)
    elif SHtype == 'geom':
        wfs = ceo.GeometricShackHartmann(nLenslet, d, N_GS=1)
    gs = ceo.Source("R",zenith=0.,azimuth=0.,
                    rays_box_size=D, rays_box_sampling=nPx, rays_origin=[0.0,0.0,25])

    # Calibrate SH (valid SAs, slope null vector)
    gs.reset()
    gmt.reset()
    gmt.propagate(gs)
    wfs.calibrate(gs,sh_thr)
    print("Total number of valid slopes: %d"%wfs.n_valid_slopes)
    plt.imshow(wfs.valid_lenslet.f.host().reshape(nLenslet,nLenslet), interpolation='None')

    if SHtype == 'diff':
        sh_fov = wfs.pixel_scale_arcsec*wfs.N_PX_IMAGE/wfs.BIN_IMAGE

        if VISU == True:
            plt.imshow(wfs.flux.host(shape=(nLenslet,nLenslet)).T,interpolation='none', origin='lower')

        print "pupil sampling: %d pixel"%nPx
        print "number of SH SAs across pupil: %d"%nLenslet
        print "number of SH pixels per SA: %d"%n
        print "SH Pixel scale: %.3farcsec"%(wfs.pixel_scale_arcsec)
        print "SH Field of view: %.3farcsec"%(sh_fov)_____no_output_____# Calibrate M2 KLs - SH Interaction Matrix and Reconstructor
if simul_onaxis_AO == True:

    fname = 'IM_SH'+SHtype+\
    '_'+onaxis_AO_modes+'_nmode'+str(gmt.M2.modes.n_mode)+'_SHthr%1.1f.npz'%sh_thr

    z_first_mode = 1   #include segment piston (to be removed later....)
    Zstroke = 10e-9 #m rms

    if os.path.isfile(fname) == False:
        D_M2_MODES = gmt.calibrate(wfs, gs, mirror="M2", mode=onaxis_AO_modes, stroke=Zstroke, first_mode=z_first_mode)
        np.savez(fname, D_M2=D_M2_MODES, first_mode=z_first_mode, Stroke=Zstroke)
    else:
        print 'Reading file: '+fname
        ftemp = np.load(fname)
        D_M2_MODES = ftemp.f.D_M2
        ftemp.close()

    nall = (D_M2_MODES.shape)[1]  ## number of zernike DoFs calibrated
    n_mode = nall/7

    print 'AO SH WFS - M2 Segment Modal IM:'
    print D_M2_MODES.shape_____no_output_____

if simul_onaxis_AO == True and AOtype == 'NGAOish':

    onps = ceo.IdealSegmentPistonSensor(gs, D, nPx, segment='full')
    gs.reset()
    gmt.reset()
    gmt.propagate(gs)
    onps_signal_ref = onps.piston(gs).ravel() #[0:6] # reference signal

    #-----> SH - M2 segment piston IM
    print("SPP - SH IM:")
    PSstroke = 200e-9 #m
    D_M2_PS_sh = gmt.calibrate(wfs, gs, mirror="M2", mode="segment piston", stroke=PSstroke)

    #-----> Ideal SPS - M2 segment Zernikes IM
    D_M2_MODES_PSideal = np.zeros((7,nall))

    #z_first_mode = 1   #include segment piston (to be removed later....)
    #Zstroke = 10e-9 #m rms
    #D_M2_MODES_PSideal = gmt.calibrate(onps, gs, mirror="M2",
    #    mode=onaxis_AO_modes, stroke=Zstroke, first_mode=z_first_mode)

    #-----> Ideal SPS - M2 segment piston IM
    print("SPP - SPS IM:")
    PSstroke = 50e-9 #m
    D_M2_PSideal = gmt.calibrate(onps, gs, mirror="M2", mode="segment piston", stroke=PSstroke)

    if VISU == True:
        fig, (ax1,ax2) = plt.subplots(ncols=2)
        fig.set_size_inches(10,5)
        imm = ax1.pcolor(D_M2_PSideal)
        ax1.grid()
        fig.colorbar(imm, ax=ax1)#, fraction=0.012)

        imm1 = ax2.pcolor(D_M2_PS_sh)
        ax2.grid()
        fig.colorbar(imm1)_____no_output_____#----> Create super-merger IM for "simplified NGAO control"
# DoFs: segment Zernikes (Z2->Zx), segment Piston
# Sensors: on-axis SH WFS, on-axis idealized Piston Sensor
if simul_onaxis_AO==True:

    if AOtype == 'NGAOish':

        D_AO_SH = np.concatenate((D_M2_MODES, D_M2_PS_sh), axis=1)
        print 'Merged (seg Z & Pist) - SH IM:'
        print D_AO_SH.shape

        D_AO_PS = np.concatenate((D_M2_MODES_PSideal, D_M2_PSideal), axis=1)
        print 'Merged (seg Z & Pist) - Ideal SPS IM:'
        print D_AO_PS.shape

        D_AO = np.concatenate((D_AO_SH, D_AO_PS), axis=0)
        print 'NGAO-ish super-merged IM:'
        print D_AO.shape

    elif AOtype == 'LTAOish':
        D_AO = D_M2_MODES
        print 'LTAOish IM:'
        print D_AO.shape_____no_output_____if simul_onaxis_AO==True:

    print 'Condition number: %f'%np.linalg.cond(D_AO)
    Uz, Sz, Vz =np.linalg.svd(D_AO)

    fig, ax = plt.subplots()
    fig.set_size_inches(7,5)

    ax.semilogy(Sz/np.max(Sz), 'o-', )
    ax.grid()
    ax.tick_params(labelsize=14)
    ax.set_xlabel('eigenmode number', fontsize=14)
    ax.set_ylabel('normalized singular value', fontsize=14)
    #ax.set_xlim([2400,2500])_____no_output_____#--- Compute the Rec Matrix (with SVD filtering)
if simul_onaxis_AO==True:

    R_AO = np.linalg.pinv(D_AO, rcond=1.0e-3)
    print 'AO SH WFS - M2 Segment Modal Rec:'
    print R_AO.shape_____no_output_____

# Initialize Phasing System_____no_output_____

#Override parameters with the ones in batch
params = np.load('batch_params.npz')
mag = float(params['mag'])
alpha_ps = float(params['alpha_ps'])
#print mag
#print alpha_ps/6013.25
9.5
#AGWS Guide Star locations
if simul_SPS==True:
    if asterism_type == 'Dummy':
        zenith_angle_agws  = np.ones(N_GS_PS)*alpha_ps*math.pi/180/3600 # in radians
        azimuth_angle_agws = np.arange(N_GS_PS)* (360.0/N_GS_PS) * (math.pi/180) # in radians
        mag = np.ones(N_GS_PS)*mag

    elif asterism_type == 'Besancon_GP':

        _asterism_zenith = []
        _asterism_azimuth = []
        _asterism_magnitude = []
        _asterism_Jmag = []

        with open('asterisms_phasing.txt', 'rb') as csvfile:
            asterisms = csv.reader(csvfile)
            for row in asterisms:
                _asterism_zenith.append( float(row[0])*ceo.constants.ARCMIN2RAD )
                _asterism_azimuth.append( float(row[1]) )
                _asterism_magnitude.append( float(row[2]) )
                _asterism_Jmag.append( float(row[3]) )

        asterism_id = np.load('asterism_id.npy')
        asterism_id = int(asterism_id.tolist())
        print "Asterism ID: %d"%asterism_id

        asterism_zenith = np.array(_asterism_zenith[asterism_id*3:(1+asterism_id)*3])
        asterism_azimuth = np.array(_asterism_azimuth[asterism_id*3:(1+asterism_id)*3])
        asterism_magnitude = np.array(_asterism_magnitude[asterism_id*3:(1+asterism_id)*3])
        asterism_Jmag = np.array(_asterism_Jmag[asterism_id*3:(1+asterism_id)*3])

        ordmag = (np.argsort(asterism_Jmag))
        mag = (asterism_Jmag[ordmag])[0:3]
        zenith_angle_agws = (asterism_zenith[ordmag])[0:3]
        azimuth_angle_agws = (asterism_azimuth[ordmag])[0:3]

    elif asterism_type == 'Besancon_GP_AB':

        _asterism_X = []
        _asterism_Y = []
        _asterism_Jmag = []

        with open('agws_sps_aster.txt', 'rb') as csvfile:
            asterisms = csv.reader(csvfile)
            headers = asterisms.next()
            for row in asterisms:
                _asterism_X.append(   np.array((float(row[0]),float(row[3]),float(row[6]))))
                _asterism_Y.append(   np.array((float(row[1]),float(row[4]),float(row[7]))))
                _asterism_Jmag.append(np.array((float(row[2]),float(row[5]),float(row[8]))))

        asterism_zenith = np.sqrt(np.array([X**2 for X in _asterism_X]) + \
                                  np.array([Y**2 for Y in _asterism_Y])).T * ceo.constants.ARCSEC2RAD

        asterism_azimuth = np.arctan2( np.array([Y for Y in _asterism_Y]),
                                       np.array([X for X in _asterism_X])).T

        asterism_Jmag = np.array(_asterism_Jmag).T
        nast= asterism_Jmag.shape[1]

        dummy, badast = np.where( asterism_Jmag == 0)
        goodast = list(set(range(nast)) - set(badast.tolist()))
        asterism_zenith  = asterism_zenith[:,goodast]
        asterism_azimuth = asterism_azimuth[:,goodast]
        asterism_Jmag = asterism_Jmag[:,goodast]
        nast= asterism_Jmag.shape[1]

        asterism_id = np.load('asterism_id.npy')
        asterism_id = int(asterism_id.tolist())
        print "Asterism ID: %d"%asterism_id

        mag = asterism_Jmag[:,asterism_id]
        zenith_angle_agws  =  asterism_zenith[:,asterism_id]
        azimuth_angle_agws = asterism_azimuth[:,asterism_id]

    print("Selected asterism: \nmag(J) Zenith(')  Azimuth(deg)")
    for x in range(N_GS_PS):
        print("%3.2f \t  %1.2f \t    %3.1f"%(mag[x], zenith_angle_agws[x]*ceo.constants.RAD2ARCMIN,
                                                azimuth_angle_agws[x]*180/math.pi))Selected asterism:
mag(J) Zenith(')  Azimuth(deg)
13.25 	  9.50 	    0.0
#---- OFF-AXIS DISPERSED FRINGE SENSORS:

if simul_SPS==True:
    gsps = ceo.Source(band, magnitude=mag,
                      zenith=zenith_angle_agws,azimuth=azimuth_angle_agws,
                      rays_box_size=D,rays_box_sampling=nPx,rays_origin=[0.0,0.0,25])

    for x in range(gsps.N_SRC):
        print('Number of SPS GS photons/s/m^2: %.1f'%(gsps.nPhoton[x]))
        print('Number of expected SPS GS photons [ph/s/m^2]: %.1f'%(e0*10**(-0.4*mag[x])/PupilArea))

    if VISU==True:
        fig = plt.figure(figsize=(8,8))
        ax  = plt.subplot(111,polar=True)
        zen = gsps.zenith*ceo.constants.RAD2ARCMIN
        azi = gsps.azimuth
        c = ax.scatter(azi,zen,c=mag,s=100,cmap=plt.cm.GnBu)
        ax.set_rmax(10)
        c.set_alpha(0.75)
        #c.set_clim([11,13.5])

        #c.set_edgecolor('None')
        plt.colorbar(c,ax=ax)Number of SPS GS photons/s/m^2: 25861.2
Number of expected SPS GS photons [ph/s/m^2]: 25604.1
if simul_SPS==True:
    if SPStype == 'ideal':
        ps = ceo.IdealSegmentPistonSensor(gsps, D, nPx, segment='edge')
    elif SPStype == 'DFS':
        ps = ceo.DispersedFringeSensor(gmt.M1, gsps, dispersion=sps_dispersion,
                                       field_of_view=sps_fov, nyquist_factor=nyquist_factor)
        ps.lobe_detection = lobe_detection
        ps.camera.photoelectron_gain = throughput
        if sps_mask_size > 0: ps.init_detector_mask(sps_mask_size)

    # Calibrate DFS
    gmt.reset()
    gsps.reset()
    if SPStype == 'DFS':
        ps.INIT_ALL_ATTRIBUTES = True
        ps.calibrate(gsps,gmt)

    if VISU == True and SPStype == 'DFS':
        fig1, (ax1,ax2) = plt.subplots(ncols=2)
        fig1.set_size_inches((14,6))
        ax1.imshow(gsps.wavefront.amplitude.host().reshape((nPx*N_GS_PS,nPx)), origin='lower')
        ax2.imshow(ps.fft_mask.f.host().reshape((ps.camera.N_PX_FRAME,ps.camera.N_PX_FRAME)),
                   origin='lower')

    # Calibrate SPS reference vector (corresponding to field-dependent aberrations)
    gmt.reset()
    ps.reset()
    gsps.reset()
    gmt.propagate(gsps)
    ph_fda = gsps.phase.host(units='micron').T
    SPSmeas_ref = ps.piston(gsps).ravel()
    """ps.propagate(gsps)
    ps.camera.noiselessReadOut(SamplingTime)
    ps.fft()
    ps.process()
    SPSmeas_ref = ps.get_measurement()"""

    if eval_perf_sps==True:
        seg_pist_sps_ref = gsps.piston(where='segments', units_exponent=-9)

    if VISU == True:
        fig2, ax = plt.subplots()
        fig2.set_size_inches(20,5)
        fig2.suptitle('Field-dependent aberrations (um)', fontsize=20)
        imm = ax.imshow(ph_fda, interpolation='None')
        fig2.colorbar(imm, orientation='horizontal', shrink=0.6)Size of DFS detector mask [pix]: 20
if simul_SPS==True:
    # Show DFS reference imagettes
    def show_sps_imagettes():
        dataCube = ps.get_data_cube(data_type='fftlet')

        fig, ax = plt.subplots(ps.camera.N_SIDE_LENSLET,ps.camera.N_SIDE_LENSLET)
        fig.set_size_inches((12,15))
        xvec = np.arange(0,ps.camera.N_PX_IMAGE,10)
        for k in range(gsps.N_SRC*12):
            (ax.ravel())[k].imshow(np.sqrt(dataCube[:,:,k]), cmap=plt.cm.gist_earth_r, origin='lower')
            (ax.ravel())[k].autoscale(False)
            if ps.INIT_ALL_ATTRIBUTES == True:
                (ax.ravel())[k].plot(xvec, xvec*ps.pl_m[k] + ps.pl_b[k], 'y')
                (ax.ravel())[k].plot(xvec, xvec*ps.pp_m[k] + ps.pp_b[k], 'y--')
                for pp in range(3):
                    c1 = plt.Circle((ps.blob_data[k,pp,1], ps.blob_data[k,pp,0]),radius=np.sqrt(2)*ps.blob_data[k,pp,2],
                                color='b', fill=False)
                    (ax.ravel())[k].add_patch(c1)
            (ax.ravel())[k].set_title('%d'%(k%12), fontsize=12)

        for k in range(ps.camera.N_SIDE_LENSLET**2):
            (ax.ravel())[k].axis('off')

    if VISU == True and SPStype == 'DFS':
        show_sps_imagettes()_____no_output_____

### Background noise initialization_____no_output_____

#----- Estimate the effective area of DFS subapertures (in number of simulation pixels)

# Temporal on-axis GS to compute GMT area in pixels^2
ongs = ceo.Source("R", zenith=0., azimuth=0.,
              rays_box_size=D, rays_box_sampling=nPx, rays_origin=[0.0,0.0,25])
gmt.reset()
ongs.reset()
gmt.propagate(ongs)
PUPnp = np.sum(ongs.wavefront.amplitude.host())

SAmasks = ps.get_data_cube('pupil_masks')

SAnp = np.zeros(12*gsps.N_SRC)
for ii in range(12*gsps.N_SRC):
    SAnp[ii] = np.sum(SAmasks[:,:,ii])
sspp_area_all = (PupilArea/PUPnp)*SAnp
sspp_area = np.mean(sspp_area_all)
print 'average area of DFS SAs [m^2]: %0.3f'%sspp_area

if simul_bkgd == True:
    bkgd_ph1 = e0 * 10**(-0.4*bkgd_mag) / PupilArea  #ph/s/m^2/arcsec^2
    bkgd_nphsspp_all = sps_fov**2. * np.sum(sspp_area_all) * bkgd_ph1 #ph/s in ALL subaps of ALL sensors

    fringeImageSize = ps.camera.N_PX_IMAGE/2
    fake_factor = 12*gsps.N_SRC * fringeImageSize**2 / throughput
    dark_nphsspp_all = dark_cur * fake_factor

    tot_bkgd_noise = bkgd_nphsspp_all + dark_nphsspp_all
else: tot_bkgd_noise=0.effective area of DFS SAs [m^2]: 1.530

### Verify SPS noise simulation

Verify background noise (sky + dark current) and read noise simulation_____no_output_____

Verify photon noise and excess noise simulation_____no_output_____

#----- Get DFS fringe images and estimate the photon noise (using fringe images)
f, ax1 = plt.subplots()
f.set_size_inches(7,7)

#-- Initialize simulation parameters:
exposureTime_test = 100e-3
RONval_test = 0.  # [e- rms]
excess_noise_test = np.sqrt(2.0)
simul_bkgd = True
bkgd_mag_test = 12.0

#ps.camera.reset_rng()

if simul_bkgd == True:
    bkgd_ph1 = e0 * 10**(-0.4*bkgd_mag_test) / PupilArea  #ph/s/m^2/arcsec^2
    bkgd_nphsspp_all = sps_fov**2. * np.sum(sspp_area_all) * bkgd_ph1   #ph/s in ALL subaps ALL sensors
else: bkgd_nphsspp_all=0.

nfr = 50
gsps.reset()
gmt.reset()
gmt.propagate(gsps)

for jj in range(nfr):
    ps.reset()
    ps.propagate(gsps)
    ps.camera.readOut(exposureTime_test,RONval_test,bkgd_nphsspp_all, excess_noise_test)
    ps.fft()
    if jj > 0: clb1.remove()
    h1 = ax1.imshow(ps.camera.frame.host(), interpolation='None',origin='lower',
                       cmap=plt.cm.gist_earth_r)#, vmin=0, vmax=1.2*maxval)
    clb1 = f.colorbar(h1, ax=ax1, shrink=0.7)
    IPython.display.clear_output(wait=True)
    IPython.display.display(f)

    #-- convert fringes to column vectors and accumulate them:
    if jj == 0: cc = ps.camera.frame.host().ravel()
    else: cc = np.vstack((cc,ps.camera.frame.host().ravel()))

plt.close()_____no_output_____

## SPS Interaction Matrices calibration_____no_output_____

#[D_M2_Z,D_M2_PS_sh, D_M2_Z_PSideal, D_M2_PSideal] = \
if simul_onaxis_AO==True and simul_SPS==True:
    gmt0.cloop_calib_init(D, nPx, onaxis_wfs_nLenslet=nLenslet, AOtype=AOtype, svd_thr=1e-2)_____no_output_____

# Calibrate Segment Piston Sensor Interaction Matrix and Reconstructor
if simul_PS_control==True:
    PSstroke = 200e-9 #m

    if simul_onaxis_AO==False:
        D_M1_PS = gmt.calibrate(ps, gsps, mirror="M1", mode="segment piston", stroke=PSstroke)
    else:
        if AOtype == 'NGAOish':
            D_M1_PS = gmt0.calibrate(ps, gsps, mirror="M1", mode="segment piston", stroke=PSstroke)
        elif AOtype == 'LTAOish':
            D_M1_PS = gmt0.calibrate(ps, gsps, mirror="M1", mode="segment piston", stroke=PSstroke,
                                    closed_loop_calib=True)

    #D_M1_PS = D_M1_PS[:,0:6]
    R_M1_PS = np.linalg.pinv(D_M1_PS)
    print 'SPS - M1 Segment Piston IM:'
    print D_M1_PS.shape

    if VISU == True:
        fig, ax = plt.subplots()
        fig.set_size_inches(6,4)
        imm = ax.pcolor(D_M1_PS*1e-6)  #in displaced pixels per micron
        ax.grid()
        ax.set_ylim([0,12*gsps.N_SRC])
        ax.set_xticklabels(['S1 $T_z$','S2 $T_z$','S3 $T_z$','S4 $T_z$','S5 $T_z$','S6 $T_z$'], ha='left',
                           fontsize=15, rotation=45, color='b')
        ax.set_yticks(np.arange(gsps.N_SRC)*12)
        ax.tick_params(axis='y', labelsize=13)
        for pp in range(gsps.N_SRC):
            ax.text(-0.4,pp*12+6,'SPS$_%d$'%(pp+1), rotation=45, ha='center', va='center', fontsize=15, color='b')
        fig.colorbar(imm)___ M1 ___ (segment piston)
Segment #:1 2 3 4 5 6
------------
SPS - M1 Segment Piston IM:
(12, 6)

# Calibrate FSDP Interaction Matrix and Reconstructor
if simul_FDSP_control==True:
    TTstroke = 50e-3 #arcsec
    D_FDSP = gmt0.calibrate(ps, gsps, mirror="M1", mode="segment tip-tilt", stroke=TTstroke*math.pi/180/3600,
                          closed_loop_calib=True)
    #R_FDSP = np.linalg.pinv(D_FDSP)
    print 'SPS - FDSP IM:'
    print D_FDSP.shape

    if VISU == True:
        fig2, ax2 = plt.subplots()
        fig2.set_size_inches(12,4)

        #Rx and Ry are in radians. We want to show IM in microns RMS SURF of tilt
        #We found using DOS that a segment tilt of 47 mas is equivalent to 0.5 microns RMS of tilt on an M1 seg.
        AngleRadians_2_tiltcoeff = 0.5 / (47e-3*math.pi/180/3600) #angle in radians to microns RMS of tilt coeff

        imm = ax2.pcolor(D_FDSP/AngleRadians_2_tiltcoeff)  #in displaced pixels per microns RMS of M1 segment tilt
        ax2.grid()
        ax2.set_ylim([0,36])
        ax2.set_xticks(range(14))
        ax2.set_xticklabels(['S1 $R_x$','S1 $R_y$','S2 $R_x$','S2 $R_y$','S3 $R_x$','S3 $R_y$',
                            'S4 $R_x$','S4 $R_y$','S5 $R_x$','S5 $R_y$','S6 $R_x$','S6 $R_y$', 'S7 $R_x$','S7 $R_y$'],
                           ha='left', fontsize=15, rotation=45, color='b')
        ax2.set_yticks([0,12,24,36])
        ax2.tick_params(axis='y', labelsize=13)
        ax2.text(-0.4,6,'SPS$_1$', rotation=45, ha='center', va='center', fontsize=15, color='b')
        ax2.text(-0.4,18,'SPS$_2$', rotation=45, ha='center', va='center', fontsize=15, color='b')
        ax2.text(-0.4,30,'SPS$_3$', rotation=45, ha='center', va='center', fontsize=15, color='b')
        fig2.colorbar(imm)_____no_output_____if simul_FDSP_control==True:

    D_FDSP = D_FDSP[:,0:12]
    UU, ss, VV = np.linalg.svd(D_FDSP, full_matrices=False)
    plt.semilogy(ss/np.max(ss), 'o-')

    R_FDSP = np.linalg.pinv(D_FDSP)_____no_output_____# Combine Interaction Matrices of M1 segment piston AND FDSP.
if simul_PS_control==True and simul_FDSP_control==True:
    D_PIST = np.concatenate((D_M1_PS, D_FDSP), axis=1)

    print 'Merged SPS - PISTON IM:'
    print D_PIST.shape

    print 'PIST Condition number: %f'%np.linalg.cond(D_M1_PS)
    print 'FDSP Condition number: %f'%np.linalg.cond(D_FDSP)
    print 'Merged Condition number: %f'%np.linalg.cond(D_PIST)

    if VISU == True:
        plt.pcolor(D_PIST)
        plt.colorbar()_____no_output_____if simul_PS_control==True and simul_FDSP_control==True:

    UU, ss, VV = np.linalg.svd(D_PIST, full_matrices=False)
    plt.semilogy(ss/np.max(ss), 'o-')_____no_output_____if simul_PS_control==True and simul_FDSP_control==True:

    R_PIST1 = np.linalg.pinv(D_PIST, rcond=1e-2)
    R_PIST = np.zeros((20,ps.get_measurement_size()))
    R_PIST[0:18,:] = R_PIST1_____no_output_____

# Initialize SH AGWS sensors_____no_output_____

# Performance Evaluation Initialization_____no_output_____### On-axis performance evaluation_____no_output_____

# Initialize on-axis star for performance evaluation
ongs = ceo.Source("R", zenith=0., azimuth=0.,
              rays_box_size=D, rays_box_sampling=nPx, rays_origin=[0.0,0.0,25])
gmt.reset()
ongs.reset()
gmt.propagate(ongs)
ph_fda_on = ongs.phase.host(units='micron')
on_rms0 = ongs.wavefront.rms()*1e9
print '--> WF RMS on-axis: %3.2f nm wf RMS'%on_rms0
seg_pist_onaxis_ref = ongs.piston(where='segments', units_exponent=-9)

if VISU == True:
    fig, ax = plt.subplots()
    fig.set_size_inches(20,5)
    fig.suptitle('Field-dependent aberrations (um wf)', fontsize=20)
    imm = ax.imshow(ph_fda_on, interpolation='None')
    fig.colorbar(imm, orientation='horizontal', shrink=0.6)

# On-axis projection of segment shapes onto Zernikes
if eval_perf_modal==True:
    proj_radial_order= np.max( (8, gmt.M1.zernike.max_n+1, gmt.M2.zernike.max_n+1) )
    Zobj = ceo.ZernikeS(proj_radial_order)
    Zobj.fitting_init(ongs)--> WF RMS on-axis: 0.76 nm wf RMS

### Off-axis (Science FoV) performance evaluation_____no_output_____

 eval_perf_field = False_____no_output_____#Initialize asterism for performance evaluation in the science field
if eval_perf_field == True:

    sfx = np.arange(-45.,55.,22.5) # in arcsec
    sfy = np.arange(-45.,55.,22.5) # in arcsec
    sfxx, sfyy = np.meshgrid(sfx, sfy)
    sf_zenith_angle = np.sqrt(sfxx**2 + sfyy**2) * ceo.constants.ARCSEC2RAD
    sf_azimuth_angle = np.arctan2(sfyy,sfxx)

    sfgs = ceo.Source("R",zenith=sf_zenith_angle.ravel(),azimuth=sf_azimuth_angle.ravel(),#*math.pi/180,
                      rays_box_size=D,rays_box_sampling=nPx,rays_origin=[0.0,0.0,25])

    if VISU == True:
        fig,ax1 = plt.subplots()
        fig.set_size_inches((4,4))
        ax1.plot(sfxx,sfyy, '*', markersize=10)
        ax1.set_xlim([-100,100])
        ax1.set_ylim([-100,100])
        ax1.grid()_____no_output_____# Field aberrations in the off-axis performance directions (sfgs)
if eval_perf_field == True:
    sfgs.reset()
    gmt.propagate(sfgs)
    sf_rms0 = sfgs.wavefront.rms()*1e9
    print '--> max WF RMS at the edge of the field: %3.2f nm WF RMS'%np.max(sf_rms0)

    if VISU == True:
        fig, ax2 = plt.subplots()
        fig.set_size_inches((5,4))
        contp = ax2.contourf(sfx, sfy, sf_rms0.reshape(len(sfx),-1))
        clb = fig.colorbar(contp, ax=ax2)
        ax2.grid()
        ax2.tick_params(labelsize=12)
        ax2.set_xlabel('field angle [arcsec]', fontsize=15)
        clb.set_label('nm WF RMS', fontsize=15)
        clb.ax.tick_params(labelsize=12)_____no_output_____if eval_perf_field == True:
    sf_pist0 = sfgs.piston(where='segments', units_exponent=-9)

    if VISU == True:
        fig, ax2 = plt.subplots()
        fig.set_size_inches((5,4))
        contp = ax2.contourf(sfx, sfy, np.std(sf_pist0, axis=1).reshape(len(sfx),-1))
        clb = fig.colorbar(contp, ax=ax2)
        ax2.grid()
        ax2.tick_params(labelsize=12)
        ax2.set_xlabel('field angle [arcsec]', fontsize=15)
        clb.set_label('nm WF', fontsize=15)
        clb.ax.tick_params(labelsize=12)_____no_output_____

# Closed-loop simulation initialization_____no_output_____### INITIAL SCRAMBLE_____no_output_____

# Reset before starting
if simul_onaxis_AO==True:
    gs.reset()
if simul_SPS==True:
    gsps.reset()
ongs.reset()
gmt.reset()

# Apply a known Tilt to a particular segment on M1
M1RotVecInit = np.array([  #arcsec
            [0,0,0] ,  # 200e-3
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0]], dtype='float32') * math.pi/180/3600

# Apply a known segment piston/translation to a particular segment on M1
M1TrVecInit = np.array([  # meters surf
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0]], dtype='float32')

# Apply a known Tilt to a particular segment on M2
M2RotVecInit = np.array([  #arcsec
            [0,0,0] ,
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0]], dtype='float32') * math.pi/180/3600

# Apply a known segment piston/translation to a particular segment on M2
M2TrVecInit = np.array([  # meters surf
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0],
            [0,0,0]], dtype='float32')

# Initial Segment TT Scramble
if scramble_tt==True:
    TTscramble = np.float32(np.random.normal(loc=0.0, scale=1, size=12))
    TTscramble *= tt_scramble_rms*(math.pi/180/3600)/np.std(TTscramble)
    TTscramble -= np.mean(TTscramble)

    """TTscramble = np.array([ -1.75567607e-07,  -3.92353201e-07,  -3.31998194e-07,
         1.23213044e-06,   7.21566380e-08,   1.17542027e-06,
        -1.53068822e-06,   1.27538101e-07,   6.87605393e-07,
        -2.37202329e-07,  -4.88611761e-07,  -1.38429471e-07], dtype='float32')"""

    M1RotVecInit[0:6,0:2] += TTscramble.reshape((6,2))
    print "Initial M1 segment TT scramble RMS [mas]: %0.2f"%(np.std(TTscramble)*ceo.constants.RAD2MAS)
    print "Initial M1 segment TT scramble mean value [mas]: %0.2f"%(np.mean(TTscramble)*ceo.constants.RAD2MAS)

# Initial Segment Piston Scramble
if scramble_pist==True:
    pistscramble = np.float32(np.random.normal(loc=0.0, scale=1, size=7))
    pistscramble *= pist_scramble_rms/np.std(pistscramble)
    pistscramble -= np.mean(pistscramble)
    pistscramble -= pistscramble[6]

    """pistscramble = np.array([ -3.25938601e-07,  -1.26658892e-06,   1.30272156e-06,
         2.35193511e-07,  -1.64358767e-06,  -1.52229950e-06,
         0.00000000e+00], dtype='float32')"""

    M1TrVecInit[:,2] += pistscramble
    print "Initial segment piston scramble RMS [micron]: %0.2f"%(np.std(pistscramble)*1e6)
    print "Initial segment piston scramble mean value [micron]: %0.2f"%(np.mean(pistscramble)*1e6)

### Set GMT M1 and M2 rigid-body scrambles
for idx in range(7): gmt.M1.update(origin=M1TrVecInit[idx,:].tolist(),
                                   euler_angles=M1RotVecInit[idx,:].tolist(),
                                   idx=idx+1)

for idx in range(7): gmt.M2.update(origin=M2TrVecInit[idx,:].tolist(),
                                   euler_angles=M2RotVecInit[idx,:].tolist(),
                                   idx=idx+1)

"""# Apply a Zernike vector to a particular segment on M1
mysegId = 2
a_M1 = np.zeros(gmt.M1.zernike.n_mode)   #zernike coeffs, from piston Z1 to n_zern
#a_M1[10] = 250e-9      # m RMS surf
gmt.M1.zernike.a[mysegId-1,:] = a_M1
#for mmmm in range(6): gmt.M1.zernike.a[mmmm,:] = a_M1
gmt.M1.zernike.update()"""

#M2gTTVecInit = np.array((50e-3,0)) * ceo.constants.ARCSEC2RAD
#gmt.M2.global_tiptilt(M2gTTVecInit[0],M2gTTVecInit[1])

if VISU==True:
    ongs.reset()
    gmt.propagate(ongs)
    fig, ax = plt.subplots()
    fig.set_size_inches(20,5)
    imm = ax.imshow(ongs.phase.host(units='micron')-ph_fda_on, interpolation='None',cmap='RdYlBu',origin='lower')
    ax.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off', right='off', left='off', labelleft='off')
    clb = fig.colorbar(imm, ax=ax, fraction=0.012, pad=0.03,format="%.1f")
    clb.set_label('$\mu m$ WF', fontsize=25)
    clb.ax.tick_params(labelsize=15)
    ongs.reset()_____no_output_____

### INIT VARIABLES AND ALLOCATE MEMORY FOR RESULTS_____no_output_____

totSimulIter = int(totSimulTime/Tsim)
timeVec = np.arange(totSimulIter)*Tsim

if simul_SPS==True:
    sps_exp_count = 0
    sps_exp_count_max = int(exposureTime/Tsim)
    sps_sampl_count = 0
    sps_sampl_count_max = int(samplingTime/exposureTime)
    kk = 1 # to keep the count of PS/FDSP commands applied.
    ps.camera.reset_rng(sps_seed)

    if eval_perf_sps==True:
        seg_pist_sps_iter = np.zeros((N_GS_PS,7,totSimulIter))

    SPSmeas_iter = np.zeros((N_GS_PS*12,sps_sampl_iter+1))
    SPSfft_images = np.zeros((ps.camera.N_PX_IMAGE,ps.camera.N_PX_IMAGE,N_GS_PS*12,sps_sampl_iter+1))

M1TrVec  = M1TrVecInit  #np.zeros((7,3))
M1RotVec = M1RotVecInit
M2TrVec  = M2TrVecInit
M2RotVec = M2RotVecInit

if simul_PS_control==True:
    M1PSresiter = np.zeros((7,sps_sampl_iter+1))
    M1PSresiter[:,0] = M1TrVec[:,2] # kk=0

if simul_FDSP_control==True:
    M1TTresiter = np.zeros((7,2,sps_sampl_iter+1))
    M1TTresiter[:,:,0] = M1RotVec[:,0:2]  # kk=0

if simul_onaxis_AO==True:
    a_M2      = np.zeros(nall)
    a_M2_iter = np.zeros((nall,totSimulIter))
    if AOtype=='NGAOish':
        myAOest1 = np.zeros(nall+7)  #7x2 segments TTs + 7 outer segment pistons
        M2PSiter = np.zeros((7,totSimulIter))
    elif AOtype=='LTAOish':
        myAOest1 = np.zeros(nall)  #delta command buffer

if eval_perf_onaxis==True:
    wfe_gs_iter      = np.zeros(totSimulIter)
    seg_pist_onaxis_iter  = np.zeros((7,totSimulIter))
    seg_wfe_onaxis_iter = np.zeros((7,totSimulIter))
    if simul_turb==True:
        wfe_gs_tur_iter = np.zeros(totSimulIter)
    if eval_perf_modal==True:
        seg_aRes_gs_iter = np.zeros((Zobj.n_mode,7,totSimulIter))
        if simul_turb==True:
            seg_aTur_gs_iter = np.zeros((Zobj.n_mode,7,totSimulIter))

if eval_perf_field == True:
    wfe_sfgs_iter  = np.zeros((sfgs.N_SRC, totSimulIter))
    pist_sfgs_iter = np.zeros((sfgs.N_SRC, totSimulIter))

elapsedTime = np.zeros(totSimulIter)

print 'Total simulation time [s]: %4.3f'%(totSimulTime)
if simul_SPS == True:
    print '  DFS Sampling time  [ms]: %4.2f'%(samplingTime*1e3)
    print '  DFS Exposure time  [ms]: %4.2f'%(exposureTime*1e3)
print 'Simulation time step [ms]: %4.2f'%(Tsim*1e3)Total simulation time [s]: 0.300
  DFS Sampling time  [ms]: 30.00
  DFS Exposure time  [ms]: 10.00
Simulation time step [ms]: 2.00

### CLOSE THE LOOP!!_____no_output_____

if simul_SPS==True: ps.reset()

for jj in range(totSimulIter):
    #print("\nElapsed time:%3.3f"%((jj+1)*Tsim))
    tid.tic()
    if simul_SPS==True: gsps.reset()
    if simul_SH==True:  gs.reset()
    if eval_perf_onaxis==True: ongs.reset()
    if eval_perf_field==True: sfgs.reset()

    #----- Update Turbulence --------------------------------------------
    if simul_turb == True:
        if simul_SPS==True:
            atm.ray_tracing(gsps, p,nPx,p,nPx, jj*Tsim)
        if simul_onaxis_AO==True:
            atm.ray_tracing(  gs, p,nPx,p,nPx, jj*Tsim)
        if eval_perf_onaxis==True:
            atm.ray_tracing(ongs, p,nPx,p,nPx, jj*Tsim)
            wfe_gs_tur_iter[jj] = ongs.wavefront.rms()
            if eval_perf_modal==True:
                PhaseTur = ongs.phase.host(units='nm')-ph_fda_on*1e3
                seg_aTur_gs_iter[:,:,jj] = Zobj.fitting(PhaseTur)
        if eval_perf_field==True:
            atm.ray_tracing(sfgs, p,nPx,p,nPx, jj*Tsim)

    #----- Update M1 positions -------------------------------------
    for idx in range(7): gmt.M1.update(origin =  M1TrVec[idx,:].tolist(),
                                 euler_angles = M1RotVec[idx,:].tolist(),
                                 idx = idx+1)

    #---- segment KL correction (on M2)
    if simul_onaxis_AO==True:
        gmt.M2.modes.a[:,z_first_mode:] -= myAOest1[0:nall].reshape((7,-1))
        gmt.M2.modes.update()
        if AOtype=='NGAOish':
            gmt.M2.motion_CS.origin[:,2] -= myAOest1[nall:]
            gmt.M2.motion_CS.update()

    #----- On-axis WFS measurement and correction -----------------------
    if simul_onaxis_AO==True:
        gmt.propagate(gs)
        wfs.reset()
        wfs.analyze(gs)
        slopevec = wfs.get_measurement()

        if AOtype=='NGAOish':
            onpsvec =  onps.piston(gs).ravel() - onps_signal_ref
            AOmeasvec = np.concatenate((slopevec, onpsvec))
        elif AOtype=='LTAOish':
            AOmeasvec = slopevec

        myAOest1 = np.dot(R_AO, AOmeasvec)
        myAOest1[0:nall] *= gAO
        a_M2_iter[:,jj] = a_M2
        a_M2 -= myAOest1[0:nall]

        if AOtype=='NGAOish':
            M2PSiter[:,jj] = M2TrVec[:,2]
            myAOest1[nall:]  *= gAO_PS
            M2TrVec[:,2] -= myAOest1[nall:]

    #----- Performance evaluation ---------------------------------
    if eval_perf_onaxis==True:
        gmt.propagate(ongs)
        wfe_gs_iter[jj] = ongs.wavefront.rms()
        seg_pist_onaxis_iter[:,jj] = ongs.piston(where='segments', units_exponent=-9) - seg_pist_onaxis_ref
        seg_wfe_onaxis_iter[:,jj] = ongs.phaseRms(where='segments', units_exponent=-9)

        if eval_perf_modal==True:
            PhaseRes = ongs.phase.host(units='nm')-ph_fda_on*1e3
            seg_aRes_gs_iter[:,:,jj] = Zobj.fitting(PhaseRes)

    if eval_perf_field==True:
        gmt.propagate(sfgs)
        wfe_sfgs_iter[:,jj]  = sfgs.wavefront.rms()
        pist_sfgs_iter[:,jj] = np.std(sfgs.piston(where='segments', units_exponent=-9) - sf_pist0, axis=1)

    #----- Segment Piston Sensors measurement and correction -------------
    if simul_SPS==True:
        gmt.propagate(gsps)

        if sps_sampl_count >= sps_sampl_delay:
            #print("Fringe Generation at %4.3f s:"%((jj+1)*Tsim))
            ps.propagate(gsps)

        if eval_perf_sps==True:
            seg_pist_sps_iter[:,:,jj] = gsps.piston(where='segments', units_exponent=-9) - seg_pist_sps_ref

        if sps_exp_count == sps_exp_count_max-1:

            if sps_sampl_count >= sps_sampl_delay:
                #print("Reading Out SPS detectors at %4.3f s:"%((jj+1)*Tsim))

                #--- Read-out the detector
                if simul_phot==True:
                    ps.camera.readOut(exposureTime,RONval,tot_bkgd_noise,excess_noise)

                #--- Compute and integrate FFTlet images:
                ps.fft()

            ps.camera.reset()
            sps_exp_count = 0
            #print("SPS exposure completed at %3.1f ms"%((jj+1)*Tsim*1e3))

            #--- Compute SPS signals:
            if sps_sampl_count == sps_sampl_count_max-1:
                #print("Applying SPP/FDSP command at %4.3f s:"%((jj+1)*Tsim))
                ps.process()
                SPSmeas = ps.measurement - SPSmeas_ref
                SPSfft_images[:,:,:,kk] = ps.get_data_cube(data_type='fftlet')
                SPSmeas_iter[:,kk] = SPSmeas

                #--- segment piston AND FDSP control:
                if simul_PS_control==True and simul_FDSP_control==True:
                    PISTvec = np.dot(R_PIST, SPSmeas)
                    myPSest1 = gPS * PISTvec[0:6]
                    M1TrVec[0:6,2] -= myPSest1

                    if gPS>0: M1PSresiter[:,kk] = M1TrVec[:,2]
                    else:   M1PSresiter[0:6,kk] = PISTvec[0:6]  #To save SPP estimates without applying them!

                    myFDSPest1 = gFDSP * PISTvec[6:].reshape((7,2))
                    #myFDSPest1 = gFDSP * PISTvec[7:].reshape((7,2))
                    M1RotVec[:,0:2] -= myFDSPest1
                    M1TTresiter[:,:,kk] = M1RotVec[:,0:2]

                #--- ONLY segment piston control:
                elif simul_PS_control==True:
                    PISTvec = np.dot(R_M1_PS, SPSmeas)
                    myPSest1 = gPS * PISTvec
                    M1TrVec[0:6,2] -= myPSest1
                    if gPS>0: M1PSresiter[:,kk] = M1TrVec[:,2]
                    else:   M1PSresiter[0:6,kk] = PISTvec #Temporary line to keep track of estimates without applying them.

                #--- ONLY FDSP control:
                elif simul_FDSP_control==True:
                    PISTvec = np.dot(R_FDSP, SPSmeas)
                    myFDSPest1 = gFDSP * PISTvec.reshape((6,2))
                    M1RotVec[0:6,0:2] -= myFDSPest1
                    M1TTresiter[:,:,kk] = M1RotVec[:,0:2]

                if simul_PS_control==True:
                    sys.stdout.write("\nseg piston RMS [microns]: %0.3f\n"%(np.std(M1PSresiter[:,kk])*1e6))

                ps.fftlet.reset()
                sps_sampl_count = 0
                sps_exp_delay_count = 0
                kk += 1
                #print("SPS command applied at %3.1f ms"%((jj+1)*Tsim*1e3))
            else: sps_sampl_count += 1
        else: sps_exp_count += 1

    tid.toc()
    elapsedTime[jj] = tid.elapsedTime
    sys.stdout.write("\r iter: %d/%d, ET: %.2f, seg piston [nm WF RMS] on-axis: %0.1f"%(jj, totSimulIter, tid.elapsedTime, np.std(seg_pist_onaxis_iter[:,jj])))
    sys.stdout.flush()

if VISU == True:
    #ongs.reset()
    #gmt.propagate(ongs)
    fig, ax = plt.subplots()
    fig.set_size_inches(20,5)
    imm = ax.imshow(ongs.phase.host(units='micron'), interpolation='None',cmap='RdYlBu',origin='lower')#, vmin=-8, vmax=8)
    ax.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off',
                   right='off', left='off', labelleft='off')
    clb = fig.colorbar(imm, ax=ax, fraction=0.012, pad=0.03,format="%.1f")
    clb.set_label('$\mu m$ WF', fontsize=25)
    clb.ax.tick_params(labelsize=15)
    #ongs.reset() iter: 149/150, ET: 162.99, seg piston [nm WF RMS] on-axis: 0.0tosave = dict(D=D, nPx=nPx, PupilArea=PupilArea, Tsim=Tsim, totSimulTime=totSimulTime, simul_turb=simul_turb,
        simul_onaxis_AO=simul_onaxis_AO, simul_PS_control=simul_PS_control,
        simul_FDSP_control=simul_FDSP_control, simul_SH=simul_SH, simul_SPS=simul_SPS,
        elapsedTime=elapsedTime, scramble_tt=scramble_tt, scramble_pist=scramble_pist, CEO_git_commit=CEO_git_commit)

if simul_turb == True:
    tosave.update(dict(r0=r0, L0=L0, atm_fname=atm_fname, atm_seed=atm_seed))

if simul_SPS == True:
    tosave.update(dict(SPStype=SPStype, asterism_type=asterism_type, N_GS_PS=N_GS_PS,
        zenith=zenith_angle_agws, azimuth=azimuth_angle_agws,
        band=band, mag=mag, bkgd_mag=bkgd_mag,sps_fov=sps_fov, sps_dispersion=sps_dispersion,
        sps_mask_size=sps_mask_size, RONval=RONval, sps_seed=sps_seed, simul_phot=simul_phot, simul_bkgd=simul_bkgd,
        dark_cur=dark_cur, excess_noise=excess_noise, e0=e0,
        SPSmeas_ref=SPSmeas_ref, exposureTime=exposureTime, samplingTime=samplingTime, sps_sampl_delay=sps_sampl_delay,
        throughput=throughput, lobe_detection=lobe_detection, nyquist_factor=nyquist_factor,
        sps_sampl_iter=sps_sampl_iter, SPSmeas_iter=SPSmeas_iter, SPSfft_images=SPSfft_images))
    if asterism_type =='Besancon_GP' or asterism_type=='Besancon_GP_AB': tosave['asterism_id']=asterism_id
    if eval_perf_sps == True: tosave['seg_pist_sps_iter']=seg_pist_sps_iter

if simul_SH == True:
    tosave.update(dict(sh_SA=nLenslet, sh_npix=n, SHtype=SHtype, sh_thr=sh_thr))
    if SHtype == 'diff':
        tosave['sh_fov'] = sh_fov

if simul_onaxis_AO == True:
    tosave.update(dict(AOtype=AOtype, gAO=gAO, onaxis_AO_modes=onaxis_AO_modes, a_M2_iter=a_M2_iter))
    if onaxis_AO_modes==u'zernikes': tosave['M2_radial_order']=M2_radial_order
    elif onaxis_AO_modes==u'Karhunen-Loeve': tosave['M2_n_modes']=M2_n_modes
    if AOtype=='NGAOish': tosave.update(dict(M2PSiter=M2PSiter, gAO_PS=gAO_PS))

if simul_PS_control == True:
    tosave.update(dict(gPS=gPS, M1PSresiter=M1PSresiter))

if simul_FDSP_control == True:
    tosave.update(dict(gFDSP=gFDSP, M1TTresiter=M1TTresiter))

if eval_perf_onaxis == True:
    tosave.update(dict(wfe_gs_iter=wfe_gs_iter, seg_pist_onaxis_iter=seg_pist_onaxis_iter,
                      seg_wfe_onaxis_iter=seg_wfe_onaxis_iter))
    if simul_turb == True: tosave['wfe_gs_tur_iter']=wfe_gs_tur_iter

    if eval_perf_modal == True:
        tosave['seg_aRes_gs_iter']=seg_aRes_gs_iter
        if simul_turb == True: tosave['seg_aTur_gs_iter']=seg_aTur_gs_iter

if eval_perf_field == True:
    #tosave['wfe_sfgs_iter'] = wfe_sfgs_iter
    #tosave['wfe_sfgs_ref'] = sf_rms0
    tosave['pist_sfgs_iter'] = pist_sfgs_iter
    tosave['pist_sfgs_ref'] = sf_pist0

if scramble_tt == True:
    tosave.update(dict(tt_scramble_rms=tt_scramble_rms, TTscramble=TTscramble))

if scramble_pist == True:
    tosave.update(dict(pist_scramble_rms=pist_scramble_rms, pistscramble=pistscramble))_____no_output_____#---- Save the results file!
#now = datetime.datetime.now()
#save_dir = './'
#filename=save_dir+'SPS'+str(int(alpha_ps/60))+'_'+clcalib+'_mag'+str(int(mag))+'_Turb'+str(atm_seed)+'_Noise'+str(sps_seed)+'_Texp'+str(int(exposureTime*1e3))+'ms'+'_Ts'+str(int(samplingTime*1e3))+'ms.npz'
#filename=save_dir+now.strftime("%Y%m%d-%H%M")+'.npz'
filename='/mnt/simul_results/simul_results.npz'
np.savez_compressed(filename, **tosave)_____no_output_____VISU=False_____no_output_____#### on-axis WFE vs. iteration number
if VISU == True and eval_perf_onaxis==True:
    fig, ax = plt.subplots()
    fig.set_size_inches(15,5)
    ax.plot(timeVec*1e3, wfe_gs_iter*1e9, '-+')
    ax.grid()
    ax.set_xlabel('Time [ms]', fontsize=20)
    ax.set_ylabel('nm WF rms', fontsize=20)
    ax.tick_params(labelsize=15)

    wfe_final = np.mean(wfe_gs_iter[100:])
    print ('Final WFE [nm RMS]: %3.1f'%(wfe_final*1e9))
    print ('Equivalent SR in K band: %1.4f'%(np.exp(-(wfe_final*2*np.pi/2.2e-6)**2)))Final WFE [nm RMS]: 0.8
Equivalent SR in K band: 1.0000
#### on-axis segment Piston vs. iteration number
if VISU == True and eval_perf_onaxis==True:
    fig, ax = plt.subplots()
    fig.set_size_inches(15,5)
    ax.plot(timeVec*1e3, seg_pist_onaxis_iter.T, '-+')
    ax.grid()
    ax.set_xlabel('Time [ms]', fontsize=20)
    ax.set_ylabel('segment piston [nm WF]', fontsize=20)
    ax.tick_params(labelsize=15)

    print "seg piston [nm WF RMS] on-axis: %0.1f"%(np.std(seg_pist_onaxis_iter[:,100:]))seg piston [nm WF RMS] on-axis: 0.0
#### on-axis segment WFE vs. iteration number
if VISU == True and eval_perf_onaxis==True:
    fig, ax = plt.subplots()
    fig.set_size_inches(15,5)
    ax.plot(timeVec*1e3, seg_wfe_onaxis_iter.T, '-+')
    ax.grid()
    ax.set_xlabel('Time [ms]', fontsize=20)
    ax.set_ylabel('segment RMS [nm WF]', fontsize=20)
    ax.tick_params(labelsize=15)

    wfe_final = np.mean(seg_wfe_onaxis_iter[:,100:])
    print ('Final WFE [nm RMS]: %3.1f'%(wfe_final))
    print ('Equivalent SR in K band: %1.4f'%(np.exp(-(wfe_final*2*np.pi/2200.0)**2)))Final WFE [nm RMS]: 0.7
Equivalent SR in K band: 1.0000
#### Residual segment piston analysis
print 'Final M1 final piston (Tz) values [nm WF]:'
print np.array_str(gmt.M1.motion_CS.origin[:,2]*1e9*2, precision=3, suppress_small=True)
print '-----'
print 'Final M2 final piston (Tz) values [nm WF]:'
print np.array_str(gmt.M2.motion_CS.origin[:,2]*1e9*2, precision=3, suppress_small=True)

if VISU == True:
    if simul_PS_control==True or simul_FDSP_control==True:
        timeCorVec = np.arange(sps_sampl_iter+1)*samplingTime
    else: timeCorVec = timeVec

    if simul_onaxis_AO==True and AOtype=='NGAOish':
        f1, (ax1,ax2) = plt.subplots(ncols=2)
        f1.set_size_inches(20,5)
    else:
        f1, ax1 = plt.subplots()
        f1.set_size_inches(10,5)

    if simul_PS_control==True:
        ax1.plot(timeCorVec, M1PSresiter.T *1e9*2, label='S', drawstyle='steps-post')
        ax1.grid()
        #ax1.set_title('Tz', size='x-large')
        ax1.set_xlabel('time elapsed [s]', size='x-large')
        ax1.set_ylabel('M1 segment piston [nm WF]', size='x-large')
        ax1.tick_params(labelsize=18)
        #ax1.legend()
    elif scramble_pist==True:
        ax1.plot(np.array([0, np.max(timeVec)]), np.array([pistscramble,pistscramble])*1e9*2)
        ax1.grid()
        ax1.set_xlabel('elapsed time [s]',size='x-large')
        ax1.set_ylabel('M1 segment piston [nm WF]', size='x-large')
        ax1.tick_params(labelsize=15)

    if simul_onaxis_AO==True and AOtype=='NGAOish':
        ax2.plot(timeVec, M2PSiter.T *1e9*2, label='S')
        ax2.grid()
        #ax1.set_title('Tz', size='x-large')
        ax2.set_xlabel('time elapsed [s]', size='x-large')
        ax2.set_ylabel('M2 segment piston [nm WF]', size='x-large')
        ax2.tick_params(labelsize=15)Final M1 final piston (Tz) values [nm WF]:
[ 0.  0.  0.  0.  0.  0.  0.]
-----
Final M2 final piston (Tz) values [nm WF]:
[ 0.  0.  0.  0.  0.  0.  0.]
if simul_PS_control==True:
    if scramble_pist==True:
        SPP_convergence_sampl_niter = 5
    else:
        SPP_convergence_sampl_niter = 1

    print("Residual M1 segment piston [nm WF RMS]: %.1f"%(np.std(M1PSresiter[:,SPP_convergence_sampl_niter:]*1e9*2)))Residual M1 segment piston [nm WF RMS]: 156.4
if VISU==True and (simul_FDSP_control==True or scramble_tt==True):
    f1, (ax3,ax4) = plt.subplots(ncols=2)
    f1.set_size_inches(20,5)

    if simul_FDSP_control==True :
        timeCorVec = np.arange(sps_sampl_iter+1)*samplingTime
        ax3.plot(timeCorVec, M1TTresiter[:,0,:].T *ceo.constants.RAD2MAS*8, drawstyle='steps-post')
        ax3.grid()
        ax3.set_title('Rx', size='x-large')
        ax3.set_xlabel('elapsed time [s]',size='x-large')
        ax3.set_ylabel('M1 segment y-tilt [mas]', size='x-large')
        ax3.tick_params(labelsize=13)
    #    ax3.set_xlim([0,60])

        ax4.plot(timeCorVec, M1TTresiter[:,1,:].T *ceo.constants.RAD2MAS*8, drawstyle='steps-post')
        ax4.grid()
        ax4.set_title('Ry', size='x-large')
        ax4.set_xlabel('elapsed time [s]', size='x-large')
        ax4.set_ylabel('M1 segment x-tilt [mas]', size='x-large')
        ax4.tick_params(labelsize=13)
    #    ax4.set_xlim([0,60])

    elif scramble_tt==True :
        TTscr = TTscramble.reshape((6,2))
        ax3.plot([0, max(timeVec)], np.array([TTscr[:,0],TTscr[:,0]])*ceo.constants.RAD2MAS*8)
        ax3.grid()
        #ax3.set_title('Rx', size='x-large')
        ax3.set_xlabel('elapsed time [s]',size='x-large')
        ax3.set_ylabel('M1 segment y-tilt [mas]', size='x-large')
        ax3.tick_params(labelsize=13)
    #    ax3.set_xlim([0,60])

        ax4.plot([0, max(timeVec)], np.array([TTscr[:,1],TTscr[:,1]])*ceo.constants.RAD2MAS*8)
        ax4.grid()
        #ax4.set_title('Rx', size='x-large')
        ax4.set_xlabel('elapsed time [s]',size='x-large')
        ax4.set_ylabel('M1 segment y-tilt [mas]', size='x-large')
        ax4.tick_params(labelsize=13)
    #    ax4.set_xlim([0,60])_____no_output_____if simul_FDSP_control==True:
    if scramble_tt==True:
        FDSP_convergence_sampl_niter = 5
    else:
        FDSP_convergence_sampl_niter = 1
    FDSP_convergence_niter = np.int32(FDSP_convergence_sampl_niter * samplingTime / Tsim)

    #print FDSP_convergence_niter
    print "M1 x-tilt residual [mas RMS]"
    print np.std(M1TTresiter[:,1,FDSP_convergence_sampl_niter:].T)*ceo.constants.RAD2MAS #* (30/7)
    print "M1 y-tilt residual [mas RMS]"
    print np.std(M1TTresiter[:,0,FDSP_convergence_sampl_niter:].T)*ceo.constants.RAD2MAS _____no_output_____

if eval_perf_sps==True and VISU==True:

    seg_pist_rms_sps_iter = np.std(seg_pist_sps_iter, axis=1)

    #### off-axis segment piston (at the three SPS directions) vs. iteration number
    fig, ax = plt.subplots(ncols=gsps.N_SRC, nrows=2)
    fig.set_size_inches(17,10)

    for kk in range(N_GS_PS):
        (ax.ravel())[kk].plot(timeVec/60, seg_pist_sps_iter[kk,:,:].T, '-+')
        (ax.ravel())[kk].grid()
        #(ax.ravel())[kk].set_xlabel('elapsed time [min]', fontsize=20)
        if kk==0: (ax.ravel())[kk].set_ylabel('nm WF', fontsize=20)
        (ax.ravel())[kk].tick_params(labelsize=15)

    for kk in range(N_GS_PS):
        (ax.ravel())[kk+N_GS_PS].plot(timeVec/60, seg_pist_rms_sps_iter[kk,:], '-+')
        (ax.ravel())[kk+N_GS_PS].grid()
        (ax.ravel())[kk+N_GS_PS].set_xlabel('elapsed time [min]', fontsize=20)
        if kk==0: (ax.ravel())[kk+N_GS_PS].set_ylabel('nm WF RMS', fontsize=20)
        (ax.ravel())[kk+N_GS_PS].tick_params(labelsize=15)

    print "seg piston [nm WF RMS] on PS GSs: %0.1f, %0.1f, %0.1f"%((np.std(seg_pist_sps_iter[0,:,75:]),
                                                                   np.std(seg_pist_sps_iter[1,:,75:]),
                                                                   np.std(seg_pist_sps_iter[2,:,75:])))_____no_output_____### Show final on-axis residual WF
if VISU == True:
    #ongs.reset()
    #gmt.propagate(ongs)
    fig, ax = plt.subplots()
    fig.set_size_inches(20,5)
    imm = ax.imshow(ongs.phase.host(units='nm')-ph_fda_on*1e3, interpolation='None',cmap='RdYlBu', origin='lower')#, vmin=-1.5, vmax=1.5)
    ax.set_title('on-axis WF')
    ax.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off', right='off', left='off', labelleft='off')
    clb = fig.colorbar(imm, ax=ax, fraction=0.012, pad=0.03,format="%.1f")
    clb.set_label('nm WF', fontsize=25)
    clb.ax.tick_params(labelsize=15)_____no_output_____


# Notebook from Blaizzy/Boring_weekends
Path: Deep_Learning/GNNs/GNNs.ipynb

Deep Graph Library (DGL)_____no_output_____

!pip install dglRequirement already satisfied: dgl in /usr/local/lib/python3.6/dist-packages (0.4.2)
Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from dgl) (1.4.1)
Requirement already satisfied: networkx>=2.1 in /usr/local/lib/python3.6/dist-packages (from dgl) (2.4)
Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from dgl) (1.17.5)
Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.1->dgl) (4.4.1)
!pip install networkxRequirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (2.4)
Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx) (4.4.1)
import dgl
import networkx as nx
import torch_____no_output_____

##**NOTE**: This notebook was developed taking into account you read my article [Can Graph Neural Networks Solve Real-World Problems ](https://hackernoon.com/can-graph-neural-networks-solve-real-world-problems-7hd636dn) or have some previous domain knowledge. If you don't know about GNNs please take a minute and read my article, it will help you get a better understanding.

![alt text](https://media.giphy.com/media/WoWm8YzFQJg5i/giphy.gif)_____no_output_____# DONE READING? NOW, HAVE FUN!!!_____no_output_____# Node classification(Cluster formation)
Here we will binary node classification to predict whether a node is going to be in the cluster 1 or cluster 2 represented by node 0 and 33 which.
A typical example of friend recommendation system, just like facebook does to you.

Credits for the dataset and example to DGL(Deep Graph Library) library [website](https://docs.dgl.ai/tutorials/basics/1_first.html).

![alt text](http://snap.stanford.edu/gnnexplainer/files/explainer-introduction.jpg)
*Image credits to: http://snap.stanford.edu/gnnexplainer/*

_____no_output_____**Libraries**:

*   **DGL** (Graph NN library built atop Pytorch endoresed by the inventor of GNNs himself Thomas Kipf)
*  **Networkx** (Python library for studying graphs and networks)
*   **Pytorch** (Most famous DL library)
*   **Matplotlib** (ploting library)
*   **pdb** (Python Debugger)

_____no_output_____

def build_graph():
    g = dgl.DGLGraph()
    g.add_nodes(34)
    edge_list = [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2),
        (4, 0), (5, 0), (6, 0), (6, 4), (6, 5), (7, 0), (7, 1),
        (7, 2), (7, 3), (8, 0), (8, 2), (9, 2), (10, 0), (10, 4),
        (10, 5), (11, 0), (12, 0), (12, 3), (13, 0), (13, 1), (13, 2),
        (13, 3), (16, 5), (16, 6), (17, 0), (17, 1), (19, 0), (19, 1),
        (21, 0), (21, 1), (25, 23), (25, 24), (27, 2), (27, 23),
        (27, 24), (28, 2), (29, 23), (29, 26), (30, 1), (30, 8),
        (31, 0), (31, 24), (31, 25), (31, 28), (32, 2), (32, 8),
        (32, 14), (32, 15), (32, 18), (32, 20), (32, 22), (32, 23),
        (32, 29), (32, 30), (32, 31), (33, 8), (33, 9), (33, 13),
        (33, 14), (33, 15), (33, 18), (33, 19), (33, 20), (33, 22),
        (33, 23), (33, 26), (33, 27), (33, 28), (33, 29), (33, 30),
        (33, 31), (33, 32)]
    # add edges two lists of nodes: src and dst
    src, dst = tuple(zip(*edge_list))
    g.add_edges(src,dst)
    # edges are bidir
    g.add_edges(dst,src)
    return g

_____no_output_____G=build_graph()
print('We have %d Nodes'% G.number_of_nodes())
print('We have %d edges'% G.number_of_edges())We have 34 Nodes
We have 156 edges
# Since the actual graph is undirected, we convert it for visualization
# purpose.
nx_G = G.to_networkx().to_undirected()_____no_output_____pos = nx.kamada_kawai_layout(nx_G) # style of the layout of the graph
nx.draw(nx_G, pos, with_labels =True, node_color=[[.7,.7,.7]])_____no_output_____

## Step 2: Example of assigning features to nodes or edges
Graph neural networks associate features with nodes and edges for training. For our classification example, we assign each node an input feature as a one-hot vector: node vi‘s feature vector is [0,…,1,…,0], where the ith position is one.
_____no_output_____

G.ndata['features'] = torch.eye(34)_____no_output_____print(G.nodes[2].data['features'])tensor([[0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])
print(G.nodes[[10,11]].data['features'])tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])

## Step 3: Define a Graph Convolutional Network (GCN)
To perform node classification, use the Graph Convolutional Network (GCN) developed by Kipf and Welling. Here is the simplest definition of a GCN framework.

*   At layer l, each node V_li (where li = layer i) carries a feature vector H_li.
*   Each layer of the GCN tries to aggregate the features from U_li where U_i‘s are neighborhood nodes to V into the next layer representation at V_l+1i.

This is followed by an affine transformation with some non-linearity.
The above definition of GCN fits into a message-passing paradigm: Each node will update its own feature with information sent from neighboring nodes. A graphical demonstration is displayed below.
![alt text](https://s3.us-east-2.amazonaws.com/dgl.ai/tutorial/1_first/mailbox.png)_____no_output_____

import torch.nn as nn
import torch.nn.functional as F_____no_output_____

**Message passing**(MP):
outlines a general framework of spatial-based ConvGNNs. It treats
graph convolutions as a message passing process in which
information(node features/datat) can be passed from one node to another along
edges directly. MP runs K-step message passing iterations
to let information propagate further, thus each node learns more about its sorrounding nodes(refered to as 1st-order neighbours), also learns about its 2nd-order neighbour and we can repeat the process until each node learns about its n-order neighbours.

![alt text](https://www.outsystems.com/blog/-/media/images/blog/posts/graph-neural-networks/nn-gif-5.gif?h=393&w=750)

**Reduce functions**(RF):  Each node has two members, data and mailbox. Data contains the node features and mailbox contains all incoming message features, stacked along the second dimension (hence the dim=1 argument). And RF takes the **`sum`** or **` mean `**of the incoming nods with the source node._____no_output_____

# Define message and reduce function
def gcn_message(edges):
    # The argument is a batch of edges
    # This computes a batch of message called 'msg' using the source node's feature 'h'
    return {'msg': edges.src['h']}

def gcn_reduce(nodes):
    # The argument is a batch of nodes
    # This computes the new 'h' features by summing received 'msg' in each node's mailbox
    return {'h': torch.sum(nodes.mailbox['msg'], dim=1)}_____no_output_____

**Use higher-level APIs for efficiency**

DGL provides many routines that combine basic send and recv in various ways. These routines are called level-2 APIs. For example:

```
# def forward(self, g, inputs):
    ...
    g.update_all(message_func = gcn_message,
                     reduce_func= gcn_reduce)
    ...
```

In addition to update_all, you can use pull, push, and send_and_recv in this level-2 category. For more information, see [API reference](https://docs.dgl.ai/api/python/function.html)._____no_output_____

# Define GCN Layer module
class GCNLayer(nn.Module):
    def __init__(self, in_fts, out_fts):
        super(GCNLayer, self).__init__()
        self.linear = nn.Linear(in_fts, out_fts)

    def forward(self, g, inputs):
        # g is the graph and the inputs is the input node features
        # first set the node features
        g.ndata['h'] = inputs
        # trigger message passing on all edges
        g.send(g.edges(), gcn_message)
        # trigger aggregation at all nodes
        g.recv(g.nodes(), gcn_reduce)
        # get the result node features(readout)
        h = g.ndata.pop('h')
        return self.linear(h)_____no_output_____# Define a 2-layer GCN
class GCN(nn.Module):
    def __init__(self, in_fts, hidden_sz, num_classes):
        super(GCN, self).__init__()
        self.gcn1 = GCNLayer(in_fts, hidden_sz)
        self.gcn2 = GCNLayer(hidden_sz, num_classes)

    def forward(self, g, inputs):
        h = self.gcn1(g, inputs)
        h = torch.relu(h)
        h = self.gcn2(g, h)
        return h _____no_output_____net = GCN(34, 5, 2) _____no_output_____net_____no_output_____

## Step 4: Data preparation and initialization
We use one-hot vectors to initialize the node features. Since this is a semi-supervised setting, only the john (node 0) and the Mary (node 33) are assigned labels. The implementation is available as follow._____no_output_____

inputs = torch.eye(34)
labeled_nodes = torch.tensor([0,33])
labels = torch.tensor([0,1])_____no_output_____

## Step 5: Train then visualize
The training loop is exactly the same as other PyTorch models. We (1) create an optimizer, (2) feed the inputs to the model, (3) calculate the loss and (4) use autograd to optimize the model._____no_output_____

optimizer= torch.optim.Adam(net.parameters(), lr=0.01)
all_logits = []
for epoch in range(30):
    optimizer.zero_grad()
    logits = net(G, inputs)
    all_logits.append(logits.detach())
    logp = F.log_softmax(logits, dim=1)
    loss = F.nll_loss(logp[labeled_nodes], labels)
    loss.backward()
    optimizer.step()

    print(f'Epoch: {epoch}', 'Loss: %.4f' % loss.item())Epoch: 0 Loss: 0.5473
Epoch: 1 Loss: 0.4169
Epoch: 2 Loss: 0.3221
Epoch: 3 Loss: 0.2385
Epoch: 4 Loss: 0.1719
Epoch: 5 Loss: 0.1197
Epoch: 6 Loss: 0.0794
Epoch: 7 Loss: 0.0505
Epoch: 8 Loss: 0.0311
Epoch: 9 Loss: 0.0186
Epoch: 10 Loss: 0.0109
Epoch: 11 Loss: 0.0064
Epoch: 12 Loss: 0.0038
Epoch: 13 Loss: 0.0023
Epoch: 14 Loss: 0.0015
Epoch: 15 Loss: 0.0009
Epoch: 16 Loss: 0.0006
Epoch: 17 Loss: 0.0004
Epoch: 18 Loss: 0.0003
Epoch: 19 Loss: 0.0002
Epoch: 20 Loss: 0.0002
Epoch: 21 Loss: 0.0001
Epoch: 22 Loss: 0.0001
Epoch: 23 Loss: 0.0001
Epoch: 24 Loss: 0.0001
Epoch: 25 Loss: 0.0000
Epoch: 26 Loss: 0.0000
Epoch: 27 Loss: 0.0000
Epoch: 28 Loss: 0.0000
Epoch: 29 Loss: 0.0000

This is a rather toy example, so it does not even have a validation or test set. Instead, Since the model produces an output feature of **size 2 for each node**(either 0,1), we can visualize by plotting the output feature in a 2D space. The following code animates the training process from initial guess (where the nodes are not classified correctly at all) to the end (where the nodes are linearly separable)._____no_output_____

all_logits[0][:5] # printing first 5 elements of the first batch of outputs _____no_output_____from matplotlib import animation, rc
import matplotlib.pyplot as plt

def draw(i):
    cls1color = '#00FFFF'
    cls2color = '#FF00FF'
    pos = {}
    colors = []
    for v in range(34): # number of nodes
        pos[v] = all_logits[i][v].numpy()
        cls = pos[v].argmax()
        colors.append(cls1color if cls else cls2color)
    ax.cla()
    ax.axis('off')
    ax.set_title('Epoch %d' % i)
    nx.draw_networkx(nx_G.to_undirected(), pos,
                     node_color=colors, node_size=300, ax = ax)

fig = plt.figure(dpi = 150)
fig.clf()
ax = fig.subplots()
draw(0)
plt.show()_____no_output_____ani = animation.FuncAnimation(fig, draw, frames = len(all_logits), interval=200)
rc('animation', html='jshtml')
ani
  _____no_output_____

# Graph Classifiction

We will work on Mini Graph Classification Dataset that has 8 classes of different graphs.
![alt text](https://s3.us-east-2.amazonaws.com/dgl.ai/tutorial/batch/dataset_overview.png)
*Image credits: DGL Docs - > https://https://docs.dgl.ai/tutorials/basics/1_first.html*

As I refered in the article: [Can Graph Neural Networks Solve Real-World Problems ](https://https://hackernoon.com/can-graph-neural-networks-solve-real-world-problems-7hd636dn)
this example here is just a teaser to what we could do with Graph Neural Networks(GNNs).
We could easily train a graph classifier on a dataset of drugs vs protein to a relatevily good accuracy that can predict whether a molecular structure is a drug or not.
I strongly believe this will catapult drug discovery to a whole new level since machines can hidden patterns within large amounts of data that us humans can't or simply would take us long to do.

**I'm working on a application of this technique on a real-world case scenario, it took me long but I finally found a website with graph datasets related to health. Stay tunned for that example.**

Credits of this tutorial to DGL [website](https://https://docs.dgl.ai/tutorials/basics/1_first.html)._____no_output_____

from dgl.data import MiniGCDataset
import matplotlib.pyplot as plt_____no_output_____dataset = MiniGCDataset(80, 10, 20) # a tuple of (graph, label)
graph, label = dataset[10] # deconstruction assignment
fig, ax = plt.subplots()
nx.draw(graph.to_networkx(), ax = ax) # network
ax.set_title('Class: {:d}'.format(label))
plt.show()_____no_output_____

##Batch samples
Batching graph inputs has two challenges:

*   Graphs are sparse.
*   Graphs can have various length. For example, number of nodes and edges.

To address this, DGL provides a `dgl.batch()` API. It leverages the idea that a batch of graphs can be viewed as a large graph that has many disjointed connected components(Adjancency of matrices).

![alt text](https://1.bp.blogspot.com/-ip-AG9a8_zk/Ux5EUHIyB7I/AAAAAAAACLw/WM6wNqupeNk/s1600/Adjacency+Matrix+Representation+of+Undirected+Graph.JPG)

dgl.batch(graph1, graph2)

```
[(adjacency of graph1, 0),
(0, adjacency of graph2)]
```

_____no_output_____

def collate(samples):
    graph, labels = map(list, zip(*samples)) # arg:  func, iterable
    graph_batch = dgl.batch(graph)
    return graph_batch, torch.tensor(labels)_____no_output_____import dgl.function as fn
import torch.nn as nn
import pdb_____no_output_____

**Graph classification process**
![alt text](https://s3.us-east-2.amazonaws.com/dgl.ai/tutorial/batch/graph_classifier.png)
From a batch of graphs, perform message passing and graph convolution for nodes to communicate with others. After message passing, compute a tensor for graph representation from node (and edge) attributes. This step might be called readout or aggregation. Finally, the graph representations are fed into a classifier g to predict the graph labels._____no_output_____**Graph convolution**

The graph convolution operation is basically the same as that for graph convolutional network (GCN). To learn more, see the GCN tutorial). The only difference is that we replace

 `h = ReLU(linear(torch.sum(nodes.mailbox['msg'], 1)))` by

  ` h =  RelU(linear(torch.mean(nodes.mailbox['msg'], 1)))`

The replacement of summation by average is to balance nodes with different degrees. This gives a better performance for this experiment.

The self edges added in the dataset initialization allows you to include the original node feature h(l)v when taking the average._____no_output_____

msg = fn.copy_u('h', 'm') # message passing

def reducer(nodes):
    accum = torch.mean(nodes.mailbox['m'], 1)
    return {'h': accum}

class NodeApplyModule(nn.Module):
    def __init__(self, in_fts, out_fts, activation):
        super(NodeApplyModule, self).__init__()
        self.linear = nn.Linear(in_fts, out_fts)
        self.activation = activation

    def forward(self, node):
        h = self.linear(node.data['h'])
        h = self.activation(h)
        return{'h' : h}

class GCN(nn.Module):
    def __init__(self, in_fts, out_fts, activation):
        super(GCN, self).__init__()
        self.apply_mod = NodeApplyModule(in_fts, out_fts, activation)

    def forward(self, g, inputs):
        # Init node features
        g.ndata['h'] = inputs
        #pdb.set_trace() uncomment this line execute and see output line-by-line
        g.update_all(msg, reducer)
        g.apply_nodes(func=self.apply_mod)
        return g.ndata.pop('h')
_____no_output_____

Consider initial node features to be **their degrees**. After 3 rounds of graph convolution, perform a graph readout by averaging over all node features for each graph in the batch and passing it to a fully connected layer._____no_output_____

import torch.nn.functional as F

class Classifier(nn.Module):
    def __init__(self, in_dim, hidden_dim, n_classes, activation=F.relu, n_extra_layers = 2):
        super(Classifier, self).__init__()
        self.gconv1 = GCN(in_dim, hidden_dim, activation)
        self.layers = nn.ModuleList([
            GCN(hidden_dim, hidden_dim, activation) for i in range(n_extra_layers)])
        self.ln = nn.Linear(hidden_dim, n_classes)

    def forward(self,g):
        # For undirected graphs, in_degree is the same as
        # out_degree
        h = g.in_degrees().view(-1, 1).float()
        # pdb.set_trace() uncomment this line execute and see output line-by-line
        h = self.gconv1(g, h)
        for gconv in self.layers:
            h = gconv(g, h)
        g.ndata['h'] = h # update node representations(data) for avg and readout
        hg = dgl.mean_nodes(g, 'h')
        return self.ln(hg) # classification_____no_output_____import torch.optim as optim
from torch.utils.data import DataLoader

# training & testing
trainset = MiniGCDataset(320, 10, 20)
testset = MiniGCDataset(80, 10, 20)
# We use the collate fn defined above with a Dataloader to create batches
data = DataLoader(trainset, 16, shuffle = True, collate_fn = collate)

# create model
# in_fts = 1 because we passing a vector of nodes indegrees
# of size .view(-1, 1) since the graphs have different number
# of nodes and edges and degres we can't fix the first dimension.
model = Classifier(1, 256, trainset.num_classes)
loss_fn = nn.CrossEntropyLoss()
opt = optim.Adam(model.parameters(), lr = 1e-2)
model.train()

epoch_losses = []
for epoch in range(80):
    epoch_loss = 0
    acc = 0
    for i,(bg, label) in enumerate(data):
        opt.zero_grad()
        # pdb.set_trace()
        pred = model(bg)
        loss = loss_fn(pred, label)
        loss.backward()
        opt.step()
        epoch_loss+=loss.detach().item()
    epoch_loss /= (i+1)
    acc /= len(data) * 100
    print("Epoch: {}, Loss: {:.4f}".format(epoch, epoch_loss))
    epoch_losses.append(epoch_loss)Epoch: 0, Loss: 2.8453
Epoch: 1, Loss: 1.7920
Epoch: 2, Loss: 1.4413
Epoch: 3, Loss: 1.2986
Epoch: 4, Loss: 1.2462
Epoch: 5, Loss: 1.1451
Epoch: 6, Loss: 1.0528
Epoch: 7, Loss: 0.9451
Epoch: 8, Loss: 0.8046
Epoch: 9, Loss: 0.7909
Epoch: 10, Loss: 0.8586
Epoch: 11, Loss: 0.7999
Epoch: 12, Loss: 0.7287
Epoch: 13, Loss: 0.8004
Epoch: 14, Loss: 0.7471
Epoch: 15, Loss: 0.7485
Epoch: 16, Loss: 0.7200
Epoch: 17, Loss: 0.7486
Epoch: 18, Loss: 0.7243
Epoch: 19, Loss: 0.7022
Epoch: 20, Loss: 0.9102
Epoch: 21, Loss: 0.8474
Epoch: 22, Loss: 0.8523
Epoch: 23, Loss: 0.7214
Epoch: 24, Loss: 0.7069
Epoch: 25, Loss: 0.6476
Epoch: 26, Loss: 0.6588
Epoch: 27, Loss: 0.6650
Epoch: 28, Loss: 0.6312
Epoch: 29, Loss: 0.5991
Epoch: 30, Loss: 0.5844
Epoch: 31, Loss: 0.6057
Epoch: 32, Loss: 0.6169
Epoch: 33, Loss: 0.6086
Epoch: 34, Loss: 0.6291
Epoch: 35, Loss: 0.6282
Epoch: 36, Loss: 0.6235
Epoch: 37, Loss: 0.6800
Epoch: 38, Loss: 0.6233
Epoch: 39, Loss: 0.5988
Epoch: 40, Loss: 0.6128
Epoch: 41, Loss: 0.7808
Epoch: 42, Loss: 0.7412
Epoch: 43, Loss: 0.7282
Epoch: 44, Loss: 0.6878
Epoch: 45, Loss: 0.6408
Epoch: 46, Loss: 0.7367
Epoch: 47, Loss: 0.7660
Epoch: 48, Loss: 0.6052
Epoch: 49, Loss: 0.5975
Epoch: 50, Loss: 0.5877
Epoch: 51, Loss: 0.6060
Epoch: 52, Loss: 0.5494
Epoch: 53, Loss: 0.5739
Epoch: 54, Loss: 0.6071
Epoch: 55, Loss: 0.5700
Epoch: 56, Loss: 0.6581
Epoch: 57, Loss: 0.6113
Epoch: 58, Loss: 0.5943
Epoch: 59, Loss: 0.5958
Epoch: 60, Loss: 0.5679
Epoch: 61, Loss: 0.5737
Epoch: 62, Loss: 0.5781
Epoch: 63, Loss: 0.5676
Epoch: 64, Loss: 0.5747
Epoch: 65, Loss: 0.5629
Epoch: 66, Loss: 0.5884
Epoch: 67, Loss: 0.6100
Epoch: 68, Loss: 0.5790
Epoch: 69, Loss: 0.6333
Epoch: 70, Loss: 0.5364
Epoch: 71, Loss: 0.6366
Epoch: 72, Loss: 0.6358
Epoch: 73, Loss: 0.7454
Epoch: 74, Loss: 0.6253
Epoch: 75, Loss: 0.6000
Epoch: 76, Loss: 0.5690
Epoch: 77, Loss: 0.5708
Epoch: 78, Loss: 0.5406
Epoch: 79, Loss: 0.5554
plt.title('cross entropy averaged over minibatches')
plt.plot(epoch_losses)
plt.show()_____no_output_____

## Testing_____no_output_____

model.eval()
all_preds = []
test_X, test_Y = collate(testset)
test_Y = test_Y.view(-1,1).float()
probs = torch.softmax(model(test_X), 1)
# The rows of input do not need to sum to one,
# but must be non-negative, finite and have a non-zero sum.
sampled_Y = torch.multinomial(probs,1)
# Test accuracy using argmax
argmax_Y = torch.max(probs,1)[1].view(-1,1)
all_preds.append(argmax_Y)
print('Accuracy of sampled predictions on the testset: {:.4f}'.format(
    (sampled_Y == test_Y).sum().item() / len(testset) * 100))
print('Accuracy of argmax prediction on the testset: {:.4f}'.format(
    (argmax_Y == test_Y).sum().item() / len(testset) * 100))Accuracy of sampled predictions on the testset: 61.2500
Accuracy of argmax prediction on the testset: 71.2500

## Visualizing results_____no_output_____

from matplotlib import animation, rc
def draw(i):
    clscolor1 = 'red'
    clscolor2 = 'green'
    colors = []
    colors.append(clscolor2 if (all_preds[0][i]==test_Y[i][0]) else clscolor1)
    ax.cla()
    ax.axis('off')
    ax.set_title('Epoch: {}, Predicted class: {}, True class: {},\nPrediction Accuracy: {:.2f} % '.format(i, all_preds[0][i].item(), int(test_Y[i][0]), torch.max(probs,1)[0][i]*100))
    nx.draw(testset[i][0].to_networkx(),
                        node_color=colors, ax = ax)

fig = plt.figure(dpi = 150)
fig.clf()
ax = fig.subplots()
draw(0)
plt.show()_____no_output_____ani = animation.FuncAnimation(fig, draw, frames = len(probs), interval = 500)
rc('animation', html = 'jshtml')
ani_____no_output_____

#Pytorch Geometric
Under Development...._____no_output_____

!pip install --verbose --no-cache-dir torch-scatter
!pip install --verbose --no-cache-dir torch-sparse
!pip install --verbose --no-cache-dir torch-cluster
!pip install torch-geometric_____no_output_____import torch
from torch_geometric.data import Data_____no_output_____edge_index  = torch.tensor([[0, 1, 1, 2],
                            [1, 0, 2, 1]], dtype= torch.long)
x = torch.tensor([[-1], [0], [1]], dtype=torch.float)
data = Data(x=x, edge_index=edge_index)_____no_output_____data_____no_output_____print(data.keys)
_____no_output_____print(data['x'])_____no_output_____data.num_nodes, data.num_edges, data.num_node_features_____no_output_____data.contains_isolated_nodes(), data.contains_self_loops(), data.is_directed()_____no_output_____# Transfer data object to GPU.
device = torch.device('cuda')
data = data.to(device)_____no_output_____

Dataset_____no_output_____

from torch_geometric.datasets import TUDataset_____no_output_____data = TUDataset(root='tmp/ENZYMES', name='ENZYMES')
data_____no_output_____data[0]_____no_output__________no_output__________no_output_____


# Notebook from rodriguezda/python-programming-104
Path: unit-6-pandas/instructor-resources/06-pandas-unit-lab/code/Pandas-Lab-starter-code.ipynb

## ![](https://ga-dash.s3.amazonaws.com/production/assets/logo-9f88ae6c9c3871690e33280fcf557f33.png) Pandas for EDA
by [@josephofiowa](https://twitter.com/josephofiowa)

_____no_output_____# Pandas Unit Lab

**Woo!** We've made it to the end of our Pandas Unit. Let's put our skills to the test.

We're going to explore data from some of the top movies according to IMDB. This is a guided question-and-response lab where some areas are specific asks and others are open ended for you to explore.

#### Important!!!
- **There are two ways to do this lab!**
  - The first way is to read in a dataset that _has already been pulled from the API and cleaned for you_ (`movies_rated.csv`). This is the recommended 'first-pass' way to do this lab.
  - _After you have completed the lab using the supplied_ `movies_rated.csv`, you can call the API yourself!
  - Calling the API yourself takes time! Be prepared to parse lots of JSON, read docs, etc. Consider this a take-home exercise if the students desire.

In this lab, we will:
- Use `movie_app.py` to obtain relevant moving rating data
- Leverage Pandas to conduct exploratory data analysis, including:
    - Assess data integrity
    - Create exploratory visualizations
    - Produce insights on top actors/actresses across films

Let's get going!_____no_output_____## The Dataset

We'll work with a dataset on the top [IMDB movies](https://www.imdb.com/search/title?count=100&groups=top_1000&sort=user_rating), as rated by IMDB.

Specifically, we have a CSV that contains:
- IMDB star rating
- Movie title
- Year
- Content rating
- Genre
- Duration
- Gross

_[Details available at the above link]_
_____no_output_____### Import our necessary libraries_____no_output_____

import pandas as pd
import numpy as np
import matplotlib as plt
import re
%matplotlib inline_____no_output_____

### Read in the dataset

First, read in the dataset, called `movies.csv` into a DataFrame called "movies." It's in the `./data` folder._____no_output_____## Check the dataset basics

Let's first explore our dataset to verify we have what we expect._____no_output_____Print the first five rows._____no_output_____How many rows and columns are in the datset?_____no_output_____What are the column names?_____no_output_____How many unique genres are there?_____no_output_____How many movies are there per genre?_____no_output_____## Only run the below cells if you've obtained an [API key!](http://www.omdbapi.com/apikey.aspx)
Otherwise, proceed to the `importing movies_rated.csv` section below._____no_output_____### Obtain more data (with an API call)!

- Let's take advantage of our `OmdbAPI` module (stored in `./OmdbAPI.py`, if you'd like to look under the hood) to obtain data from OMDB API on movie ratings. This will enable us to answer the question: **How do other publication's scores compare to IMDB ratings?** Specifically, where do Rotten Tomato critics most disagree with IMDB reviews?
- Using the OmdbAPI module, we will obtain the `Internet Movie Database`, the `Rotten Tomatoes`, and the `Metacritic` reviews on the top rated IMDB movies. We will store these ratings in new columns in a new `movies_rated` DataFrame. We have also stored the file locally at `./movies_rated.csv`._____no_output_____

import OmdbAPI_____no_output_____# replace e54ad9e7 with your API key
# this may take a minute
movies_rated = OmdbAPI.Omdb(movies, 'e54ad9e7').get_ratings()_____no_output_____

Just in case there were movies that the API was unable to get, let's drop nulls._____no_output_____Let's get the ratings in the same float format using an apply function with some regular expressions. Note the use of .copy() when writing and reading from the same dataframe as a best practice._____no_output_____Finally, let's write the cleaned result to a local file so we don't have to call the API again and risk exceeding our daily limit._____no_output_____## Importing `movies_rated.csv`

If you just called the API in the previous section, you can skip this and proceed to the `exploratory data analysis` section._____no_output_____Let's read in the cleaned, rated `movies_rated.csv` file, which was included with this repo just in case you couldn't call the API._____no_output_____Check our datatypes. Notice anything potentially problematic?_____no_output_____## Exploratory data analysis

Let's transition to asking and answering some questions with our data._____no_output_____What are the top five R-Rated movies?

*hint: Boolean filters needed! Then sorting!*_____no_output_____What is the average Rotten Tomato score for the top IMDB films?_____no_output_____What is the Five Number Summary like for top rated films as per IMDB? Is it skewed?_____no_output_____The average is *slightly* higher than the median, so there's a small positive skew._____no_output_____Create your own question...then answer it!_____no_output_____

# correlation between star rating and Rotten Tomato rating?
_____no_output_____

**Challenge:** Create a dataframe that is the ratio between Rotten Tomato rating vs IMDB rating. What film has the highest IMDB : Rotten Tomato ratio? The lowest?

*[skip this if you are low on time]*_____no_output_____## Exploratory data analysis with visualizations

For each of these prompts, create a plot to visualize the answer. Consider what plot is *most appropriate* to explore the given prompt.
_____no_output_____What is the relationship between IMDB ratings and Rotten Tomato ratings?_____no_output_____What is the relationship between IMDB rating and movie duration?_____no_output_____How many movies are there in each genre category? (Remember to create a plot here)_____no_output_____What does the distribution of Rotten Tomatoes ratings look like?_____no_output_____## Bonus

There are many things left unexplored! Consider investigating something about gross revenue and genres._____no_output_____

# histogram of gross sales_____no_output_____# top 10 grossing films_____no_output_____