# Notebook from adeethyiashankar/Wavelet-Based-Machine-Learning-Approaches-Towards-Precision-Medicine-in-Diabetes-Mellitus
Path: Code/Preprocessing/Preprocess HMP Data.ipynb

from scipy import stats
import pandas as pd
import numpy as np
import math
import os
import multiprocessing
import progressbar
from time import time

data_path = 'gdrive/My Drive/Summer Research/hmp2-data-stanford/'
preprocessed_data_path = 'gdrive/My Drive/Summer Research/hmp2-data-stanford/Preprocessed/'
wt_data_path = 'gdrive/My Drive/Summer Research/hmp2-data-stanford/Preprocessed/Wavelet Transform/'
paths = [preprocessed_data_path, wt_data_path+'Denoised/',
         wt_data_path+'WT Domain/']

hmp_datas = ['cytokine_abundance','gut_16s_abundance','Lipidomics',
          'metabolome_abundance','Metabolomics','nares_16s_abundance',
          'proteome_abundance','Proteomics','RNAseq_abundance',
          'Targ.proteomics','Transcriptomics_VST_excl_3participants']

from google.colab import drive
drive.mount('/content/gdrive')Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
def get_subjects():
  df = pd.read_csv(data_path+'Subjects.csv', index_col=False)
  subjects = dict()
  for i in range(len(df)):
    subject_data = dict(df.iloc[i,:])
    subject_id = subject_data.pop('SubjectID')
    subjects[subject_id] = subject_data
  return subjects_____no_output_____def get_class_from_visit_id():
  df = pd.read_csv(data_path+'Visit.csv', index_col=False,
                   usecols=['VisitID', 'SubjectID'])
  subjects = get_subjects()
  return dict((df.iloc[i,0],subjects[df.iloc[i,1]]) for i in range(len(df)))_____no_output_____def get_data(f):
  df = pd.read_csv(data_path+f+'.csv', index_col=False)
  gc = get_class_from_visit_id()
  race = list()
  sex = list()
  age = list()
  bmi = list()
  sspg = list()
  classifications = list()
  indices_to_drop = list()
  nan_values = [i for i in range(len(df)) if df.isnull().any(axis=1)[i]]

  bools = list(df.astype('bool').mean(axis=1) < 0.75)
  zero_values = [i for i in range(len(bools)) if bools[i] == True]
  nan_values.extend(zero_values)

  for i in range(len(df)):
    v_id = df.iloc[i,0]
    try:
      if i not in nan_values:
        c = gc[v_id]
        race.append(c['Race'])
        sex.append(c['Sex'])
        age.append(c['Age'])
        bmi.append(c['BMI'])
        sspg.append(c['SSPG'])
        classifications.append(c['IR_IS_classification'])
      else:
        indices_to_drop.append(i)
    except KeyError:
      indices_to_drop.append(i)

  df = df.drop(indices_to_drop)
  df.insert(1, 'IR_IS_classification', classifications, False)
  df.insert(1, 'SSPG', sspg, False)
  df.insert(1, 'BMI', bmi, False)
  df.insert(1, 'Age', age, False)
  df.insert(1, 'Sex', sex, False)
  df.insert(1, 'Race', race, False)
  return df.reset_index(drop=True)_____no_output_____def save_all_data():
  for i in hmp_datas:
    df = get_data(i)
    df.to_csv(preprocessed_data_path+i+'.csv', index=False)_____no_output_____#save_all_data()_____no_output_____def check_for_nan():
  for i in hmp_datas:
    df = get_data(i)
    print(i+': '+str(df.isnull().values.sum()))_____no_output_____def matrix_size():
  for i in hmp_datas:
      df = pd.read_csv(preprocessed_data_path+i+'.csv', index_col=False)
      print(i+': '+str(df.shape))_____no_output_____

Normalization_____no_output_____

def normalization():
  widgets = [' [',
        progressbar.Timer(format= 'elapsed time: %(elapsed)s'),
        '] ',
          progressbar.Bar('#'),' (',
          progressbar.ETA(), ') ',
          progressbar.Counter(format='%(value)d/%(max_value)d')
          ]
  bar = progressbar.ProgressBar(max_value=len(hmp_datas)-2, widgets=widgets).start()
  count = 0

  for j in hmp_datas:
    if j not in ['metabolome_abundance', 'proteome_abundance']:
      count += 1
      bar.update(count)

      sid_and_class = pd.read_csv(preprocessed_data_path+j+'.csv', index_col=False,
                        usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
      df = pd.read_csv(preprocessed_data_path+j+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)
      cols = ['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(df.columns)

      add = 0.1
      df += add
      df = np.log2(df)
      df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)
      df.columns = pd.Index(cols)

      df.to_csv(preprocessed_data_path+'Normalized/'+j+'.csv', index=False)_____no_output_____

Wavelet Transform_____no_output_____

def fourWTM(n):
  #Filter banks
  h0 = np.array([0.2697890,0.3947890,0.5197890,0.6447890,0.2302110,0.1052110,-0.0197890,-0.1447890])
  h1 = np.array([-0.2825435,0.5553379,0.2385187,-0.0783004, -0.5834819,-0.2666627,0.0501564,0.3669755])
  h2 = np.array([0.4125840,-0.6279376,0.3727824,0.1487574, -0.4125840,-0.1885590,0.0354659,0.2594909])
  h3 = np.array([0.2382055,0.1088646,-0.7275830,0.5572896, -0.2382055,-0.1088646,0.0204763,0.1498171])
  #Matrix of filter banks created for convenience
  h = np.array([h0,h1,h2,h3])

  k = int(n/4)
  T = np.zeros((n,n))
  for j in range(4):
    for i in range(k):
      if 4*i+8 > 4*k:
        T[k*j+i,range((4*i),(4*i+4))] = h[j,range(4)]
        T[k*j+i,range(4)] = h[j,range(4,8)]
      else:
        T[k*j+i,range((4*i),(4*i+8))] = h[j,range(8)]
  return T_____no_output_____def four_Wavelet_Transform(data_path, f):
  #Import HMP data
  sid_and_class = pd.read_csv(data_path+f+'.csv', index_col=False,
                      usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
  hmp = pd.read_csv(data_path+f+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)

  n = hmp.shape[1]
  n = n + 4 - (n % 4)
  z = np.zeros((hmp.shape[0], n-hmp.shape[1]))
  t = fourWTM(n)
  s = np.zeros((hmp.shape[0],n))

  cols = ['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(hmp.columns)

  hmp_array = np.concatenate((np.array(hmp), z), axis=1)

  ts = np.matmul(hmp_array, t.T)

  #Create WT Domain DataFrame
  wt_type = 'WT Domain'
  df = pd.DataFrame(data=ts)
  df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)

  for i in range(n - hmp.shape[1]):
    cols.append('')

  df.columns = pd.Index(cols)

  df.to_csv(data_path+wt_type+'/'+f+'.csv', index=False)

  #Create Denoised DataFrame
  wt_type = 'Denoised'
  dim = int(n/4)
  A1 = np.matmul(ts[:, 0:dim], t[0:dim, :])
  d = np.zeros((hmp.shape[0],dim,3))
  D = np.zeros((hmp.shape[0],n))
  for j in range(3):
    d[:,:,j] = ts[:, (j+1)*dim:(j+2)*dim]
    for k in range(hmp.shape[0]):
      #Denoise details
      lbda = np.std(d[k,:,j])*math.sqrt(2*math.log(dim))
      for i in range(dim):
        if abs(d[k,i,j]) < lbda:
          d[k,i,j] = 0
    D += np.matmul(d[:,:,j], t[(j+1)*dim:(j+2)*dim, :])

  s = A1+D
  df = pd.DataFrame(data=s)
  df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)
  df.columns = pd.Index(cols)

  df.to_csv(data_path+wt_type+'/'+f+'.csv', index=False)_____no_output_____def fourWT_just_wt_domain(data_path, f):
  #Import HMP data
  sid_and_class = pd.read_csv(data_path+f+'.csv', index_col=False,
                      usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
  hmp = pd.read_csv(data_path+f+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)

  n = hmp.shape[1]
  n = n + 4 - (n % 4)
  z = np.zeros((hmp.shape[0], n-hmp.shape[1]))
  t = fourWTM(n)
  s = np.zeros((hmp.shape[0],n))

  cols = ['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(hmp.columns)

  hmp_array = np.concatenate((np.array(hmp), z), axis=1)

  ts = np.matmul(hmp_array, t.T)

  #Create WT Domain DataFrame
  wt_type = 'WT Domain'
  df = pd.DataFrame(data=ts)
  df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)

  for i in range(n - hmp.shape[1]):
    cols.append('')

  df.columns = pd.Index(cols)

  df.to_csv(data_path+wt_type+'/'+f+'.csv', index=False)_____no_output_____def rna_just_wt_domain():
  count = 0
  hold_list_of_processes = list()
  for j in [preprocessed_data_path, preprocessed_data_path+'Normalized/']:
    for i in ['Transcriptomics_VST_excl_3participants','RNAseq_abundance']:
      hold_list_of_processes.append(multiprocessing.Process(target=fourWT_just_wt_domain, args=(j, i)))

  for p in hold_list_of_processes:
    p.start()

  for p in hold_list_of_processes:
    p.join()
    count += 1
    print(str(count) + '/' + str(len(hold_list_of_processes)))

  print('Done')_____no_output_____def transform_small_data():
  widgets = [' [',
        progressbar.Timer(format= 'elapsed time: %(elapsed)s'),
        '] ',
          progressbar.Bar('#'),' (',
          progressbar.ETA(), ') ',
          progressbar.Counter(format='%(value)d/%(max_value)d')
          ]
  bar = progressbar.ProgressBar(max_value=2*(len(hmp_datas)-4), widgets=widgets).start()
  count = 0

  processes = list()
  for j in [preprocessed_data_path, preprocessed_data_path+'Normalized/']:
    for i in hmp_datas:
      if i not in ['metabolome_abundance', 'proteome_abundance',
                  'Transcriptomics_VST_excl_3participants','RNAseq_abundance']:
        p = multiprocessing.Process(target=four_Wavelet_Transform, args=(j, i))
        processes.append(p)
        p.start()

  for p in processes:
    count += 1
    bar.update(count)
    p.join()_____no_output_____def four_Wavelet_Transform_big_denoised(data_path, f):
  #Import HMP data
  sid_and_class = pd.read_csv(data_path+f+'.csv', index_col=False,
                      usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
  hmp = pd.read_csv(data_path+f+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)

  n = hmp.shape[1]
  n = n + 4 - (n % 4)
  z = np.zeros((hmp.shape[0], n-hmp.shape[1]))
  t = fourWTM(n)
  s = np.zeros((hmp.shape[0],n))

  cols = ['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(hmp.columns)

  hmp_array = np.concatenate((np.array(hmp), z), axis=1)

  ts = np.matmul(hmp_array, t.T)

  widgets = [' [',
        progressbar.Timer(format= 'elapsed time: %(elapsed)s'),
        '] ',
          progressbar.Bar('#'),' (',
          progressbar.ETA(), ') ',
          progressbar.Counter(format='%(value)d/%(max_value)d')
          ]
  bar = progressbar.ProgressBar(max_value=3, widgets=widgets).start()

  #Create Denoised DataFrame
  wt_type = 'Denoised'
  dim = int(n/4)
  A1 = np.matmul(ts[:, 0:dim], t[0:dim, :])
  d = np.zeros((hmp.shape[0],3*dim))
  D = np.zeros((hmp.shape[0],n))

  count = 0
  for j in range(3):
    count += 1
    bar.update(count)
    d[:, (j)*dim:(j+1)*dim] = ts[:, (j+1)*dim:(j+2)*dim]
    for k in range(hmp.shape[0]):
      #Denoise details
      lbda = np.std(d[k, (j)*dim:(j+1)*dim])*math.sqrt(2*math.log(dim))
      for i in range(dim):
        if abs(d[k, j*dim+i]) < lbda:
          d[k, j*dim+i] = 0
  D = np.matmul(d, t[dim:, :])

  s = A1+D
  df = pd.DataFrame(data=s)
  df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)
  for i in range(n - hmp.shape[1]):
    cols.append('')
  df.columns = pd.Index(cols)

  df.to_csv(data_path+wt_type+'/'+f+'.csv', index=False)_____no_output_____def transform_big_data():
  processes = list()
  for j in [preprocessed_data_path, preprocessed_data_path+'Normalized/']:
    for i in ['Transcriptomics_VST_excl_3participants','RNAseq_abundance']:
      p = multiprocessing.Process(target=four_Wavelet_Transform_big_denoised, args=(j, i))
      processes.append(p)
      p.start()

  for p in processes:
    p.join()_____no_output_____transform_big_data() [elapsed time: 0:00:13] |###############################| (ETA:  00:00:00) 3/3

MAD Dimension Reduction_____no_output_____

def reduce_dims_mad(data_path, rna_data, threshold):
  sid_and_class = pd.read_csv(data_path+rna_data+'.csv', index_col=False,
                    usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
  df = pd.read_csv(data_path+rna_data+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)
  cols = df.columns

  mad = dict()
  for i in df.columns:
    mad[i] = stats.median_absolute_deviation(df.loc[:,i], scale=1) / np.median(df.loc[:,i])
  mad_df = pd.DataFrame.from_dict(data=mad, orient='index').dropna()
  mad_df.columns = pd.Index(['MAD'])
  mad_df.sort_values(by='MAD', ascending=False)

  count = 1
  mad_sum = mad_df.loc[:, 'MAD'].sum()
  while mad_df.iloc[0:count, 0].sum() < threshold*mad_sum:
    count += 1

  df = pd.concat([sid_and_class, df.iloc[:, 0:count]], axis=1, ignore_index=True)
  df.columns = pd.Index(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(cols[0:count]))
  return df_____no_output_____def only_genus():
  gut_data = 'gut_16s_abundance'
  nares_data = 'nares_16s_abundance'

  for data_path in ['', 'WT Domain/', 'Denoised/', 'Normalized/',
                    'Normalized/WT Domain/', 'Normalized/Denoised/']:
    try:
      os.mkdir(preprocessed_data_path+'Genus only/')
    except FileExistsError:
      pass

    for j in [gut_data, nares_data]:
      sid_and_class = pd.read_csv(preprocessed_data_path+data_path+j+'.csv', index_col=False,
                        usecols=['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'])
      df = pd.read_csv(preprocessed_data_path+data_path+j+'.csv', index_col=False).drop(['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'], axis=1)
      df_cols_to_keep = [i for i in df.columns if i[0:5] == 'genus']

      df = df.loc[:,df_cols_to_keep]
      cols = ['SampleID', 'Race', 'Sex', 'Age', 'BMI', 'SSPG', 'IR_IS_classification'] + list(df_cols_to_keep)
      df = pd.concat([sid_and_class, df], axis=1, ignore_index=True)
      df.columns = pd.Index(cols)

      os.makedirs(preprocessed_data_path+'Genus only/'+data_path, exist_ok=True)
      df.to_csv(preprocessed_data_path+'Genus only/'+data_path+j+'.csv', index=False)_____no_output_____


# Notebook from eduardojdiniz/CompNeuro
Path: tutorials/W1D5_DimensionalityReduction/ED_W1D5_Tutorial1.ipynb

_____no_output_____# Tutorial 1: Geometric view of data
**Week 1, Day 5: Dimensionality Reduction**

**By Neuromatch Academy**

__Content creators:__ Alex Cayco Gajic, John Murray

__Content reviewers:__ Roozbeh Farhoudi, Matt Krause, Spiros Chavlis, Richard Gao, Michael Waskom, Siddharth Suresh, Natalie Schaworonkow, Ella Batty
_____no_output_____**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

_____no_output_____---
# Tutorial Objectives

*Estimated timing of tutorial: 50 minutes*

In this notebook we'll explore how multivariate data can be represented in different orthonormal bases. This will help us build intuition that will be helpful in understanding PCA in the following tutorial.

Overview:
 - Generate correlated multivariate data.
 - Define an arbitrary orthonormal basis.
 - Project the data onto the new basis._____no_output_____

# @title Tutorial slides

# @markdown These are the slides for the videos in all tutorials today
from IPython.display import IFrame
IFrame(src=f"https://mfr.ca-1.osf.io/render?url=https://osf.io/kaq2x/?direct%26mode=render%26action=download%26mode=render", width=854, height=480)_____no_output_____# @title Video 1: Geometric view of data
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1Af4y1R78w", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="THu9yHnpq9I", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

---
# Setup_____no_output_____

# Imports

import numpy as np
import matplotlib.pyplot as plt_____no_output_____# @title Figure Settings
import ipywidgets as widgets  # interactive display
%config InlineBackend.figure_format = 'retina'
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")_____no_output_____# @title Plotting Functions

def plot_data(X):
  """
  Plots bivariate data. Includes a plot of each random variable, and a scatter
  plot of their joint activity. The title indicates the sample correlation
  calculated from the data.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable

  Returns:
    Nothing.
  """

  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(X[:, 0], color='k')
  plt.ylabel('Neuron 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(X[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(X[:, 1], color='k')
  plt.xlabel('Sample Number')
  plt.ylabel('Neuron 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(X[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(X[:, 0], X[:, 1], '.', markerfacecolor=[.5, .5, .5],
           markeredgewidth=0)
  ax3.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(X[:, 0], X[:, 1])[0, 1]))
  plt.show()

def plot_basis_vectors(X, W):
  """
  Plots bivariate data as well as new basis vectors.

  Args:
    X (numpy array of floats) :   Data matrix each column corresponds to a
                                  different random variable
    W (numpy array of floats) :   Square matrix representing new orthonormal
                                  basis each column represents a basis vector

  Returns:
    Nothing.
  """

  plt.figure(figsize=[4, 4])
  plt.plot(X[:, 0], X[:, 1], '.', color=[.5, .5, .5], label='Data')
  plt.axis('equal')
  plt.xlabel('Neuron 1 activity')
  plt.ylabel('Neuron 2 activity')
  plt.plot([0, W[0, 0]], [0, W[1, 0]], color='r', linewidth=3,
           label='Basis vector 1')
  plt.plot([0, W[0, 1]], [0, W[1, 1]], color='b', linewidth=3,
           label='Basis vector 2')
  plt.legend()
  plt.show()

def plot_data_new_basis(Y):
  """
  Plots bivariate data after transformation to new bases.
  Similar to plot_data but with colors corresponding to projections onto
  basis 1 (red) and basis 2 (blue). The title indicates the sample correlation
  calculated from the data.

  Note that samples are re-sorted in ascending order for the first
  random variable.

  Args:
    Y (numpy array of floats):   Data matrix in new basis each column
                                 corresponds to a different random variable

  Returns:
    Nothing.
  """
  fig = plt.figure(figsize=[8, 4])
  gs = fig.add_gridspec(2, 2)
  ax1 = fig.add_subplot(gs[0, 0])
  ax1.plot(Y[:, 0], 'r')
  plt.xlabel
  plt.ylabel('Projection \n basis vector 1')
  plt.title('Sample var 1: {:.1f}'.format(np.var(Y[:, 0])))
  ax1.set_xticklabels([])
  ax2 = fig.add_subplot(gs[1, 0])
  ax2.plot(Y[:, 1], 'b')
  plt.xlabel('Sample number')
  plt.ylabel('Projection \n basis vector 2')
  plt.title('Sample var 2: {:.1f}'.format(np.var(Y[:, 1])))
  ax3 = fig.add_subplot(gs[:, 1])
  ax3.plot(Y[:, 0], Y[:, 1], '.', color=[.5, .5, .5])
  ax3.axis('equal')
  plt.xlabel('Projection basis vector 1')
  plt.ylabel('Projection basis vector 2')
  plt.title('Sample corr: {:.1f}'.format(np.corrcoef(Y[:, 0], Y[:, 1])[0, 1]))
  plt.show()_____no_output_____

---
# Section 1: Generate correlated multivariate data_____no_output_____

# @title Video 2: Multivariate data
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1xz4y1D7ES", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="jcTq2PgU5Vw", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

This video describes the covariance matrix and the multivariate normal distribution.

 Click here for text recap of video

To gain intuition, we will first use a simple model to generate multivariate data. Specifically, we will draw random samples from a *bivariate normal distribution*. This is an extension of the one-dimensional normal distribution to two dimensions, in which each $x_i$ is marginally normal with mean $\mu_i$ and variance $\sigma_i^2$:

\begin{align}
x_i \sim \mathcal{N}(\mu_i,\sigma_i^2).
\end{align}

Additionally, the joint distribution for $x_1$ and $x_2$ has a specified correlation coefficient $\rho$. Recall that the correlation coefficient is a normalized version of the covariance, and ranges between -1 and +1:

\begin{align}
\rho = \frac{\text{cov}(x_1,x_2)}{\sqrt{\sigma_1^2 \sigma_2^2}}.
\end{align}

For simplicity, we will assume that the mean of each variable has already been subtracted, so that $\mu_i=0$ for both $i=1$ and $i=2$. The remaining parameters can be summarized in the covariance matrix, which for two dimensions has the following form:

\begin{align}
{\bf \Sigma} =
\begin{pmatrix}
 \text{var}(x_1) & \text{cov}(x_1,x_2) \\
 \text{cov}(x_1,x_2) &\text{var}(x_2)
\end{pmatrix}.
\end{align}

In general, $\bf \Sigma$ is a symmetric matrix with the variances $\text{var}(x_i) = \sigma_i^2$ on the diagonal, and the covariances on the off-diagonal. Later, we will see that the covariance matrix plays a key role in PCA.

_____no_output_____## Coding Exercise 1: Draw samples from a distribution

We have provided code to draw random samples from a zero-mean bivariate normal distribution with a specified covariance matrix (`get_data`). Throughout this tutorial, we'll imagine these samples represent the activity (firing rates) of two recorded neurons on different trials. Fill in the function below to calculate the covariance matrix given the desired variances and correlation coefficient. The covariance can be found by rearranging the equation above:

\begin{align}
\text{cov}(x_1,x_2) = \rho \sqrt{\sigma_1^2 \sigma_2^2}.
\end{align}

_____no_output_____

# @markdown Execute this cell to get helper function `get_data`

def get_data(cov_matrix):
  """
  Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian.

  Note that samples are sorted in ascending order for the first random variable

  Args:
    cov_matrix (numpy array of floats): desired covariance matrix

  Returns:
    (numpy array of floats) : samples from the bivariate Gaussian, with each
                              column corresponding to a different random
                              variable
  """

  mean = np.array([0, 0])
  X = np.random.multivariate_normal(mean, cov_matrix, size=1000)
  indices_for_sorting = np.argsort(X[:, 0])
  X = X[indices_for_sorting, :]

  return X

help(get_data)Help on function get_data in module __main__:

get_data(cov_matrix)
    Returns a matrix of 1000 samples from a bivariate, zero-mean Gaussian.

    Note that samples are sorted in ascending order for the first random variable

    Args:
      cov_matrix (numpy array of floats): desired covariance matrix

    Returns:
      (numpy array of floats) : samples from the bivariate Gaussian, with each
                                column corresponding to a different random
                                variable

def calculate_cov_matrix(var_1, var_2, corr_coef):
  """
  Calculates the covariance matrix based on the variances and correlation
  coefficient.

  Args:
    var_1 (scalar)          : variance of the first random variable
    var_2 (scalar)          : variance of the second random variable
    corr_coef (scalar)      : correlation coefficient

  Returns:
    (numpy array of floats) : covariance matrix
  """

  #################################################
  ## TODO for students: calculate the covariance matrix
  # Fill out function and remove
  #raise NotImplementedError("Student excercise: calculate the covariance matrix!")
  #################################################

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

# Set parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data with this covariance matrix
X = get_data(cov_matrix)

# Visualize
plot_data(X)_____no_output_____# to_remove solution

def calculate_cov_matrix(var_1, var_2, corr_coef):
  """
  Calculates the covariance matrix based on the variances and correlation
  coefficient.

  Args:
    var_1 (scalar)          : variance of the first random variable
    var_2 (scalar)          : variance of the second random variable
    corr_coef (scalar)      : correlation coefficient

  Returns:
    (numpy array of floats) : covariance matrix
  """

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

# Set parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data with this covariance matrix
X = get_data(cov_matrix)

# Visualize
with plt.xkcd():
  plot_data(X)findfont: Font family ['xkcd', 'xkcd Script', 'Humor Sans', 'Comic Neue', 'Comic Sans MS'] not found. Falling back to DejaVu Sans.
findfont: Font family ['xkcd', 'xkcd Script', 'Humor Sans', 'Comic Neue', 'Comic Sans MS'] not found. Falling back to DejaVu Sans.
findfont: Font family ['xkcd', 'xkcd Script', 'Humor Sans', 'Comic Neue', 'Comic Sans MS'] not found. Falling back to DejaVu Sans.

## Interactive Demo 1: Correlation effect on data

We'll use the function you just completed but now we can change the correlation coefficient via slider. You should get a feel for how changing the correlation coefficient affects the geometry of the simulated data.

1.   What effect do negative correlation coefficient values have?
2.   What correlation coefficient results in a circular data cloud?

Note that we sort the samples according to neuron 1's firing rate, meaning the plot of neuron 1 firing rate over sample number looks clean and pretty unchanging when compared to neuron 2.
_____no_output_____

# @markdown Execute this cell to enable widget

def _calculate_cov_matrix(var_1, var_2, corr_coef):

  # Calculate the covariance from the variances and correlation
  cov = corr_coef * np.sqrt(var_1 * var_2)

  cov_matrix = np.array([[var_1, cov], [cov, var_2]])

  return cov_matrix

@widgets.interact(corr_coef = widgets.FloatSlider(value=.2, min=-1, max=1, step=0.1))
def visualize_correlated_data(corr_coef=0):
  variance_1 = 1
  variance_2 = 1

  # Compute covariance matrix
  cov_matrix = _calculate_cov_matrix(variance_1, variance_2, corr_coef)

  # Generate data with this covariance matrix
  X = get_data(cov_matrix)

  # Visualize
  plot_data(X)_____no_output_____# to_remove explanation

"""

1) Negative correlation coefficients reverse the direction of the relationship: now higher neuron 1
activities are associated with lower neuron 2 activities.

2) A correlation coefficient of 0 (no correlation) results in a circular looking data cloud.

"""_____no_output_____

---
# Section 2: Define a new orthonormal basis

*Estimated timing to here from start of tutorial: 20 min*
_____no_output_____

# @title Video 3: Orthonormal bases
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1wT4y1E71g", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="PC1RZELnrIg", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

This video shows that data can be represented in many ways using different bases. It also explains how to check if your favorite basis is orthonormal.

 Click here for text recap of video

Next, we will define a new orthonormal basis of vectors ${\bf u} = [u_1,u_2]$ and ${\bf w} = [w_1,w_2]$. As we learned in the video, two vectors are orthonormal if:

1.   They are orthogonal (i.e., their dot product is zero):
\begin{align}
{\bf u\cdot w} = u_1 w_1 + u_2 w_2 = 0
\end{align}
2.   They have unit length:
\begin{align}
||{\bf u} || = ||{\bf w} || = 1
\end{align}

In two dimensions, it is easy to make an arbitrary orthonormal basis. All we need is a random vector ${\bf u}$, which we have normalized. If we now define the second basis vector to be ${\bf w} = [-u_2,u_1]$, we can check that both conditions are satisfied:
\begin{align}
{\bf u\cdot w} = - u_1 u_2 + u_2 u_1 = 0
\end{align}
and
\begin{align}
{|| {\bf w} ||} = \sqrt{(-u_2)^2 + u_1^2} = \sqrt{u_1^2 + u_2^2} = 1,
\end{align}
where we used the fact that ${\bf u}$ is normalized. So, with an arbitrary input vector, we can define an orthonormal basis, which we will write in matrix by stacking the basis vectors horizontally:

\begin{align}
{{\bf W} } =
\begin{pmatrix}
 u_1 & w_1 \\
 u_2 & w_2
\end{pmatrix}.
\end{align}
_____no_output_____## Coding Exercise 2: Find an orthonormal basis

In this exercise you will fill in the function below to define an orthonormal basis, given a single arbitrary 2-dimensional vector as an input.

**Steps**
* Modify the function `define_orthonormal_basis` to first normalize the first basis vector $\bf u$.
* Then complete the function by finding a basis vector $\bf w$ that is orthogonal to $\bf u$.
* Test the function using initial basis vector ${\bf u} = [3,1]$. Plot the resulting basis vectors on top of the data scatter plot using the function `plot_basis_vectors`. (For the data, use  $\sigma_1^2 =1$, $\sigma_2^2 =1$, and $\rho = .8$)._____no_output_____

def define_orthonormal_basis(u):
  """
  Calculates an orthonormal basis given an arbitrary vector u.

  Args:
    u (numpy array of floats) : arbitrary 2-dimensional vector used for new
                                basis

  Returns:
    (numpy array of floats)   : new orthonormal basis
                                columns correspond to basis vectors
  """

  #################################################
  ## TODO for students: calculate the orthonormal basis
  # Fill out function and remove
  #raise NotImplementedError("Student excercise: implement the orthonormal basis function")
  #################################################

  # Normalize vector u
  u = u / np.linalg.norm(u)

  # Calculate vector w that is orthogonal to w
  w = np.array([-u[1], u[0]])

  # Put in matrix form
  W = np.column_stack([u, w])

  return W

# Set up parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8
u = np.array([3, 1])

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data
X = get_data(cov_matrix)

# Get orthonomal basis
W = define_orthonormal_basis(u)

# Visualize
plot_basis_vectors(X, W)_____no_output_____# to_remove solution

def define_orthonormal_basis(u):
  """
  Calculates an orthonormal basis given an arbitrary vector u.

  Args:
    u (numpy array of floats) : arbitrary 2-dimensional vector used for new
                                basis

  Returns:
    (numpy array of floats)   : new orthonormal basis
                                columns correspond to basis vectors
  """

  # Normalize vector u
  u = u / np.sqrt(u[0] ** 2 + u[1] ** 2)

  # Calculate vector w that is orthogonal to w
  w = np.array([-u[1], u[0]])

  # Put in matrix form
  W = np.column_stack([u, w])

  return W

# Set up parameters
np.random.seed(2020)  # set random seed
variance_1 = 1
variance_2 = 1
corr_coef = 0.8
u = np.array([3, 1])

# Compute covariance matrix
cov_matrix = calculate_cov_matrix(variance_1, variance_2, corr_coef)

# Generate data
X = get_data(cov_matrix)

# Get orthonomal basis
W = define_orthonormal_basis(u)

# Visualize
with plt.xkcd():
  plot_basis_vectors(X, W)_____no_output_____

---
# Section 3: Project data onto new basis

*Estimated timing to here from start of tutorial: 35 min*_____no_output_____

# @title Video 4: Change of basis
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="BV1LK411J7NQ", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="Mj6BRQPKKUc", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

Finally, we will express our data in the new basis that we have just found. Since $\bf W$ is orthonormal, we can project the data into our new basis using simple matrix multiplication :

\begin{align}
{\bf Y = X W}.
\end{align}

We will explore the geometry of the transformed data $\bf Y$ as we vary the choice of basis._____no_output_____## Coding Exercise 3: Change to orthonormal basis
In this exercise you will fill in the function below to change data to a orthonormal basis.

**Steps**
* Complete the function `change_of_basis` to project the data onto the new basis.
* Plot the projected data using the function `plot_data_new_basis`.
* What happens to the correlation coefficient in the new basis? Does it increase or decrease?
* What happens to variance?

_____no_output_____

def change_of_basis(X, W):
  """
  Projects data onto new basis W.

  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors

  Returns:
    (numpy array of floats)    : Data matrix expressed in new basis
  """

  #################################################
  ## TODO for students: project the data onto o new basis W
  # Fill out function and remove
  #raise NotImplementedError("Student excercise: implement change of basis")
  #################################################

  # Project data onto new basis described by W
  Y = X @ W

  return Y

# Project data to new basis
Y = change_of_basis(X, W)

# Visualize
plot_data_new_basis(Y)_____no_output_____# to_remove solution

def change_of_basis(X, W):
  """
  Projects data onto new basis W.

  Args:
    X (numpy array of floats) : Data matrix each column corresponding to a
                                different random variable
    W (numpy array of floats) : new orthonormal basis columns correspond to
                                basis vectors

  Returns:
    (numpy array of floats)    : Data matrix expressed in new basis
  """

  # Project data onto new basis described by W
  Y = X @ W

  return Y

# Project data to new basis
Y = change_of_basis(X, W)

# Visualize
with plt.xkcd():
  plot_data_new_basis(Y)_____no_output_____

## Interactive Demo 3: Play with the basis vectors
To see what happens to the correlation as we change the basis vectors, run the cell below. The parameter $\theta$ controls the angle of $\bf u$ in degrees. Use the slider to rotate the basis vectors.

1.   What happens to the projected data as you rotate the basis?
2.   How does the correlation coefficient change? How does the variance of the projection onto each basis vector change?
3.   Are you able to find a basis in which the projected data is **uncorrelated**?_____no_output_____

# @markdown Make sure you execute this cell to enable the widget!

def refresh(theta=0):
  u = [1, np.tan(theta * np.pi / 180)]
  W = define_orthonormal_basis(u)
  Y = change_of_basis(X, W)
  plot_basis_vectors(X, W)
  plot_data_new_basis(Y)

_ = widgets.interact(refresh, theta=(0, 90, 5))_____no_output_____# to_remove explanation

"""

1) As you rotate the basis vectors, the look of the cloud of projected data changes.
Specifically, the correlations and variances of the projected data change.

2) As the angle increases, the correlation coefficient decreases, goes to 0, then starts
to become negative. It changes from 0.8 to -.8 when the angle is changed by 90 degrees.

With theta of 0, the data projected onto the two basis vectors has equal variances of 1.
As theta increases, these variances become unequal: the projected data has larger and larger variance
for basis vector 1 and smaller variance for basis vector 2. Past theta of 45, the trend reverses and
the variances start becoming more similar.

3) If theta is 45 degrees, the projected data is uncorrelated.

"""_____no_output_____

---
# Summary

*Estimated timing of tutorial: 50 minutes*

- In this tutorial, we learned that multivariate data can be visualized as a cloud of points in a high-dimensional vector space. The geometry of this cloud is shaped by the covariance matrix.

- Multivariate data can be represented in a new orthonormal basis using the dot product. These new basis vectors correspond to specific mixtures of the original variables - for example, in neuroscience, they could represent different ratios of activation  across a population of neurons.

- The projected data (after transforming into the new basis) will generally have a different geometry from the original data. In particular, taking basis vectors that are aligned with the spread of cloud of points decorrelates the data.

* These concepts - covariance, projections, and orthonormal bases - are key for understanding PCA, which we be our focus in the next tutorial._____no_output_____---
# Notation

\begin{align}
x_i &\quad \text{data point for dimension } i\\
\mu_i &\quad \text{mean along dimension } i\\
\sigma_i^2 &\quad \text{variance along dimension } i \\
\bf u, \bf w &\quad \text{orthonormal basis vectors}\\
\rho &\quad \text{correlation coefficient}\\
\bf \Sigma &\quad \text{covariance matrix}\\
\bf X &\quad \text{original data matrix}\\
\bf W &\quad \text{projection matrix}\\
\bf Y &\quad \text{transformed data}\\
\end{align}_____no_output_____


# Notebook from lennartvansluijs/Spectroscopic-Transit-Search
Path: scripts/plot_fig6.ipynb

# Figure 6_____no_output_____Run the steps below to generate the data and plot of Figure 6.

**Lennart van Sluijs** // 2019 Jan 8 // Leiden Observatory // vansluijs@strw.leidenuniv.nl_____no_output_____

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
from astropy import units as u
from astropy import constants as const
from matplotlib import pyplot as plt
from aux import get_flist
from aux import tableau20
import os_____no_output_____

# Calculate period completeness
First define some functions to perform the calculation._____no_output_____

def transit_duration(Rs, Rp, a, P, b):
    """
    Description:
        Analytical estimate of the transit duration.
    Input:
        Rs - radius of the star
        Rp - radius of the planet
        a - semi-major axis
        P - orbital period
        b - impact parameter
    Output:
        Tdur - transit duration
    """

    # caclulate transit duration
    Tdur = P/(np.pi * u.rad) \
    * np.arcsin( np.sqrt( (Rs + Rp)**2 - (b*Rs)**2 )/a )

    return Tdur

def a2P(a, Ms):
    """
    Description:
        Convert the semi-major axis into a period using Kepler's third law.
    Input:
        a - semi-major axis
        Ms - stellar mass
    Output:
        P - period of the planet
    """

    # get period from Kepler's third law
    P = np.sqrt((4*np.pi**2*a**3)/(const.G * Ms))

    return P

def init_time(inputfolder):
    """
    Description:
        Create a list of times for all time frames of all observations.
    Input:
        inputfolder - contains time data for all observations
    Output:
        time - list of times of all observations
    """

    # get all files containing timestamp data
    flist = get_flist(inputfolder)

    # create list of times
    time = []
    for fname in flist:
        fname_time = np.loadtxt(os.path.join(inputfolder, fname))
        time.append(fname_time)
    time = np.array(time)

    return time

def init_obs_mask(time, osf):
    """
    Description:
        Create an oversampled mask of times when observations where taken.
    Input:
        time - numpy array with times for every observed night
        osf - oversampling factor
    Output:
        t - time series corresponding with observational mask
        obs_mask - observational mask
    """

    # create oversampled time grid
    t = np.linspace(time[0][0], time[-1][-1], osf)
    obs_mask = np.zeros(len(t))

    # create observation window
    for n in range(len(time)):
        obs_mask[np.where(np.logical_and(t > time[n][0],
                                         t < time[n][-1]))] = 1.0

    return t, obs_mask

def init_transit_mask(Tdur, t):
    """
    Description:
        Create a mask of the duration of a transit with shared x-axis
        with the observation mask.
    Input:
        Tdur - duration of transit
        t - time series corresponding with observational mask
    Output:
        transit_mask - transit mask
    """

    # mask of length of the transit duration for the specified time grid
    t -= t[0]
    transit_mask = t[np.where(t < Tdur.to(u.d).value)]
    transit_mask[:] = 1.0

    return transit_mask

def convolve_masks(obs_mask, transit_mask):
    """
    Description:
        Convolve the in-transit mask with the observation mask.
    Input:
        transit_mask - transit mask
        obs_mask - observational mask
    Output:
        conv_masks - convolution of both input masks
    """

    # convolve observation window with transit duration window
    conv_masks = np.convolve(obs_mask, transit_mask, mode = 'same')
    conv_masks[np.where(conv_masks > 0.)] = 1. # set same weight everywhere

    return conv_masks

def fold_lc(t, f, per):
    """
    Description:
        Fold light curve.
    Input:
        t - list of times
        f - list of fluxes
        per - period for which to fold data
    Output:
        tf - list of folded times
        ff - list of folded fluxes
    """

    # fold light curve data
    folded = (t % per)
    ind = np.array(folded).argsort()
    tf = folded[ind]
    ff = f[ind]

    return tf, ff

def bin_array(x, nbins, median = False):
    """
    Description:
        Bin data.
    Input:
        x - list of values
        nbins - number of bins to use
    Ouput:
        xb - list of binned values
    """

    # bin with equal binsize
    xs = np.array_split(x, nbins)
    xb = [np.mean(xs[j]) for j in range(0, nbins)]

    return xb

def calculate_completeness(t, P, conv_masks):
    """
    Description:
        Get period completeness for the specified exoplanet orbital period
        and observation window.
    Input:
        t - oversampled time grid to use
        P - exoplanet orbital period
        conv_masks - observation window convolved with the transit duration
    Output:
        completeness - period completeness
    """

    # two situation: (1) period > observation window, for which the observation
    # window over the gaps is the completeness
    dt = t[-1] - t[0]
    Np = int(dt/P.to(u.d).value) + 1
    if (t[-1] - t[0]) - P.to(u.d).value < 0:
        completeness = float(len(np.where(conv_masks > 0)[0]))/ \
        float(len(conv_masks) * (P.to(u.d).value/(t[-1] - t[0])))
    # (2) period < observation window, for which we fold first and apply the
    # same routine
    else:
        for n in range(Np):
            ind = np.where(np.logical_and(t >= n * P.to(u.d).value,
                                          t < (n+1) * P.to(u.d).value))[0]
            if n == 0:
                xf = np.linspace(0, P, len(ind))
                yf = np.zeros(len(xf))
            yf[0:len(ind)] += conv_masks[ind]
        completeness = float(len(np.where(yf > 0)[0]))/float(len(yf))

    return completeness_____no_output_____

The following function calculates the period completeness._____no_output_____

def get_completeness(a, Rp, Rs, Ms, b, osf):
    """
    Description:
        Calculate the period completeness.
    Input:
        a - list of semi-major axes to explore
        Rp - list of planet radii to explore
        Ms - stellar mass
        Rs - stellar radius
        b - impact parameter
        osf - oversampling factor to use for the masks
    Output:
        completeness - completeness(a)
    """

    # get time
    time = init_time(inputfolder)

    # explore the whole parameter space
    completeness = np.zeros(len(a))
    for i in range(len(a)):

        print('Calculate completeness (' + str(i+1) + '/' + str(len(a)) + ')')

        # get period and transit duration
        P = a2P(a[i], Ms)
        Tdur = transit_duration(Rs, Rp, a[i], P, b)

        # get observation window and convolve it with the transit duration
        t, obs_mask = init_obs_mask(time, osf)
        transit_mask = init_transit_mask(Tdur, t)
        conv_masks = convolve_masks(obs_mask, transit_mask)

        # calculate the completeness
        completeness[i] = calculate_completeness(t, P, conv_masks)

    return completeness_____no_output_____

Perform the calculation. This takes several minutes on my personal laptop. Reduce the oversampling factor for faster speed, but larger error on the observational window duration._____no_output_____

# load time stamps for observation window
inputfolder = '../data/frame_times'
time = init_time(inputfolder)

# set system parameters
Rs = 1.8 * u.solRad
Rp = 1.0 * u.R_jup
Ms = 1.73 * u.solMass
b = 0
osf = 2000000 # used for the oversampled time grid, higher is more accurate, but
# computationally heavy
a = np.linspace(0.01, 3., 200) * u.au

# find period completeness
completeness = get_completeness(a, Rp, Rs, Ms, b, osf)Calculate completeness (1/200)
Calculate completeness (2/200)
Calculate completeness (3/200)
Calculate completeness (4/200)
Calculate completeness (5/200)
Calculate completeness (6/200)
Calculate completeness (7/200)
Calculate completeness (8/200)
Calculate completeness (9/200)
Calculate completeness (10/200)
Calculate completeness (11/200)
Calculate completeness (12/200)
Calculate completeness (13/200)
Calculate completeness (14/200)
Calculate completeness (15/200)
Calculate completeness (16/200)
Calculate completeness (17/200)
Calculate completeness (18/200)
Calculate completeness (19/200)
Calculate completeness (20/200)
Calculate completeness (21/200)
Calculate completeness (22/200)
Calculate completeness (23/200)
Calculate completeness (24/200)
Calculate completeness (25/200)
Calculate completeness (26/200)
Calculate completeness (27/200)
Calculate completeness (28/200)
Calculate completeness (29/200)
Calculate completeness (30/200)
Calculate completeness (31/200)
Calculate completeness (32/200)
Calculate completeness (33/200)
Calculate completeness (34/200)
Calculate completeness (35/200)
Calculate completeness (36/200)
Calculate completeness (37/200)
Calculate completeness (38/200)
Calculate completeness (39/200)
Calculate completeness (40/200)
Calculate completeness (41/200)
Calculate completeness (42/200)
Calculate completeness (43/200)
Calculate completeness (44/200)
Calculate completeness (45/200)
Calculate completeness (46/200)
Calculate completeness (47/200)
Calculate completeness (48/200)
Calculate completeness (49/200)
Calculate completeness (50/200)
Calculate completeness (51/200)
Calculate completeness (52/200)
Calculate completeness (53/200)
Calculate completeness (54/200)
Calculate completeness (55/200)
Calculate completeness (56/200)
Calculate completeness (57/200)
Calculate completeness (58/200)
Calculate completeness (59/200)
Calculate completeness (60/200)
Calculate completeness (61/200)
Calculate completeness (62/200)
Calculate completeness (63/200)
Calculate completeness (64/200)
Calculate completeness (65/200)
Calculate completeness (66/200)
Calculate completeness (67/200)
Calculate completeness (68/200)
Calculate completeness (69/200)
Calculate completeness (70/200)
Calculate completeness (71/200)
Calculate completeness (72/200)
Calculate completeness (73/200)
Calculate completeness (74/200)
Calculate completeness (75/200)
Calculate completeness (76/200)
Calculate completeness (77/200)
Calculate completeness (78/200)
Calculate completeness (79/200)
Calculate completeness (80/200)
Calculate completeness (81/200)
Calculate completeness (82/200)
Calculate completeness (83/200)
Calculate completeness (84/200)
Calculate completeness (85/200)
Calculate completeness (86/200)
Calculate completeness (87/200)
Calculate completeness (88/200)
Calculate completeness (89/200)
Calculate completeness (90/200)
Calculate completeness (91/200)
Calculate completeness (92/200)
Calculate completeness (93/200)
Calculate completeness (94/200)
Calculate completeness (95/200)
Calculate completeness (96/200)
Calculate completeness (97/200)
Calculate completeness (98/200)
Calculate completeness (99/200)
Calculate completeness (100/200)
Calculate completeness (101/200)
Calculate completeness (102/200)
Calculate completeness (103/200)
Calculate completeness (104/200)
Calculate completeness (105/200)
Calculate completeness (106/200)
Calculate completeness (107/200)
Calculate completeness (108/200)
Calculate completeness (109/200)
Calculate completeness (110/200)
Calculate completeness (111/200)
Calculate completeness (112/200)
Calculate completeness (113/200)
Calculate completeness (114/200)
Calculate completeness (115/200)
Calculate completeness (116/200)
Calculate completeness (117/200)
Calculate completeness (118/200)
Calculate completeness (119/200)
Calculate completeness (120/200)
Calculate completeness (121/200)
Calculate completeness (122/200)
Calculate completeness (123/200)
Calculate completeness (124/200)
Calculate completeness (125/200)
Calculate completeness (126/200)
Calculate completeness (127/200)
Calculate completeness (128/200)
Calculate completeness (129/200)
Calculate completeness (130/200)
Calculate completeness (131/200)
Calculate completeness (132/200)
Calculate completeness (133/200)
Calculate completeness (134/200)
Calculate completeness (135/200)
Calculate completeness (136/200)
Calculate completeness (137/200)
Calculate completeness (138/200)
Calculate completeness (139/200)
Calculate completeness (140/200)
Calculate completeness (141/200)
Calculate completeness (142/200)
Calculate completeness (143/200)
Calculate completeness (144/200)
Calculate completeness (145/200)
Calculate completeness (146/200)
Calculate completeness (147/200)
Calculate completeness (148/200)
Calculate completeness (149/200)
Calculate completeness (150/200)
Calculate completeness (151/200)
Calculate completeness (152/200)
Calculate completeness (153/200)
Calculate completeness (154/200)
Calculate completeness (155/200)
Calculate completeness (156/200)
Calculate completeness (157/200)
Calculate completeness (158/200)
Calculate completeness (159/200)
Calculate completeness (160/200)
Calculate completeness (161/200)
Calculate completeness (162/200)
Calculate completeness (163/200)
Calculate completeness (164/200)
Calculate completeness (165/200)
Calculate completeness (166/200)
Calculate completeness (167/200)
Calculate completeness (168/200)
Calculate completeness (169/200)
Calculate completeness (170/200)
Calculate completeness (171/200)
Calculate completeness (172/200)
Calculate completeness (173/200)
Calculate completeness (174/200)
Calculate completeness (175/200)
Calculate completeness (176/200)
Calculate completeness (177/200)
Calculate completeness (178/200)
Calculate completeness (179/200)
Calculate completeness (180/200)
Calculate completeness (181/200)
Calculate completeness (182/200)
Calculate completeness (183/200)
Calculate completeness (184/200)
Calculate completeness (185/200)
Calculate completeness (186/200)
Calculate completeness (187/200)
Calculate completeness (188/200)
Calculate completeness (189/200)
Calculate completeness (190/200)
Calculate completeness (191/200)
Calculate completeness (192/200)
Calculate completeness (193/200)
Calculate completeness (194/200)
Calculate completeness (195/200)
Calculate completeness (196/200)
Calculate completeness (197/200)
Calculate completeness (198/200)
Calculate completeness (199/200)
Calculate completeness (200/200)

# Plot the coverage_____no_output_____

def plot_coverage(a, completeness, outputfolder):
    """
    Description:
        Create a plot of the coverage for a set different planet radii
        and semi-major axes.
    Input:
        a - list of semi-major axes to explore
        completeness - list of corresponding period completeness
        outputfolder - save plot here
    """

    # create folder if it does not exist yet
    if not os.path.exists(outputfolder):
            os.makedirs(outputfolder)

    # explore the whole parameter space
    fig = plt.figure()
    ax = fig.add_subplot(111)

    # plot sensitivity for the white noise simulation
    sensitivity = [0.7428229665, 0.3588516746, 0.053827751]
    yoffsets = [2.5, 2.5, -1]
    labels = ['Jupiter', 'Saturn', 'Neptune']
    ls_list = ['-', '--', ':']
    for i in range(3):
        coverage = sensitivity[i] * completeness * 100
        ax.plot(a, coverage, color = tableau20[i*2], ls = ls_list[i],
                lw = 1., zorder = len(labels)-i)
        ax.text(0.3, sensitivity[i]*100 - yoffsets[i], s = labels[i])

    # set xticks
    xticks = np.linspace(a[0].to(u.au).value, a[-1].to(u.au).value, 10)
    xticklabels = np.round(np.linspace(a[0].to(u.au).value,
                                       a[-1].to(u.au).value,
                                       10), 2)
    ax.set_xticks(xticks)
    ax.set_xticklabels(xticklabels)
    ax.set_xlabel('a [au]', size = 15)
    ax.set_ylabel('Coverage [%]', size = 15)
    ax.axhline(y=50, ls = '--', color = 'k', lw = 0.5)
    ax.set_xlim(a[0].to(u.au).value, a[-1].to(u.au).value)
    ax.set_ylim(0., 100.0)

    # set xticks above plot for period range
    ax2 = ax.twiny()
    xticks = np.linspace(0, 1, 10)
    xticklabels = np.round(np.linspace(a2P(a[0], Ms).to(u.d).value,
                                       a2P(a[-1], Ms).to(u.d).value,
                                       10), 1)
    ax2.set_xticks(xticks)
    ax2.set_xticklabels(xticklabels)
    ax2.set_xlabel('Period [d]', size = 15)

    # set right spine unvisble
    ax.spines['right'].set_visible(False)
    ax2.spines['right'].set_visible(False)

    # plot tight layout
    plt.tight_layout()

# create a plot of the coverage
outputfolder = '../output/figure6'
plot_coverage(a, completeness, outputfolder)

# save the plot
plt.savefig(os.path.join(outputfolder, 'figure6.png'), dpi = 300)
plt.savefig(os.path.join(outputfolder, 'figure6.pdf'))
plt.show()_____no_output_____


# Notebook from cristina-iacob/kickstarter_analysis
Path: DSI_kickstarterscrape_dataset.ipynb

#!pip install nb_black
%reload_ext nb_black
# autosave every 3 minutes
%autosave 180_____no_output_____#!pip install plotly
#!pip install cufflinks_____no_output_____import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import PercentFormatter
import seaborn as sns

import plotly.tools as tls
import plotly
import plotly.offline as py
from plotly.offline import init_notebook_mode, iplot, plot
import plotly.graph_objs as go

init_notebook_mode(connected=True)

%matplotlib inline_____no_output_____# ks_df = pd.read_csv("DSI_kickstarterscrape_dataset.csv", encoding="utf-8")
ks_df = pd.read_csv("data/DSI_kickstarterscrape_dataset.csv", encoding="ISO-8859-1")
ks_df.head()_____no_output_____ks_df.shape_____no_output_____ks_df.info()<class 'pandas.core.frame.DataFrame'>
RangeIndex: 45957 entries, 0 to 45956
Data columns (total 17 columns):
 #   Column             Non-Null Count  Dtype
---  ------             --------------  -----
 0   project id         45957 non-null  int64
 1   name               45957 non-null  object
 2   url                45957 non-null  object
 3   category           45957 non-null  object
 4   subcategory        45957 non-null  object
 5   location           44635 non-null  object
 6   status             45957 non-null  object
 7   goal               45957 non-null  float64
 8   pledged            45945 non-null  float64
 9   funded percentage  45957 non-null  float64
 10  backers            45957 non-null  int64
 11  funded date        45957 non-null  object
 12  levels             45957 non-null  int64
 13  reward levels      45898 non-null  object
 14  updates            45957 non-null  int64
 15  comments           45957 non-null  int64
 16  duration           45957 non-null  float64
dtypes: float64(4), int64(5), object(8)
memory usage: 6.0+ MB
ks_df.isnull().sum()_____no_output_____

There are 12 missing values for pledged data, 59 for reward_levels and 1322 for location._____no_output_____

ks_df.describe()_____no_output_____

#### 1. What is the mean (total) pledge that projects get?_____no_output_____

pledged_mean = round(ks_df["pledged"].mean(), 2)
pledged_mean_____no_output_____

#### 2. Create a histogram that shows the distribution for number of backers. What is the skew of the distribution?_____no_output_____

plt.figure(figsize=(12, 5))
sns.histplot(ks_df["backers"], kde=True, bins=100)
plt.xlabel("Backers")
plt.ylabel("Frequency")
plt.title("Distribution of Project Backers")
plt.show()_____no_output_____ks_df["backers"].describe()_____no_output_____ks_df.skew(axis=0, skipna=True)_____no_output_____backers_skewness = ks_df["backers"].skew()

print("Skewness:")
print(round(backers_skewness, 2))Skewness:
87.47

There are lots of otliers which I will eliminate usint the IQR method._____no_output_____

# calculate 25% and 75% quartiles and IQR = Q3-Q1
Q1 = ks_df.backers.quantile(0.25)
Q3 = ks_df.backers.quantile(0.75)
IQR = Q3 - Q1
Q1, Q3, IQR_____no_output_____# calculate min and max thresholds
lower_limit = Q1 - 1.5 * IQR
upper_limit = Q3 + 1.5 * IQR
lower_limit, upper_limit_____no_output_____# outliers
# ks_df[(ks_df.backers < lower_limit) | (ks_df.backers > upper_limit)]

# dataframe w/o outliers
df_no_outlier = ks_df[(ks_df.backers > lower_limit) & (ks_df.backers < upper_limit)]
df_no_outlier.head(2)_____no_output_____df_no_outlier.describe()_____no_output_____sns.boxplot(x=df_no_outlier["backers"])
plt.xlabel("Backers")
plt.title("Boxplot of Project Backers (outliers excluded)")
plt.show()_____no_output_____plt.figure(figsize=(12, 5))
sns.histplot(df_no_outlier["backers"], kde=True, bins=50, color="orange")
plt.xlabel("Backers")
plt.ylabel("Frequency")
plt.title("Distribution of Project Backers (outliers removed)")
plt.show()_____no_output_____backers_skewness_n = df_no_outlier["backers"].skew()

print("Skewness_wo_outliers:")
print(round(backers_skewness_n, 2))Skewness_wo_outliers:
1.26

#### 3. Is the 'duration' variable normally distributed?_____no_output_____

plt.figure(figsize=(10, 5))
sns.histplot(ks_df["duration"], kde=True, bins=30)
plt.xlabel("Duration")
plt.ylabel("Frequecny")
plt.title("Distribution of Project Duration")
plt.show()_____no_output_____

Duration variable is not normal distributed._____no_output_____#### 4. If you could collect data on another attribute of these projects, what would it be and why?
_____no_output_____There is nothing about marketing presence so a possible new variable can be if there is a FB or Twitter account which can contribute to the marketing campaign. Also of interest might be knowing the targeted audience or even the expected number of customers._____no_output_____## Qualitative Analysis_____no_output_____

ks_df.status.value_counts(normalize=True).plot(kind="pie", figsize=(9, 6))
# colors = ["orange", "red", "grey", "darkturquoise", "salmon"]
# status = round(ks_df["status"].value_counts() / len(ks_df["status"]) * 100, 2)

# labels = list(status.index)
# values = list(status.values)

# trace1 = go.Pie(
#     labels=labels,
#     values=values,
#     marker=dict(colors=colors),
#     #    line=dict(color="rgb(100,100,100)", width=1),
# )

# layout = go.Layout(title="Distribuition of Status", legend=dict(orientation="h"))

# fig = go.Figure(data=[trace1], layout=layout)
# iplot(fig)_____no_output_____

### What's the best length of time to run a campaign?_____no_output_____

# average succsessful project duration
success_duration_avg = ks_df[ks_df["status"] == "successful"].duration.mean()

print("Successful campaign average duration:")
print(round(success_duration_avg, 1))Successful campaign average duration:
38.0
plt.figure(figsize=(12, 5))
sns.histplot(
    data=ks_df[ks_df["status"] == "successful"],
    x="duration",
    bins=20,
    color="orange",
    kde=True,
)_____no_output_____

### What's the ideal pledge goal?_____no_output_____

# average successful project pledge
success_pledge_goal_avg = ks_df[ks_df["status"] == "successful"].goal.mean()

print("Successful campaign average goal: ")
print(round(success_pledge_goal_avg, 1))Successful campaign average goal:
5483.8
# Plot showing rates of success by duration

plt.figure(figsize=(12, 6))

sns.catplot(x="status", y="goal", data=ks_df, kind="bar", ci=None)
plt.xlabel("Status")
plt.ylabel("Pledge Goal")
plt.title("Pledge Goals by Status")
plt.show()_____no_output_____

### What type of projects would be most successful at getting funded?_____no_output_____

# Print categories
ks_df["category"].value_counts()_____no_output_____

Film & video apperas 2 times, one of them is misspelled, needs to be fixed_____no_output_____

ks_df["category"] = ks_df["category"].replace(["Film & Video"], "Film & Video")
ks_df["category"].value_counts()_____no_output_____

There are 14 categories_____no_output_____

ks_df.category.value_counts(normalize=True).plot(kind="pie", figsize=(9, 6))
# colors = ["orange", "red", "grey", "darkturquoise", "salmon"]
# category = round(ks_df["category"].value_counts() / len(ks_df["category"]) * 100, 2)

# labels = list(category.index)
# values = list(category.values)

# trace1 = go.Pie(
#     labels=labels,
#     values=values,
#     marker=dict(colors=colors),
#     #    line=dict(color="rgb(100,100,100)", width=1),
# )

# layout = go.Layout(title="Distribuition of Category", legend=dict(orientation="h"))

# fig = go.Figure(data=[trace1], layout=layout)
# iplot(fig)_____no_output_____ks_df[ks_df["status"] == "successful"].category.value_counts(normalize=True).plot(kind="pie", figsize=(9, 6))
#df[df["status"] == "successful"]["category"]

# colors = ["orange", "red", "grey", "darkturquoise", "salmon"]
# category = round(ks_df[ks_df["status"] == "successful"]["category"].value_counts() / len(ks_df[ks_df["status"] == "successful"]) * 100, 2)

# labels = list(category.index)
# values = list(category.values)

# trace1 = go.Pie(
#     labels=labels,
#     values=values,
#     marker=dict(colors=colors),
#     #    line=dict(color="rgb(100,100,100)", width=1),
# )

# layout = go.Layout(title="Distribuition of Category", legend=dict(orientation="h"))

# fig = go.Figure(data=[trace1], layout=layout)
# iplot(fig)
_____no_output_____plt.figure(figsize=(12,6))
#sns.countplot(x="category", data=ks_df_success, palette="coolwarm")
#plt.title("Count of Successful Campaigns Based on Category")
#plt.show()

plt.figure(figsize=(15,6))
chart=sns.countplot(
    x='category',
    data=ks_df[ks_df["status"] == "successful"],
    order=ks_df[ks_df["status"] == "successful"]['category'].value_counts().index[:13]
)

chart.set_xticklabels(chart.get_xticklabels(),
    rotation=30,
    horizontalalignment='right',
    fontweight='light',
    fontsize='large')
plt.title('Count of Successful Campaigns Based on Category')

plt.tight_layout()
_____no_output_____

### Is there an ideal month/day/time to launch a campaign?_____no_output_____

# Changing the datetime object
ks_df["funded date"] = pd.to_datetime(ks_df["funded date"])

# Extracting Timestamp to generate new columns as Month, Day, Hour and Weekday
# ad_df["Timestamp"].dt.hour
ks_df["funded hour"] = ks_df["funded date"].dt.hour
ks_df["funded month"] = ks_df["funded date"].dt.month
ks_df["funded day no"] = ks_df["funded date"].dt.weekday
ks_df["funded day"] = ks_df["funded date"].dt.day_name()
ks_df["funded year"] = ks_df["funded date"].dt.year_____no_output_____ks_df.head(2)_____no_output_____# Hourly distribution of succsessful campaigns
plt.figure(figsize=(12, 6))
sns.countplot(
    x="funded hour", data=ks_df[ks_df["status"] == "successful"], palette="magma"
)
plt.title("Successful Campaigns by funded hour distribution")_____no_output_____

The most uccessful campaigns were funded at 3-4am _____no_output_____

# Dayly distribution of succsessful campaigns
plt.figure(figsize=(12, 6))
sns.countplot(
    x="funded day", data=ks_df[ks_df["status"] == "successful"], palette="magma"
)
plt.title("Successful Campaigns by funded day distribution")_____no_output_____

Not a big difference but he most uccessful campaigns were funded Saturdays._____no_output_____

# Monthly distribution of succsessful campaigns
plt.figure(figsize=(12, 6))
sns.countplot(
    x="funded month", data=ks_df[ks_df["status"] == "successful"], palette="magma"
)
plt.title("Successful Campaigns by funded month distribution")_____no_output_____

The most successful campaigns were funded in spring, March - May._____no_output_____

ks_df["funded year"].value_counts()_____no_output_____

Future analysis might include the location of the campaign- continents, countries, states for US_____no_output_____

# average successful project updates
success_pledge_updates_avg = ks_df[ks_df["status"] == "successful"].updates.mean()

print("Successful campaign average updates: ")
print(round(success_pledge_updates_avg, 1))Successful campaign average updates:
6.7
# average successful project comments
success_pledge_comments_avg = ks_df[ks_df["status"] == "successful"].comments.mean()

print("Successful campaign average comments: ")
print(round(success_pledge_comments_avg, 1))Successful campaign average comments:
14.4


# Notebook from muma7490/TRANS
Path: Gaia/Teutsch55/Teutsch55.ipynb

<code>
import numpy as np
import matplotlib.pyplot as plt
import os
from os.path import dirname
import csv_____no_output_____datadir = os.getcwd()

suffix = ['1-20', '21-40', '41-60', '61-80', '81-100', '101-120', '121-140', '141-160', '161-163']

datafile_input = []
for i in range(0 , len(suffix)):
    temp = '/ids_{0}.csv'.format(suffix[i])

    with open(datadir+temp, 'r') as f:
        reader = csv.reader(f)
        input_1_20 = list(reader)
    datafile_input.append(input_1_20)

datafile_output = []
for i in range(0 , len(suffix)):
    temp = '/{0}.csv'.format(suffix[i])

    with open(datadir+temp, 'r') as f:
        reader = csv.reader(f)
        output_1_20 = list(reader)
    datafile_output.append(output_1_20)   _____no_output_____#extract gaia source IDs from the input files
input_ids = []

for j in range(0, len(datafile_input)):
    input_idss = []
    for i in range(0, len(datafile_input[j])):
        input_idss.append(int(datafile_input[j][i][0].split(" ")[2]))
    input_ids.append(input_idss)_____no_output_____#extract gaia source IDs from the output files

output_ids = []

for j in range(0, len(datafile_output)):
    temp = [int(datafile_output[j][i][0]) for i in range(1,len(datafile_output[j]))]
    output_ids.append(temp)_____no_output_____#check if every pair of files (resp. first input and first output file) contain same IDs

for i in range(0, len(output_ids)):
    print(set(output_ids[i]) == set(input_ids[i]))True
True
True
True
True
True
True
True
True
#now extract all data into lists

output_info = datafile_output[0][0]_____no_output_____output_all = []
for j in range(0, len(datafile_output)):
    #print(j)
    for i in range(0, len(datafile_output[j])-1):
        #print(i)
        temp = datafile_output[j][1:][i]
        output_all.append(temp)_____no_output_____len(output_all)_____no_output_____output_info_____no_output_____sid = np.array([int(output_all[i][0]) for i in range(0, len(output_all))])
ra = np.array([float(output_all[i][1]) for i in range(0, len(output_all))])
dec = np.array([float(output_all[i][3]) for i in range(0, len(output_all))])

magg = np.array([float(output_all[i][11]) for i in range(0, len(output_all))])
fluxg = 10**(-0.4*np.array(magg))_____no_output_____max(magg)_____no_output_____plt.subplots(1,1,figsize=(16,14))
plt.scatter(ra, dec, s=fluxg*5e5)

plt.gca().invert_xaxis()

plt.xlabel('RA (°)')
plt.ylabel('DEC (°)')

plt.show()_____no_output_____#because an (or some) elements in the following lists is not a number we cant convert it yet into floats...
pax = np.asarray([output_all[i][5] for i in range(0, len(output_all))])
pmra = np.asarray([output_all[i][7] for i in range(0, len(output_all))])
pmdec = np.asarray([output_all[i][9] for i in range(0, len(output_all))])_____no_output_____pax_____no_output_____'' in pax_____no_output_____for i in range(0, len(pax)):
    if pax[i] == '':
        print(i)
    if pmra[i] == '':
        print(i)
    if pmdec[i] == '':
        print(i)

# star 133 resp. element 132 has no pax, pmra & pmdec!
# so the star will be removed from all lists67
67
67
94
94
94
113
113
113
#remove element 132:

msk_lst = [67, 94, 113]

sid_new = np.delete(sid, msk_lst)
ra_new = np.delete(ra, msk_lst)
dec_new = np.delete(dec, msk_lst)

magg_new = np.delete(magg, msk_lst)
fluxg_new = np.delete(fluxg, msk_lst)

pax_new = np.delete(pax, msk_lst).astype(float)
pmra_new = np.delete(pmra, msk_lst).astype(float)
pmdec_new = np.delete(pmdec, msk_lst).astype(float)_____no_output_____pax_new_____no_output_____def arrows(x, y, pm_x, pm_y, scale):
    temp = []
    for i in range(0, len(x)):
        temp2 = [x[i], y[i], scale * pm_x[i], scale * pm_y[i]]
        temp.append(temp2)
    return np.array(temp)_____no_output_____soa = arrows(ra_new, dec_new, pmra_new, pmdec_new, 0.005)

X, Y, U, V = zip(*soa)
plt.subplots(1,1,figsize=(16,14))
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1, width=0.001, alpha=0.5, color='r')
ax.scatter(ra_new, dec_new, s=np.array(fluxg_new)*5e5)

ax.invert_xaxis()

ax.margins(0.25)

ax.set_xlabel('RA (°)')
ax.set_ylabel('DEC (°)')

plt.savefig('Teutsch55_pm.png')

plt.draw()
plt.show()_____no_output_____# convert parallaxes into parsecs
parcs = 1000./np.array(pax_new)_____no_output_____fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1, figsize=(16,14))
bottom = 0.1
ax1.hist(parcs, bins=np.arange(0,max(parcs)+10,200))
ax2.hist(parcs, bins=np.arange(0,10000,100))
ax3.hist(pmra_new, bins=np.arange(min(pmra_new),max(pmra_new),0.5))
ax4.hist(pmdec_new, bins=np.arange(min(pmdec_new),max(pmdec_new),0.5))

ax1.set_title('distance')
ax2.set_title('distance zoom')
ax3.set_title('pm ra')
ax4.set_title('pm dec')

ax1.set_xlabel('parsecs')
ax2.set_xlabel('parsecs')
ax3.set_xlabel('mas/yr')
ax4.set_xlabel('mas/yr')

plt.subplots_adjust(hspace=0.5)

plt.show()_____no_output_____mask_dist = []
mask_pmra = []
mask_pmde = []

for i in range(len(parcs)):
    #mask_dist.append(400 <= parcs[i] <= 600)
    mask_dist.append(5000 <= parcs[i] <= 8000)

for j in range(len(pmra_new)):
    #mask_pmra.append(-5 <= pmra_new[j] <= 5)
    mask_pmra.append(-10 <= pmra_new[j] <= 10)

for k in range(len(pmdec_new)):
    #mask_pmde.append(-10 <= pmdec_new[k] <= 0)
    mask_pmde.append(-10 <= pmdec_new[k] <= 10)

mask_dist = np.array(mask_dist)
mask_pmra = np.array(mask_pmra)
mask_pmde = np.array(mask_pmde)

mask_cluster = []

for ind in range(max(len(mask_dist),len(mask_pmra),len(mask_pmde))):
    if mask_dist[ind] and mask_pmra[ind] and mask_pmde[ind]:
        mask_cluster.append(True)
    else:
        mask_cluster.append(False)

mask_cluster = np.array(mask_cluster)_____no_output_____mask_cluster_____no_output_____ra_cl = ra_new[mask_cluster]
dec_cl =dec_new[mask_cluster]

pmra_new_cl = pmra_new[mask_cluster]
pmdec_new_cl = pmdec_new[mask_cluster]
parcs_cl = parcs[mask_cluster]

fluxg_cl = fluxg_new[mask_cluster]_____no_output_____soa = arrows(ra_cl, dec_cl, pmra_new_cl, pmdec_new_cl, 0.005)

X, Y, U, V = zip(*soa)
plt.subplots(1,1,figsize=(16,14))
ax = plt.gca()
ax.quiver(X, Y, U, V, angles='xy', scale_units='xy', scale=1, width=0.001, alpha=0.5, color='r')
ax.scatter(ra_new, dec_new, s=np.array(fluxg_new)*5e5)
ax.scatter(ra_cl, dec_cl, s=np.array(fluxg_cl)*5e5,c='k')

ax.invert_xaxis()

ax.margins(0.25)

ax.set_xlabel('RA (°)')
ax.set_ylabel('DEC (°)')

plt.draw()
plt.show()_____no_output_____fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(16,14))

_,bins, __ = ax1.hist(parcs_cl, bins='auto')#np.arange(400,600,20)
ax2.hist(pmra_new_cl, bins='auto') #np.arange(min(pmra_new_cl),max(pmra_new_cl),0.2)
ax3.hist(pmdec_new_cl, bins='auto') #np.arange(min(pmdec_new_cl),max(pmdec_new_cl),0.2)
ax1.set_xlabel('parsecs')
ax2.set_xlabel('mas/yr')
ax3.set_xlabel('mas/yr')
#plt.subplots_adjust(hspace=0.5)
plt.show()_____no_output_____
</code>