# Notebook from jaemyungkim/Vitis-AI
Path: examples/DPUCZDX8G/ofa_resnet50/ofa_resnet50_search_byOps.ipynb

This is End-to-End example how to search a subnet from ofa/resnet50 design space with constraint of Ops, then quantizing/compiling it with target device of ZCU102 board. _____no_output_____## build ofa resnet50 design space_____no_output_____

from ofa.model_zoo import ofa_net
ofa_network = ofa_net('ofa_resnet50', pretrained=True)_____no_output_____

##  build accuracy predictor_____no_output_____

import torch
from ofa.nas.accuracy_predictor import ResNetArchEncoder
from ofa.nas.accuracy_predictor import AccuracyPredictor
from ofa.utils import download_url

image_size_list = [128, 144, 160, 176, 192, 224, 240, 256]
arch_encoder = ResNetArchEncoder(
	image_size_list=image_size_list, depth_list=ofa_network.depth_list, expand_list=ofa_network.expand_ratio_list,
    width_mult_list=ofa_network.width_mult_list, base_depth_list=ofa_network.BASE_DEPTH_LIST)

#ofa/utils/common_tools.py
acc_predictor_checkpoint_path = download_url(
    'https://hanlab.mit.edu/files/OnceForAll/tutorial/ofa_resnet50_acc_predictor.pth',
    model_dir='~/.ofa/',
)

device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
acc_predictor = AccuracyPredictor(arch_encoder, 400, 3,
                                  checkpoint_path=acc_predictor_checkpoint_path, device=device)

print('The accuracy predictor is ready!')
print(acc_predictor)Loaded checkpoint from /home/vitis-ai-user/.ofa/ofa_resnet50_acc_predictor.pth
The accuracy predictor is ready!
AccuracyPredictor(
  (layers): Sequential(
    (0): Sequential(
      (0): Linear(in_features=82, out_features=400, bias=True)
      (1): ReLU(inplace=True)
    )
    (1): Sequential(
      (0): Linear(in_features=400, out_features=400, bias=True)
      (1): ReLU(inplace=True)
    )
    (2): Sequential(
      (0): Linear(in_features=400, out_features=400, bias=True)
      (1): ReLU(inplace=True)
    )
    (3): Linear(in_features=400, out_features=1, bias=False)
  )
)

## build efficiency predictor_____no_output_____

from ofa.nas.efficiency_predictor import ResNet50FLOPsModel

efficiency_predictor = ResNet50FLOPsModel(ofa_network)

print(efficiency_predictor)<ofa.nas.efficiency_predictor.ResNet50FLOPsModel object at 0x7fa50867ccc0>

## build evolution finder_____no_output_____

import argparse
parser = argparse.ArgumentParser()

args = parser.parse_args(args=[])
args.arch_mutate_prob = 0.1
args.resolution_mutate_prob = 0.5
args.population_size = 100
args.max_time_budget = 50
args.parent_ratio = 0.25
args.mutation_ratio = 0.5

from ofa.nas.search_algorithm import EvolutionFinder

evolution_finder = EvolutionFinder(efficiency_predictor, acc_predictor, **args.__dict__)_____no_output_____

## search best subnet_____no_output_____

# get best subnet with constraint(Mflops)
# constraint : Mega flops
_, best_info = evolution_finder.run_evolution_search(constraint=2000, verbose=True)
print(best_info)
predicted_acc, arch_dict, efficiency = best_infoSearching with constraint (2000):   0%|          | 0/50 [00:00<?, ?it/s, acc=0.816]

## save net_config for the subnet_____no_output_____

import pickle

with open('./models/net_config_resnet50_fp2000.pickle', 'wb') as handle:
    pickle.dump(arch_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
    _____no_output_____

## upload the saved subnet (when you need to load the saved model)_____no_output_____

import pickle

with open('./models/net_config_resnet50_fp2000.pickle', 'rb') as handle:
    arch_dict = pickle.load(handle)_____no_output_____

## extract subnet's weight_____no_output_____

ofa_network.set_active_subnet(**arch_dict)
subnet = ofa_network.get_active_subnet(preserve_weight=True)

from ofa.imagenet_classification.run_manager import ImagenetRunConfig
from ofa.imagenet_classification.run_manager import RunManager

run_config = ImagenetRunConfig(test_batch_size=200, n_worker=4, image_size=image_size_list, valid_size=1000)
run_manager = RunManager('.tmp/eval_subnet', subnet, run_config, init=False)

run_manager.run_config.data_provider.assign_active_img_size(arch_dict['image_size'])
run_manager.reset_running_statistics(subnet, subset_size=1000, subset_batch_size=250)

# evaluate subnet on validation dataset
_, (top1, _) = run_manager.validate(is_test=True)
#print('Test acc: %.3f,\t best_info: %s' % (top1, best_info))
print('Test acc: %.3f' % (top1))Color jitter: tf, resize_scale: 0.08, img_size: [128, 144, 160, 176, 192, 224, 240, 256]
Use MyRandomResizedCrop: [128, 144, 160, 176, 192, 224, 240, 256], 	 None sync=True, continuous=False
ResNets(
  (input_stem): ModuleList(
    (0): ConvLayer(
      (conv): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (act): ReLU(inplace=True)
    )
    (1): ConvLayer(
      (conv): Conv2d(24, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (act): ReLU(inplace=True)
    )
  )
  (max_pooling): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (blocks): ModuleList(
    (0): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(48, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(40, 40, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(40, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=1, stride=1, padding=0)
        (conv): Conv2d(48, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (1): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(168, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(40, 40, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(40, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (2): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(168, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(64, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(168, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (3): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 88, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(88, 88, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(88, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (4): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 88, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(88, 88, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(88, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (5): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(160, 160, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(160, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(336, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (6): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (7): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (8): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (9): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(160, 160, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(160, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (10): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 408, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(408, 408, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(408, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(816, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (11): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(1640, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(576, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (12): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(1640, 408, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(408, 408, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(408, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
  )
  (global_avg_pool): AdaptiveAvgPool2d(output_size=1)
  (classifier): LinearLayer(
    (linear): Linear(in_features=1640, out_features=1000, bias=True)
  )
)
Total training params: 17.05M
Total FLOPs: 2555.37M

## save the subnet's weight _____no_output_____

print(subnet)
torch.save(subnet.state_dict(), './models/resnet50_fp2000.pth', _use_new_zipfile_serialization=False)        ResNets(
  (input_stem): ModuleList(
    (0): ConvLayer(
      (conv): Conv2d(3, 24, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (act): ReLU(inplace=True)
    )
    (1): ConvLayer(
      (conv): Conv2d(24, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (act): ReLU(inplace=True)
    )
  )
  (max_pooling): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (blocks): ModuleList(
    (0): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(48, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(40, 40, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(40, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=1, stride=1, padding=0)
        (conv): Conv2d(48, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (1): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(168, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(40, 40, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(40, 168, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(168, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (2): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(168, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(64, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(168, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (3): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 88, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(88, 88, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(88, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (4): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 88, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(88, 88, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(88, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(88, 336, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (5): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(336, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(160, 160, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(160, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(336, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (6): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (7): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (8): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 208, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(208, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(208, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (9): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(160, 160, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(160, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (10): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(816, 408, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(408, 408, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(408, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): Sequential(
        (avg_pool): AvgPool2d(kernel_size=2, stride=2, padding=0)
        (conv): Conv2d(816, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (final_act): ReLU(inplace=True)
    )
    (11): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(1640, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(576, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
    (12): ResNetBottleneckBlock(
      (conv1): Sequential(
        (conv): Conv2d(1640, 408, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv2): Sequential(
        (conv): Conv2d(408, 408, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn): BatchNorm2d(408, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (act): ReLU(inplace=True)
      )
      (conv3): Sequential(
        (conv): Conv2d(408, 1640, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(1640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (downsample): IdentityLayer()
      (final_act): ReLU(inplace=True)
    )
  )
  (global_avg_pool): AdaptiveAvgPool2d(output_size=1)
  (classifier): LinearLayer(
    (linear): Linear(in_features=1640, out_features=1000, bias=True)
  )
)

## quantize the model (subnet)_____no_output_____

# find the input image size of the subnet
print(arch_dict['image_size'])192
# quantization with finetune
!python ofa_quant.py --model_name 'resnet50_fp2000' --image_size 192 --quant_mode calib --fast_finetune

# general quantization
#!python ofa_quant.py --model_name 'resnet50_fp2000' --image_size 192 --quant_mode calib --subset_len 200
[NNDCT_NOTE]: Loading NNDCT kernels...
-------- Start resnet50_fp2000 test

[NNDCT_NOTE]: Quantization calibration process start up...

[NNDCT_NOTE]: =>Quant Module is in 'cuda'.

[NNDCT_NOTE]: =>Parsing ResNets...

[NNDCT_NOTE]: =>Doing weights equalization...

[NNDCT_NOTE]: =>Quantizable module is generated.(quantize_result/ResNets.py)

[NNDCT_NOTE]: =>Get module with quantization.

[NNDCT_NOTE]: =>Preparing data for fast finetuning module parameters ...
100%|███████████████████████████████████████████| 32/32 [00:09<00:00,  3.37it/s]
100%|███████████████████████████████████████████| 32/32 [00:15<00:00,  2.08it/s]

[NNDCT_NOTE]: =>Fast finetuning module parameters for better quantization accuracy...
100%|███████████████████████████████████████████| 58/58 [04:38<00:00,  4.80s/it]

[NNDCT_NOTE]: =>Export fast finetuned parameters ...

[NNDCT_NOTE]: =>Exporting quant model parameters.(quantize_result/param.pth)
100%|███████████████████████████████████████████| 94/94 [00:46<00:00,  2.00it/s]
loss: 0.0365178
top-1 / top-5 accuracy: 77 / 93.9667

[NNDCT_NOTE]: =>Exporting quant config.(quantize_result/quant_info.json)
-------- End of resnet50_fp2000 test

## evaluate the quantized model_____no_output_____

# general evaluation
#!python ofa_quant.py --model_name 'resnet50_fp2000' --image_size 192 --quant_mode test

# evaluation with finetune
!python ofa_quant.py --model_name 'resnet50_fp2000' --image_size 192 --quant_mode test --fast_finetune
[NNDCT_NOTE]: Loading NNDCT kernels...
-------- Start resnet50_fp2000 test

[NNDCT_NOTE]: Quantization test process start up...

[NNDCT_NOTE]: =>Quant Module is in 'cuda'.

[NNDCT_NOTE]: =>Parsing ResNets...

[NNDCT_NOTE]: =>Doing weights equalization...

[NNDCT_NOTE]: =>Quantizable module is generated.(quantize_result/ResNets.py)

[NNDCT_NOTE]: =>Get module with quantization.

[NNDCT_NOTE]: =>Loading quant model parameters.(quantize_result/param.pth)
100%|███████████████████████████████████████████| 94/94 [00:26<00:00,  3.56it/s]
loss: 0.0364622
top-1 / top-5 accuracy: 77.1667 / 94.3667
-------- End of resnet50_fp2000 test

## Export the quantized model as xmodel_____no_output_____

!python ofa_quant.py --model_name 'resnet50_fp2000' --image_size 192 --quant_mode test --subset_len 1 --batch_size=1 --deploy

[NNDCT_NOTE]: Loading NNDCT kernels...
-------- Start resnet50_fp2000 test

[NNDCT_NOTE]: Quantization test process start up...

[NNDCT_NOTE]: =>Quant Module is in 'cuda'.

[NNDCT_NOTE]: =>Parsing ResNets...

[NNDCT_NOTE]: =>Doing weights equalization...

[NNDCT_NOTE]: =>Quantizable module is generated.(quantize_result/ResNets.py)

[NNDCT_NOTE]: =>Get module with quantization.
100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  6.69it/s]
loss: 0.367108
top-1 / top-5 accuracy: 100 / 100

[NNDCT_NOTE]: =>Converting to xmodel ...

[NNDCT_NOTE]: =>Successfully convert 'ResNets' to xmodel.(quantize_result/ResNets_int.xmodel)
-------- End of resnet50_fp2000 test

## Compile the model_____no_output_____

# target hardware: ZCu102
!vai_c_xir -x quantize_result/ResNets_int.xmodel -a /opt/vitis_ai/compiler/arch/DPUCZDX8G/ZCU102/arch.json -o compiled -n resnet50_fp2000
**************************************************
* VITIS_AI Compilation - Xilinx Inc.
**************************************************
[UNILOG][INFO] The compiler log will be dumped at "/tmp/vitis-ai-user/log/xcompiler-20210401-175142-10774"
[UNILOG][INFO] Target architecture: DPUCZDX8G_ISA0_B4096_MAX_BG2
[UNILOG][INFO] Compile mode: dpu
[UNILOG][INFO] Debug mode: function
[UNILOG][INFO] Target architecture: DPUCZDX8G_ISA0_B4096_MAX_BG2
[UNILOG][INFO] Graph name: ResNets, with op num: 453
[UNILOG][INFO] Begin to compile...
[UNILOG][WARNING] xir::Op{name = ResNets__ResNets_ResNetBottleneckBlock_Sequential_downsample__AvgPool2d_avg_pool__input_7, type = pool-fix}'s input and output is unchanged, so it will be removed.
[UNILOG][INFO] Total device subgraph number 3, DPU subgraph number 1
[UNILOG][INFO] Compile done.
[UNILOG][INFO] The meta json is saved to "/workspace/automl/compiled/meta.json"
[UNILOG][INFO] The compiled xmodel is saved to "/workspace/automl/compiled/resnet50_fp2000.xmodel"
[UNILOG][INFO] The compiled xmodel's md5sum is 8b91249aad96c211390ca06c9e37a5df, and been saved to "/workspace/automl/compiled/md5sum.txt"


# Notebook from jupyrdf/ipykgb
Path: notebooks/NER_Evaluation.ipynb

# Named Entity Recognition (NER)_____no_output_____NER is a task in information extraction that locates and classifies entities in a body of text. This capability can be used to identify people based on their names, locations, time, numerical values, etc. The first step in knowledge graph construction is to identify the named entities in a text and use them as anchors to build relationships off of to other entities in the graph. In this notebook, we evaluate various methods of entity extraction and justify the usage of the best performing method._____no_output_____We will use the following text example in this notebook:_____no_output_____

starwars_text = 'Darth_Vader, also known by his birth name Anakin Skywalker, is a fictional character in the Star Wars franchise. Darth Vader appears in the original film trilogy as a pivotal antagonist whose actions drive the plot, while his past as Anakin Skywalker and the story of his corruption are central to the narrative of the prequel trilogy. The character was created by George Lucas and has been portrayed by numerous actors. His appearances span the first six Star Wars films, as well as Rogue One, and his character is heavily referenced in Star Wars: The Force Awakens. He is also an important character in the Star Wars expanded universe of television series, video games, novels, literature and comic books. Originally a Jedi who was prophesied to bring balance to the Force, he falls to the dark side of the Force and serves the evil Galactic Empire at the right hand of his Sith master, Emperor Palpatine (also known as Darth Sidious).'
starwars_text_____no_output_____

# spaCy_____no_output_____

import spacy
import pandas as pd
import nltk
nlp = spacy.load('en_core_web_lg')_____no_output_____

### Small Text Example_____no_output_____

doc = nlp('darthvader is also known by his birth name anakinskywalker.')
results = pd.DataFrame(columns=['Text', 'Start', 'End', 'Label'])

for ent in doc.ents:
    results = results.append({'Text':ent.text, 'Start':ent.start_char, 'End':ent.end_char, 'Label':ent.label_}, ignore_index=True)
results_____no_output_____

### Large Text Example_____no_output_____

doc = nlp(starwars_text)
results = pd.DataFrame(columns=['Text', 'Start', 'End', 'Label'])

for ent in doc.ents:
    results = results.append({'Text':ent.text, 'Start':ent.start_char, 'End':ent.end_char, 'Label':ent.label_}, ignore_index=True)
results_____no_output_____

For larger bodies of text, spaCy does a good job of identifying named entities of various types. We can compare this performance with Stanford NER._____no_output_____# Stanford NER with NLTK Tokenizers_____no_output_____### Small Text Example_____no_output_____

sentences = nltk.sent_tokenize('Darth Vader is also known by his birth name Anakin Skywalker.')
ner_tagger = nltk.tag.StanfordNERTagger("../stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz", "../stanford-ner-2018-10-16/stanford-ner.jar")

ner_dict = {}
results = []
for sent in sentences:
    words = [token for token in nltk.word_tokenize(sent)]
    tagged = ner_tagger.tag(words)
    results += tagged

for res in results:
    ner_dict[res[0]] = res[1]
ner_dict_____no_output_____

### Large Text Example_____no_output_____

sentences = nltk.sent_tokenize(starwars_text)
ner_tagger = nltk.tag.StanfordNERTagger("../stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz", "../stanford-ner-2018-10-16/stanford-ner.jar")

ner_dict = {}
results = []
for sent in sentences:
    words = [token for token in nltk.word_tokenize(sent)]
    tagged = ner_tagger.tag(words)
    results += tagged

for res in results:
    ner_dict[res[0]] = res[1]
ner_dict_____no_output_____

# Stanford NER with spaCy Tokenizers_____no_output_____### Small Text Example_____no_output_____

nlp = spacy.lang.en.English()
nlp.add_pipe(nlp.create_pipe('sentencizer'))
doc = nlp('Darth Vader is also known by his birth name Anakin Skywalker.')
sentences = [sent.string.strip() for sent in doc.sents]_____no_output_____ner_tagger = nltk.tag.StanfordNERTagger("../stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz", "../stanford-ner-2018-10-16/stanford-ner.jar")

ner_dict = {}
results = []

nlp = spacy.lang.en.English()
tokenizer = spacy.tokenizer.Tokenizer(nlp.vocab)
for sent in sentences:
    words = [token.orth_ for token in tokenizer(sent)]
    tagged = ner_tagger.tag(words)
    results += tagged

for res in results:
    ner_dict[res[0]] = res[1]
ner_dict_____no_output_____

### Large Text Example_____no_output_____

nlp = spacy.lang.en.English()
nlp.add_pipe(nlp.create_pipe('sentencizer'))
doc = nlp(starwars_text)
sentences = [sent.string.strip() for sent in doc.sents]

ner_tagger = nltk.tag.StanfordNERTagger("../stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz", "../stanford-ner-2018-10-16/stanford-ner.jar")

ner_dict = {}
results = []

nlp = spacy.lang.en.English()
tokenizer = spacy.tokenizer.Tokenizer(nlp.vocab)
for sent in sentences:
    words = [token.orth_ for token in tokenizer(sent)]
    tagged = ner_tagger.tag(words)
    results += tagged

for res in results:
    ner_dict[res[0]] = res[1]
ner_dict_____no_output_____

Based on these results, Stanford NER plus spaCy tokenizers offers a slightly better performance on larger bodies of text as compared to using nltk tokenizers. But overall, Stanford NER does not perform as well as spaCy NER as it recogizes PERSONs but sometimes in partial fragments. Other entities that it misses out on include LOCATION, WORK_OF_ART, DATE, etc. Because of this, we opted to use spaCy NER for the knowledge graph construction as it provided a **simpler** and **better performing** interface._____no_output_____


# Notebook from singfie/Data-Driven-Optimization
Path: Finite horizon Beta-Bernoulli bandit problems.ipynb

Implementation of finite horizon Beta-Bernoulli bandit problems
1. Greedy sampling
2. Thompson sampling_____no_output_____

<code>
import numpy as np_____no_output_____def greedy_epsilon_sampling(probabilities, N, T, epsilon):

    # initialize

    # define play # holder
    K = np.zeros([N+1,T]).astype(int) # no of arms is rows + 1, no of stages is columns
    K[0:,0] = 0

    # define probability estimate holder
    phat = np.zeros([N+1,T]) # no of arms is rows, no of stages is columns
    alphas = np.zeros([N+1,T])
    betas = np.zeros([N+1,T])

    # initialize
    alphas[0:,0] = np.array([0,1,1,1]) #np.concatenate(([0], np.random.randint(1,10,N)), axis = 0)
    betas[0:,0] = np.array([0,1,1,1]) # np.concatenate(([0], np.random.randint(1,10,N)), axis = 0)

    # populate arms with initial estimates
    for t in range(T-1):

        for i in range(1,N+1):
            phat[i,t] = (alphas[i,t])/(alphas[i,t] + betas[i,t]) # assign probabilities to arms

        # flip a coin to decide whether to explore or exploit
        decider = np.random.binomial(size = 1, n = 1, p = epsilon)
        #print(decider)

        # choose index of arm to pull
        if decider == 1:
            # if true play random arm
            at = np.random.randint(1,N+1,1)[0]
            #print("Explore new arm:", at)
        else:
            # if false play the arm with the highest probability
            at = np.random.choice(np.where(phat[:,t] == np.amax(phat[:,t]))[0]) # tie braker rule: random

        # update pulled arm
        zt = np.random.binomial(size=1, n=1, p = probabilities[at-1])
        alphas[at,t+1] = alphas[at,t] + zt
        betas[at,t+1] = betas[at,t] + (1 - zt)
        K[at,t] = K[at, t-1] + 1

        # update non-pulled arms
        for i in range(1,N+1):
            if i != at:
                alphas[i,t+1] = alphas[i,t]
                betas[i,t+1] = betas[i,t]
                K[i,t] = K[i, t-1]

    return(K)_____no_output_____def Thompson_sampling(probabilities, N, T):

    # initialize

    # define play # holder
    K = np.zeros([N+1,T]).astype(int) # no of arms is rows + 1, no of stages is columns
    K[0:,0] = 0

    # define probability estimate holder
    phat = np.zeros([N+1,T]) # no of arms is rows, no of stages is columns
    alphas = np.zeros([N+1,T])
    betas = np.zeros([N+1,T])

    # initialize
    alphas[0:,0] = np.array([0,1,1,1]) #np.concatenate(([0], np.random.randint(1,10,N)), axis = 0)
    betas[0:,0] = np.array([0,1,1,1]) # np.concatenate(([0], np.random.randint(1,10,N)), axis = 0)

    # populate arms with initial estimates
    for t in range(T-1):

        for i in range(1,N+1):
            #print("alpha", np.absolute(alphas[i,t]))
            #print("beta", np.absolute(betas[i,t]))
            phat[i,t] = np.random.beta(np.absolute(alphas[i,t]), np.absolute(betas[i,t]), size = 1) # assign probabilities to arms

        # if false play the arm with the highest probability
        at = np.random.choice(np.where(phat[:,t] == np.amax(phat[:,t]))[0]) # break ties randomly

        # update pulled arm
        zt = np.random.binomial(size=1, n=1, p = probabilities[at-1])
        alphas[at,t+1] = alphas[at,t] + zt
        betas[at,t+1] = betas[at,t] + (1 - zt)
        K[at,t] = K[at, t-1] + 1

        # update non-pulled arms
        for i in range(1,N+1):
            if i != at:
                alphas[i,t+1] = alphas[i,t]
                betas[i,t+1] = betas[i,t]
                K[i,t] = K[i, t-1]

    return(K)_____no_output_____N = 3
T = 1000
probabilities = np.array([0.9, 0.6, 0.3])_____no_output_____M = 10000
for i in range(M):
    np.random.seed(i)
    if i == 0:
        GE0 = greedy_epsilon_sampling(probabilities, N, T, 0) # greedy
        GE001 = greedy_epsilon_sampling(probabilities, N, T, 0.01) # epsilon greedy with 001
        GE01 = greedy_epsilon_sampling(probabilities, N, T, 0.1) # espilon greedy with 0.1
        TK = Thompson_sampling(probabilities, N, T)

    else:
        GE0 = GE0 + greedy_epsilon_sampling(probabilities, N, T, 0) # greedy
        GE001 = GE001 + greedy_epsilon_sampling(probabilities, N, T, 0.01) # greedy wiht 0.01
        GE01 = GE01 + greedy_epsilon_sampling(probabilities, N, T, 0.1) # epsilon greedy with 0.1
        TK = TK + Thompson_sampling(probabilities, N, T)

    #print(i)_____no_output_____greedy0K = np.true_divide(GE0, M)
greedy001K = np.true_divide(GE001, M)
greedy01K = np.true_divide(GE01, M)
ThompsonK = np.true_divide(TK, M)_____no_output_____greedy0K = np.hstack((np.array([[0,0,0,0]]).T, greedy0K))
greedy001K = np.hstack((np.array([[0,0,0,0]]).T, greedy001K))
greedy01K = np.hstack((np.array([[0,0,0,0]]).T, greedy01K))
ThompsonK = np.hstack((np.array([[0,0,0,0]]).T, ThompsonK))_____no_output_____greedy0K[0,:] = range(T+1)
greedy001K[0,:] = range(T+1)
greedy01K[0,:] = range(T+1)
ThompsonK[0,:] = range(T+1)_____no_output_____from matplotlib import pyplot as plt

plt.style.use('seaborn-whitegrid')

plt.figure(figsize=[12,5])

arm1 = greedy0K[1,:T]/greedy0K[0,:T]
arm2 = greedy0K[2,:T]/greedy0K[0,:T]
arm3 = greedy0K[3,:T]/greedy0K[0,:T]
t = range(T)

plot1 = plt.subplot(121)
plt.plot(t, arm1, label = 'arm 1')
plt.plot(t, arm2, label = 'arm 2')
plt.plot(t, arm3, label = 'arm 3')
plt.xlabel("Stage")
plt.ylabel("Fraction of pulls over 10000 reps")
plt.legend(loc="center right")
plot1.set_title("Evolution of arm pulls for greedy sampling")

arm11 = greedy001K[1,:T]/greedy001K[0,:T]
arm21 = greedy001K[2,:T]/greedy001K[0,:T]
arm31 = greedy001K[3,:T]/greedy001K[0,:T]
# t = range(T)

plot2 = plt.subplot(122)
plt.plot(t, arm11, label = 'arm 1')
plt.plot(t, arm21, label = 'arm 2')
plt.plot(t, arm31, label = 'arm 3')
plt.xlabel("Stage")
plt.ylabel("Fraction of pulls over 10000 reps")
plt.legend(loc="center right")
plot2.set_title("Evolution of arm pulls for epsilon greedy sampling at epsilon = 0.01")

plt.savefig('Q5 Results 1.png', dpi = 800)

plt.show()

greedy0K[1,:]
greedy0K[0,:]/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: RuntimeWarning: invalid value encountered in true_divide
  import sys
/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:8: RuntimeWarning: invalid value encountered in true_divide

/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:9: RuntimeWarning: invalid value encountered in true_divide
  if __name__ == '__main__':
/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:21: RuntimeWarning: invalid value encountered in true_divide
/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:22: RuntimeWarning: invalid value encountered in true_divide
/Users/fietekrutein/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:23: RuntimeWarning: invalid value encountered in true_divide
from matplotlib import pyplot as plt

plt.style.use('seaborn-whitegrid')

plt.figure(figsize=[12,5])

arm1 = greedy01K[1,1:T-1]/greedy01K[0,1:T-1]
arm2 = greedy01K[2,1:T-1]/greedy01K[0,1:T-1]
arm3 = greedy01K[3,1:T-1]/greedy01K[0,1:T-1]
t = range(T-2)

plot1 = plt.subplot(121)
plt.plot(t, arm1, label = 'arm 1')
plt.plot(t, arm2, label = 'arm 2')
plt.plot(t, arm3, label = 'arm 3')
plt.xlabel("Stage")
plt.ylabel("Fraction of pulls over 10000 reps")
plt.legend(loc="center right")
plot1.set_title("Evolution of arm pulls for greedy sampling at epsilon = 0.1")

arm11 = ThompsonK[1,1:T-1]/ThompsonK[0,1:T-1]
arm21 = ThompsonK[2,1:T-1]/ThompsonK[0,1:T-1]
arm31 = ThompsonK[3,1:T-1]/ThompsonK[0,1:T-1]
# t = range(T)

plot2 = plt.subplot(122)
plt.plot(t, arm11, label = 'arm 1')
plt.plot(t, arm21, label = 'arm 2')
plt.plot(t, arm31, label = 'arm 3')
plt.xlabel("Stage")
plt.ylabel("Fraction of pulls over 10000 reps")
plt.legend(loc="center right")
plot2.set_title("Evolution of arm pulls for Thompson sampling")

plt.savefig('Q5 Results 2.png', dpi = 800)

plt.show()_____no_output_____
</code>


# Notebook from trevordavid/case-studies
Path: docs/tutorials/joint.ipynb

(joint)=

# Joint RV & transit fits_____no_output_____

import exoplanet

exoplanet.utils.docs_setup()
print(f"exoplanet.__version__ = '{exoplanet.__version__}'")_____no_output_____

In this tutorial, we will combine many of the previous tutorials to perform a fit of the K2-24 system using the K2 transit data and the RVs from [Petigura et al. (2016)](https://arxiv.org/abs/1511.04497).
This is the same system that we fit in {ref}`rv` and we'll combine that model with the transit model from {ref}`transit` and the Gaussian Process noise model from {ref}`stellar-variability`.

## Datasets and initializations

To get started, let's download the relevant datasets.
First, the transit light curve from [Everest](https://rodluger.github.io/everest/):_____no_output_____

import numpy as np
import matplotlib.pyplot as plt

from astropy.io import fits
from scipy.signal import savgol_filter

# Download the data
lc_url = "https://archive.stsci.edu/hlsps/everest/v2/c02/203700000/71098/hlsp_everest_k2_llc_203771098-c02_kepler_v2.0_lc.fits"
with fits.open(lc_url) as hdus:
    lc = hdus[1].data
    lc_hdr = hdus[1].header

# Work out the exposure time
texp = lc_hdr["FRAMETIM"] * lc_hdr["NUM_FRM"]
texp /= 60.0 * 60.0 * 24.0

# Mask bad data
m = (
    (np.arange(len(lc)) > 100)
    & np.isfinite(lc["FLUX"])
    & np.isfinite(lc["TIME"])
)
bad_bits = [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17]
qual = lc["QUALITY"]
for b in bad_bits:
    m &= qual & 2 ** (b - 1) == 0

# Convert to parts per thousand
x = lc["TIME"][m]
y = lc["FLUX"][m]
mu = np.median(y)
y = (y / mu - 1) * 1e3

# Identify outliers
m = np.ones(len(y), dtype=bool)
for i in range(10):
    y_prime = np.interp(x, x[m], y[m])
    smooth = savgol_filter(y_prime, 101, polyorder=3)
    resid = y - smooth
    sigma = np.sqrt(np.mean(resid ** 2))
    m0 = np.abs(resid) < 3 * sigma
    if m.sum() == m0.sum():
        m = m0
        break
    m = m0

# Only discard positive outliers
m = resid < 3 * sigma

# Shift the data so that the K2 data start at t=0. This tends to make the fit
# better behaved since t0 covaries with period.
x_ref = np.min(x[m])
x -= x_ref

# Plot the data
plt.plot(x, y, "k", label="data")
plt.plot(x, smooth)
plt.plot(x[~m], y[~m], "xr", label="outliers")
plt.legend(fontsize=12)
plt.xlim(x.min(), x.max())
plt.xlabel("time")
plt.ylabel("flux")

# Make sure that the data type is consistent
x = np.ascontiguousarray(x[m], dtype=np.float64)
y = np.ascontiguousarray(y[m], dtype=np.float64)
smooth = np.ascontiguousarray(smooth[m], dtype=np.float64)_____no_output_____

Then the RVs from [RadVel](https://radvel.readthedocs.io):_____no_output_____

import pandas as pd

url = "https://raw.githubusercontent.com/California-Planet-Search/radvel/master/example_data/epic203771098.csv"
data = pd.read_csv(url, index_col=0)

# Don't forget to remove the time offset from above!
x_rv = np.array(data.t) - x_ref
y_rv = np.array(data.vel)
yerr_rv = np.array(data.errvel)

plt.errorbar(x_rv, y_rv, yerr=yerr_rv, fmt=".k")
plt.xlabel("time [days]")
_ = plt.ylabel("radial velocity [m/s]")_____no_output_____

We can initialize the transit parameters using [the box least squares periodogram from AstroPy](http://docs.astropy.org/en/latest/timeseries/bls.html).
(Note: you'll need AstroPy v3.1 or more recent to use this feature.)
A full discussion of transit detection and vetting is beyond the scope of this tutorial so let's assume that we know that there are two periodic transiting planets in this dataset._____no_output_____

from astropy.timeseries import BoxLeastSquares

m = np.zeros(len(x), dtype=bool)
period_grid = np.exp(np.linspace(np.log(5), np.log(50), 50000))
bls_results = []
periods = []
t0s = []
depths = []

# Compute the periodogram for each planet by iteratively masking out
# transits from the higher signal to noise planets. Here we're assuming
# that we know that there are exactly two planets.
for i in range(2):
    bls = BoxLeastSquares(x[~m], y[~m] - smooth[~m])
    bls_power = bls.power(period_grid, 0.1, oversample=20)
    bls_results.append(bls_power)

    # Save the highest peak as the planet candidate
    index = np.argmax(bls_power.power)
    periods.append(bls_power.period[index])
    t0s.append(bls_power.transit_time[index])
    depths.append(bls_power.depth[index])

    # Mask the data points that are in transit for this candidate
    m |= bls.transit_mask(x, periods[-1], 0.5, t0s[-1])_____no_output_____

Let's plot the initial transit estimates based on these periodograms:_____no_output_____

fig, axes = plt.subplots(len(bls_results), 2, figsize=(15, 10))

for i in range(len(bls_results)):
    # Plot the periodogram
    ax = axes[i, 0]
    ax.axvline(np.log10(periods[i]), color="C1", lw=5, alpha=0.8)
    ax.plot(np.log10(bls_results[i].period), bls_results[i].power, "k")
    ax.annotate(
        "period = {0:.4f} d".format(periods[i]),
        (0, 1),
        xycoords="axes fraction",
        xytext=(5, -5),
        textcoords="offset points",
        va="top",
        ha="left",
        fontsize=12,
    )
    ax.set_ylabel("bls power")
    ax.set_yticks([])
    ax.set_xlim(np.log10(period_grid.min()), np.log10(period_grid.max()))
    if i < len(bls_results) - 1:
        ax.set_xticklabels([])
    else:
        ax.set_xlabel("log10(period)")

    # Plot the folded transit
    ax = axes[i, 1]
    p = periods[i]
    x_fold = (x - t0s[i] + 0.5 * p) % p - 0.5 * p
    m = np.abs(x_fold) < 0.4
    ax.plot(x_fold[m], y[m] - smooth[m], ".k")

    # Overplot the phase binned light curve
    bins = np.linspace(-0.41, 0.41, 32)
    denom, _ = np.histogram(x_fold, bins)
    num, _ = np.histogram(x_fold, bins, weights=y - smooth)
    denom[num == 0] = 1.0
    ax.plot(0.5 * (bins[1:] + bins[:-1]), num / denom, color="C1")

    ax.set_xlim(-0.4, 0.4)
    ax.set_ylabel("relative flux [ppt]")
    if i < len(bls_results) - 1:
        ax.set_xticklabels([])
    else:
        ax.set_xlabel("time since transit")

_ = fig.subplots_adjust(hspace=0.02)_____no_output_____

The discovery paper for K2-24 ([Petigura et al. (2016)](https://arxiv.org/abs/1511.04497)) includes the following estimates of the stellar mass and radius in Solar units:_____no_output_____

M_star_petigura = 1.12, 0.05
R_star_petigura = 1.21, 0.11_____no_output_____

Finally, using this stellar mass, we can also estimate the minimum masses of the planets given these transit parameters._____no_output_____

import exoplanet as xo
import astropy.units as u

msini = xo.estimate_minimum_mass(
    periods, x_rv, y_rv, yerr_rv, t0s=t0s, m_star=M_star_petigura[0]
)
msini = msini.to(u.M_earth)
print(msini)_____no_output_____

## A joint transit and radial velocity model in PyMC3

Now, let's define our full model in *PyMC3*.
There's a lot going on here, but I've tried to comment it and most of it should be familiar from the other tutorials and case studies.
In this case, I've put the model inside a model "factory" function because we'll do some sigma clipping below._____no_output_____

import pymc3 as pm
import aesara_theano_fallback.tensor as tt

import pymc3_ext as pmx
from celerite2.theano import terms, GaussianProcess

# These arrays are used as the times/phases where the models are
# evaluated at higher resolution for plotting purposes
t_rv = np.linspace(x_rv.min() - 5, x_rv.max() + 5, 500)
phase_lc = np.linspace(-0.3, 0.3, 100)

def build_model(mask=None, start=None):
    if mask is None:
        mask = np.ones(len(x), dtype=bool)
    with pm.Model() as model:

        # Parameters for the stellar properties
        mean = pm.Normal("mean", mu=0.0, sd=10.0)
        u_star = xo.QuadLimbDark("u_star")
        star = xo.LimbDarkLightCurve(u_star)
        BoundedNormal = pm.Bound(pm.Normal, lower=0, upper=3)
        m_star = BoundedNormal(
            "m_star", mu=M_star_petigura[0], sd=M_star_petigura[1]
        )
        r_star = BoundedNormal(
            "r_star", mu=R_star_petigura[0], sd=R_star_petigura[1]
        )

        # Orbital parameters for the planets
        t0 = pm.Normal("t0", mu=np.array(t0s), sd=1, shape=2)
        log_m_pl = pm.Normal("log_m_pl", mu=np.log(msini.value), sd=1, shape=2)
        log_period = pm.Normal("log_period", mu=np.log(periods), sd=1, shape=2)

        # Fit in terms of transit depth (assuming b<1)
        b = pm.Uniform("b", lower=0, upper=1, shape=2)
        log_depth = pm.Normal(
            "log_depth", mu=np.log(depths), sigma=2.0, shape=2
        )
        ror = pm.Deterministic(
            "ror",
            star.get_ror_from_approx_transit_depth(
                1e-3 * tt.exp(log_depth), b
            ),
        )
        r_pl = pm.Deterministic("r_pl", ror * r_star)

        m_pl = pm.Deterministic("m_pl", tt.exp(log_m_pl))
        period = pm.Deterministic("period", tt.exp(log_period))

        ecs = pmx.UnitDisk("ecs", shape=(2, 2), testval=0.01 * np.ones((2, 2)))
        ecc = pm.Deterministic("ecc", tt.sum(ecs ** 2, axis=0))
        omega = pm.Deterministic("omega", tt.arctan2(ecs[1], ecs[0]))
        xo.eccentricity.vaneylen19(
            "ecc_prior", multi=True, shape=2, fixed=True, observed=ecc
        )

        # RV jitter & a quadratic RV trend
        log_sigma_rv = pm.Normal(
            "log_sigma_rv", mu=np.log(np.median(yerr_rv)), sd=5
        )
        trend = pm.Normal(
            "trend", mu=0, sd=10.0 ** -np.arange(3)[::-1], shape=3
        )

        # Transit jitter & GP parameters
        log_sigma_lc = pm.Normal(
            "log_sigma_lc", mu=np.log(np.std(y[mask])), sd=10
        )
        log_rho_gp = pm.Normal("log_rho_gp", mu=0.0, sd=10)
        log_sigma_gp = pm.Normal(
            "log_sigma_gp", mu=np.log(np.std(y[mask])), sd=10
        )

        # Orbit models
        orbit = xo.orbits.KeplerianOrbit(
            r_star=r_star,
            m_star=m_star,
            period=period,
            t0=t0,
            b=b,
            m_planet=xo.units.with_unit(m_pl, msini.unit),
            ecc=ecc,
            omega=omega,
        )

        # Compute the model light curve
        light_curves = (
            star.get_light_curve(orbit=orbit, r=r_pl, t=x[mask], texp=texp)
            * 1e3
        )
        light_curve = pm.math.sum(light_curves, axis=-1) + mean
        resid = y[mask] - light_curve

        # GP model for the light curve
        kernel = terms.SHOTerm(
            sigma=tt.exp(log_sigma_gp),
            rho=tt.exp(log_rho_gp),
            Q=1 / np.sqrt(2),
        )
        gp = GaussianProcess(kernel, t=x[mask], yerr=tt.exp(log_sigma_lc))
        gp.marginal("transit_obs", observed=resid)

        # And then include the RVs as in the RV tutorial
        x_rv_ref = 0.5 * (x_rv.min() + x_rv.max())

        def get_rv_model(t, name=""):
            # First the RVs induced by the planets
            vrad = orbit.get_radial_velocity(t)
            pm.Deterministic("vrad" + name, vrad)

            # Define the background model
            A = np.vander(t - x_rv_ref, 3)
            bkg = pm.Deterministic("bkg" + name, tt.dot(A, trend))

            # Sum over planets and add the background to get the full model
            return pm.Deterministic(
                "rv_model" + name, tt.sum(vrad, axis=-1) + bkg
            )

        # Define the model
        rv_model = get_rv_model(x_rv)
        get_rv_model(t_rv, name="_pred")

        # The likelihood for the RVs
        err = tt.sqrt(yerr_rv ** 2 + tt.exp(2 * log_sigma_rv))
        pm.Normal("obs", mu=rv_model, sd=err, observed=y_rv)

        # Compute and save the phased light curve models
        pm.Deterministic(
            "lc_pred",
            1e3
            * tt.stack(
                [
                    star.get_light_curve(
                        orbit=orbit, r=r_pl, t=t0[n] + phase_lc, texp=texp
                    )[..., n]
                    for n in range(2)
                ],
                axis=-1,
            ),
        )

        # Fit for the maximum a posteriori parameters, I've found that I can get
        # a better solution by trying different combinations of parameters in turn
        if start is None:
            start = model.test_point
        map_soln = pmx.optimize(start=start, vars=[trend])
        map_soln = pmx.optimize(start=map_soln, vars=[log_sigma_lc])
        map_soln = pmx.optimize(start=map_soln, vars=[log_depth, b])
        map_soln = pmx.optimize(start=map_soln, vars=[log_period, t0])
        map_soln = pmx.optimize(
            start=map_soln, vars=[log_sigma_lc, log_sigma_gp]
        )
        map_soln = pmx.optimize(start=map_soln, vars=[log_rho_gp])
        map_soln = pmx.optimize(start=map_soln)

        extras = dict(
            zip(
                ["light_curves", "gp_pred"],
                pmx.eval_in_model([light_curves, gp.predict(resid)], map_soln),
            )
        )

    return model, map_soln, extras

model0, map_soln0, extras0 = build_model()_____no_output_____

Now let's plot the map radial velocity model._____no_output_____

def plot_rv_curve(soln):
    fig, axes = plt.subplots(2, 1, figsize=(10, 5), sharex=True)

    ax = axes[0]
    ax.errorbar(x_rv, y_rv, yerr=yerr_rv, fmt=".k")
    ax.plot(t_rv, soln["vrad_pred"], "--k", alpha=0.5)
    ax.plot(t_rv, soln["bkg_pred"], ":k", alpha=0.5)
    ax.plot(t_rv, soln["rv_model_pred"], label="model")
    ax.legend(fontsize=10)
    ax.set_ylabel("radial velocity [m/s]")

    ax = axes[1]
    err = np.sqrt(yerr_rv ** 2 + np.exp(2 * soln["log_sigma_rv"]))
    ax.errorbar(x_rv, y_rv - soln["rv_model"], yerr=err, fmt=".k")
    ax.axhline(0, color="k", lw=1)
    ax.set_ylabel("residuals [m/s]")
    ax.set_xlim(t_rv.min(), t_rv.max())
    ax.set_xlabel("time [days]")

_ = plot_rv_curve(map_soln0)_____no_output_____

That looks pretty similar to what we got in {ref}`rv`.
Now let's also plot the transit model._____no_output_____

def plot_light_curve(soln, extras, mask=None):
    if mask is None:
        mask = np.ones(len(x), dtype=bool)

    fig, axes = plt.subplots(3, 1, figsize=(10, 7), sharex=True)

    ax = axes[0]
    ax.plot(x[mask], y[mask], "k", label="data")
    gp_mod = extras["gp_pred"] + soln["mean"]
    ax.plot(x[mask], gp_mod, color="C2", label="gp model")
    ax.legend(fontsize=10)
    ax.set_ylabel("relative flux [ppt]")

    ax = axes[1]
    ax.plot(x[mask], y[mask] - gp_mod, "k", label="de-trended data")
    for i, l in enumerate("bc"):
        mod = extras["light_curves"][:, i]
        ax.plot(x[mask], mod, label="planet {0}".format(l))
    ax.legend(fontsize=10, loc=3)
    ax.set_ylabel("de-trended flux [ppt]")

    ax = axes[2]
    mod = gp_mod + np.sum(extras["light_curves"], axis=-1)
    ax.plot(x[mask], y[mask] - mod, "k")
    ax.axhline(0, color="#aaaaaa", lw=1)
    ax.set_ylabel("residuals [ppt]")
    ax.set_xlim(x[mask].min(), x[mask].max())
    ax.set_xlabel("time [days]")

    return fig

_ = plot_light_curve(map_soln0, extras0)_____no_output_____

There are still a few outliers in the light curve and it can be useful to remove those before doing the full fit because both the GP and transit parameters can be sensitive to this.

## Sigma clipping

To remove the outliers, we'll look at the empirical RMS of the residuals away from the GP + transit model and remove anything that is more than a 7-sigma outlier._____no_output_____

mod = (
    extras0["gp_pred"]
    + map_soln0["mean"]
    + np.sum(extras0["light_curves"], axis=-1)
)
resid = y - mod
rms = np.sqrt(np.median(resid ** 2))
mask = np.abs(resid) < 7 * rms

plt.plot(x, resid, "k", label="data")
plt.plot(x[~mask], resid[~mask], "xr", label="outliers")
plt.axhline(0, color="#aaaaaa", lw=1)
plt.ylabel("residuals [ppt]")
plt.xlabel("time [days]")
plt.legend(fontsize=12, loc=4)
_ = plt.xlim(x.min(), x.max())_____no_output_____

That looks better. Let's re-build our model with this sigma-clipped dataset._____no_output_____

model, map_soln, extras = build_model(mask, map_soln0)
_ = plot_light_curve(map_soln, extras, mask)_____no_output_____

Great! Now we're ready to sample.

## Sampling

The sampling for this model is the same as for all the previous tutorials, but it takes a bit longer.
This is partly because the model is more expensive to compute than the previous ones and partly because there are some non-affine degeneracies in the problem (for example between impact parameter, eccentricity, and radius/radius ratio).
It might be worth thinking about reparameterizations (in terms of duration instead of eccentricity), but that's beyond the scope of this tutorial.
Besides, using more traditional MCMC methods, this would have taken a lot longer to get thousands of effective samples!_____no_output_____

import multiprocessing

with model:
    trace = pm.sample(
        tune=1500,
        draws=1000,
        start=map_soln,
        cores=2,
        chains=2,
        target_accept=0.95,
        return_inferencedata=True,
        random_seed=[203771098, 203775000],
        mp_ctx=multiprocessing.get_context("fork"),
        init="adapt_full",
    )_____no_output_____

Let's look at the convergence diagnostics for some of the key parameters:_____no_output_____

import arviz as az

az.summary(
    trace,
    var_names=[
        "period",
        "r_pl",
        "m_pl",
        "ecc",
        "omega",
        "b",
        "log_sigma_gp",
        "log_rho_gp",
    ],
)_____no_output_____

As you see, the effective number of samples for the impact parameters and eccentricites are lower than for the other parameters.
This is because of the correlations that I mentioned above:_____no_output_____

import corner

_ = corner.corner(trace, var_names=["b", "ecc", "r_pl"])_____no_output_____

## Phase plots

Finally, we can make folded plots of the transits and the radial velocities and compare to the posterior model predictions. (Note: planets b and c in this tutorial are swapped compared to the labels from [Petigura et al. (2016)](https://arxiv.org/abs/1511.04497))_____no_output_____

flat_samps = trace.posterior.stack(sample=("chain", "draw"))
gp_mod = extras["gp_pred"] + map_soln["mean"]

for n, letter in enumerate("bc"):
    plt.figure()

    # Get the posterior median orbital parameters
    p = np.median(flat_samps["period"][n])
    t0 = np.median(flat_samps["t0"][n])

    # Plot the folded data
    x_fold = (x[mask] - t0 + 0.5 * p) % p - 0.5 * p
    m = np.abs(x_fold) < 0.3
    plt.plot(
        x_fold[m], y[mask][m] - gp_mod[m], ".k", label="data", zorder=-1000
    )

    # Plot the folded model
    pred = np.percentile(flat_samps["lc_pred"][:, n, :], [16, 50, 84], axis=-1)
    plt.plot(phase_lc, pred[1], color="C1", label="model")
    art = plt.fill_between(
        phase_lc, pred[0], pred[2], color="C1", alpha=0.5, zorder=1000
    )
    art.set_edgecolor("none")

    # Annotate the plot with the planet's period
    txt = "period = {0:.4f} +/- {1:.4f} d".format(
        np.mean(flat_samps["period"][n].values),
        np.std(flat_samps["period"][n].values),
    )
    plt.annotate(
        txt,
        (0, 0),
        xycoords="axes fraction",
        xytext=(5, 5),
        textcoords="offset points",
        ha="left",
        va="bottom",
        fontsize=12,
    )

    plt.legend(fontsize=10, loc=4)
    plt.xlabel("time since transit [days]")
    plt.ylabel("de-trended flux")
    plt.title("K2-24{0}".format(letter))
    plt.xlim(-0.3, 0.3)_____no_output_____for n, letter in enumerate("bc"):
    plt.figure()

    # Get the posterior median orbital parameters
    p = np.median(flat_samps["period"][n])
    t0 = np.median(flat_samps["t0"][n])

    # Compute the median of posterior estimate of the background RV
    # and the contribution from the other planet. Then we can remove
    # this from the data to plot just the planet we care about.
    other = np.median(flat_samps["vrad"][:, (n + 1) % 2], axis=-1)
    other += np.median(flat_samps["bkg"], axis=-1)

    # Plot the folded data
    x_fold = (x_rv - t0 + 0.5 * p) % p - 0.5 * p
    plt.errorbar(x_fold, y_rv - other, yerr=yerr_rv, fmt=".k", label="data")

    # Compute the posterior prediction for the folded RV model for this
    # planet
    t_fold = (t_rv - t0 + 0.5 * p) % p - 0.5 * p
    inds = np.argsort(t_fold)
    pred = np.percentile(
        flat_samps["vrad_pred"][inds, n], [16, 50, 84], axis=-1
    )
    plt.plot(t_fold[inds], pred[1], color="C1", label="model")
    art = plt.fill_between(
        t_fold[inds], pred[0], pred[2], color="C1", alpha=0.3
    )
    art.set_edgecolor("none")

    plt.legend(fontsize=10)
    plt.xlim(-0.5 * p, 0.5 * p)
    plt.xlabel("phase [days]")
    plt.ylabel("radial velocity [m/s]")
    plt.title("K2-24{0}".format(letter))_____no_output_____

We can also compute the posterior constraints on the planet densities._____no_output_____

volume = 4 / 3 * np.pi * flat_samps["r_pl"].values ** 3
density = u.Quantity(
    flat_samps["m_pl"].values / volume, unit=u.M_earth / u.R_sun ** 3
)
density = density.to(u.g / u.cm ** 3).value

bins = np.linspace(0, 1.1, 45)
for n, letter in enumerate("bc"):
    plt.hist(
        density[n],
        bins,
        histtype="step",
        lw=2,
        label="K2-24{0}".format(letter),
        density=True,
    )
plt.yticks([])
plt.legend(fontsize=12)
plt.xlim(bins[0], bins[-1])
plt.xlabel("density [g/cc]")
_ = plt.ylabel("posterior density")_____no_output_____

## Citations

As described in the [citation tutorial](https://docs.exoplanet.codes/en/stable/tutorials/citation/), we can use [citations.get_citations_for_model](https://docs.exoplanet.codes/en/stable/user/api/#exoplanet.citations.get_citations_for_model) to construct an acknowledgement and BibTeX listing that includes the relevant citations for this model._____no_output_____

with model:
    txt, bib = xo.citations.get_citations_for_model()
print(txt)_____no_output_____print(bib.split("\n\n")[0] + "\n\n...")_____no_output_____


# Notebook from JanGin/pydata-book-learning
Path: pydata-learning/ch7.ipynb

# 《利用Python进行数据分析》1st Edition chapter 7_____no_output_____

import numpy as np
import pandas as pd
from pandas import Series, DataFrame_____no_output_____

## 数据库风格的DataFrame 合并_____no_output_____

df1 = DataFrame({'key':['b','b','a','c','a','a','b'], 'data1':np.arange(7)})
df2 = DataFrame({'key':['a','b','c'], 'data2':np.arange(3)})_____no_output_____df1_____no_output_____df2_____no_output_____pd.merge(df1, df2)_____no_output_____

显式指定合并时用于连接的列， merge默认将重叠列的列名当做键_____no_output_____

pd.merge(df1, df2, on='key')_____no_output_____

merge默认是作 "inner"连接_____no_output_____

df3 = DataFrame({'lkey':['b','b','a','c','a','a','b'], 'data1':np.arange(7)})
df4 = DataFrame({'rkey':['a','b','d'], 'data2':np.arange(3)})_____no_output_____pd.merge(df3, df4, left_on='lkey', right_on='rkey')_____no_output_____df3_____no_output_____df4_____no_output_____

指定使用外连接_____no_output_____

pd.merge(df1, df2, how='outer')_____no_output_____pd.merge(df1, df2, on= 'key', how='left')_____no_output_____df5 = DataFrame({'key':['b','b','a','c','a','b'], 'data1':np.arange(6)})
df6 = DataFrame({'key':['a','b','a','b','d'], 'data2':np.arange(5)})_____no_output_____df5_____no_output_____df6_____no_output_____pd.merge(df5, df6, on= 'key', how='left')_____no_output_____pd.merge(df5, df6, on= 'key', how='inner')_____no_output_____left = DataFrame({'key1': ['foo', 'foo', 'bar'],
                  'key2': ['one', 'two', 'one'],
                  'lval': [1, 2, 3]})
right = DataFrame({'key1': ['foo', 'foo', 'bar', 'bar'],
                   'key2': ['one', 'one', 'one', 'two'],
                   'rval': [4, 5, 6, 7]})
pd.merge(left, right, on=['key1', 'key2'], how='outer')_____no_output_____pd.merge(left, right, on='key1')_____no_output_____

处理列名重复问题，使用 suffixes 选项_____no_output_____

pd.merge(left, right, on='key1', suffixes=('_left', '_right'))_____no_output_____

显然，没有prefixes 选项_____no_output_____

pd.merge(left, right, on='key1', prefixes=('left_', 'right_'))_____no_output_____

## 索引上的合并_____no_output_____

left1 = DataFrame({'key': ['a', 'b', 'a', 'a', 'b', 'c'],
                  'value': np.arange(6)})
right1 = DataFrame({'group_val': [3.5, 7]}, index=['a', 'b'])_____no_output_____left1_____no_output_____right1_____no_output_____pd.merge(left1, right1, left_on='key', right_index=True)_____no_output_____pd.merge(left1, right1, left_on='key', right_on='group_val')_____no_output_____

默认的merge方法是求取连接键的交集，若想获取并集，则应使用外连接的方式_____no_output_____

pd.merge(left1, right1, left_on='key', right_index=True, how='outer')_____no_output_____lefth = DataFrame({'key1': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],
                   'key2': [2000, 2001, 2002, 2001, 2002],
                   'data': np.arange(5.)})
righth = DataFrame(np.arange(12).reshape((6, 2)),
                   index=[['Nevada', 'Nevada', 'Ohio', 'Ohio', 'Ohio', 'Ohio'],
                          [2001, 2000, 2000, 2000, 2001, 2002]],
                   columns=['event1', 'event2'])_____no_output_____lefth_____no_output_____lefth=DataFrame({'key1': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],
                   'key2': [2000, 2001, 2002, 2001, 2002],
                   'data': np.arange(5.)}, columns=['key1', 'key2', 'data'])
lefth_____no_output_____righth_____no_output_____# 使用lefth的 key列表 和 righth 的索引列 进行连接_____no_output_____pd.merge(lefth, righth, left_on=['key1', 'key2'], right_index=True)_____no_output_____pd.merge(lefth, righth, left_on=['key1', 'key2'], right_index=True, how='outer')_____no_output_____left2 = DataFrame([[1., 2.], [3., 4.], [5., 6.]], index=['a', 'c', 'e'],
                 columns=['Ohio', 'Nevada'])
right2 = DataFrame([[7., 8.], [9., 10.], [11., 12.], [13, 14]],
                   index=['b', 'c', 'd', 'e'], columns=['Missouri', 'Alabama'])_____no_output_____left2_____no_output_____right2_____no_output_____pd.merge(left2, right2, left_index=True, right_index=True, how='outer')_____no_output_____left2.join(right2, how='outer')_____no_output_____

## 轴向连接_____no_output_____

arr = np.arange(12).reshape((3, 4))
arr_____no_output_____np.concatenate([arr, arr], axis=1)_____no_output_____np.concatenate([arr, arr], axis=0)_____no_output_____s1 = Series([0, 1], index=['a', 'b'])
s2 = Series([2, 3, 4], index=['c', 'd', 'e'])
s3 = Series([5, 6], index=['f', 'g'])_____no_output_____pd.concat([s1, s2, s3])_____no_output_____pd.concat([s1, s2, s3], axis=1)D:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  """Entry point for launching an IPython kernel.
s4 = pd.concat([s1 *5, s3])
s4_____no_output_____pd.concat([s1, s4], axis=1)D:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  """Entry point for launching an IPython kernel.
pd.concat([s1, s4], axis=1, sort='True')_____no_output_____pd.concat([s1, s4], axis=1, join='inner', sort=True)_____no_output_____s4_____no_output_____s1_____no_output_____pd.concat([s1, s4], axis=1, join_axes=[['a', 'c', 'b', 'e', 'g']])_____no_output_____result = pd.concat([s1, s1, s3], keys=['one', 'two', 'three'])      #keys 为层次化索引_____no_output_____result_____no_output_____result.unstack()_____no_output_____result = pd.concat([s1, s1, s3], axis=1, keys=['one', 'two', 'three']) #指定合并方向为 axis=1, keys将成为列名
resultD:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  """Entry point for launching an IPython kernel.
df1 = DataFrame(np.arange(6).reshape(3, 2), index=['a', 'b', 'c'],
                columns=['one', 'two'])
df2 = DataFrame(5 + np.arange(4).reshape(2, 2), index=['a', 'c'],
                columns=['three', 'four'])
pd.concat([df1, df2], axis=1, keys=['level1', 'level2'], sort=True)_____no_output_____df1_____no_output_____df2_____no_output_____pd.concat([df1, df2], keys=['level1', 'level2'], sort=True)_____no_output_____pd.concat({'level1' : df1, 'level2' : df2}, axis=1)D:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  """Entry point for launching an IPython kernel.
pd.concat({'level1' : df1, 'level2' : df2}, axis=1, names=['upper', 'lower'], sort=True)_____no_output_____df1 = DataFrame(np.random.randn(3, 4), columns=['a', 'b', 'c', 'd'])
df2 = DataFrame(np.random.randn(2, 3), columns=['b', 'd', 'a'])_____no_output_____df1_____no_output_____df2_____no_output_____pd.concat([df1, df2], axis=1, ignore_index=True)_____no_output_____pd.concat([df1, df2], ignore_index=True)D:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  """Entry point for launching an IPython kernel.

## 合并重叠数据_____no_output_____

a = Series([np.nan, 2.5, np.nan, 3.5, 4.5, np.nan], index=['f','e','d','c','b','a'])
b = Series(np.arange(len(a), dtype=np.float64), index=['f','e','d','c','b','a'])_____no_output_____b_____no_output_____b[-1] = np.nan_____no_output_____np.where(pd.isnull(a), b, a):
df1 = DataFrame({'a': [1., np.nan, 5., np.nan],
                 'b': [np.nan, 2., np.nan, 6.],
                 'c': range(2, 18, 4)})
df2 = DataFrame({'a': [5., 4., np.nan, 3., 7.],
                 'b': [np.nan, 3., 4., 6., 8.]})_____no_output_____b[:-2]_____no_output_____a[2:]_____no_output_____b[:-2].combine_first(a[2:])_____no_output_____df1 = DataFrame({'a': [1., np.nan, 5., np.nan],
                 'b': [np.nan, 2., np.nan, 6.],
                 'c': range(2, 18, 4)})
df2 = DataFrame({'a': [5., 4., np.nan, 3., 7.],
                 'b': [np.nan, 3., 4., 6., 8.]})_____no_output_____df1_____no_output_____df2_____no_output_____df1.combine_first(df2)_____no_output_____

## 重塑和轴向旋转_____no_output_____stack 将数据的列旋转为行， unstack 将数据的行旋转为列_____no_output_____

data = DataFrame(np.arange(6).reshape((2, 3)),
                 index=pd.Index(['Ohio', 'Colorado'], name='state'),
                 columns=pd.Index(['one', 'two', 'three'], name='number'))
data_____no_output_____result = data.stack()
result
type(result)_____no_output_____result_____no_output_____result.unstack()_____no_output_____result.unstack(0)_____no_output_____result_____no_output_____result.unstack('state')_____no_output_____result.unstack('number')_____no_output_____s1 = Series([0, 1, 2, 3], index=['a', 'b', 'c', 'd'])
s2 = Series([4, 5, 6], index=['c', 'd', 'e'])
data2 = pd.concat([s1, s2], keys=['one', 'two'])
data2_____no_output_____data2.unstack()_____no_output_____data2.unstack().stack()_____no_output_____data2.unstack().stack(dropna=False)    #stack默认会过滤缺失数据_____no_output_____df = DataFrame({'left': result, 'right': result + 5},
               columns=pd.Index(['left', 'right'], name='side'))
df_____no_output_____df.unstack('state')_____no_output_____ df.unstack('number')_____no_output_____df.unstack('state').stack('side')_____no_output_____

## 将长格式旋转为宽格式_____no_output_____

data = pd.read_csv('G:/dataAnalysis/data/macrodata.csv')
periods = pd.PeriodIndex(year=data.year, quarter=data.quarter, name='date')
data = DataFrame(data.to_records(),
                 columns=pd.Index(['realgdp', 'infl', 'cpi', 'unemp', 'pop'], name='item'),
                 index=periods.to_timestamp('D', 's'))_____no_output_____data_____no_output_____ldata = data.stack().reset_index().rename(columns={0: 'value'})_____no_output_____ldata[:10]_____no_output_____pivoted = ldata.pivot('date', 'item', 'value')
pivoted.head()_____no_output_____# pivot(row_index_name, col_index_name, col_name_which_is_to_fill_in_the_DataFrame)_____no_output_____ldata['value2'] = np.random.randn(len(ldata))_____no_output_____ldata[:10]_____no_output_____pivoted = ldata.pivot('date', 'item')_____no_output_____pivoted[:10]_____no_output_____pivoted['value'][:5]_____no_output_____unstacked = ldata.set_index(['date', 'item']).unstack('item')    #pivot() 等同于 先 set_index() 再  unstack()_____no_output_____unstacked[:10]_____no_output_____

## 数据转换_____no_output_____

data = DataFrame({'k1': ['one'] * 3 + ['two'] * 4,
                  'k2': [1, 1, 2, 3, 3, 4, 4]})
data_____no_output_____data.duplicated()_____no_output_____data.drop_duplicates()_____no_output_____data['v1'] = np.arange(7)
data_____no_output_____data.drop_duplicates(['k1'])_____no_output_____data.duplicated()_____no_output_____data_____no_output_____data.drop_duplicates(['k1', 'k2'], take_last=True)_____no_output_____data.drop_duplicates(['k1', 'k2'], keep='last')_____no_output_____

## 利用函数或映射进行数据转换_____no_output_____

data = DataFrame({'food': ['bacon', 'pulled pork', 'bacon', 'Pastrami',
                           'corned beef', 'Bacon', 'pastrami', 'honey ham',
                           'nova lox'],
                  'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
data_____no_output_____

map() 方法接受一个函数或者含有映射关系的字典型对象_____no_output_____

meat_to_animal = {
  'bacon': 'pig',
  'pulled pork': 'pig',
  'pastrami': 'cow',
  'corned beef': 'cow',
  'honey ham': 'pig',
  'nova lox': 'salmon'
}
data['animal'] = data['food'].map(str.lower).map(meat_to_animal)_____no_output_____data_____no_output_____data['animal'] = data['food'].map(lambda x : meat_to_animal[x.lower()])
data_____no_output_____

## 离散化和面元划分_____no_output_____

ages = [20, 22, 25, 27, 21, 23, 37, 31, 62, 45, 43, 32]
bins = [18, 25, 35, 60 ,100]
cats = pd.cut(ages, bins)
cats_____no_output_____type(cats)_____no_output_____cats.labels_____no_output_____# 上述异常表明属性名称的更新  categories 替换levels
cats.categories_____no_output_____cats.codes     # codes 替换 labels_____no_output_____pd.value_counts(cats)_____no_output_____# 自己设置面元名称
group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
pd.cut(ages, bins, labels=group_names)_____no_output_____data = np.random.rand(20)_____no_output_____data_____no_output_____pd.cut(data, 4, precision=2)     #传入面元数量则根据数据的最小和最大值等值计算等长面元_____no_output_____data = np.random.rand(1000)
data_____no_output_____cats = pd.qcut(data, 4)
cats_____no_output_____pd.value_counts(cats)     # 使用qcut() 可以得到大小基本相等的面元_____no_output_____

## 检测和过滤异常值_____no_output_____

np.random.seed(12345)_____no_output_____data = DataFrame(np.random.randn(1000, 4))
data.describe()_____no_output_____# 找出第4列中绝对值大小超过3的值：
col = data[3]
col[np.abs(col) > 3]_____no_output_____# 找出全部含有 “大小超过3或 -3”的行
data[(np.abs(data) > 3).any(1)]_____no_output_____# 将取值限制在 [-3, 3] 之间
data[np.abs(data) > 3] = np.sign(data) * 3_____no_output_____data.describe()_____no_output_____

## 排列和随机采样_____no_output_____

df = DataFrame(np.arange(5 * 4).reshape(5, 4))
df_____no_output_____sampler = np.random.permutation(5)
sampler_____no_output_____df.take(sampler)_____no_output_____len(df)_____no_output_____df.take(np.random.permutation(len(df))[:3])_____no_output_____np.random.permutation(len(df))[:3]_____no_output_____np.random.permutation(len(df))[:3]_____no_output_____

## 计算指标/哑变量_____no_output_____

df = DataFrame({'key':['a','b','b','c','a','b'], 'data1' : range(6)})
pd.get_dummies(df['key'])_____no_output_____dummies = pd.get_dummies(df['key'], prefix="key")
dummies_____no_output_____df_with_dummy = df[['data1']].join(dummies)
df_with_dummy_____no_output_____df_____no_output_____df['data1']_____no_output_____df[['data1']]_____no_output_____

##  正则表达式_____no_output_____

import re_____no_output_____text = "foo bar\t baz   \tqux"
re.split('\s+', text)_____no_output_____#等同于：
regex = re.compile('\s+')
regex.split(text)_____no_output_____regex.findall(text)_____no_output_____text = """Dave dave@google.com
Steve steve@gmail.com
Rob rob@gmail.com
Ryan ryan@yahoo.com
"""
pattern = r'[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}'_____no_output_____regex = re.compile(pattern, flags=re.IGNORECASE)_____no_output_____regex.findall(text)_____no_output_____print regex.match(text)   #match() 只匹配出现在字符串开头的模式None
print regex.sub('REDACTED', text)   # sub()方法将替换匹配到的模式为指定的字符串Dave REDACTED
Steve REDACTED
Rob REDACTED
Ryan REDACTED

## pandas 中矢量化的字符串函数_____no_output_____

data = {'Dave': 'dave@google.com', 'Steve': 'steve@gmail.com',
        'Rob': 'rob@gmail.com', 'Wes': np.nan}
data = Series(data)
data_____no_output_____data.isnull()_____no_output_____data.str.contains('gmail')_____no_output_____pattern_____no_output_____data.str.findall(pattern, flags=re.IGNORECASE)_____no_output_____matches = data.str.match(pattern, flags=re.IGNORECASE)
matches_____no_output_____matches.str_____no_output_____matches.str.get(1)_____no_output_____matches.str[0]_____no_output_____data.str[:5]_____no_output_____data.index.contains('Dave')_____no_output_____

## USDA 食品数据库示例_____no_output_____

import json_____no_output_____db = json.load(open('G:/dataAnalysis/data/foods-2011-10-03.json'))
len(db)_____no_output_____db[:10]_____no_output_____db[0].keys()_____no_output_____db[0]['nutrients']_____no_output_____nutrients = DataFrame(db[0]['nutrients'])_____no_output_____nutrients[:5]_____no_output_____nutrients[:10]_____no_output_____info_keys = ['description', 'group', 'id', 'manufacturer']
info = DataFrame(db, columns=info_keys)_____no_output_____info[:5]_____no_output_____pd.value_counts(info.group)[:10]_____no_output_____nutrients = []

for rec in db:
    fnuts = DataFrame(rec['nutrients'])
    fnuts['id'] = rec['id']
    nutrients.append(fnuts)

nutrients = pd.concat(nutrients, ignore_index=True)_____no_output_____nutrients _____no_output_____nutrients.duplicated().sum()_____no_output_____nutrients = nutrients.drop_duplicates()_____no_output_____nutrients_____no_output_____info_____no_output_____col_mappings = {'description': 'food', 'group' : 'fgourp'}
info = info.rename(columns=col_mappings, copy=False)_____no_output_____info_____no_output_____col_mapping = {'description' : 'nutrient',
               'group' : 'nutgroup'}
nutrients = nutrients.rename(columns=col_mapping, copy=False)
nutrients_____no_output_____ndata = pd.merge(info, nutrients, on='id', how='outer')_____no_output_____ndata_____no_output_____ndata.ix[30000]D:\Anaconda\folder\lib\site-packages\ipykernel_launcher.py:1: DeprecationWarning:
.ix is deprecated. Please use
.loc for label based indexing or
.iloc for positional indexing

See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated
  """Entry point for launching an IPython kernel.
ndata.loc[30000]_____no_output_____