# Notebook from albeckr/Thesis_CoRoDro
Path: Exp_algorithms/Exp_DQN.ipynb

# MultiAgentEnvironment: simple Map_____no_output_____

#%autosave 30
import glob
import sys
from operator import itemgetter
import numpy as np
import random
import time
import math

import networkx as nx
from networkx.algorithms.shortest_paths.generic import shortest_path_length

import ray
from ray import tune
#from ray.tune.logger import pretty_print
from ray.rllib.policy.policy import Policy, PolicySpec
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
from ray.rllib.models.tf.fcnet import FullyConnectedNetwork

#from ray.rllib.agents.callbacks import DefaultCallbacks
from ray.rllib.env import MultiAgentEnv
from ray.rllib.utils.framework import try_import_tf

from gym.spaces import Discrete, Box, Tuple, MultiDiscrete, Dict, MultiBinary
from ray.rllib.utils.spaces.repeated import Repeated

import matplotlib.pyplot as plt

tf1, tf, tfv = try_import_tf()  # prefered TF import for Ray.io

from threading import Thread, Event

print("Imports successful")Imports successful
######## Utility Classes ########
class BoolTimer(Thread):
    """A boolean value that toggles after a specified number of seconds:

    Example:
        bt = BoolTimer(30.0, False)
        bt.start()

    Used in the Centrality Baseline to limit the computation time.
    """

    def __init__(self, interval, initial_state=True):
        Thread.__init__(self)
        self.interval = interval
        self.state = initial_state
        self.finished = Event()

    def __bool__(self):
        return bool(self.state)

    def run(self):
        self.finished.wait(self.interval)
        if not self.finished.is_set():
            self.state = not self.state
        self.finished.set()

######## Static helper functions ########
def shuffle_actions(action_dict, check_space = False):
    """
        Used to shuffle the action dict to ensure that agents with a lower id are not always preferred
        when picking up parcels over other agents that chose the same action.
        For debugging: Can also be used to check if all actions are in the action_space.
    """
    keys = list(action_dict)
    random.shuffle(keys)
    shuffled = {}

    for agent in keys:
        if check_space:  #assert actions are in action space -> disable for later trainig, extra check while development
            assert self.action_space.contains(action_dict[agent]),f"Action {action_dict[agent]} taken by agent {agent} not in action space"
        shuffled[agent] = action_dict[agent]

    return shuffled

def load_graph(data):
    """Loads topology (map) from json file into a networkX Graph and returns the graph"""

    nodes = data["nodes"]
    edges = data["edges"]

    g = nx.DiGraph()  # directed graph
    g.add_nodes_from(nodes)
    g.edges(data=True)

    for node in nodes:  # add attribute values
        g.nodes[node]["id"] = nodes[node]["id"]
        g.nodes[node]["type"] = nodes[node]["type"]

    for edge in edges:  # add edges with attributes
        f = edges[edge]["from"]
        t = edges[edge]["to"]

        weight_road, weight_air, _type = sys.maxsize, sys.maxsize, None

        if edges[edge]["road"] >= 0:
            weight_road = edges[edge]["road"]
            _type = 'road'
        if edges[edge]["air"] >= 0:
            weight_air = edges[edge]["air"]
            _type = 'both' if _type == 'road' else 'air'

        weight = min(weight_road, weight_air)  # needed for optimality baseline

        g.add_edge(f, t, type=_type, road= weight_road, air=weight_air, weight=weight)

    return g_____no_output_____

## Environment Code_____no_output_____

# Map definition
topology = {
    'nodes': {
            0: {'id': 1, 'type': 'parking'},
            1: {'id': 1, 'type': 'parking'},
            2: {'id': 2, 'type': 'parking'},
            3: {'id': 3, 'type': 'parking'},
            4: {'id': 4, 'type': 'parking'},
            5: {'id': 5, 'type': 'air'},
            6: {'id': 6, 'type': 'parking'},
            7: {'id': 7, 'type': 'air'},
            8: {'id': 8, 'type': 'parking'},
            9: {'id': 9, 'type': 'parking'},
            10: {'id': 10, 'type': 'parking'},
            11: {'id': 11, 'type': 'air'}
    },
    'edges': {
    ## Outer Ring    --> Road much (!) faster than drones
    "e01":{"from": 0,"to": 1,"road": 10, "air": 40},
    "e02":{"from": 1,"to": 0,"road": 10, "air": 40},
    "e03":{"from": 1,"to": 4,"road": 8, "air": 6},
    "e04":{"from": 4,"to": 1,"road": 8, "air": 6},
    "e03":{"from": 4,"to": 2,"road": 8, "air": 6},
    "e04":{"from": 2,"to": 4,"road": 8, "air": 6},
    "e05":{"from": 2,"to": 3,"road": 10, "air": 30},
    "e06":{"from": 3,"to": 2,"road": 10, "air": 30},
    "e07":{"from": 3,"to": 0,"road": 10, "air": 40},
    "e08":{"from": 0,"to": 3,"road": 8, "air": 35},
    ## Inner Nodes  --> Reinfahren langsamer als rausfahren
    ##              -->
    "e11":{"from": 4,"to": 6,"road": 2, "air": 5},
    "e12":{"from": 6,"to": 4,"road": 2, "air": 5},
    "e13":{"from": 0,"to": 6,"road": 6, "air": -1},
    "e14":{"from": 6,"to": 0,"road": 6, "air": -1},
    "e15":{"from": 3,"to": 6,"road": 6, "air": -1},
    "e16":{"from": 6,"to": 3,"road": 6, "air": -1},
    ## Outliers   --> Distance about equal if both exist!
    "e17":{"from": 4,"to": 5,"road": 2, "air": 2},
    "e18":{"from": 5,"to": 4,"road": 2, "air": 2},
    "e19":{"from": 6,"to": 7,"road": -1, "air": 2},
    "e20":{"from": 7,"to": 6,"road": -1, "air": 2},
    "e21":{"from": 3,"to": 11,"road": -1, "air": 3},
    "e22":{"from": 11,"to": 3,"road": -1, "air": 3},
    ## Upper left square  --> Hier besser mit den Drohnen agieren, Falls Sie da sind!!
    "e23":{"from": 0,"to": 8,"road": -1, "air": 4},
    "e24":{"from": 8,"to": 0,"road": -1, "air": 4},
    "e25":{"from": 8,"to": 9,"road": 3, "air": 3},
    "e26":{"from": 9,"to": 8,"road": 3, "air": 3},
    "e27":{"from": 9,"to": 10,"road": 3, "air": 3},
    "e28":{"from": 10,"to": 9,"road": 3, "air": 3},
    "e29":{"from": 0,"to": 10,"road": 3, "air": 3},
    "e30":{"from": 10,"to": 0,"road": 3, "air": 3}
    }
}_____no_output_____class Map_Environment(MultiAgentEnv):

    def __init__(self, env_config: dict = {}):
        # ensure config file includes all necessary settings
        assert 'NUMBER_STEPS_PER_EPISODE' in env_config
        assert 'NUMBER_OF_DRONES' in env_config
        assert 'NUMBER_OF_CARS' in env_config
        assert 'INIT_NUMBER_OF_PARCELS' in env_config
        assert 'TOPOLOGY' in env_config
        assert 'MAX_NUMBER_OF_PARCELS' in env_config
        assert 'THRESHOLD_ADD_NEW_PARCEL' in env_config
        assert 'BASELINE_FLAG' in env_config
        assert 'BASELINE_TIME_CONSTRAINT' in env_config
        assert 'BASELINE_OPT_CONSTANT' in env_config
        assert 'CHARGING_STATION_NODES' in env_config
        assert 'REWARDS' in env_config

        self.graph = load_graph(topology)

        # Map config
        self.NUMBER_OF_DRONES = env_config['NUMBER_OF_DRONES']
        self.NUMBER_OF_CARS = env_config['NUMBER_OF_CARS']
        self.NUMBER_OF_EDGES = self.graph.number_of_edges()
        self.NUMBER_OF_NODES = self.graph.number_of_nodes()
        self.CHARGING_STATION_NODES = env_config['CHARGING_STATION_NODES']
        self.MAX_BATTERY_POWER = env_config['MAX_BATTERY_POWER']

        # Simulation config
        self.NUMBER_STEPS_PER_EPISODE = env_config['NUMBER_STEPS_PER_EPISODE']
        self.INIT_NUMBER_OF_PARCELS = env_config['INIT_NUMBER_OF_PARCELS']
        self.RANDOM_SEED = env_config.get('RANDOM_SEED', None)
        self.MAX_NUMBER_OF_PARCELS = env_config['MAX_NUMBER_OF_PARCELS']
        self.THRESHOLD_ADD_NEW_PARCEL = env_config['THRESHOLD_ADD_NEW_PARCEL']
        self.BASELINE_FLAG = env_config['BASELINE_FLAG']
        self.BASELINE_TIME_CONSTRAINT = env_config['BASELINE_TIME_CONSTRAINT']
        self.BASELINE_OPT_CONSTANT = env_config['BASELINE_OPT_CONSTANT']

        self.DEBUG_LOG = env_config.get('DEBUG_LOGS', False)
        # Some Sanity Checks on the settings
        if self.DEBUG_LOG: assert self.MAX_NUMBER_OF_PARCELS >= self.INIT_NUMBER_OF_PARCELS, "Number of initial parcels exceeds max parcel limit"

        #Reward constants
        self.STEP_PENALTY = env_config['REWARDS']['STEP_PENALTY']
        self.PARCEL_DELIVERED = env_config['REWARDS']['PARCEL_DELIVERED']
        # compute other rewards
        self.BATTERY_DIED = self.STEP_PENALTY * self.NUMBER_STEPS_PER_EPISODE
        self.BATTERY_DIED_WITH_PARCEL = self.BATTERY_DIED * 2
        #self.DELIVERY_CONTRIBUTION: depends on active agents --> computed when given in prepare_global_reward()
        #self.ALL_DELIVERED_CONTRIB: depends on active agents --> computed when given in prepare_global_reward(episode_success=True)

        # Computed constants
        self.NUMBER_OF_AGENTS = self.NUMBER_OF_DRONES + self.NUMBER_OF_CARS
        self.PARCEL_STATE_DELIVERED = self.NUMBER_OF_AGENTS + self.NUMBER_OF_NODES
        self.NUMBER_OF_ACTIONS = 1 + self.NUMBER_OF_NODES + 1 + self.MAX_NUMBER_OF_PARCELS
        self.ACTION_DROPOFF = 1 + self.NUMBER_OF_NODES     # First Action NOOP is 0

        # seed RNGs
        self.seed(self.RANDOM_SEED)

        self.state = None
        self.current_step = None
        self.blocked_agents = None
        self.parcels_delivered = None
        self.done_agents = None
        self.all_done = None
        self.allowed_actions = None

        # baseline related
        self.baseline_missions = None
        self.agents_base = None
        self.o_employed = None

        # metrics for the evaluation
        self.parcel_delivered_steps = None # --> {p1: 20, p2: 240, p:140}
        self.parcel_added_steps = None     # --> {p1: 0, p2: 0, p: 50}
        self.agents_crashed = None         # --> {c_2: 120, d_0: 242}
        self.metrics = None

        self.agents = [*["d_" + str(i) for i in range(self.NUMBER_OF_DRONES)],*["c_" + str(i) for i in range(self.NUMBER_OF_DRONES, self.NUMBER_OF_DRONES + self.NUMBER_OF_CARS)]]

        # Define observation and action spaces for individual agents
        self.action_space = Discrete(self.NUMBER_OF_ACTIONS)

        #---- Repeated Obs Space: Represents a parcel with (id, location, destination)  --> # parcel_id starts at 1
        parcel_space = Dict({'id': Discrete(self.MAX_NUMBER_OF_PARCELS+1),
                             'location': Discrete(self.NUMBER_OF_NODES + self.NUMBER_OF_AGENTS + 1),
                             'destination': Discrete(self.NUMBER_OF_NODES)
                            })
        self.observation_space = Dict({'obs': Dict({'state':
                                                 Dict({ 'position': Discrete(self.NUMBER_OF_NODES),
                                                        'battery': Discrete(self.MAX_BATTERY_POWER + 1), #[0-100]
                                                        'has_parcel': Discrete(self.MAX_NUMBER_OF_PARCELS + 1),
                                                        'current_step': Discrete(self.NUMBER_STEPS_PER_EPISODE + 1)}),
                                                 'parcels': Repeated(parcel_space, max_len=self.MAX_NUMBER_OF_PARCELS)
                                                }),
                                        'allowed_actions': MultiBinary(self.NUMBER_OF_ACTIONS)
                                      })
        #TODO: why is reset() not called by env?
        self.reset()

    def step(self, action_dict):
        """conduct the state transitions caused by actions in action_dict
        :returns:
            - observation_dict: observations for agents that need to act in the next round
            - rewards_dict: rewards for agents following their chosen actions
            - done_dict: indicates end of episode if max_steps reached or all parcels delivered
            - info_dict: pass data to custom logger
        """

        if self.DEBUG_LOG: print(f"Debug log flag set to {self.DEBUG_LOG}")

        # ensure no disadvantage for agents with higher IDs if action conflicts with that taken by other agent
        action_dict = shuffle_actions(action_dict)

        self.current_step += 1

        # grant step penalty reward
        agent_rewards = {agent: self.STEP_PENALTY for agent in self.agents}

        # setting an agent done twice might cause crash when used with tune... -> https://github.com/ray-project/ray/issues/10761
        dones = {}

        self.metrics['step'] = self.current_step

        # dynamically add parcel
        if random.random() <= self.THRESHOLD_ADD_NEW_PARCEL and len(self.state['parcels']) < self.MAX_NUMBER_OF_PARCELS:

            p_id, parcel = self.generate_parcel()
            assert p_id not in self.state['parcels'], "Duplicate parcel ID generated"
            self.state['parcels'][p_id] = parcel

            if self.BASELINE_FLAG:
                self.metrics["optimal"] = self.compute_optimality_baseline(p_id, extra_charge=self.BASELINE_OPT_CONSTANT)

            for agent in self.agents:
                self.allowed_actions[agent][self.ACTION_DROPOFF + p_id] = np.array([1]).astype(bool)

        if self.BASELINE_FLAG:
            old_actions = action_dict
            action_dict = {}
            # Replace actions with actions recommended by the central baseline
            for agent, action in old_actions.items():
                if len(self.baseline_missions[agent]) > 0:
                    new_action = self.baseline_missions[agent][0]

                    if type(new_action) is tuple: # dropoff action with minimal time

                        if self.current_step >= new_action[1]:
                            new_action = new_action[0]
                            self.baseline_missions[agent].pop(0)
                        else: #agent has to wait for previous subroute agent
                            new_action = 0
                    else: # move or pickup or charge
                        self.baseline_missions[agent].pop(0)

                    action_dict[agent] = new_action

                else: # agent has no baseline mission -> Noop
                    action_dict[agent] = 0

        # carry out State Transition

        # handel NOP actions: -> action == 0
        noop_agents =  {agent: action for agent, action in action_dict.items() if action == 0}
        effectual_agents_items = {agent: action for agent, action in action_dict.items() if action > 0}.items()

        # now: transaction between agents modelled as pickup of just offloaded (=dropped) parcel --> handle dropoff first
        moving_agents = {agent: action for agent, action in effectual_agents_items if 0 < action and action <= self.NUMBER_OF_NODES}
        dropoff_agents = {agent: action for agent, action in effectual_agents_items if action == self.ACTION_DROPOFF}
        pickup_agents = {agent: action for agent, action in effectual_agents_items if action > self.ACTION_DROPOFF}

        # handle noop / charge decisions:
        for agent, action in noop_agents.items():
            # check if recharge is possible
            current_pos = self.state['agents'][agent]['position']
            if current_pos in self.CHARGING_STATION_NODES:
                self.state['agents'][agent]['battery'] = self.MAX_BATTERY_POWER

        # handle Movement actions:
        for agent, action in moving_agents.items():
            # get Current agent position from state
            self.state['agents'][agent]['battery'] += -1
            current_pos = self.state['agents'][agent]['position']

            # networkX: use node instead of edge:
            destination = action - 1

            if self.graph.has_edge(current_pos, destination):
                # Agent chose existing edge! -> check if type is suitable

                agent_type = 'road' if agent[:1] == 'c' else 'air'
                if self.graph[current_pos][destination]["type"] in [agent_type, 'both']:
                    # Edge has correct type
                    self.state['agents'][agent]['position'] = destination
                    self.state['agents'][agent]['battery'] += -(self.graph[current_pos][destination][agent_type] +1)
                    if self.state['agents'][agent]['battery'] < 0:        # ensure negative battery value does not break obs_space
                        #Battery below 0 --> reset to 0 (stay in obs space)
                        self.state['agents'][agent]['battery'] = 0

                    self.blocked_agents[agent] = self.graph[current_pos][destination][agent_type]
                    self.update_allowed_actions_nodes(agent)

        # handle Dropoff Decision: -> action == self.NUMBER_OF_NODES + 2
        for agent, action in dropoff_agents.items():
            self.state['agents'][agent]['battery'] += -1

            if self.state['agents'][agent]['has_parcel'] > 0:   # agent has parcel
                parcel_id = self.state['agents'][agent]['has_parcel']
                self.state['agents'][agent]['has_parcel'] = 0
                self.state['parcels'][parcel_id][0] = self.state['agents'][agent]['position']
                if self.state['parcels'][parcel_id][0] ==  self.state['parcels'][parcel_id][1]:
                    # Delivery successful

                    agent_rewards[agent] += self.PARCEL_DELIVERED  # local reward
                    # global contribution rewards
                    active_agents, reward = self.prepare_global_reward()
                    for a_id in active_agents: agent_rewards[a_id] += reward

                    self.state['parcels'][parcel_id][0] = self.PARCEL_STATE_DELIVERED
                    self.parcels_delivered[int(parcel_id) -1] = True                    # Parcel_ids start at 1

                    self.metrics['delivered'].update({"p_" + str(parcel_id): self.current_step})

                self.update_allowed_actions_parcels(agent)

        # handle Pickup Decision:
        for agent, action in pickup_agents.items():
            self.state['agents'][agent]['battery'] += -1

            # agent has free capacity
            if self.state['agents'][agent]['has_parcel'] == 0:  #free parcel capacity
                # convert action_id to parcel_id
                parcel_id = action - self.ACTION_DROPOFF

                if self.DEBUG_LOG: assert parcel_id in self.state['parcels'] # parcel {parcel_id} already in ENV ??

                elif self.state['parcels'][parcel_id][0] == self.state['agents'][agent]['position']:
                    # Successful pickup operation
                    self.state['parcels'][parcel_id][0] = self.NUMBER_OF_NODES + int(agent[2:])
                    self.state['agents'][agent]['has_parcel'] = int(parcel_id)

                    self.update_allowed_actions_parcels(agent)

        # unblock agents for next round
        self.blocked_agents = {agent: remaining_steps -1 for agent, remaining_steps in self.blocked_agents.items() if remaining_steps > 1}

        # handle dones - out of battery or max_steps or goal reached
        for agent in action_dict.keys():
            if agent not in self.done_agents and self.state['agents'][agent]['battery'] <= 0:
                agent_rewards[agent] = self.BATTERY_DIED_WITH_PARCEL if self.state['agents'][agent]['has_parcel'] != 0 else self.BATTERY_DIED
                dones[agent] = True
                self.done_agents.append(agent)

                self.metrics['crashed'].update({agent: self.current_step})

                if len(self.done_agents) == self.NUMBER_OF_AGENTS:
                    # all agents dead
                    self.all_done = True

        # check if episode terminated because of goal reached or all agents crashed -> avoid setting done twice
        if self.current_step >= self.NUMBER_STEPS_PER_EPISODE or (all(self.parcels_delivered) and len(self.parcels_delivered) == self.MAX_NUMBER_OF_PARCELS):
            # check if episode success:
            if self.current_step < self.NUMBER_STEPS_PER_EPISODE:
                    # grant global reward for all parcels delivered
                    active_agents, reward = self.prepare_global_reward(_episode_success=True)
                    for a_id in active_agents: agent_rewards[a_id] += reward

            self.all_done = True

        dones['__all__'] = self.all_done

        parcel_obs = self.get_parcel_obs()

        # obs \ rewards \ dones\ info
        return  {agent: { 'obs': {'state': {'position': self.state['agents'][agent]['position'], 'battery': self.state['agents'][agent]['battery'],
                                    'has_parcel': self.state['agents'][agent]['has_parcel'],'current_step': self.current_step},
                                    'parcels': parcel_obs},
                         'allowed_actions': self.allowed_actions[agent]} for agent in self.agents if agent not in self.blocked_agents and agent not in self.done_agents}, \
                { agent: agent_rewards[agent] for agent in self.agents}, \
                dones, \
                {}

    def seed(self, seed=None):
        tf.random.set_seed(seed)
        np.random.seed(seed)
        random.seed(seed)

    def reset(self):
        """resets variables; returns dict with observations, keys are agent_ids"""
        self.current_step = 0
        self.blocked_agents = {}
        self.parcels_delivered = [False for _ in range(self.MAX_NUMBER_OF_PARCELS)]
        self.done_agents = []
        self.all_done = False

        self.metrics = { "step": self.current_step,
            "delivered": {},
            "crashed": {},
            "added": {},
            "optimal": None
        }

        #Baseline
        self.baseline_missions = {agent: [] for agent in self.agents}
        self.o_employed = [0 for _ in range(self.NUMBER_OF_AGENTS)]

        self.agents_base = None  # env.agents in the pseudocode
        self.allowed_actions = { agent: np.array([1 for act in range(self.NUMBER_OF_ACTIONS)]) for agent in self.agents}

        #Reset State
        self.state = {'agents': {},
                      'parcels': {}}
        #generate initial parcels
        for _ in range(self.INIT_NUMBER_OF_PARCELS):
            p_id, parcel = self.generate_parcel()
            self.state['parcels'][p_id] = parcel

        parcel_obs = self.get_parcel_obs()

        # init agents
        self.state['agents'] = {agent: {'position': self._random_feasible_agent_position(agent),
                                        'battery': self.MAX_BATTERY_POWER, 'has_parcel': 0} for agent in self.agents}

        if self.BASELINE_FLAG:
            for parcel in self.state['parcels']:
                self.compute_central_delivery(parcel, debug_log=False)

                # TODO really return something here??
                self.metrics['optimal'] = self.compute_optimality_baseline(parcel, extra_charge=self.BASELINE_OPT_CONSTANT, debug_log=False)

        # compute allowed actions per agent --> move to function
        for agent in self.agents:
            self.update_allowed_actions_nodes(agent)
            self.update_allowed_actions_parcels(agent)

        agent_obs = {agent: {'obs': {'state':  {'position': state['position'], 'battery': state['battery'],
                                        'has_parcel': state['has_parcel'],'current_step': self.current_step},
                            'parcels': parcel_obs
                            },
                            'allowed_actions': self.allowed_actions[agent]
                            } for agent, state in self.state['agents'].items()
                    }

        return {**agent_obs}

    def _random_feasible_agent_position(self, agent_id):
        """Needed to avoid car agents being initialized at nodes only reachable by drones
        and thus trapped from the beginning. Ensures that car agents start at a node of type 'road' or 'parking'.
        """
        position = random.randrange(self.NUMBER_OF_NODES)
        if agent_id[0] == 'c':    # agent is car
            while self.graph.nodes[position]['type'] == 'air':      # position not reachable by car
                position = random.randrange(self.NUMBER_OF_NODES)
        return position

    def update_allowed_actions_nodes(self, agent):
        new_pos = self.state['agents'][agent]['position']
        next_steps = list(self.graph.neighbors(new_pos))
        agent_type = 'air' if agent[0]=='d' else 'road'

        allowed_nodes = np.zeros(self.NUMBER_OF_NODES)
        for neighbor in next_steps:
            if self.graph[new_pos][neighbor]['type'] in [agent_type, 'both']:
                allowed_nodes[neighbor] = 1
        self.allowed_actions[agent][1:self.NUMBER_OF_NODES+1] = np.array(allowed_nodes).astype(bool)

    def update_allowed_actions_parcels(self, agent):
        """ Allow only the Dropoff or Pickup actions, depending on the has_parcel value of the agent.
        Pickup is not concerned with the parcel actually being at the agents current location, only with free capacity
        and the parcel already being added to the ENV"""
        num_parcels = len(self.state['parcels'])

        allowed_parcels = np.zeros(self.MAX_NUMBER_OF_PARCELS)
        dropoff = 1
        if self.state['agents'][agent]['has_parcel'] == 0:
            dropoff = 0
            allowed_parcels = np.concatenate([np.ones(num_parcels), np.zeros(self.MAX_NUMBER_OF_PARCELS - num_parcels)])

        self.allowed_actions[agent][self.NUMBER_OF_NODES+1:] = np.array([dropoff, *allowed_parcels]).astype(bool)

    def get_parcel_obs(self):
        parcel_obs = [{'id':pid, 'location': parcel[0], 'destination':parcel[1]} for (pid, parcel) in self.state['parcels'].items()]
        return parcel_obs

    def generate_parcel(self):
        """generate new parcel id and new parcel with random nodes for location and destination.
            p_ids (int) start at 1, later nodes for parcel will be sampled to avoid parcels that already spawn at their destination"""
        p_id = len(self.state['parcels']) + 1

        parcel = random.sample(range(self.NUMBER_OF_NODES), 2)  # => initial location != destination
        self.metrics['added'].update({p_id: self.current_step})

        return p_id, parcel

    def prepare_global_reward(self, _episode_success=False):
        """ computes a global reward for all active agents still in the environment.
            If _episode_success is set to True, all parcels have been delivered an the ALL_DELIVERED reward is granted.
            :Returns: list of active agents and the reward value
        """
        agents_alive = list(set(self.agents).difference(set(self.done_agents)))
        if self.DEBUG_LOG: assert len(agents_alive) > 0
        reward = self.PARCEL_DELIVERED * (self.NUMBER_STEPS_PER_EPISODE - self.current_step) / self.NUMBER_STEPS_PER_EPISODE if _episode_success else self.PARCEL_DELIVERED / len(agents_alive)

        return agents_alive, reward

    #------ BASELINE related methods ------###
    def compute_optimality_baseline(self, parcel_id, extra_charge=2.5, debug_log=False):
        """Used in the optimality baseline
            Input: parcel_id, (extra_charge)
            Output: new total delivery rounds needed for all parcels
        """
        parcel = self.state['parcels'][parcel_id]
        path_time = 2 + shortest_path_length(self.graph, parcel[0], parcel[1], 'weight')

        _time = math.ceil(path_time * extra_charge) # round to next higher integer

        min_index = self.o_employed.index(min(self.o_employed))
        self.o_employed[min_index] += _time

        return max(self.o_employed)

    def compute_central_delivery(self, p_id, debug_log = False):
        """Used in the central baseline, iteratively tries to find a good delivery route
        with the available agents in the time specified in BASELINE_TIME_CONSTRAINT
            Input: parcel_id
            Output: Dict: {agent_id: [actions], ...}  --> update that dict! (merge in this function with prev actions!)
        """

        if self.agents_base is None:
            self.agents_base = {a_id: (a['position'], 0) for (a_id, a) in self.state["agents"].items()}  # last instructed pos + its step count

        min_time = None
        new_missions = {}   # key: agent, value: [actions]

        source = self.state["parcels"][p_id][0]
        target = self.state["parcels"][p_id][1]
        shortest_paths_generator = nx.shortest_simple_paths(self.graph, source, target, weight='weight')

        running = BoolTimer(self.BASELINE_TIME_CONSTRAINT)
        running.start()

        while running:
            # Default --> assign full route to nearest drone
            if min_time is None:

                air_route = nx.shortest_path(self.graph, source= source, target= target, weight="air")
                air_route_time = nx.shortest_path_length(self.graph, source= source, target= target, weight="air")
                air_route.pop(0)  # remove source node from path

                # Assign most suitable Drone
                best_drone = None
                for (a_id, a_tp) in self.agents_base.items():
                    if a_id[0] != 'd':                      # filter for correct agent type
                        continue

                    journey_time = nx.shortest_path_length(self.graph, source=a_tp[0], target=source, weight="air") + a_tp[1]
                    if min_time is None or journey_time < min_time:
                        min_time = journey_time
                        best_drone = (a_id, a_tp)

                # construct path for agent
                drone_route = nx.shortest_path(self.graph, source= best_drone[1][0], target= source, weight="air")

                drone_route_actions = [x+1 for x in drone_route[1:]] + [(self.ACTION_DROPOFF + p_id, 0)] + [x+1 for x in air_route[1:]] + [self.ACTION_DROPOFF]   # increment node_ids by one to get corresponding action
                min_time += air_route_time + 2 # add 2 steps for pick & drop

                self._add_charging_stops_to_route(drone_route_actions, debug_log=debug_log)

                new_missions[best_drone[0]] = (drone_route_actions, min_time)

            else:   # try to improve the existing base mission
                try:
                    shortest_route = next(shortest_paths_generator)
                except StopIteration:
                    break               # all existing shortest paths already tried

                subroutes = self._path_to_subroutes(shortest_route, debug_log=debug_log)
                duration, min_agents = self._find_best_agents(subroutes, min_time, debug_log=debug_log)

                if duration < min_time:
                    # faster delivery route found!
                    assert duration < min_time, "Central Baseline prefered a longer route..."

                    # update min_time
                    min_time = duration
                    new_missions = self._build_missions(min_agents, subroutes, p_id, debug_log=debug_log)

        #---- end while = Timer
        # now save the best mission found in the ENV
        for agent in new_missions.keys():
            # retrieve target node from mission, depends on charging stops and case no move necessary
            target = None
            if isinstance(new_missions[agent][0][-2], int):
                target = new_missions[agent][0][-2]
                if target == 0: target = new_missions[agent][0][-3]    # charging action was added before dropoff
                target -= 1                                            # action-1 = node_id
            else:  # handle case no move necessary -> pick action before dropoff
                target = self.agents_base[agent][0]

            self.agents_base[agent] = (target, self.agents_base[agent][1] + new_missions[agent][1])
            self.baseline_missions[agent].extend(new_missions[agent][0])

        return new_missions

    def _find_best_agents(self, subroutes, min_time, debug_log=False):
        """ For use in centrality_baseline. Finds best available agents for traversing a set of subroutes
            and returns these with the total duration for doing so.
            Input: subroutes = [ edge_type, [actions]]
            """

        min_agents = {}
        temp_agents_base = {k:v for k,v in self.agents_base.items()}  # deep copy for temporary planning transfer or stick with agent??

        for i,r in enumerate(subroutes):

            # init some helper vars
            a_type = "d" if r[0] == 'air' else 'c'
            min_time_sub = None    # best time (min) for this subroute (closest agent)
            best_agent_sub = None

            # iterate over agents of correct type!  --> later: Busy / unbusy
            for (a_id, a_tp) in temp_agents_base.items():
                #reminder: a_tp is tuple of latest future position (node, timestep)

                # filter for correct agent type - even if type is 'both' one agent can still take only its edge type
                weight_agent = r[0]
                if r[0] == 'both':
                    weight_agent = 'road' if a_id[0] == 'c' else 'air'
                else:                     # wrong agent type
                    if a_id[0] != a_type:  # todo replace with parameter variable in function!
                        continue

                journey_time = nx.shortest_path_length(self.graph, source=a_tp[0], target=r[1][0], weight=weight_agent) + a_tp[1] # earliest time agent can be there
                if min_time_sub is None or journey_time < min_time_sub:
                    min_time_sub = journey_time
                    best_agent_sub = (a_id, a_tp)    # a_id, a_tp = (latest location, timestep)

            # closest available agent found
            best_agent_weight = weight_agent = 'road' if best_agent_sub[0] == 'c' else 'air'
            duration_sub = min_time_sub + nx.shortest_path_length(self.graph, source=r[1][0], target=r[1][-1], weight=best_agent_weight) + 1

            # update agent state in temporary planning
            temp_agents_base[best_agent_sub[0]] = (r[1][-1], duration_sub)

            # agent_tuple, duration_subroutes_until_then
            min_agents[i] = (best_agent_sub, duration_sub)

            if debug_log: assert duration_sub < sys.maxsize, "Non existent edge taken somewhere..."

            # check if current subroute already longer than the min one
            if duration_sub > min_time:
                break # already worse, check next simple path

        return duration_sub, min_agents

    def _build_missions(self, min_agents, subroutes, parcel_id, debug_log = False):
        """For use in centrality_baseline. Computes list of actions for delivery of parcel parcel_id
        and necessary duration for execution from list of agents, subroutes"""
        new_mission = {}

        for i, s in enumerate(subroutes):
            best_agent_pos = min_agents[i][0][1][0]
            #earliest time to start that action
            time_pickup = min_agents[i-1][1] if i > 0 else 0 # First subroute pickup as soon as possible

            # construct the actual delivery path to pickup
            delivery_route = nx.shortest_path(self.graph, source=best_agent_pos, target=s[1][0], weight=s[0])
            delivery_route_actions = [x+1 for x in delivery_route[1:]] + [(self.ACTION_DROPOFF + parcel_id, time_pickup)] + [x+1 for x in s[1][1:]] + [self.ACTION_DROPOFF]

            if debug_log: assert min_agents[i][0] not in new_mission  #I see no preferable case where agent picks-up 1 parcel 2 times --> holding always better than following

            self._add_charging_stops_to_route(delivery_route_actions, debug_log=debug_log)
            new_mission[min_agents[i][0][0]] = (delivery_route_actions, min_agents[i][1])

        return new_mission

    def _add_charging_stops_to_route(self, route_actions, debug_log=False):
        """For use in centrality_baseline. Iterates through a list of actions and inserts a charging action
        after every move action to a node with a charging station.
        Tuples representing Dropoff actions with minimal executions time are updated to account for eventual delays. """

        delay = 0
        for i, n in enumerate(route_actions):
            if type(n) is tuple:
                if delay > 0: route_actions[i] = (n[0], n[1] + delay)
            else:
                if n-1 in self.CHARGING_STATION_NODES:
                    delay += 1
                    route_actions.insert(i+1, 0)

    def _path_to_subroutes(self, path, debug_log= False):
        """For use in centrality_baseline. Takes path in the graph as input and returns list of subroutes
        split at changes of the edge types with the type"""

        # get subroutes by their edge_types
        e_type_prev = None
        e_type_changes = [] # save indices of source nodes before new edge type
        subroutes = []
        _subroute = [path[0]]

        if len(path) > 1:
            e_type_prev = self.graph.edges[path[0], path[1]]['type']

            for i, node in enumerate(path[1:-1], start=1):

                e_type_next = self.graph.edges[node, path[i+1]]['type']

                _subroute.append(node)
                if e_type_next != e_type_prev:
                    subroutes.append((e_type_prev, _subroute))
                    _subroute = [node]
                    e_type_prev = e_type_next

            _subroute.append(path[-1])
        subroutes.append((e_type_prev, _subroute))   # don't forget last subroute

        return subroutes
_____no_output_____

## Agent Model and Experiment Evaluation Code_____no_output_____

from gym.spaces import Discrete, Box, Tuple, MultiDiscrete, Dict, MultiBinary
#from ray.rllib.utils.spaces.space_utils import flatten_space
#from ray.rllib.models.preprocessors import DictFlatteningPreprocessor

# Parametric-action agent model  --> apply Action Masking!
class ParametricAgentModel(TFModelV2):
    def __init__(self, obs_space, action_space, num_outputs, model_config, name, *args, **kwargs):
        super(ParametricAgentModel, self).__init__(obs_space, action_space, num_outputs, model_config, name, *args, **kwargs)

        assert isinstance(action_space, Discrete), f'action_space is a {type(action_space)}, but should be Discrete!'

        # Adjust for number of agents/parcels/Nodes!! -> Simply copy found shape from the thrown exception
        true_obs_shape = (1750, )
        action_embed_size = action_space.n

        self.action_embed_model = FullyConnectedNetwork(
            Box(0, 1, shape=true_obs_shape),
            action_space,
            action_embed_size,
            model_config,
            name + '_action_embedding')

    def forward(self, input_dict, state, seq_lens):

        action_mask = input_dict['obs']['allowed_actions']
        action_embedding, _ = self.action_embed_model.forward({'obs_flat': input_dict["obs_flat"]}, state, seq_lens)
        intent_vector = tf.expand_dims(action_embedding, 1)
        action_logits = tf.reduce_sum(intent_vector, axis=1)
        inf_mask = tf.maximum(tf.math.log(action_mask), tf.float32.min)

        return action_logits + inf_mask, state

    def value_function(self):
        return self.action_embed_model.value_function()
    _____no_output_____## Proposed way to train / evaluate MARL policy from github Issues: --> https://github.com/ray-project/ray/issues/9123 and  https://github.com/ray-project/ray/issues/9208

def train(config, name, save_dir, stop_criteria, num_samples, verbosity=1):
    """
    Train an RLlib agent using tune until any of the configured stopping criteria is met.
    :param stop_criteria: Dict with stopping criteria.
        See https://docs.ray.io/en/latest/tune/api_docs/execution.html#tune-run
    :return: Return the path to the saved agent (checkpoint) and tune's ExperimentAnalysis object
        See https://docs.ray.io/en/latest/tune/api_docs/analysis.html#experimentanalysis-tune-experimentanalysis
    """

    print("Start training")
    analysis = ray.tune.run(DQNTrainer, verbose=verbosity, config=config, local_dir=save_dir,
                            stop=stop_criteria, name=name, num_samples=num_samples,
                            checkpoint_at_end=True, resume=True)

    # list of lists: one list per checkpoint; each checkpoint list contains 1st the path, 2nd the metric value
    checkpoints = analysis.get_trial_checkpoints_paths(trial=analysis.get_best_trial('episode_reward_mean', mode='max'),
                                                       metric='episode_reward_mean')
    # retriev the checkpoint path; we only have a single checkpoint, so take the first one
    checkpoint_path = checkpoints[0][0]
    print(f"Saved trained model in checkpoint {checkpoint_path} - achieved episode_reward_mean: {checkpoints[0][1]}")
    return checkpoint_path, analysis

def load(config, path):
    """
    Load a trained RLlib agent from the specified path. Call this before testing the trained agent.
    """
    agent = DQNTrainer(config=config) #, env=env_class)
    agent.restore(path)
    return agent

def test(env_class, env_config, policy_mapping_fcn, agent):
    """Test trained agent for a single episode. Return the retrieved env metrics for this episode and the episode reward"""
    # instantiate env class
    env = env_class(env_config)
    episode_reward = 0
    done = False
    obs = env.reset()

    while not done: # run until episode ends
        actions = {}
        for agent_id, agent_obs in obs.items():
            # Here: policy_id == agent_id - added this to avoid confusion for other policy mappings
            policy_id = policy_mapping_fcn(agent_id, episode=None, worker=None)
            actions[agent_id] = agent.compute_action(agent_obs, policy_id=policy_id)
        obs, reward, done, info = env.step(actions)
        done = done['__all__']

        # sum up reward for all agents
        episode_reward += sum(reward.values())

    # Retrieve custom metrics from ENV
    return env.metrics, episode_reward
_____no_output_____#  Import the corresponding Trainer class for your algorithm of choice here - needed for the evaluation.
from ray.rllib.agents.dqn import DQNTrainer

def train_and_test_scenarios(config, seeds=None):
    """ Trains for a single scenario indicated by a seed """
    # TODO how to distinguish between the different algos ?
    print("Starte: run_function_trainer!")

    # prepare the config dicts
    NAME = config['NAME']
    SAVE_DIR = config['SAVE_DIR']
    ENVIRONMENT = config['ENV']

    # Simulations
    NUMBER_STEPS_PER_EPISODE = config['NUMBER_STEPS_PER_EPISODE']
    STOP_CRITERIA = config['STOP_CRITERIA']
    NUMBER_OF_SAMPLES = config['NUMBER_OF_SAMPLES']

    #MAP / ENV
    NUMBER_OF_DRONES = config['NUMBER_OF_DRONES']
    NUMBER_OF_CARS = config['NUMBER_OF_CARS']
    NUMBER_OF_AGENTS = NUMBER_OF_DRONES + NUMBER_OF_CARS
    MAX_NUMBER_OF_PARCELS = config['MAX_NUMBER_OF_PARCELS']

    # TESTING
    SEEDS = config['SEEDS']

    env_config = {
            'DEBUG_LOGS':False,
            'TOPOLOGY': config['TOPOLOGY'],
            # Simulation config
            'NUMBER_STEPS_PER_EPISODE': NUMBER_STEPS_PER_EPISODE,
            #'NUMBER_OF_TIMESTEPS': NUMBER_OF_TIMESTEPS,
            'RANDOM_SEED': None, # 42
            # Map
            'CHARGING_STATION_NODES': config['CHARGING_STATION_NODES'],
            # Entities
            'NUMBER_OF_DRONES': NUMBER_OF_DRONES,
            'NUMBER_OF_CARS': NUMBER_OF_CARS,
            'MAX_BATTERY_POWER': config['MAX_BATTERY_POWER'],  # TODO split this for drone and car??
            'INIT_NUMBER_OF_PARCELS': config['INIT_NUMBER_OF_PARCELS'],
            'MAX_NUMBER_OF_PARCELS': config['MAX_NUMBER_OF_PARCELS'],
            'THRESHOLD_ADD_NEW_PARCEL': config['THRESHOLD_ADD_NEW_PARCEL'],
            # Baseline settings
            'BASELINE_FLAG': False,  # is set True in the test function when needed
            'BASELINE_OPT_CONSTANT': config['BASELINE_OPT_CONSTANT'],
            'BASELINE_TIME_CONSTRAINT': config['BASELINE_TIME_CONSTRAINT'],
            # TODO
            #Rewards
            'REWARDS': config['REWARDS']
        }

    run_config = {
        'num_gpus': config['NUM_GPUS'],
        'num_workers': config['NUM_WORKERS'],
        'env': ENVIRONMENT,
        'env_config': env_config,
        'multiagent': {
            'policies': {
                # tuple values: policy, obs_space, action_space, config
                **{a: (None, None, None, { 'model': {'custom_model': ParametricAgentModel }, 'framework': 'tf'}) for a in ['d_'+ str(j) for j in range(NUMBER_OF_DRONES)] + ['c_'+ str(i) for i in range(NUMBER_OF_DRONES, NUMBER_OF_CARS + NUMBER_OF_DRONES)]}
            },
            'policy_mapping_fn': policy_mapping_fn,
            'policies_to_train': ['d_'+ str(i) for i in range(NUMBER_OF_DRONES)] + ['c_'+ str(i) for i in range(NUMBER_OF_DRONES, NUMBER_OF_CARS + NUMBER_OF_DRONES)]
        },
        #'log_level': "INFO",
        "hiddens": [],     # For DQN
        "dueling": False,  # For DQN
    }

    # Train and Evaluate the agents !
    checkpoint, analysis = train(run_config, NAME, SAVE_DIR, STOP_CRITERIA, NUMBER_OF_SAMPLES)
    print("Training finished - Checkpoint: ", checkpoint)

    env_class = ENVIRONMENT
    # Restore trained policies for evaluation
    agent = load(run_config, checkpoint)
    print("Agent loaded - Agent: ", agent)

    # Run the test cases for the specified seeds
    seeds = SEEDS#[10, 91, 32, 75, 6]
    runs = {'max_steps': NUMBER_STEPS_PER_EPISODE, 'max_parcels': MAX_NUMBER_OF_PARCELS, 'max_agents': NUMBER_OF_AGENTS}

    for seed in seeds:
        env_config['RANDOM_SEED'] = seed
        # TODO check if
        assert run_config['env_config']['RANDOM_SEED'] == seed
        result = test_scenario(run_config, agent)
        runs.update(result)

    return runs

def test_scenario(config, agent):
    """
    Loads a pretrained agent, initializes an environment from the seed
    and then evaluates it over one episode with the Marl agents and the central baseline.

    Returns: dict with results for graph creation for both evaluation / inference runs
    """
    #TODO unterscheidung run_config > env_config
    # Wie die beiden Runs abspeichern --> {Seed + [marl / base] : result }

    env_class = config['env']
    env_config = config['env_config']
    seed = env_config['RANDOM_SEED']
    policy_mapping_fn = config['multiagent']['policy_mapping_fn']

    # Test with MARL

    metrics_marl, reward_marl = test(env_class, env_config, policy_mapping_fn, agent)

    # Test with CentralBase
    env_config['BASELINE_FLAG'] = True
    metrics_central, reward_central = test(env_class, env_config, policy_mapping_fn, agent)
    env_config['BASELINE_FLAG'] = False

    # ASSERT that both optimal values are equal
    #assert metrics_marl['optimal'] == metrics_central['optimal']

    return {"M_" + str(seed): metrics_marl, "C_" + str(seed): metrics_central}

_____no_output_____

def policy_mapping_fn(agent_id, episode, worker, **kwargs):
    return agent_id

basic_config = {
        # experiment
        'NAME': 'dqn',
        'SAVE_DIR': 'Exp_algorithms',
        'ALGO': DQNTrainer,
        'ENV': Map_Environment,
        'DEBUG_LOGS':False,
        'NUM_GPUS': 2,
        'NUM_WORKERS': 32,
        'NUMBER_OF_SAMPLES': 2,
        # Simulation config
        'NUMBER_STEPS_PER_EPISODE': 1200,
        'RANDOM_SEED': None, # 42
        # Map
        'TOPOLOGY': topology,
        'CHARGING_STATION_NODES': [0,1,2,3,4,6,9],
        # Entities
        'NUMBER_OF_DRONES': 2,
        'NUMBER_OF_CARS': 2,
        'INIT_NUMBER_OF_PARCELS': 10,
        'MAX_NUMBER_OF_PARCELS': 10,
        'THRESHOLD_ADD_NEW_PARCEL': 0.1, # 10% chance
        'MAX_BATTERY_POWER': 100,
        #Baseline
        'BASELINE_TIME_CONSTRAINT': 10,
        'BASELINE_OPT_CONSTANT': 2.5,
        #TESTING
        'SEEDS': [72, 21, 44, 66, 86, 14],
        #Rewards
        'REWARDS': {
            'PARCEL_DELIVERED': 200,
            'STEP_PENALTY': -0.1,
        },
        'STOP_CRITERIA': {
            'timesteps_total': 24_000_000,
        }
}

print("Test evaluation functions")
experiment_results = train_and_test_scenarios(basic_config)
print("Test evaluation finished ;)")_____no_output_____ experiment_results_____no_output_____saved_results = {'max_steps': 1200,
 'max_parcels': 10,
 'max_agents': 4,
 'M_72': {'step': 1200,
  'delivered': {},
  'crashed': {'c_3': 604},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_72': {'step': 1200,
  'delivered': {'p_4': 49,
   'p_1': 67,
   'p_3': 67,
   'p_7': 95,
   'p_5': 126,
   'p_10': 224,
   'p_9': 403},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 75},
 'M_21': {'step': 1200,
  'delivered': {},
  'crashed': {'c_3': 220},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_21': {'step': 1200,
  'delivered': {'p_1': 47,
   'p_6': 76,
   'p_3': 135,
   'p_9': 215,
   'p_7': 350,
   'p_8': 539},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 105},
 'M_44': {'step': 1200,
  'delivered': {},
  'crashed': {'d_0': 78, 'c_3': 711},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_44': {'step': 1200,
  'delivered': {'p_7': 253, 'p_8': 258},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 103},
 'M_66': {'step': 1200,
  'delivered': {'p_10': 1116},
  'crashed': {'d_1': 186},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_66': {'step': 1200,
  'delivered': {'p_1': 16, 'p_3': 18, 'p_5': 70},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 85},
 'M_86': {'step': 1200,
  'delivered': {},
  'crashed': {'c_3': 195},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_86': {'step': 1200,
  'delivered': {'p_8': 66,
   'p_1': 90,
   'p_9': 110,
   'p_4': 112,
   'p_7': 168,
   'p_2': 185,
   'p_6': 282},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 90},
 'M_14': {'step': 1200,
  'delivered': {},
  'crashed': {'c_3': 926},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': None},
 'C_14': {'step': 1200,
  'delivered': {'p_1': 12, 'p_3': 12, 'p_6': 66},
  'crashed': {},
  'added': {1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, 8: 0, 9: 0, 10: 0},
  'optimal': 108}}_____no_output_____
def create_chart_bars(results_dict):
    """ Function that plots a bar graph with the duration of one episode
        run with MARL agents/ Centrality Baseline/ Optimality Baseline, recorded in the :param results_dict:.
    """

    # Design choices
    colors = {
        "marl": 'blue',
        "central": 'green',
        "optimal": 'red'
    }

    # Retrieve settings values from results dict
    max_steps = results_dict['max_steps']
    max_parcels = results_dict['max_parcels']
    max_agents = results_dict['max_agents']

    # Deep copy for further computations
    scenario_results = {k:v for k,v in results_dict.items() if k[0:3] != 'max'}

    # filter
    runs = scenario_results.keys()
    values = scenario_results.values()

    # TODO statt all dead => einfach not all delivered marker
    merged = {}  # key is seed as str, value is a dict with [marl, central, optimal, all_dead_marl, all_dead_cent]

    # Retrieve the data
    for run_id, res in scenario_results.items():

        split_id = run_id.split('_')  # --> type, seed
        key_type, key_seed = split_id[0], split_id[1]
        # merge data from the runs with same seed (marl + baselines)

        # add new dict if seed not encountered yet
        if key_seed not in merged:
            merged[key_seed] = {}

        _key_delivered = 'marl'
        #_key_crashed = 'all_dead_marl'
        if key_type == 'C':
            # Baseline run
            merged[key_seed]['optimal'] = res['optimal']
            _key_delivered = 'central'
            #_key_crashed = 'all_dead_central'

        # Retrieve number of steps in run
        last_step = res['step']

        # old code for plotting the number of steps needed in the episode
        merged[key_seed][_key_delivered] = last_step
        merged[key_seed][_key_delivered + '_all'] = len(res['delivered'])

#        all_parcels_delivered = len(res['delivered']) == max_parcels     # were all parcels delivered
#        merged[key_seed][_key_delivered + '_all'] = all_parcels_delivered
#        if not all_parcels_delivered:
#            merged[key_seed][_key_delivered] = max_steps

    print("Merged: ", merged)

    #  example data = [[30, 25, 50, 20],
#    [40, 23, 51, 17],
#    [35, 22, 45, 19]]

    data = [[],[],[],[], []]
    labels = []

    # split data into type of run
    for seed,values in merged.items():
        labels.append('S_'+seed)
        data[0].append(values['marl'])
        data[1].append(values['central'])
        data[2].append(values['optimal'])
        data[3].append(values['marl_all'])
        data[4].append(values['central_all'])

    print("Data: ", data)

    X = np.arange(len(labels))
    #print(X)

    fig = plt.figure()
    ax = fig.add_axes([0,0,1,1])

    #ax.bar(X + 0.00, data[2], color = colors['optimal'], label="Optimality Baseline", width = 0.25)
    ax.bar(X + 0.25, data[3], color = colors['marl'], label="MARL System", width = 0.25, alpha=0.8)
    ax.bar(X + 0.50, data[4], color = colors['central'], label="Centrality Baseline", width = 0.25)

    plt.xlabel("Experiments")
    plt.ylabel("Parcels_delivered")
    plt.ylim(bottom=0, top=max_parcels)

    # y axis as percentage
    yticks = mtick.PercentFormatter(max_parcels)
    ax.yaxis.set_major_formatter(yticks)

    # Add experiment identifiers x-Axis
    plt.xticks(X + 0.37, labels)
    # Add legend
    ax.legend()
_____no_output_____# Plot duration bar graphs from the results
create_chart_bars(experiment_results)_____no_output_____import matplotlib.ticker as mtick

def create_chart_episode_events(results_dict, draw_crashes = False):
    """ Function that plots a graph with either the deliveries of parcels or the crashes of agents
        over the course of an episode, recorded in the :param results_dict:.
        Set the :param draw_crashes: flag for plotting crashes, default are deliveries.
    """
    # TODO Graphen auch abspeichern oder nur hier anzeigen ??
    # TODO - Parcel additions ??
    # TODO - overthink fill with max_value for mean computation

    # Design choices
    colors = {
        "marl": 'blue',
        "central": 'green',
        "optimal": 'red'
    }
    alpha = 0.6  # Opacity of individual value lines
    alpha_opt = 0.2
    opt_marker_size = 15
    line_width_mean = 5
    line_width_optimal = 2

    fig = plt.figure()
    ax = fig.add_subplot(111)

    # Retrieve settings values from results dict
    max_steps = results_dict['max_steps']
    max_parcels = results_dict['max_parcels']
    max_agents = results_dict['max_agents']

    _len = max_agents if draw_crashes else max_parcels
    _len += 1     # start plot at origin

    _key = 'crashed' if draw_crashes else 'delivered'

    # Deep copy for further computations
    scenario_results = {k:v for k,v in results_dict.items() if k[0:3] != 'max'}

    # for computation of mean
    m_values, c_values, o_values = [], [], []
    Y = [str(i) for i in range(0, _len)]

    # iterate over configs
    for scenario, results in scenario_results.items():

        # Default settings -> MARL run
        color = colors["marl"]
        _type = m_values
        label= "marl_system"

        if scenario[0] == 'C':
            # Baseline run
            color = colors["central"]
            _type = c_values
            label = "centrality_baseline"

            # Retrieve and plot optimality baseline
            optimal_time = results['optimal']
            assert optimal_time is not None
            if not draw_crashes: ax.plot(optimal_time, 0, "*", color = colors["optimal"], label="optimality_baseline", markersize=opt_marker_size, alpha= alpha_opt, clip_on=False)
            o_values.append(optimal_time)

        _num_steps = results['step']

        X = [0] + list(results[_key].values())

        X = X + [max_steps]*(_len - len(X))           # Fill X up with max_step values for not delivered parcels / not crashed agents

        _type.append(X)  # add X to the respective mean list
        #Y = [str(i) for i in range(0, len(results[_key].values())+1)]

        #print("Data: ", results[_key].values())
        #print("new X: ", X)
        #print("new Y: ", Y)

        ax.step(X, Y, label=label, where='post', color=color, alpha=alpha)

        # Attempt to improve the filling mess in the plot...
        #X = X + [max_steps]*(_len - len(X))           # Fill X up with max_step values for not delivered parcels / not crashed agents

        #_type.append(X)  # add X to the respective mean list

    # compute mean values
    m_mean = np.mean(np.array(m_values), axis=0)
    c_mean = np.mean(np.array(c_values), axis=0)
    o_mean = np.mean(np.array(o_values), axis=0)

    ax.step(m_mean, Y, label="marl_system", where='post', color=colors["marl"], linewidth=line_width_mean)
    ax.step(c_mean, Y, label="centrality_baseline", where='post', color=colors["central"], linewidth=line_width_mean)

    # star for opt:    if not draw_crashes: plt.plot(o_mean, 0, "*", label="optimality_baseline", color="r", markersize=opt_marker_size, clip_on=False)
    # better?: vertical line for opt
    if not draw_crashes: ax.axvline(o_mean, label="optimality_baseline", color=colors["optimal"], linewidth=2, alpha=alpha_opt+0.3)

    # y axis as percentage
    max_percent = max_agents if draw_crashes else max_parcels
    xticks = mtick.PercentFormatter(max_percent)
    ax.yaxis.set_major_formatter(xticks)

    # Lables and Legend
    plt.xlabel("steps")
    ylabel = "% of crashed agents" if draw_crashes else "% of delivered parcels"
    plt.ylabel(ylabel)
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())

    # Margins
    plt.ylim(bottom=0)
    plt.xlim()
    plt.margins(x=0, y=0)

    plt.show()
    _____no_output_____# Plot delivery graphs from the results
create_chart_episode_events(experiment_results, draw_crashes=False)
_____no_output_____# Plot crash graphs from the results
create_chart_episode_events(experiment_results, draw_crashes=True)_____no_output_____

### Manual Actions for debugging_____no_output_____

##################
env_config = {
            'DEBUG_LOGS':False,
            'TOPOLOGY': topology,
            # Simulation config
            'NUMBER_STEPS_PER_EPISODE': 1000,
            #'NUMBER_OF_TIMESTEPS': NUMBER_OF_TIMESTEPS,
            'RANDOM_SEED': None, # 42
            # Map
            'CHARGING_STATION_NODES': [0,1,2,3,4],
            # Entities
            'NUMBER_OF_DRONES': 2,
            'NUMBER_OF_CARS': 2,
            'MAX_BATTERY_POWER': 100,  # TODO split this for drone and car??
            'INIT_NUMBER_OF_PARCELS': 3,
            'MAX_NUMBER_OF_PARCELS': 3,
            'THRESHOLD_ADD_NEW_PARCEL': 0.01,
            # Baseline settings
            'BASELINE_FLAG': False,  # is set True in the test function when needed
            'BASELINE_OPT_CONSTANT': 2.5,
            'BASELINE_TIME_CONSTRAINT': 5,
            # TODO
            #Rewards
            'REWARDS': {
                'PARCEL_DELIVERED': 200,
                'STEP_PENALTY': -0.1,
            },
        }

env = Map_Environment(env_config)
env.state
#env.ACTION_DROPOFF_____no_output_____# TODO select actions to give agent 0 reward!!
#print(env.action_space)

actions_1 = {'d_0': 6, 'd_1': 2, 'c_2': 3, 'c_3': 4}
#actions_1 = {'d_0': 0, 'd_1': 0, 'c_2': 5, 'c_3': 5}
actions_2 = {'d_0': 2, 'd_1': 8, 'c_2': 7, 'c_3':0}
actions_3 = {'d_0': 5, 'd_1': 5, 'c_2':2 }

new_obs, rewards, dones, infos = env.step(actions_1)
print(infos)
print(f"New Obs are: {new_obs}")
print(rewards)
print("------------------")
new_obs2, rewards2, dones2, infos2 = env.step(actions_2)
print(f"New Obs are: {new_obs2}")
print(rewards2)
print("------------------")
new_obs3, rewards3, dones3, infos3 = env.step(actions_3)
print(f"New Obs are: {new_obs3}")
print(rewards3)

actions_4 = {'d_0': 0, 'd_1': 0, 'c_2': 1, 'c_3': 0}
actions_5 = {'d_0': 0, 'd_1': 0, 'c_2': 0, 'c_3': 0}
actions_6 = {'d_0': 0, 'd_1': 0, 'c_2': 5, 'c_3': 0}

new_obs, rewards, dones, infos = env.step(actions_4)
print(f"New Obs are: {new_obs}")
print(rewards)
print("------------------")
new_obs2, rewards2, dones2, infos2 = env.step(actions_5)
print(f"New Obs are: {new_obs2}")
print(rewards2)
print("------------------")
new_obs3, rewards3, dones3, infos3 = env.step(actions_6)
print(f"New Obs are: {new_obs3}")
print(rewards3)

print("------------------")
print(dones3)_____no_output_____# TENSORBOARD
# Load the TensorBoard notebook extension
%load_ext tensorboard
#Start tensorboard below the notebook
#%tensorboard --logdir logs_____no_output_____


# Notebook from satrio-hw/pymc3
Path: docs/source/notebooks/cox_model.ipynb

# Cox model_____no_output_____

import arviz as az
import numpy as np
import pymc3 as pm
import scipy as sp
import theano.tensor as tt

from pymc3 import (NUTS, Gamma, Metropolis, Model, Normal, Poisson, find_MAP,
                   forestplot, sample, starting, traceplot)
from theano import function as fn
from theano import printing

print(f'Running on PyMC3 v{pm.__version__}')Running on PyMC3 v3.9.0
%config InlineBackend.figure_format = 'retina'
az.style.use('arviz-darkgrid')_____no_output_____

Here is the original model, implemented in BUGS:

```R
model
{
    # Set up data
        for(i in 1:Nsubj) {
            for(j in 1:T) {
    # risk set = 1 if obs.t >= t
                Y[i,j] <- step(obs.t[i] - t[j] + eps)
    # counting process jump = 1 if obs.t in [ t[j], t[j+1] )
    #                      i.e. if t[j] <= obs.t < t[j+1]
                dN[i, j] <- Y[i, j] * step(t[j + 1] - obs.t[i] - eps) * FAIL[i]
            }
        }

    # Model
        for(j in 1:T) {
            for(i in 1:Nsubj) {
                dN[i, j]   ~ dpois(Idt[i, j])              # Likelihood
                Idt[i, j] <- Y[i, j] * exp(beta[1]*pscenter[i] + beta[2]*
                hhcenter[i] + beta[3]*ncomact[i] + beta[4]*rleader[i] + beta[5]*dleader[i] + beta[6]*inter1[i] + beta[7]*inter2[i]) * dL0[j]    # Intensity
            }
            dL0[j] ~ dgamma(mu[j], c)
            mu[j] <- dL0.star[j] * c    # prior mean hazard
        }

    c ~ dgamma(0.0001, 0.00001)
    r ~ dgamma(0.001, 0.0001)

    for (j in 1 : T) {  dL0.star[j] <- r * (t[j + 1] - t[j])  }
    # next line indicates number of covariates and is for the corresponding betas
    for(i in 1:7) {beta[i] ~ dnorm(0.0,0.00001)}

}
```_____no_output_____

dta = dict(T=73, Nsubj=430, eps=0.0,  t=[1, 21, 85, 128, 129, 148, 178, 204,
                                         206, 210, 211, 212, 225, 238, 241,
                                         248, 259, 273, 275, 281, 286, 289,
                                         301, 302, 303, 304, 313, 317, 323,
                                         344, 345, 349, 350, 351, 355, 356,
                                         359, 364, 385, 386, 389, 390, 391,
                                         392, 394, 395, 396, 397, 398, 399,
                                         400, 406, 415, 416, 426, 427, 434,
                                         435, 437, 441, 447, 448, 449, 450,
                                         451, 453, 455, 456, 458, 459, 460,
                                         461, 462, 463],
obs_t = [460, 313, 435, 350, 435, 350, 350, 460, 460, 448, 225, 225, 396, 435, 396, 396, 453, 396, 456, 397, 397, 396, 395, 275, 449, 395, 395, 462, 302, 302, 458, 461, 396, 241, 389, 458, 304, 304, 395, 395, 364, 460, 415, 463, 396, 459, 441, 435, 396, 458, 437, 396, 356, 356, 396, 455, 396, 462, 399, 400, 350, 350, 395, 395, 441, 355, 85, 458, 128, 396, 386, 386, 386, 462, 458, 390, 390, 396, 396, 396, 427, 458, 395, 275, 275, 395, 359, 395, 395, 441, 395, 463, 178, 275, 463, 396, 396, 259, 396, 396, 458, 441, 396, 463, 396, 463, 435, 396, 437, 396, 398, 463, 460, 462, 460, 460, 210, 396, 435, 458, 385, 323, 323, 359, 396, 396, 460, 238, 441, 450, 392, 458, 396, 458, 396, 396, 462, 435, 396, 394, 396, 435, 458, 1, 395, 395, 451, 462, 458, 462, 396, 286, 396, 349, 449, 462, 455, 21, 463, 461, 461, 456, 435, 396, 460, 462, 462, 435, 435, 460, 386, 396, 458, 386, 461, 441, 435, 435, 463, 456, 396, 275, 460, 406, 460, 406, 317, 406, 461, 396, 359, 458, 463, 435, 462, 458, 396, 396, 273, 396, 435, 281, 275, 396, 447, 225, 447, 396, 435, 416, 396, 248, 396, 435, 435, 396, 461, 385, 396, 458, 458, 396, 461, 396, 448, 396, 396, 460, 455, 456, 463, 462, 458, 463, 396, 462, 395, 456, 396, 463, 396, 435, 459, 396, 396, 396, 395, 435, 455, 395, 461, 344, 396, 395, 396, 317, 396, 395, 426, 461, 396, 289, 441, 395, 396, 458, 396, 396, 435, 396, 395, 396, 441, 345, 396, 359, 435, 435, 396, 396, 395, 458, 461, 458, 212, 301, 458, 456, 395, 396, 395, 435, 396, 396, 303, 458, 460, 400, 396, 462, 359, 458, 396, 206, 441, 396, 458, 396, 462, 396, 396, 275, 396, 395, 435, 435, 462, 225, 458, 462, 396, 396, 289, 396, 303, 455, 400, 400, 359, 461, 396, 462, 460, 463, 463, 463, 204, 435, 435, 396, 396, 396, 463, 458, 396, 455, 435, 396, 396, 463, 396, 461, 463, 460, 441, 460, 435, 435, 460, 455, 460, 395, 460, 460, 460, 435, 449, 463, 462, 129, 391, 396, 391, 391, 434, 356, 462, 396, 349, 225, 396, 435, 461, 391, 391, 351, 211, 461, 212, 434, 148, 356, 458, 456, 455, 435, 463, 463, 462, 435, 463, 437, 460, 396, 406, 451, 460, 435, 396, 460, 455, 396, 398, 456, 458, 396, 456, 449, 396, 128, 396, 462, 463, 396, 396, 396, 435, 460, 396, 458],
FAIL= [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
pscenter= [
 -.01434325, -.01460965, .01322687, .00971885, -.03223412, -.01113493, -.01359567, -.03357866, -.0387039, -.0553269, -.03238896, -.07464545, -.07325128, -.07062459, -.07464545, -.07032613, -.0703005, .00965232, -.01408955, .00577483, -.00219072, -.00084567, .01643198, .06509522, .06824313, .07300876, .07300876, .01394272, .06824313, .02063087, .00383186, -.02573045, -.02410864, -.02272752, .05120398, -.00997729, -.00550709, -.02062663, -.03077685, -.01688493, .01035959, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .01149963, .0034338, .0376236, .00733331, .01520069, .03832785, .03832785, -.02622275, -.02622275, -.02622275, -.01492678, -.02897806, -.02897806, -.02897806, -.02847666, -.031893, -.03919478, -.04224754, -.04743705, -.0510477, -.031893, -.01129093, .01706207, .00193999, -.01503116, .003101, -.00083466, .02395027, -.07952866, -.08559135, -.07251801, -.06586029, -.08432532, -.0613939, -.081205, -.07540084, -.08488011, -.08488011, -.08488011, -.07492433, -.08907269, -.09451609, -.05301854, -.08980743, -.0771635, -.0771635, -.08650947, -.07856082, -.0771635, -.08204606, -.08178245, -.05263504, -.05355574, -.05109092, -.04696729, -.04696729, -.04696729, -.05257489, -.05303248, -.05348096, -.04983674, -.04699414, .00584956, -.00792241, -.01719816, -.02138029, -.01576016, -.04274812, -.04014061, .0471441, .0471441, .0471441, .0471441, .0471441, .0471441, .0471441, .04233112, .0471441, .04233112, .050568, .07388823, .0493324, .04512087, .03205975, .02913185, .06010427, .05324252, .06973204, .05579907, .01212243, .07962459, .05054695, .06672142, .14026688, .01734403, .06078221, .06543709, .06438115, .20126908, -.03138622, -.02180659, .01637333, -.02415774, .01828684, .03106104, .04268495, .01897239, .01591935, -.02367065, -.0619156, -.06403028, -.06851645, -.04821694, -.03889525, -.05023452, -.05013452, -.01557191, -.01171948, -.01362136, -.01174715, -.02707938, -.02634164, -.02634164, -.02634164, -.00692153, -.02381614, -.00890537, -.00611669, -.00894752, -.03551984, -.0252678, -.01513384, -.01016569, -.03551984, -.03773227, -.01978032, .06803483, .06706496, .10551275, .15091534, .03092981, .06556855, .10781559, .12671031, .0936299, .09362991, .09362991, .08294538, .09362991, .09362991, .09362991, .01177025, .02610553, .03546937, .03546937, .03546937, .034415, -.00305626, .04973665, .05103208, .07546701, .05306436, .00824125, .01961115, .01202359, -.02919447, -.01016712, .01756074, -.04035511, -.04753104, -.04463152, -.04845615, -.05010044, .00031411, -.07911871, -.08799869, -.07980882, -.09393142, -.08000018, -.07666632, -.07817401, -.07444922, -.07226554, -.08216553, -.0777643, -.07752042, -.05767992, -.04727952, -.03774814, -.06870384, -.05999847, -.05947695, .02989959, .04627543, .02772475, .02883079, .03642944, .02871235, .04148949, .04240279, .07747082, .07626323, .04268012, .03225577, .06468724, -.05140995, -.05399637, -.05351515, .07302427, .02432223, .0490674, .0490674, .0490674, .0490674, .09013112, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .10476315, .07008056, .08666077, .01546215, .01667466, .03417671, .05253941, .04293926, .01496588, .02692172, -.03827151, .04809769, .08742411, .04533176, .01455173, .01831875, .02710811, .09834951, .09952456, .06993483, .02945534, .038731, .1181948, .04435538, .04435538, -.02357505, .05824019, .05820741, -.02357505, .09324722, .15534712, .07207468, .04692869, -.03490683, -.04404809, -.05054474, -.05325826, -.0474724, -.04905931, .01068221, .02879751, .00852646, .02693032, .01835589, .02989959, .02989959, .02989959, .04976377, .04439012, .03397319, .02989959, .02989959, .05468828, .04463226, .05886378, .06311052, .02989959, .04595331, .04203459, .01231324, -.01399783, .04595331, .00145386, .04601278, .06459354, -.0007196, .00012216, -.07614055, -.08435525, -.07957162, -.10299519, -.08156988, -.08225659, -.07449063, -.00210284, -.00797183, -.025355, -.01258251, -.04372031, -.03985972, -.03545086, -.03384566, -.04025533, -.07523724, -.05947702, -.061286, -.07666647, -.07663169, -.05902354, -.07652324, -.07645561, -.06258684, -.09604834, -.08813326, -.03292062, -.07848112, -.08239502, -.08316891, -.07244316, -.075417, -.07652324, -.07922532, -.08755959, -.08583414, -.07450142, -.08066016, -.06057205, -.07652324, -.06249051, -.08781742, -.086076, -.07652324, -.07696518, -.0618688, -.06073988, -.06524737, -.04419825, -.04489509, -.04390368, -.04358438, -.04489509, -.04520512, -.04187583, -.03653955, -.03973426, -.03753508, -.03569439, -.06789339, .06689456, .05526327, .05139003, .02641841, .04891529, .07078697, .06862645, .06832582, .04104258, -.00120631, .01947345, .04891779, .04891779, .03561932, .02576244, .03158225, .03608047, .08685057, .04632537, .06841581, -.02899643],
hhcenter= [ -.78348798, -.63418788, -.91218799, -.98388809, -.23518796, .11481193, -1.415588, -1.2535881, -.55738801, -.88128799, -1.109488, .05721192, -1.045788, -.30888793, .29651192, -.36688802, -.50058788, .02271203, -.59088796, -.04198809, .50561196, -.07418796, .98481184, .78921205, .09431199, -.06488796, 2.1662121, .08891205, 1.4004121, 1.316112, 1.9362121, 2.0107121, 1.150712, .31951192, -.23918791, -.1562881, -.9575879, -.07728811, .29641202, 1.2273121, 1.7717118, 1.5764117, .14181189, .72131211, 1.279212, .68241197, -.72808808, -.00488802, -.23938794, -1.000788, .55081207, -.52348799, 1.780612, -.35888812, .36481193, 1.5480118, -.03078791, 1.389112, .30211189, .70901209, -.16668792, 1.435812, .47001198, 2.0838118, 1.1673121, .18461208, -.30608794, 1.4470119, .23301201, -.58458799, .44011191, -.61948794, -.41388795, .263212, .66171199, .92451197, .78081208, .90991193, 1.6920118, 1.334012, 1.2101121, .41591194, -.48498794, -.73278803, -1.093588, .09911207, -.93418807, -.46908805, .0205119, .0535119, -.14228792, -.55708808, -.45498797, -.54008788, -.30998799, -.10958811, -.0960879, -.01338812, -.88168806, -.51788801, .36801198, .46621206, .13271193, -.11208793, -.76768798, -.54508799, -1.2773881, .16641192, .95871216, -.48238799, 1.6281118, -.18848796, -.49718806, -.41348812, -.31628796, -.59528798, -.11718794, -.57058805, -.59488791, -.21248789, -.65658802, -.56298798, -.52698797, -.65758795, -.04988809, .55341202, -.76328796, .254612, 1.3500118, -.54958791, 1.665812, .14671211, 1.963912, .29161194, -.56838793, 1.9371119, .90991193, -.39558789, .39521196, -.55208796, -.05268808, -.77368802, -.45428798, .05841212, -.45308802, -.12458798, .01431207, -.28228804, .79281193, -.26358792, -.54738802, -.38158795, -.54118794, -.72828788, -.58128804, .355912, -.24078794, -1.0384881, -.75038809, -.41018793, -.43538806, -1.566388, -.53388804, -.28388807, -1.2348881, -.69028801, -1.620088, -.78128809, -.54648799, -.92738789, .11871199, .26851204, .61571199, .82891208, 1.1985121, 1.012012, 1.0602121, -.02988811, .79301196, .67731196, .43991187, .9404121, .5254119, 1.0365119, 1.6220121, .61671191, -.50318807, 2.6073117, .02361206, -.60438794, -.79278797, -.18108793, -.48178813, -.44038793, -.22628804, -.07398792, .519512, .40211204, .582012, 1.830512, .80441195, .58801204, -.56368798, -1.5451881, .45991209, -.23448797, -.36918804, 1.3247118, .19541197, -.20818801, 1.163012, -.78228801, -.6048879, -.575288, 1.3241119, .0147119, -.76518792, -.37478802, -.35508797, -.90038794, -1.250888, -.46608803, -.98488802, -1.5185881, -.90908808, -1.048188, -.90138787, -.77278799, -1.248988, -.34448811, -.61628789, .38531187, -.51728791, -.00878807, -.60078806, -.45358798, .46301201, -.22048803, -.71518797, -.76478809, -.75028795, -.4952881, .01731209, -.83718795, .57951194, .54291207, .45341209, .16941194, 1.054112, .61721212, 2.2717118, 1.1593118, 2.0280118, .92281204, 1.0100121, -.1866879, 2.6503119, 2.3914118, -.19948788, -.36418793, -.9259879, -.71058792, -.1104879, .16971211, 1.474812, 1.9360118, 2.5344119, 2.0171118, 1.9387121, .55071193, -.03918811, .20681195, .40421203, -.75518793, -.45678803, -1.0271881, .77211195, 1.146812, -1.147788, -1.565588, -.34888789, 1.303812, 1.952312, 1.639112, .07731203, .25901201, -.45608804, -.5028879, .03641204, -.03808804, .38571194, .31831196, -.17648788, -.44528791, -.55918807, -.53108805, .39721206, -.06328794, -.34038803, -.05988808, -.89548796, -.03518792, .045512, -.1859879, -.039288, -.82568806, .01431207, .40091208, -.2531881, .030412, -.31918809, -.54958791, -.79078788, .36691192, -.324388, -1.0082881, -1.232188, -.53248805, -.23678799, -.89188808, .25111201, -.6766879, -.3565881, -.61228794, -.21078797, -1.0343881, -.58358806, -.15588804, -.39238808, -.67818803, -.19498797, 1.099412, 1.2767119, -.64068788, -.50678796, -.64058799, -.86918801, 1.4048119, -.59648794, .23331194, .68371207, .11251191, -.17128797, .17081194, -.44218799, -.48708794, .09591202, .20131211, -.20108791, -.02158805, -.48188803, -.3012881, -.55008787, -1.146188, -.82128805, -.87638801, -.54488796, -.60288805, -1.003088, -.25078794, -.14818807, -.14738794, -.80938786, -.85988802, -.90188807, -.94998807, -.75718802, -.37418792, -.66708797, 1.0981121, 1.1441121, .47381189, -.12958808, -.34358808, -.84328789, -.33498809, -.98088807, -.6903879, -1.284988, -.80838794, -.91838807, -.81848806, -.34488794, -.83438796, .12971191, .99381214, -.91608804, -.31808802, -.01018806, .98171192, -.91638798, -1.043988, -1.0103881, 1.451612, -.01528808, .02441196, -.41458794, .25691202, .18601207, -.815988, -.02908798, -.59088796, -.35608789, .79691201, 1.8123121, -.98588794, 1.548912, 2.3653121, -.09238812, .96741205, .05891208, -.15618797, -.5660879, -.28338811, -.10088798, 1.1663117, .21981196, .07151202, -.009088, -.49578807, .15441208, -.44488809, -.2677879, -.54388803, -.25468799, .68631202, -.88128799, -.84628791, -1.2549881, -.36198804],
ncomact= [ 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1],
rleader= [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dleader= [ 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
inter1= [ -.01434325, -.01460965, 0, 0, 0, -.01113493, 0, 0, 0, -.0553269, -.03238896, 0, 0, -.07062459, -.07464545, -.07032613, 0, 0, -.01408955, 0, -.00219072, 0, 0, 0, 0, 0, .07300876, .01394272, 0, 0, 0, 0, 0, 0, .05120398, 0, -.00550709, -.02062663, -.03077685, -.01688493, 0, .01149963, 0, .01149963, .01149963, 0, 0, 0, 0, 0, 0, 0, 0, 0, .01149963, .0034338, .0376236, .00733331, 0, .03832785, .03832785, -.02622275, -.02622275, -.02622275, -.01492678, 0, 0, -.02897806, -.02847666, 0, 0, -.04224754, -.04743705, -.0510477, -.031893, 0, 0, 0, -.01503116, .003101, -.00083466, .02395027, -.07952866, 0, 0, -.06586029, 0, -.0613939, -.081205, -.07540084, -.08488011, -.08488011, 0, -.07492433, -.08907269, -.09451609, 0, -.08980743, 0, -.0771635, 0, 0, -.0771635, -.08204606, 0, -.05263504, 0, -.05109092, -.04696729, 0, -.04696729, 0, -.05303248, -.05348096, 0, 0, .00584956, -.00792241, -.01719816, 0, -.01576016, 0, -.04014061, 0, 0, 0, 0, 0, .0471441, 0, .04233112, 0, .04233112, 0, 0, .0493324, .04512087, .03205975, .02913185, 0, .05324252, 0, 0, 0, 0, .05054695, 0, .14026688, .01734403, .06078221, 0, 0, 0, -.03138622, 0, .01637333, 0, 0, 0, 0, .01897239, .01591935, 0, -.0619156, 0, -.06851645, 0, -.03889525, -.05023452, -.05013452, 0, 0, -.01362136, 0, 0, -.02634164, 0, 0, 0, 0, -.00890537, -.00611669, 0, 0, 0, -.01513384, 0, -.03551984, 0, -.01978032, 0, .06706496, .10551275, 0, .03092981, .06556855, 0, 0, 0, .09362991, 0, 0, 0, 0, 0, 0, .02610553, .03546937, 0, 0, .034415, 0, 0, 0, .07546701, 0, 0, 0, 0, -.02919447, -.01016712, 0, 0, 0, 0, -.04845615, -.05010044, 0, 0, 0, 0, 0, 0, -.07666632, 0, 0, -.07226554, -.08216553, -.0777643, 0, 0, -.04727952, 0, -.06870384, -.05999847, 0, 0, 0, .02772475, .02883079, .03642944, 0, .04148949, 0, 0, 0, .04268012, .03225577, 0, -.05140995, -.05399637, 0, 0, .02432223, 0, .0490674, .0490674, .0490674, 0, 0, 0, 0, 0, 0, 0, 0, .10476315, 0, 0, 0, 0, 0, .07008056, 0, 0, .01667466, 0, .05253941, .04293926, 0, .02692172, 0, 0, .08742411, .04533176, 0, .01831875, 0, .09834951, .09952456, 0, .02945534, .038731, 0, .04435538, 0, -.02357505, 0, 0, -.02357505, .09324722, 0, 0, 0, -.03490683, 0, -.05054474, 0, -.0474724, -.04905931, 0, .02879751, 0, 0, 0, 0, 0, 0, 0, .04439012, 0, .02989959, .02989959, .05468828, .04463226, 0, 0, 0, 0, 0, .01231324, -.01399783, .04595331, .00145386, 0, .06459354, -.0007196, 0, -.07614055, -.08435525, 0, -.10299519, 0, 0, 0, -.00210284, -.00797183, 0, 0, 0, 0, -.03545086, 0, 0, 0, 0, -.061286, -.07666647, 0, -.05902354, -.07652324, -.07645561, 0, 0, 0, -.03292062, 0, 0, 0, 0, -.075417, 0, -.07922532, 0, -.08583414, -.07450142, -.08066016, 0, 0, -.06249051, 0, 0, 0, 0, -.0618688, 0, -.06524737, -.04419825, -.04489509, 0, 0, 0, -.04520512, -.04187583, 0, 0, -.03753508, 0, 0, 0, 0, 0, 0, 0, 0, .06862645, 0, 0, -.00120631, .01947345, 0, 0, .03561932, 0, .03158225, .03608047, 0, 0, 0, -.02899643],

inter2= [-.78348798, -.63418788, 0, 0, 0, .11481193, 0, 0, 0, -.88128799, -1.109488, 0, 0, -.30888793, .29651192, -.36688802, 0, 0, -.59088796, 0, .50561196, 0, 0, 0, 0, 0, 2.1662121, .08891205, 0, 0, 0, 0, 0, 0, -.23918791, 0, -.9575879, -.07728811, .29641202, 1.2273121, 0, 1.5764117, 0, .72131211, 1.279212, 0, 0, 0, 0, 0, 0, 0, 0, 0, .36481193, 1.5480118, -.03078791, 1.389112, 0, .70901209, -.16668792, 1.435812, .47001198, 2.0838118, 1.1673121, 0, 0, 1.4470119, .23301201, 0, 0, -.61948794, -.41388795, .263212, .66171199, 0, 0, 0, 1.6920118, 1.334012, 1.2101121, .41591194, -.48498794, 0, 0, .09911207, 0, -.46908805, .0205119, .0535119, -.14228792, -.55708808, 0, -.54008788, -.30998799, -.10958811, 0, -.01338812, 0, -.51788801, 0, 0, .13271193, -.11208793, 0, -.54508799, 0, .16641192, .95871216, 0, 1.6281118, 0, -.49718806, -.41348812, 0, 0, -.11718794, -.57058805, -.59488791, 0, -.65658802, 0, -.52698797, 0, 0, 0, 0, 0, 1.3500118, 0, 1.665812, 0, 1.963912, 0, 0, 1.9371119, .90991193, -.39558789, .39521196, 0, -.05268808, 0, 0, 0, 0, -.12458798, 0, -.28228804, .79281193, -.26358792, 0, 0, 0, -.72828788, 0, .355912, 0, 0, 0, 0, -.43538806, -1.566388, 0, -.28388807, 0, -.69028801, 0, -.78128809, -.54648799, -.92738789, 0, 0, .61571199, 0, 0, 1.012012, 0, 0, 0, 0, .43991187, .9404121, 0, 0, 0, .61671191, 0, 2.6073117, 0, -.60438794, 0, -.18108793, -.48178813, 0, -.22628804, -.07398792, 0, 0, 0, 1.830512, 0, 0, 0, 0, 0, 0, -.36918804, 1.3247118, 0, 0, 1.163012, 0, 0, 0, 1.3241119, 0, 0, 0, 0, -.90038794, -1.250888, 0, 0, 0, 0, -1.048188, -.90138787, 0, 0, 0, 0, 0, 0, -.00878807, 0, 0, .46301201, -.22048803, -.71518797, 0, 0, -.4952881, 0, -.83718795, .57951194, 0, 0, 0, 1.054112, .61721212, 2.2717118, 0, 2.0280118, 0, 0, 0, 2.6503119, 2.3914118, 0, -.36418793, -.9259879, 0, 0, .16971211, 0, 1.9360118, 2.5344119, 2.0171118, 0, 0, 0, 0, 0, 0, 0, 0, .77211195, 0, 0, 0, 0, 0, 1.952312, 0, 0, .25901201, 0, -.5028879, .03641204, 0, .38571194, 0, 0, -.44528791, -.55918807, 0, .39721206, 0, -.34038803, -.05988808, 0, -.03518792, .045512, 0, -.039288, 0, .01431207, 0, 0, .030412, -.31918809, 0, 0, 0, -.324388, 0, -1.232188, 0, -.23678799, -.89188808, 0, -.6766879, 0, 0, 0, 0, 0, 0, 0, -.67818803, 0, 1.099412, 1.2767119, -.64068788, -.50678796, 0, 0, 0, 0, 0, .68371207, .11251191, -.17128797, .17081194, 0, -.48708794, .09591202, 0, -.20108791, -.02158805, 0, -.3012881, 0, 0, 0, -.87638801, -.54488796, 0, 0, 0, 0, -.14738794, 0, 0, 0, 0, -.75718802, -.37418792, 0, 1.0981121, 1.1441121, .47381189, 0, 0, 0, -.33498809, 0, 0, 0, 0, -.91838807, 0, -.34488794, 0, .12971191, .99381214, -.91608804, 0, 0, .98171192, 0, 0, 0, 0, -.01528808, 0, -.41458794, .25691202, .18601207, 0, 0, 0, -.35608789, .79691201, 0, 0, 1.548912, 0, 0, 0, 0, 0, 0, 0, 0, 1.1663117, 0, 0, -.009088, -.49578807, 0, 0, -.2677879, 0, -.25468799, .68631202, 0, 0, 0, -.36198804])_____no_output_____def load_data_cox(dta):
    array = lambda x : np.array(dta[x], dtype=float)
    t = array('t')
    obs_t = array('obs_t')
    pscenter = array('pscenter')
    hhcenter = array('hhcenter')
    ncomact = array('ncomact')
    rleader = array('rleader')
    dleader = array('dleader')
    inter1 = array('inter1')
    inter2 = array('inter2')
    fail = array('FAIL')
    return (t, obs_t, pscenter, hhcenter, ncomact,
            rleader, dleader, inter1, inter2, fail)_____no_output_____(t, obs_t, pscenter, hhcenter, ncomact, rleader,
     dleader, inter1, inter2, fail) = load_data_cox(dta)_____no_output_____X = np.array([pscenter, hhcenter, ncomact, rleader, dleader, inter1, inter2])_____no_output_____X.shape_____no_output_____with Model() as model:

    T = len(t) - 1
    nsubj = len(obs_t)

    # risk set equals one if obs_t >= t
    Y = np.array([[int(obs >= time) for time in t] for obs in obs_t])
    # counting process. jump = 1 if obs_t \in [t[j], t[j+1])
    dN = np.array([[Y[i,j]*int(t[j+1] >= obs_t[i])*fail[i] for j in range(T)] for i in
                    range(nsubj)])

    c = Gamma('c', .0001, .00001)
    r = Gamma('r', .001, .0001)

    dL0_star = r*np.diff(t)

    # prior mean hazard
    mu = dL0_star * c

    dL0 = Gamma('dL0', mu, c, shape=T)

    beta = Normal('beta', np.zeros(7),
                  np.ones(7)*100, shape=7)

    linear_model = tt.exp(tt.dot(X.T, beta))
    idt = Y[:, :-1] * tt.outer(linear_model, dL0)

    dn_like = Poisson('dn_like', idt, observed=dN)_____no_output_____with model:
    trace = sample(2000, n_init=10000, init='advi_map')Auto-assigning NUTS sampler...
Initializing NUTS using advi_map...
traceplot(trace, var_names=['c', 'r']);/dependencies/arviz/arviz/data/io_pymc3.py:89: FutureWarning: Using `from_pymc3` without the model will be deprecated in a future release. Not using the model will return less accurate and less useful results. Make sure you use the model argument or call from_pymc3 within a model context.
  FutureWarning,
forestplot(trace, var_names=['beta']);/dependencies/arviz/arviz/data/io_pymc3.py:89: FutureWarning: Using `from_pymc3` without the model will be deprecated in a future release. Not using the model will return less accurate and less useful results. Make sure you use the model argument or call from_pymc3 within a model context.
  FutureWarning,
%load_ext watermark
%watermark -n -u -v -iv -wscipy 1.4.1
arviz 0.8.3
pymc3 3.9.0
numpy 1.18.5
last updated: Mon Jun 15 2020

CPython 3.7.7
IPython 7.15.0
watermark 2.0.2