{ "architectures": [ "Metis16ForCausalLM" ], "auto_map": { "AutoConfig": "configuration_metis16.Metis16Config", "AutoModelForCausalLM": "modeling_metis16.Metis16ForCausalLM" }, "model_type": "metis16", "family": "praxis", "vocab_size": 65536, "hidden_size": 2048, "num_hidden_layers": 12, "attention_layer_indices": [ 4, 8 ], "num_attention_heads": 32, "num_key_value_heads": 8, "head_dim": 64, "attention_pass_lora_rank": 8, "mamba_d_state": 128, "mamba_d_conv": 4, "mamba_expand": 2, "mamba_head_dim": 64, "mamba_n_groups": 8, "mamba_chunk_size": 256, "mamba_norm_eps": 1e-05, "n_streams": 4, "mhc_pass_embedding_dim": 64, "mhc_sinkhorn_iterations": 8, "latent_dim": 1024, "n_routed_experts": 128, "num_experts_per_tok": 4, "min_routed_k": 1, "max_routed_k": 8, "expert_intermediate_size": 512, "shared_expert_intermediate_size": 512, "route_feature_dim": 256, "memory_dim": 256, "num_passes": 2, "max_passes": 5, "continuation_logit_limit": 4.0, "rms_norm_eps": 1e-06, "max_position_embeddings": 131072, "tie_word_embeddings": true, "use_cache": true, "pad_token_id": 1, "bos_token_id": null, "eos_token_id": [ 4, 0 ], "torch_dtype": "bfloat16" }