| { | |
| "_checkpoint_path": "../checkpoints_ablations/albertina_M6_c_20251129_154628_last.ckpt", | |
| "_matformer_config_dict": { | |
| "_checkpoint_path": "../checkpoints_ablations/albertina_M6_c_20251129_154628_last.ckpt", | |
| "_model_class": "BERTModel", | |
| "_tokenizer_name": "mrinaldi/Gettone", | |
| "attention_type": [], | |
| "bias": false, | |
| "block_size_for_attention": 128, | |
| "bos_token_id": 5, | |
| "cloze_probability": 0.8, | |
| "compile_flexattn": false, | |
| "custom_layers": {}, | |
| "decoder": null, | |
| "default_layer": { | |
| "attn_impl": "flash", | |
| "ffn_activation": "swiglu", | |
| "hooks": {}, | |
| "normalization": "rmsnorm", | |
| "normalization_position": "pre", | |
| "positional_encoding": [ | |
| "alibi", | |
| "rope" | |
| ], | |
| "sliding_window_size": null | |
| }, | |
| "encoder": null, | |
| "entropy": null, | |
| "eos_token_id": 6, | |
| "ffn_factor": 4.0, | |
| "has_entropy_model": null, | |
| "has_text_autoencoder": null, | |
| "hidden_size": 768, | |
| "is_causal": null, | |
| "loss_type": "normal", | |
| "mask_token_id": 4, | |
| "masked_substitution_rate": 0.15, | |
| "max_position_embeddings": 1024, | |
| "model_class": null, | |
| "name": "Albertina_M6_c", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "num_labels": 2, | |
| "pad_token_id": 0, | |
| "random_probability": 0.1, | |
| "rms_norm_eps": 1e-06, | |
| "rope_theta": 10000.0, | |
| "same_probability": 0.1, | |
| "sliding_type": null, | |
| "tie_word_embeddings": false, | |
| "tokenizer_name": null, | |
| "tokenizer_type": null, | |
| "training_objective": null, | |
| "vocab_size": 32768 | |
| }, | |
| "_model_class": "BERTModel", | |
| "_tokenizer_name": "mrinaldi/Gettone", | |
| "attention_type": [], | |
| "auto_map": { | |
| "AutoConfig": "modeling_matformer.MatformerConfig", | |
| "AutoModel": "modeling_matformer.MatformerModel", | |
| "AutoModelForMaskedLM": "modeling_matformer.MatformerForMaskedLM" | |
| }, | |
| "bias": false, | |
| "block_size_for_attention": 128, | |
| "bos_token_id": 5, | |
| "cloze_probability": 0.8, | |
| "compile_flexattn": false, | |
| "custom_layers": {}, | |
| "decoder": null, | |
| "default_layer": { | |
| "attn_impl": "flash", | |
| "ffn_activation": "swiglu", | |
| "hooks": {}, | |
| "normalization": "rmsnorm", | |
| "normalization_position": "pre", | |
| "positional_encoding": [ | |
| "alibi", | |
| "rope" | |
| ], | |
| "sliding_window_size": null | |
| }, | |
| "encoder": null, | |
| "entropy": null, | |
| "eos_token_id": 6, | |
| "ffn_factor": 4.0, | |
| "has_entropy_model": null, | |
| "has_text_autoencoder": null, | |
| "hidden_size": 768, | |
| "is_causal": null, | |
| "loss_type": "normal", | |
| "mask_token_id": 4, | |
| "masked_substitution_rate": 0.15, | |
| "max_position_embeddings": 1024, | |
| "model_class": null, | |
| "model_type": "matformer", | |
| "name": "Albertina_M6_c", | |
| "num_attention_heads": 12, | |
| "num_hidden_layers": 12, | |
| "pad_token_id": 0, | |
| "random_probability": 0.1, | |
| "rms_norm_eps": 1e-06, | |
| "rope_theta": 10000.0, | |
| "same_probability": 0.1, | |
| "sliding_type": null, | |
| "tokenizer_name": null, | |
| "tokenizer_type": null, | |
| "training_objective": null, | |
| "transformers_version": "4.57.1", | |
| "use_cache": true, | |
| "vocab_size": 32768 | |
| } | |