Text Generation
Transformers
Safetensors
English
bolmo
custom_code
benjamin commited on
Commit
49b278a
·
verified ·
1 Parent(s): 474e6ac

Update modeling_bolmo.py

Browse files
Files changed (1) hide show
  1. modeling_bolmo.py +61 -9
modeling_bolmo.py CHANGED
@@ -1,7 +1,11 @@
1
  import copy
 
 
2
  from typing import Callable, Optional, Union, cast
3
  import math
4
 
 
 
5
  import torch
6
  import torch.nn as nn
7
  from torch.nn import functional as F
@@ -21,7 +25,38 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
21
  from transformers.processing_utils import Unpack
22
  from transformers.utils import can_return_tuple
23
  from transformers.utils.deprecation import deprecate_kwarg
24
- from transformers.utils.generic import check_model_inputs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
  from .configuration_bolmo import BolmoConfig
27
  from .tokenization_bolmo import BolmoTokenizerConfig
@@ -882,12 +917,18 @@ class BolmoRotaryEmbedding(nn.Module):
882
  self.original_max_seq_len = config.max_position_embeddings
883
 
884
  self.config = config
885
- self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
 
 
 
886
 
887
  inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
888
  self.register_buffer("inv_freq", inv_freq, persistent=False)
889
  self.original_inv_freq = self.inv_freq
890
 
 
 
 
891
  @torch.no_grad()
892
  @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
893
  def forward(self, x, position_ids):
@@ -1031,18 +1072,23 @@ class BolmoModel(BolmoPreTrainedModel):
1031
  # Prepare mask arguments
1032
  mask_kwargs = {
1033
  "config": self.config,
1034
- "input_embeds": h_patch,
 
1035
  "attention_mask": attention_mask,
1036
- "cache_position": cache_position,
1037
  "past_key_values": past_key_values,
1038
  "position_ids": position_ids,
1039
  }
1040
  # Create the masks. Models whose global blocks are all full attention (OLMo 2,
1041
  # Llama 3, Qwen 3) have no `sliding_window` to build a sliding mask from.
1042
- causal_mask_mapping = {"full_attention": create_causal_mask(**mask_kwargs)}
 
 
 
 
1043
  if self.has_sliding_layers:
1044
  causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
1045
- **mask_kwargs
1046
  )
1047
 
1048
  position_embeddings_mapping = {
@@ -1195,9 +1241,15 @@ class BolmoForCausalLM(BolmoPreTrainedModel, GenerationMixin):
1195
  ) -> Union[GenerateOutput, torch.Tensor]:
1196
  # generic preprocessing
1197
 
1198
- generation_config, model_kwargs = self._prepare_generation_config(
1199
- generation_config, use_model_defaults, **kwargs
1200
- )
 
 
 
 
 
 
1201
  self._prepare_special_tokens(generation_config, device=self.model.device)
1202
 
1203
  logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
 
1
  import copy
2
+ import inspect
3
+ from importlib.metadata import version as _package_version
4
  from typing import Callable, Optional, Union, cast
5
  import math
6
 
7
+ from packaging.version import Version
8
+
9
  import torch
10
  import torch.nn as nn
11
  from torch.nn import functional as F
 
25
  from transformers.processing_utils import Unpack
26
  from transformers.utils import can_return_tuple
27
  from transformers.utils.deprecation import deprecate_kwarg
28
+ from transformers.utils.generic import check_model_inputs as _check_model_inputs
29
+
30
+ TRANSFORMERS_V5 = Version(_package_version("transformers")) >= Version("5.0.0")
31
+
32
+ _CAUSAL_MASK_PARAMS = frozenset(inspect.signature(create_causal_mask).parameters)
33
+ _SLIDING_MASK_PARAMS = frozenset(inspect.signature(create_sliding_window_causal_mask).parameters)
34
+
35
+ if TRANSFORMERS_V5:
36
+ # A plain decorator from 5.0.0 on (a deprecated alias of merge_with_config_defaults).
37
+ def check_model_inputs(**kwargs):
38
+ """Swallow 4.x's factory kwargs (e.g. tie_last_hidden_states); 5.x has no slot for them."""
39
+ return _check_model_inputs
40
+
41
+ else:
42
+ # A decorator factory before 5.0.0.
43
+ check_model_inputs = _check_model_inputs
44
+
45
+
46
+ def _compute_default_rope_parameters(config, device=None, **kwargs):
47
+ """Inverse frequencies for the original RoPE.
48
+
49
+ 5.0.0 dropped both the "default" entry of ROPE_INIT_FUNCTIONS and the module-level
50
+ helper, relocating it to a staticmethod on the rotary-embedding classes. Reads
51
+ either the 4.x (`rope_theta`) or 5.x (`rope_parameters`) config layout.
52
+ """
53
+ base = getattr(config, "rope_theta", None)
54
+ if base is None:
55
+ base = config.rope_parameters["rope_theta"]
56
+ dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
57
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
58
+ return inv_freq.to(device), 1.0
59
+
60
 
61
  from .configuration_bolmo import BolmoConfig
62
  from .tokenization_bolmo import BolmoTokenizerConfig
 
917
  self.original_max_seq_len = config.max_position_embeddings
918
 
919
  self.config = config
920
+ if TRANSFORMERS_V5 and self.rope_type == "default":
921
+ self.rope_init_fn = _compute_default_rope_parameters
922
+ else:
923
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
924
 
925
  inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
926
  self.register_buffer("inv_freq", inv_freq, persistent=False)
927
  self.original_inv_freq = self.inv_freq
928
 
929
+ # 5.x `_init_weights` looks this up on the rotary module itself.
930
+ compute_default_rope_parameters = staticmethod(_compute_default_rope_parameters)
931
+
932
  @torch.no_grad()
933
  @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
934
  def forward(self, x, position_ids):
 
1072
  # Prepare mask arguments
1073
  mask_kwargs = {
1074
  "config": self.config,
1075
+ "input_embeds": h_patch, # renamed `inputs_embeds` in 5.2.0
1076
+ "inputs_embeds": h_patch,
1077
  "attention_mask": attention_mask,
1078
+ "cache_position": cache_position, # dropped in 5.9.0
1079
  "past_key_values": past_key_values,
1080
  "position_ids": position_ids,
1081
  }
1082
  # Create the masks. Models whose global blocks are all full attention (OLMo 2,
1083
  # Llama 3, Qwen 3) have no `sliding_window` to build a sliding mask from.
1084
+ causal_mask_mapping = {
1085
+ "full_attention": create_causal_mask(
1086
+ **{k: v for k, v in mask_kwargs.items() if k in _CAUSAL_MASK_PARAMS}
1087
+ )
1088
+ }
1089
  if self.has_sliding_layers:
1090
  causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
1091
+ **{k: v for k, v in mask_kwargs.items() if k in _SLIDING_MASK_PARAMS}
1092
  )
1093
 
1094
  position_embeddings_mapping = {
 
1241
  ) -> Union[GenerateOutput, torch.Tensor]:
1242
  # generic preprocessing
1243
 
1244
+ # 5.0.0 dropped the positional `use_model_defaults`.
1245
+ if TRANSFORMERS_V5:
1246
+ generation_config, model_kwargs = self._prepare_generation_config(
1247
+ generation_config, **kwargs
1248
+ )
1249
+ else:
1250
+ generation_config, model_kwargs = self._prepare_generation_config(
1251
+ generation_config, use_model_defaults, **kwargs
1252
+ )
1253
  self._prepare_special_tokens(generation_config, device=self.model.device)
1254
 
1255
  logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()