Instructions to use allenai/Bolmo-1B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use allenai/Bolmo-1B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="allenai/Bolmo-1B", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("allenai/Bolmo-1B", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use allenai/Bolmo-1B with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "allenai/Bolmo-1B" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "allenai/Bolmo-1B", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/allenai/Bolmo-1B
- SGLang
How to use allenai/Bolmo-1B with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "allenai/Bolmo-1B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "allenai/Bolmo-1B", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "allenai/Bolmo-1B" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "allenai/Bolmo-1B", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use allenai/Bolmo-1B with Docker Model Runner:
docker model run hf.co/allenai/Bolmo-1B
Update modeling_bolmo.py
Browse files- modeling_bolmo.py +61 -9
modeling_bolmo.py
CHANGED
|
@@ -1,7 +1,11 @@
|
|
| 1 |
import copy
|
|
|
|
|
|
|
| 2 |
from typing import Callable, Optional, Union, cast
|
| 3 |
import math
|
| 4 |
|
|
|
|
|
|
|
| 5 |
import torch
|
| 6 |
import torch.nn as nn
|
| 7 |
from torch.nn import functional as F
|
|
@@ -21,7 +25,38 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
|
|
| 21 |
from transformers.processing_utils import Unpack
|
| 22 |
from transformers.utils import can_return_tuple
|
| 23 |
from transformers.utils.deprecation import deprecate_kwarg
|
| 24 |
-
from transformers.utils.generic import check_model_inputs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
from .configuration_bolmo import BolmoConfig
|
| 27 |
from .tokenization_bolmo import BolmoTokenizerConfig
|
|
@@ -882,12 +917,18 @@ class BolmoRotaryEmbedding(nn.Module):
|
|
| 882 |
self.original_max_seq_len = config.max_position_embeddings
|
| 883 |
|
| 884 |
self.config = config
|
| 885 |
-
|
|
|
|
|
|
|
|
|
|
| 886 |
|
| 887 |
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
| 888 |
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 889 |
self.original_inv_freq = self.inv_freq
|
| 890 |
|
|
|
|
|
|
|
|
|
|
| 891 |
@torch.no_grad()
|
| 892 |
@dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
|
| 893 |
def forward(self, x, position_ids):
|
|
@@ -1031,18 +1072,23 @@ class BolmoModel(BolmoPreTrainedModel):
|
|
| 1031 |
# Prepare mask arguments
|
| 1032 |
mask_kwargs = {
|
| 1033 |
"config": self.config,
|
| 1034 |
-
"input_embeds": h_patch,
|
|
|
|
| 1035 |
"attention_mask": attention_mask,
|
| 1036 |
-
"cache_position": cache_position,
|
| 1037 |
"past_key_values": past_key_values,
|
| 1038 |
"position_ids": position_ids,
|
| 1039 |
}
|
| 1040 |
# Create the masks. Models whose global blocks are all full attention (OLMo 2,
|
| 1041 |
# Llama 3, Qwen 3) have no `sliding_window` to build a sliding mask from.
|
| 1042 |
-
causal_mask_mapping = {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1043 |
if self.has_sliding_layers:
|
| 1044 |
causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
|
| 1045 |
-
**mask_kwargs
|
| 1046 |
)
|
| 1047 |
|
| 1048 |
position_embeddings_mapping = {
|
|
@@ -1195,9 +1241,15 @@ class BolmoForCausalLM(BolmoPreTrainedModel, GenerationMixin):
|
|
| 1195 |
) -> Union[GenerateOutput, torch.Tensor]:
|
| 1196 |
# generic preprocessing
|
| 1197 |
|
| 1198 |
-
|
| 1199 |
-
|
| 1200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1201 |
self._prepare_special_tokens(generation_config, device=self.model.device)
|
| 1202 |
|
| 1203 |
logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
|
|
|
|
| 1 |
import copy
|
| 2 |
+
import inspect
|
| 3 |
+
from importlib.metadata import version as _package_version
|
| 4 |
from typing import Callable, Optional, Union, cast
|
| 5 |
import math
|
| 6 |
|
| 7 |
+
from packaging.version import Version
|
| 8 |
+
|
| 9 |
import torch
|
| 10 |
import torch.nn as nn
|
| 11 |
from torch.nn import functional as F
|
|
|
|
| 25 |
from transformers.processing_utils import Unpack
|
| 26 |
from transformers.utils import can_return_tuple
|
| 27 |
from transformers.utils.deprecation import deprecate_kwarg
|
| 28 |
+
from transformers.utils.generic import check_model_inputs as _check_model_inputs
|
| 29 |
+
|
| 30 |
+
TRANSFORMERS_V5 = Version(_package_version("transformers")) >= Version("5.0.0")
|
| 31 |
+
|
| 32 |
+
_CAUSAL_MASK_PARAMS = frozenset(inspect.signature(create_causal_mask).parameters)
|
| 33 |
+
_SLIDING_MASK_PARAMS = frozenset(inspect.signature(create_sliding_window_causal_mask).parameters)
|
| 34 |
+
|
| 35 |
+
if TRANSFORMERS_V5:
|
| 36 |
+
# A plain decorator from 5.0.0 on (a deprecated alias of merge_with_config_defaults).
|
| 37 |
+
def check_model_inputs(**kwargs):
|
| 38 |
+
"""Swallow 4.x's factory kwargs (e.g. tie_last_hidden_states); 5.x has no slot for them."""
|
| 39 |
+
return _check_model_inputs
|
| 40 |
+
|
| 41 |
+
else:
|
| 42 |
+
# A decorator factory before 5.0.0.
|
| 43 |
+
check_model_inputs = _check_model_inputs
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _compute_default_rope_parameters(config, device=None, **kwargs):
|
| 47 |
+
"""Inverse frequencies for the original RoPE.
|
| 48 |
+
|
| 49 |
+
5.0.0 dropped both the "default" entry of ROPE_INIT_FUNCTIONS and the module-level
|
| 50 |
+
helper, relocating it to a staticmethod on the rotary-embedding classes. Reads
|
| 51 |
+
either the 4.x (`rope_theta`) or 5.x (`rope_parameters`) config layout.
|
| 52 |
+
"""
|
| 53 |
+
base = getattr(config, "rope_theta", None)
|
| 54 |
+
if base is None:
|
| 55 |
+
base = config.rope_parameters["rope_theta"]
|
| 56 |
+
dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
|
| 57 |
+
inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
|
| 58 |
+
return inv_freq.to(device), 1.0
|
| 59 |
+
|
| 60 |
|
| 61 |
from .configuration_bolmo import BolmoConfig
|
| 62 |
from .tokenization_bolmo import BolmoTokenizerConfig
|
|
|
|
| 917 |
self.original_max_seq_len = config.max_position_embeddings
|
| 918 |
|
| 919 |
self.config = config
|
| 920 |
+
if TRANSFORMERS_V5 and self.rope_type == "default":
|
| 921 |
+
self.rope_init_fn = _compute_default_rope_parameters
|
| 922 |
+
else:
|
| 923 |
+
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
| 924 |
|
| 925 |
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
| 926 |
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 927 |
self.original_inv_freq = self.inv_freq
|
| 928 |
|
| 929 |
+
# 5.x `_init_weights` looks this up on the rotary module itself.
|
| 930 |
+
compute_default_rope_parameters = staticmethod(_compute_default_rope_parameters)
|
| 931 |
+
|
| 932 |
@torch.no_grad()
|
| 933 |
@dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
|
| 934 |
def forward(self, x, position_ids):
|
|
|
|
| 1072 |
# Prepare mask arguments
|
| 1073 |
mask_kwargs = {
|
| 1074 |
"config": self.config,
|
| 1075 |
+
"input_embeds": h_patch, # renamed `inputs_embeds` in 5.2.0
|
| 1076 |
+
"inputs_embeds": h_patch,
|
| 1077 |
"attention_mask": attention_mask,
|
| 1078 |
+
"cache_position": cache_position, # dropped in 5.9.0
|
| 1079 |
"past_key_values": past_key_values,
|
| 1080 |
"position_ids": position_ids,
|
| 1081 |
}
|
| 1082 |
# Create the masks. Models whose global blocks are all full attention (OLMo 2,
|
| 1083 |
# Llama 3, Qwen 3) have no `sliding_window` to build a sliding mask from.
|
| 1084 |
+
causal_mask_mapping = {
|
| 1085 |
+
"full_attention": create_causal_mask(
|
| 1086 |
+
**{k: v for k, v in mask_kwargs.items() if k in _CAUSAL_MASK_PARAMS}
|
| 1087 |
+
)
|
| 1088 |
+
}
|
| 1089 |
if self.has_sliding_layers:
|
| 1090 |
causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
|
| 1091 |
+
**{k: v for k, v in mask_kwargs.items() if k in _SLIDING_MASK_PARAMS}
|
| 1092 |
)
|
| 1093 |
|
| 1094 |
position_embeddings_mapping = {
|
|
|
|
| 1241 |
) -> Union[GenerateOutput, torch.Tensor]:
|
| 1242 |
# generic preprocessing
|
| 1243 |
|
| 1244 |
+
# 5.0.0 dropped the positional `use_model_defaults`.
|
| 1245 |
+
if TRANSFORMERS_V5:
|
| 1246 |
+
generation_config, model_kwargs = self._prepare_generation_config(
|
| 1247 |
+
generation_config, **kwargs
|
| 1248 |
+
)
|
| 1249 |
+
else:
|
| 1250 |
+
generation_config, model_kwargs = self._prepare_generation_config(
|
| 1251 |
+
generation_config, use_model_defaults, **kwargs
|
| 1252 |
+
)
|
| 1253 |
self._prepare_special_tokens(generation_config, device=self.model.device)
|
| 1254 |
|
| 1255 |
logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
|