# Sequence Classification Models Fail to Train on Simple Classification Problem! Hints?

**URL:** <https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619>\
**Category:** Models\
**Created:** [June 17, 2024, 5:47pm UTC](https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619 "2024-06-17T17:47:45Z")\
**Posts on this page:** 4\
**Page:** 1

<div class="post-metadata">

**Author:** ![rotentomato](https://avatars.discourse-cdn.com/v4/letter/r/cdc98d/32.png) [@rotentomato](https://discuss.huggingface.co/u/rotentomato)\
**Post date:** [June 17, 2024, 5:47pm UTC](https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619/1 "2024-06-17T17:47:45Z")

</div>

I’ve tried so far a couple of sequence classifiers like Bert and Roberta and I’m unable to make them overfit or even make them learn the following toy dataset problem. Any suggestions on what/why they don’t train on this simple toy task? It’s a simple binary classification problems shouldn’t this be easy to learn for such models?

```auto
import torch
import random
import numpy as np
import pandas as pd
from datasets import Dataset
from torch.utils.data import DataLoader, TensorDataset
from transformers.trainer import Trainer, TrainingArguments
from transformers.data.data_collator import DataCollatorWithPadding
from transformers.models.auto import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification

def generate_addition_data(num_samples):
    data = []
    for _ in range(num_samples):
        a = random.randint(0, 999)
        b = random.randint(0, 999)
        question = f"{a} + {b}"
        correct_answer = a + b
        
        is_correct = random.choice([True, False])
        
        if is_correct:
            answer = correct_answer
        else:
            # Generate an incorrect answer by adding or subtracting a random value
            answer = correct_answer + random.choice([-10, -5, -1, 1, 5, 10])

        data.append({
            'question': question,
            'answer': str(answer),
            'label': 1 if is_correct else 0
        })
    
    return data

data = generate_addition_data(10_000) # Generate 10,000 samples
df = pd.DataFrame(data)
dataset = Dataset.from_pandas(df)

checkpoint = "FacebookAI/roberta-base"

def preprocess_function(examples):
    inputs = [f"Calculate: {q} = {a}" for q, a in zip(examples["question"], examples['answer'])]
    labels = [a for a in examples['label']]
    model_inputs = tokenizer(inputs, max_length=128, padding="max_length", truncation=True, return_tensors='pt')
    model_inputs['labels'] = labels
    return model_inputs

tokenizer = AutoTokenizer.from_pretrained(checkpoint)

tokenized_datasets = dataset.map(preprocess_function, batched=True)

# pos_weights = len(df) / (2 * df.label.value_counts()[1])
# neg_weights = len(df) / (2 * df.label.value_counts()[0])

train_test_split = tokenized_datasets.train_test_split(test_size=2000, seed=2024)

pos_weights = len(train_test_split['train'].to_pandas()) / (2 * train_test_split['train'].to_pandas().label.value_counts()[1])
neg_weights = len(train_test_split['train'].to_pandas()) / (2 * train_test_split['train'].to_pandas().label.value_counts()[0])

print(f"Pos weights: {pos_weights}, Neg weights: {neg_weights}")

config = AutoConfig.from_pretrained(checkpoint)
config.num_labels = 2
model = AutoModelForSequenceClassification.from_config(config)
data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding=True)

training_args = TrainingArguments(
    output_dir="./tseqclassifier_checkpoints/",
    evaluation_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=140,
    per_device_eval_batch_size=140,
    num_train_epochs=200,
    weight_decay=0.01,
    warmup_steps=500,
    # use_cpu=True
    load_best_model_at_end=True,
)

class WeightedCELossTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.pop("labels")
        # Get model's predictions
        outputs = model(**inputs)
        logits = outputs.get("logits")
        # Compute custom loss
        loss_fct = torch.nn.CrossEntropyLoss(weight=torch.tensor([pos_weights, neg_weights], device=model.device, dtype=logits.dtype))
       # loss_fct = torch.nn.CrossEntropyLoss(pos_weights=torch.tensor([0.9785, 10.24], device=model.device, dtype=logits.dtype))
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss

trainer = WeightedCELossTrainer(
    model=model,
    args=training_args,
    train_dataset=train_test_split['train'],
    eval_dataset=train_test_split['test'],
    tokenizer = None,
    compute_metrics =compute_metrics,
)

trainer.train()

def predict(question):
    inputs = tokenizer(f"Calculate: {question}", return_tensors="pt").input_ids.cuda()
    outputs = model(inputs)
    print(f"outputs = {outputs.logits.argmax(dim=1).item()}")
    answer = tokenizer.decode(outputs.logits.argmax(dim=1).cpu().numpy(), skip_special_tokens=True)
    return answer

question1 = "15 + 27 = 43"
question2 = "15 + 27 = 42"
print(f"Question: {question1}")
print(f"Predicted Answer: {predict(question1)}")
print(f"Question: {question2}")
print(f"Predicted Answer: {predict(question2)}")

```

---

<div class="post-metadata">

**Author:** ![MattiLinnanvuori](https://avatars.discourse-cdn.com/v4/letter/m/ec9cab/32.png) [@MattiLinnanvuori](https://discuss.huggingface.co/u/MattiLinnanvuori)\
**Post date:** [June 18, 2024, 3:42pm UTC](https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619/2 "2024-06-18T15:42:00Z")

</div>

Language models are not great at arithmetics. That is not their forte. They have been trained for language understanding that does not include many examples of arithmetics.

---

<div class="post-metadata">

**Author:** ![rotentomato](https://avatars.discourse-cdn.com/v4/letter/r/cdc98d/32.png) [@rotentomato](https://discuss.huggingface.co/u/rotentomato)\
**Post date:** [June 18, 2024, 6:27pm UTC](https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619/3 "2024-06-18T18:27:56Z")

</div>

Which models from hugging face can I use for this kind of toy task model that may work well?

---

<div class="post-metadata">

**Author:** ![MattiLinnanvuori](https://avatars.discourse-cdn.com/v4/letter/m/ec9cab/32.png) [@MattiLinnanvuori](https://discuss.huggingface.co/u/MattiLinnanvuori)\
**Post date:** [June 21, 2024, 6:26am UTC](https://discuss.huggingface.co/t/sequence-classification-models-fail-to-train-on-simple-classification-problem-hints/92619/4 "2024-06-21T06:26:55Z")

</div>

[upaya07/Arithmo2-Mistral-7B](https://huggingface.co/upaya07/Arithmo2-Mistral-7B) You can search for Mathematical Reasoning models like above.
