add model

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

README.md

@@ -0,0 +1,90 @@
+---
+license: apache-2.0
+tags:
+- generated_from_trainer
+metrics:
+- rouge
+model-index:
+- name: barthez-deft-sciences_de_l_information
+  results:
+  - task:
+      name: Summarization
+      type: summarization
+    metrics:
+    - name: Rouge1
+      type: rouge
+      value: 34.5672
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# barthez-deft-sciences_de_l_information
+
+This model is a fine-tuned version of [moussaKam/barthez](https://huggingface.co/moussaKam/barthez) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.0258
+- Rouge1: 34.5672
+- Rouge2: 16.7861
+- Rougel: 27.5573
+- Rougelsum: 27.6099
+- Gen Len: 17.8857
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 20.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rouge1  | Rouge2  | Rougel  | Rougelsum | Gen Len |
+|:-------------:|:-----:|:----:|:---------------:|:-------:|:-------:|:-------:|:---------:|:-------:|
+| 3.3405        | 1.0   | 106  | 2.3682          | 31.3511 | 12.1973 | 25.6977 | 25.6851   | 14.9714 |
+| 2.4219        | 2.0   | 212  | 2.1891          | 30.1154 | 13.3459 | 25.4854 | 25.5403   | 14.0429 |
+| 2.0789        | 3.0   | 318  | 2.0994          | 32.153  | 15.3865 | 26.1859 | 26.1672   | 15.2    |
+| 1.869         | 4.0   | 424  | 2.0258          | 34.5797 | 16.4194 | 27.6909 | 27.7201   | 16.9857 |
+| 1.6569        | 5.0   | 530  | 2.0417          | 34.3854 | 16.5237 | 28.7036 | 28.8258   | 15.2429 |
+| 1.5414        | 6.0   | 636  | 2.0503          | 33.1768 | 15.4851 | 27.2818 | 27.2884   | 16.0143 |
+| 1.4461        | 7.0   | 742  | 2.0293          | 35.4273 | 16.118  | 27.3622 | 27.393    | 16.6857 |
+| 1.3435        | 8.0   | 848  | 2.0336          | 35.3471 | 15.9695 | 27.668  | 27.6749   | 17.2    |
+| 1.2624        | 9.0   | 954  | 2.0779          | 35.9201 | 17.2547 | 27.409  | 27.3293   | 17.1857 |
+| 1.1807        | 10.0  | 1060 | 2.1301          | 35.7061 | 15.9138 | 27.3968 | 27.4716   | 17.1286 |
+| 1.0972        | 11.0  | 1166 | 2.1726          | 34.3194 | 16.1313 | 27.0367 | 27.0737   | 17.1429 |
+| 1.0224        | 12.0  | 1272 | 2.1704          | 34.9278 | 16.7958 | 27.8754 | 27.932    | 16.6571 |
+| 1.0181        | 13.0  | 1378 | 2.2458          | 34.472  | 15.9111 | 28.2938 | 28.2946   | 16.7571 |
+| 0.9769        | 14.0  | 1484 | 2.3405          | 35.1592 | 16.3135 | 29.0956 | 29.0858   | 16.5429 |
+| 0.8866        | 15.0  | 1590 | 2.3303          | 34.8732 | 15.6709 | 27.5858 | 27.6169   | 16.2429 |
+| 0.8888        | 16.0  | 1696 | 2.2976          | 35.3034 | 16.8011 | 27.7988 | 27.7569   | 17.5143 |
+| 0.8358        | 17.0  | 1802 | 2.3349          | 35.505  | 16.8851 | 28.3651 | 28.413    | 16.8143 |
+| 0.8026        | 18.0  | 1908 | 2.3738          | 35.2328 | 17.0358 | 28.544  | 28.6211   | 16.6143 |
+| 0.7487        | 19.0  | 2014 | 2.4103          | 34.0793 | 15.4468 | 27.8057 | 27.8586   | 16.7286 |
+| 0.7722        | 20.0  | 2120 | 2.3991          | 34.8116 | 15.8706 | 27.9173 | 27.983    | 16.9286 |
+
+
+### Framework versions
+
+- Transformers 4.10.2
+- Pytorch 1.7.1+cu110
+- Datasets 1.11.0
+- Tokenizers 0.10.3

all_results.json

@@ -0,0 +1,18 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 17.8857,
+    "eval_loss": 2.025848865509033,
+    "eval_rouge1": 34.5672,
+    "eval_rouge2": 16.7861,
+    "eval_rougeL": 27.5573,
+    "eval_rougeLsum": 27.6099,
+    "eval_runtime": 4.465,
+    "eval_samples": 70,
+    "eval_samples_per_second": 15.677,
+    "eval_steps_per_second": 4.031,
+    "train_loss": 1.328409050995449,
+    "train_runtime": 326.8588,
+    "train_samples": 424,
+    "train_samples_per_second": 25.944,
+    "train_steps_per_second": 6.486
+}

config.json

@@ -0,0 +1,59 @@
+{
+  "_name_or_path": "moussaKam/barthez",
+  "activation_dropout": 0.1,
+  "activation_function": "gelu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "MBartForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.1,
+  "classifier_dropout": 0.0,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 2,
+  "do_blenderbot_90_layernorm": false,
+  "dropout": 0.1,
+  "early_stopping": true,
+  "encoder_attention_heads": 12,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 2,
+  "extra_pos_embeddings": 2,
+  "force_bos_token_to_be_generated": false,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_position_embeddings": 1024,
+  "model_type": "mbart",
+  "no_repeat_ngram_size": 3,
+  "normalize_before": true,
+  "normalize_embedding": true,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "scale_embedding": false,
+  "static_position_embeddings": false,
+  "tokenizer_class": "BarthezTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.10.2",
+  "use_cache": true,
+  "vocab_size": 50002
+}

eval_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 20.0,
+    "eval_gen_len": 17.8857,
+    "eval_loss": 2.025848865509033,
+    "eval_rouge1": 34.5672,
+    "eval_rouge2": 16.7861,
+    "eval_rougeL": 27.5573,
+    "eval_rougeLsum": 27.6099,
+    "eval_runtime": 4.465,
+    "eval_samples": 70,
+    "eval_samples_per_second": 15.677,
+    "eval_steps_per_second": 4.031
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a11a473155ce756476f9948f84f95c32181d3695ee9dc75cb9e337c657ace606
+size 557190519

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582403da7cb525c4cac90b495e553cb766b818be71755e6bd885072a1067314a
+size 1115393

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "moussaKam/barthez", "tokenizer_class": "BarthezTokenizer"}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 20.0,
+    "train_loss": 1.328409050995449,
+    "train_runtime": 326.8588,
+    "train_samples": 424,
+    "train_samples_per_second": 25.944,
+    "train_steps_per_second": 6.486
+}

trainer_state.json

@@ -0,0 +1,411 @@
+{
+  "best_metric": 2.025848865509033,
+  "best_model_checkpoint": "./checkpoints/barthez-deft-sciences_de_l_information/checkpoint-424",
+  "epoch": 20.0,
+  "global_step": 2120,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.869811320754717e-05,
+      "loss": 3.3405,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_gen_len": 14.9714,
+      "eval_loss": 2.368244171142578,
+      "eval_rouge1": 31.3511,
+      "eval_rouge2": 12.1973,
+      "eval_rougeL": 25.6977,
+      "eval_rougeLsum": 25.6851,
+      "eval_runtime": 3.4096,
+      "eval_samples_per_second": 20.53,
+      "eval_steps_per_second": 5.279,
+      "step": 106
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 2.728301886792453e-05,
+      "loss": 2.4219,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_gen_len": 14.0429,
+      "eval_loss": 2.1890599727630615,
+      "eval_rouge1": 30.1154,
+      "eval_rouge2": 13.3459,
+      "eval_rougeL": 25.4854,
+      "eval_rougeLsum": 25.5403,
+      "eval_runtime": 3.216,
+      "eval_samples_per_second": 21.766,
+      "eval_steps_per_second": 5.597,
+      "step": 212
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 2.586792452830189e-05,
+      "loss": 2.0789,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_gen_len": 15.2,
+      "eval_loss": 2.0993804931640625,
+      "eval_rouge1": 32.153,
+      "eval_rouge2": 15.3865,
+      "eval_rougeL": 26.1859,
+      "eval_rougeLsum": 26.1672,
+      "eval_runtime": 3.3607,
+      "eval_samples_per_second": 20.829,
+      "eval_steps_per_second": 5.356,
+      "step": 318
+    },
+    {
+      "epoch": 3.77,
+      "learning_rate": 2.4452830188679248e-05,
+      "loss": 1.869,
+      "step": 400
+    },
+    {
+      "epoch": 4.0,
+      "eval_gen_len": 16.9857,
+      "eval_loss": 2.025848865509033,
+      "eval_rouge1": 34.5797,
+      "eval_rouge2": 16.4194,
+      "eval_rougeL": 27.6909,
+      "eval_rougeLsum": 27.7201,
+      "eval_runtime": 3.3358,
+      "eval_samples_per_second": 20.985,
+      "eval_steps_per_second": 5.396,
+      "step": 424
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 2.3037735849056604e-05,
+      "loss": 1.6569,
+      "step": 500
+    },
+    {
+      "epoch": 5.0,
+      "eval_gen_len": 15.2429,
+      "eval_loss": 2.0417497158050537,
+      "eval_rouge1": 34.3854,
+      "eval_rouge2": 16.5237,
+      "eval_rougeL": 28.7036,
+      "eval_rougeLsum": 28.8258,
+      "eval_runtime": 3.3091,
+      "eval_samples_per_second": 21.154,
+      "eval_steps_per_second": 5.44,
+      "step": 530
+    },
+    {
+      "epoch": 5.66,
+      "learning_rate": 2.162264150943396e-05,
+      "loss": 1.5414,
+      "step": 600
+    },
+    {
+      "epoch": 6.0,
+      "eval_gen_len": 16.0143,
+      "eval_loss": 2.050337791442871,
+      "eval_rouge1": 33.1768,
+      "eval_rouge2": 15.4851,
+      "eval_rougeL": 27.2818,
+      "eval_rougeLsum": 27.2884,
+      "eval_runtime": 3.4107,
+      "eval_samples_per_second": 20.524,
+      "eval_steps_per_second": 5.278,
+      "step": 636
+    },
+    {
+      "epoch": 6.6,
+      "learning_rate": 2.020754716981132e-05,
+      "loss": 1.4461,
+      "step": 700
+    },
+    {
+      "epoch": 7.0,
+      "eval_gen_len": 16.6857,
+      "eval_loss": 2.029313802719116,
+      "eval_rouge1": 35.4273,
+      "eval_rouge2": 16.118,
+      "eval_rougeL": 27.3622,
+      "eval_rougeLsum": 27.393,
+      "eval_runtime": 3.3635,
+      "eval_samples_per_second": 20.812,
+      "eval_steps_per_second": 5.352,
+      "step": 742
+    },
+    {
+      "epoch": 7.55,
+      "learning_rate": 1.879245283018868e-05,
+      "loss": 1.3435,
+      "step": 800
+    },
+    {
+      "epoch": 8.0,
+      "eval_gen_len": 17.2,
+      "eval_loss": 2.033561944961548,
+      "eval_rouge1": 35.3471,
+      "eval_rouge2": 15.9695,
+      "eval_rougeL": 27.668,
+      "eval_rougeLsum": 27.6749,
+      "eval_runtime": 3.4311,
+      "eval_samples_per_second": 20.402,
+      "eval_steps_per_second": 5.246,
+      "step": 848
+    },
+    {
+      "epoch": 8.49,
+      "learning_rate": 1.737735849056604e-05,
+      "loss": 1.2624,
+      "step": 900
+    },
+    {
+      "epoch": 9.0,
+      "eval_gen_len": 17.1857,
+      "eval_loss": 2.077875852584839,
+      "eval_rouge1": 35.9201,
+      "eval_rouge2": 17.2547,
+      "eval_rougeL": 27.409,
+      "eval_rougeLsum": 27.3293,
+      "eval_runtime": 3.4,
+      "eval_samples_per_second": 20.588,
+      "eval_steps_per_second": 5.294,
+      "step": 954
+    },
+    {
+      "epoch": 9.43,
+      "learning_rate": 1.5962264150943395e-05,
+      "loss": 1.1807,
+      "step": 1000
+    },
+    {
+      "epoch": 10.0,
+      "eval_gen_len": 17.1286,
+      "eval_loss": 2.130077600479126,
+      "eval_rouge1": 35.7061,
+      "eval_rouge2": 15.9138,
+      "eval_rougeL": 27.3968,
+      "eval_rougeLsum": 27.4716,
+      "eval_runtime": 3.3605,
+      "eval_samples_per_second": 20.83,
+      "eval_steps_per_second": 5.356,
+      "step": 1060
+    },
+    {
+      "epoch": 10.38,
+      "learning_rate": 1.4547169811320755e-05,
+      "loss": 1.0972,
+      "step": 1100
+    },
+    {
+      "epoch": 11.0,
+      "eval_gen_len": 17.1429,
+      "eval_loss": 2.1725852489471436,
+      "eval_rouge1": 34.3194,
+      "eval_rouge2": 16.1313,
+      "eval_rougeL": 27.0367,
+      "eval_rougeLsum": 27.0737,
+      "eval_runtime": 3.3905,
+      "eval_samples_per_second": 20.646,
+      "eval_steps_per_second": 5.309,
+      "step": 1166
+    },
+    {
+      "epoch": 11.32,
+      "learning_rate": 1.3132075471698114e-05,
+      "loss": 1.0224,
+      "step": 1200
+    },
+    {
+      "epoch": 12.0,
+      "eval_gen_len": 16.6571,
+      "eval_loss": 2.1703763008117676,
+      "eval_rouge1": 34.9278,
+      "eval_rouge2": 16.7958,
+      "eval_rougeL": 27.8754,
+      "eval_rougeLsum": 27.932,
+      "eval_runtime": 3.3798,
+      "eval_samples_per_second": 20.711,
+      "eval_steps_per_second": 5.326,
+      "step": 1272
+    },
+    {
+      "epoch": 12.26,
+      "learning_rate": 1.1716981132075474e-05,
+      "loss": 1.0181,
+      "step": 1300
+    },
+    {
+      "epoch": 13.0,
+      "eval_gen_len": 16.7571,
+      "eval_loss": 2.245802879333496,
+      "eval_rouge1": 34.472,
+      "eval_rouge2": 15.9111,
+      "eval_rougeL": 28.2938,
+      "eval_rougeLsum": 28.2946,
+      "eval_runtime": 3.367,
+      "eval_samples_per_second": 20.79,
+      "eval_steps_per_second": 5.346,
+      "step": 1378
+    },
+    {
+      "epoch": 13.21,
+      "learning_rate": 1.030188679245283e-05,
+      "loss": 0.9769,
+      "step": 1400
+    },
+    {
+      "epoch": 14.0,
+      "eval_gen_len": 16.5429,
+      "eval_loss": 2.3404934406280518,
+      "eval_rouge1": 35.1592,
+      "eval_rouge2": 16.3135,
+      "eval_rougeL": 29.0956,
+      "eval_rougeLsum": 29.0858,
+      "eval_runtime": 3.3904,
+      "eval_samples_per_second": 20.647,
+      "eval_steps_per_second": 5.309,
+      "step": 1484
+    },
+    {
+      "epoch": 14.15,
+      "learning_rate": 8.900943396226416e-06,
+      "loss": 0.8866,
+      "step": 1500
+    },
+    {
+      "epoch": 15.0,
+      "eval_gen_len": 16.2429,
+      "eval_loss": 2.3303470611572266,
+      "eval_rouge1": 34.8732,
+      "eval_rouge2": 15.6709,
+      "eval_rougeL": 27.5858,
+      "eval_rougeLsum": 27.6169,
+      "eval_runtime": 3.4313,
+      "eval_samples_per_second": 20.401,
+      "eval_steps_per_second": 5.246,
+      "step": 1590
+    },
+    {
+      "epoch": 15.09,
+      "learning_rate": 7.485849056603774e-06,
+      "loss": 0.8888,
+      "step": 1600
+    },
+    {
+      "epoch": 16.0,
+      "eval_gen_len": 17.5143,
+      "eval_loss": 2.297647476196289,
+      "eval_rouge1": 35.3034,
+      "eval_rouge2": 16.8011,
+      "eval_rougeL": 27.7988,
+      "eval_rougeLsum": 27.7569,
+      "eval_runtime": 3.3934,
+      "eval_samples_per_second": 20.628,
+      "eval_steps_per_second": 5.304,
+      "step": 1696
+    },
+    {
+      "epoch": 16.04,
+      "learning_rate": 6.070754716981133e-06,
+      "loss": 0.8194,
+      "step": 1700
+    },
+    {
+      "epoch": 16.98,
+      "learning_rate": 4.6556603773584905e-06,
+      "loss": 0.8358,
+      "step": 1800
+    },
+    {
+      "epoch": 17.0,
+      "eval_gen_len": 16.8143,
+      "eval_loss": 2.334933042526245,
+      "eval_rouge1": 35.505,
+      "eval_rouge2": 16.8851,
+      "eval_rougeL": 28.3651,
+      "eval_rougeLsum": 28.413,
+      "eval_runtime": 3.4202,
+      "eval_samples_per_second": 20.467,
+      "eval_steps_per_second": 5.263,
+      "step": 1802
+    },
+    {
+      "epoch": 17.92,
+      "learning_rate": 3.240566037735849e-06,
+      "loss": 0.8026,
+      "step": 1900
+    },
+    {
+      "epoch": 18.0,
+      "eval_gen_len": 16.6143,
+      "eval_loss": 2.373809814453125,
+      "eval_rouge1": 35.2328,
+      "eval_rouge2": 17.0358,
+      "eval_rougeL": 28.544,
+      "eval_rougeLsum": 28.6211,
+      "eval_runtime": 3.4477,
+      "eval_samples_per_second": 20.303,
+      "eval_steps_per_second": 5.221,
+      "step": 1908
+    },
+    {
+      "epoch": 18.87,
+      "learning_rate": 1.8254716981132076e-06,
+      "loss": 0.7487,
+      "step": 2000
+    },
+    {
+      "epoch": 19.0,
+      "eval_gen_len": 16.7286,
+      "eval_loss": 2.4102871417999268,
+      "eval_rouge1": 34.0793,
+      "eval_rouge2": 15.4468,
+      "eval_rougeL": 27.8057,
+      "eval_rougeLsum": 27.8586,
+      "eval_runtime": 3.4317,
+      "eval_samples_per_second": 20.398,
+      "eval_steps_per_second": 5.245,
+      "step": 2014
+    },
+    {
+      "epoch": 19.81,
+      "learning_rate": 4.1037735849056606e-07,
+      "loss": 0.7722,
+      "step": 2100
+    },
+    {
+      "epoch": 20.0,
+      "eval_gen_len": 16.9286,
+      "eval_loss": 2.3990561962127686,
+      "eval_rouge1": 34.8116,
+      "eval_rouge2": 15.8706,
+      "eval_rougeL": 27.9173,
+      "eval_rougeLsum": 27.983,
+      "eval_runtime": 3.3605,
+      "eval_samples_per_second": 20.83,
+      "eval_steps_per_second": 5.356,
+      "step": 2120
+    },
+    {
+      "epoch": 20.0,
+      "step": 2120,
+      "total_flos": 1197078338174976.0,
+      "train_loss": 1.328409050995449,
+      "train_runtime": 326.8588,
+      "train_samples_per_second": 25.944,
+      "train_steps_per_second": 6.486
+    }
+  ],
+  "max_steps": 2120,
+  "num_train_epochs": 20,
+  "total_flos": 1197078338174976.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b8554949943341288a5f8fbe5c1759a3f86610b662ecbd6166bf6c357c397a8
+size 2927