MMayr commited on
Commit
9526ac7
路
verified 路
1 Parent(s): f893798

Upload 7 files

Browse files
Fevbench_MASE_Results.csv ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_name,win_rate,skill_score,median_training_time_s_per100,median_inference_time_s_per100,median_e2e_time_s_per100,training_corpus_overlap,num_failures
2
+ Chronos-2,80.13468013468012,34.05843892112963,0.0,0.5141568780729167,0.5141568780729167,0.0,0.0
3
+ TiRex-2,77.44107744107744,32.57913762975083,0.0,0.34000146667364917,0.34000146667364917,0.0,0.0
4
+ Toto-2.0-2.5B,74.91582491582491,29.937868284764193,0.0,5.700191622859636,5.700191622859636,0.0,0.0
5
+ TiRex,68.18181818181817,28.68106360865612,0.0,0.46261289781250003,0.46261289781250003,0.0,0.0
6
+ pcfm,66.16161616161617,30.26513236307694,0.0,1.2517927399994733,1.2517927399994733,0.0,0.0
7
+ TimesFM-2.5,65.74074074074075,28.452518662110805,0.0,1.2560686380729167,1.2560686380729167,0.10606060606060606,0.0
8
+ Toto-2.0-22m,65.48821548821549,28.233373754068104,0.0,0.37732213339285714,0.37732213339285714,0.0,0.0
9
+ FlowState,65.15151515151516,28.065639820714992,0.0,2.201140597232143,2.201140597232143,0.09090909090909091,0.0
10
+ TabPFN-TS-3,63.69248035914702,29.873250212839018,0.0,220.55960054847804,220.55960054847804,0.0,0.0
11
+ TS-ICL,63.07519640852974,28.978862157803732,0.0,1.9648320217590551,1.9648320217590551,0.0,0.0
12
+ citras-fm,63.01907968574635,29.21136041370903,0.0,0.6132267211186921,0.6132267211186921,0.0,0.0
13
+ TabPFN-TS,57.856341189674524,29.551448334139284,0.0,78.25665314439985,78.25665314439985,0.0,0.0
14
+ Chronos-Bolt,55.02244668911336,26.150233001517663,0.0,0.5483385895714286,0.5483385895714286,0.0,0.0
15
+ Toto-1.0,54.93827160493827,26.113828254811477,0.0,15.731708019241072,15.731708019241072,0.09090909090909091,0.0
16
+ Moirai-2.0,53.59147025813693,25.972958084443544,0.0,0.412112282578125,0.412112282578125,0.3333333333333333,0.0
17
+ LightGBM,47.755331088664434,23.292690165445418,1.718187238432971,0.33695880428571423,2.3141756472596158,0.0,0.0
18
+ CatBoost,46.96969696969697,23.671631171773335,20.083596527083333,0.412382839047619,20.487343512291666,0.0,0.0
19
+ Stat. Ensemble,44.725028058361396,18.93949833366555,0.0,221.13803209946576,221.13803209946576,0.0,3.0303030303030303
20
+ Sundial-Base,44.44444444444446,23.469956497971502,0.0,8.290017479345238,8.290017479345238,0.0,0.0
21
+ TFT,35.26936026936027,17.24917569865889,1438.8169064961608,1.1147377061875,1441.4044634433035,0.0,0.0
22
+ AutoETS,33.16498316498317,1.7631190747578684,0.0,4.641626233452381,4.641626233452381,0.0,0.0
23
+ AutoARIMA,32.49158249158249,11.853068606846007,0.0,50.02205090699999,50.02205090699999,0.0,3.0303030303030303
24
+ DeepAR,31.677890011223347,16.917486519598757,1480.172810988125,1.310540413617216,1481.5843365067708,0.0,3.0303030303030303
25
+ PatchTST,29.713804713804713,16.01624542705552,1139.8450167440624,1.0059683552922078,1140.34517027125,0.0,0.0
26
+ AutoTheta,29.180695847362514,12.382017276017109,0.0,3.70417375938172,3.70417375938172,0.0,0.0
27
+ Seasonal Naive,18.069584736251407,0.0,0.0,0.7687975575806452,0.7687975575806452,0.0,0.0
28
+ Drift,16.498316498316502,-15.37605918715066,0.0,0.7859875494731183,0.7859875494731183,0.0,0.0
29
+ Naive,15.628507295173966,-16.44367020429607,0.0,0.7430134275,0.7430134275,0.0,0.0
Fevbench_MASE_Skillscore.pdf ADDED
Binary file (40.3 kB). View file
 
Fevbench_SQL_Results.csv ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model_name,win_rate,skill_score,median_training_time_s_per100,median_inference_time_s_per100,median_e2e_time_s_per100,training_corpus_overlap,num_failures
2
+ Chronos-2,84.5679012345679,41.49561003263285,0.0,0.5141568780729167,0.5141568780729167,0.0,0.0
3
+ TiRex-2,82.09876543209879,39.726840860248245,0.0,0.34000146667364917,0.34000146667364917,0.0,0.0
4
+ Toto-2.0-2.5B,78.73176206509541,37.62121801350736,0.0,5.700191622859636,5.700191622859636,0.0,0.0
5
+ TiRex,74.29854096520764,36.42117491653264,0.0,0.46261289781250003,0.46261289781250003,0.0,0.0
6
+ pcfm,71.38047138047138,37.63070857535661,0.0,1.2517927399994733,1.2517927399994733,0.0,0.0
7
+ Toto-2.0-22m,69.7530864197531,35.79257186163601,0.0,0.37732213339285714,0.37732213339285714,0.0,0.0
8
+ TimesFM-2.5,68.88327721661054,35.615548872060096,0.0,1.2560686380729167,1.2560686380729167,0.10606060606060606,0.0
9
+ TabPFN-TS-3,66.44219977553311,37.285243190831785,0.0,220.55960054847804,220.55960054847804,0.0,0.0
10
+ TS-ICL,66.16161616161617,36.13313907131952,0.0,1.9648320217590551,1.9648320217590551,0.0,0.0
11
+ citras-fm,65.82491582491582,36.077169184486934,0.0,0.6132267211186921,0.6132267211186921,0.0,0.0
12
+ FlowState,65.20763187429854,34.78787448348153,0.0,2.201140597232143,2.201140597232143,0.09090909090909091,0.0
13
+ TabPFN-TS,62.79461279461279,36.44133507151782,0.0,78.25665314439985,78.25665314439985,0.0,0.0
14
+ Toto-1.0,59.20314253647588,33.810131812128056,0.0,15.731708019241072,15.731708019241072,0.09090909090909091,0.0
15
+ Chronos-Bolt,58.10886644219978,33.60298151266301,0.0,0.5483385895714286,0.5483385895714286,0.0,0.0
16
+ Moirai-2.0,56.22895622895624,33.005611281969664,0.0,0.412112282578125,0.412112282578125,0.3333333333333333,0.0
17
+ Stat. Ensemble,41.63860830527497,19.539505493760178,0.0,221.13803209946576,221.13803209946576,0.0,3.0303030303030303
18
+ TFT,39.19753086419753,23.71745133754385,1438.8169064961608,1.1147377061875,1441.4044634433035,0.0,0.0
19
+ Sundial-Base,36.58810325476992,25.929082836285154,0.0,8.290017479345238,8.290017479345238,0.0,0.0
20
+ DeepAR,35.606060606060616,23.02731940802546,1480.172810988125,1.310540413617216,1481.5843365067708,0.0,3.0303030303030303
21
+ AutoARIMA,34.792368125701465,16.527724469020054,0.0,50.02205090699999,50.02205090699999,0.0,3.0303030303030303
22
+ PatchTST,33.67003367003367,21.81912071043972,1139.8450167440624,1.0059683552922078,1140.34517027125,0.0,0.0
23
+ AutoETS,31.874298540965214,-29.758015412074013,0.0,4.641626233452381,4.641626233452381,0.0,0.0
24
+ CatBoost,26.879910213243548,15.870429909677352,20.083596527083333,0.412382839047619,20.487343512291666,0.0,0.0
25
+ LightGBM,26.599326599326602,15.452758938302269,1.718187238432971,0.33695880428571423,2.3141756472596158,0.0,0.0
26
+ AutoTheta,23.849607182940517,3.899049739602356,0.0,3.70417375938172,3.70417375938172,0.0,0.0
27
+ Seasonal Naive,16.41414141414142,0.0,0.0,0.7687975575806452,0.7687975575806452,0.0,0.0
28
+ Drift,12.065095398428731,-37.02667380967708,0.0,0.7859875494731183,0.7859875494731183,0.0,0.0
29
+ Naive,11.139169472502806,-39.27251982221045,0.0,0.7430134275,0.7430134275,0.0,0.0
Fevbench_SQL_Skillscore.pdf ADDED
Binary file (40.4 kB). View file
 
Gifteval_CRPS_Results.txt ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==================================================================================
2
+ LEADERBOARD | metric=CRPS agg=geometric baseline=Seasonal_Naive
3
+ datasets: 97 common (0 dropped) models: 20 uni/multi datasets: 54/43
4
+ relative < 1.000 beats the baseline; lower is better
5
+ ==================================================================================
6
+
7
+ [1] FULL RANKED (all 97 datasets)
8
+ | model | Rel_CRPS | Abs_CRPS |
9
+ |:------------------------------------------|-----------:|-----------:|
10
+ | TiRex-2 (NX-AI) | 0.478 | 0.121 |
11
+ | Chronos 2 (AWS AI Labs) | 0.485 | 0.122 |
12
+ | TiRex (NX-AI) | 0.488 | 0.123 |
13
+ | TimesFM-2.5 (Google Research) | 0.490 | 0.124 |
14
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.493 | 0.124 |
15
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.495 | 0.125 |
16
+ | Chronos 2 Synth (AWS AI Labs) | 0.496 | 0.125 |
17
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.508 | 0.128 |
18
+ | Tab-PFN (PriorLabs) | 0.544 | 0.137 |
19
+ | TimesFM-2.0 (Google Research) | 0.550 | 0.139 |
20
+ | YingLong (Alibaba) | 0.567 | 0.143 |
21
+ | MOIRAI-Base (SalesforceAI) | 0.610 | 0.154 |
22
+ | iTransformer (Tsinghua University) | 0.620 | 0.156 |
23
+ | MOIRAI-Small (SalesforceAI) | 0.650 | 0.164 |
24
+ | Chronos-Base (AWS AI Labs) | 0.652 | 0.164 |
25
+ | TimesFM (Google Research) | 0.680 | 0.172 |
26
+ | DLinear (Chinese University of Hong Kong) | 0.846 | 0.213 |
27
+ | Lag-Llama (Morgan Stanley) | 0.880 | 0.222 |
28
+ | TTM (IBM Research) | 0.891 | 0.225 |
29
+ | Seasonal_Naive | 1.000 | 0.252 |
30
+
31
+ [2] UNIVARIATE vs MULTIVARIATE
32
+ | model | Univariate | Multivariate |
33
+ |:------------------------------------------|-------------:|---------------:|
34
+ | TiRex-2 (NX-AI) | 0.490 | 0.464 |
35
+ | Chronos 2 (AWS AI Labs) | 0.503 | 0.464 |
36
+ | TiRex (NX-AI) | 0.510 | 0.463 |
37
+ | TimesFM-2.5 (Google Research) | 0.511 | 0.466 |
38
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.532 | 0.448 |
39
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.530 | 0.454 |
40
+ | Chronos 2 Synth (AWS AI Labs) | 0.524 | 0.462 |
41
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.532 | 0.480 |
42
+ | Tab-PFN (PriorLabs) | 0.544 | 0.544 |
43
+ | TimesFM-2.0 (Google Research) | 0.549 | 0.553 |
44
+ | YingLong (Alibaba) | 0.598 | 0.530 |
45
+ | MOIRAI-Base (SalesforceAI) | 0.589 | 0.636 |
46
+ | iTransformer (Tsinghua University) | 0.647 | 0.589 |
47
+ | MOIRAI-Small (SalesforceAI) | 0.659 | 0.640 |
48
+ | Chronos-Base (AWS AI Labs) | 0.627 | 0.685 |
49
+ | TimesFM (Google Research) | 0.652 | 0.717 |
50
+ | DLinear (Chinese University of Hong Kong) | 0.869 | 0.817 |
51
+ | Lag-Llama (Morgan Stanley) | 0.952 | 0.798 |
52
+ | TTM (IBM Research) | 0.920 | 0.856 |
53
+ | Seasonal_Naive | 1.000 | 1.000 |
54
+
55
+ [3] VARIATE TYPE x HORIZON (Short/Medium/Long)
56
+ | model | Uni路Short | Uni路Medium | Uni路Long | Mul路Short | Mul路Medium | Mul路Long |
57
+ |:------------------------------------------|------------:|-------------:|-----------:|------------:|-------------:|-----------:|
58
+ | TiRex-2 (NX-AI) | 0.533 | 0.433 | 0.397 | 0.444 | 0.463 | 0.498 |
59
+ | Chronos 2 (AWS AI Labs) | 0.526 | 0.483 | 0.438 | 0.444 | 0.462 | 0.500 |
60
+ | TiRex (NX-AI) | 0.533 | 0.488 | 0.445 | 0.449 | 0.464 | 0.485 |
61
+ | TimesFM-2.5 (Google Research) | 0.540 | 0.475 | 0.437 | 0.441 | 0.470 | 0.505 |
62
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.564 | 0.494 | 0.453 | 0.437 | 0.438 | 0.475 |
63
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.562 | 0.491 | 0.452 | 0.436 | 0.448 | 0.491 |
64
+ | Chronos 2 Synth (AWS AI Labs) | 0.541 | 0.513 | 0.471 | 0.449 | 0.440 | 0.508 |
65
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.564 | 0.493 | 0.454 | 0.454 | 0.473 | 0.533 |
66
+ | Tab-PFN (PriorLabs) | 0.555 | 0.540 | 0.506 | 0.530 | 0.545 | 0.565 |
67
+ | TimesFM-2.0 (Google Research) | 0.546 | 0.568 | 0.543 | 0.492 | 0.585 | 0.628 |
68
+ | YingLong (Alibaba) | 0.619 | 0.579 | 0.538 | 0.501 | 0.548 | 0.559 |
69
+ | MOIRAI-Base (SalesforceAI) | 0.616 | 0.560 | 0.520 | 0.574 | 0.693 | 0.687 |
70
+ | iTransformer (Tsinghua University) | 0.701 | 0.574 | 0.526 | 0.619 | 0.547 | 0.587 |
71
+ | MOIRAI-Small (SalesforceAI) | 0.682 | 0.636 | 0.597 | 0.637 | 0.635 | 0.648 |
72
+ | Chronos-Base (AWS AI Labs) | 0.608 | 0.698 | 0.640 | 0.573 | 0.788 | 0.788 |
73
+ | TimesFM (Google Research) | 0.638 | 0.694 | 0.671 | 0.629 | 0.792 | 0.800 |
74
+ | DLinear (Chinese University of Hong Kong) | 0.897 | 0.835 | 0.796 | 0.828 | 0.797 | 0.820 |
75
+ | Lag-Llama (Morgan Stanley) | 1.035 | 0.831 | 0.783 | 0.841 | 0.774 | 0.756 |
76
+ | TTM (IBM Research) | 0.978 | 0.853 | 0.779 | 0.779 | 0.934 | 0.912 |
77
+ | Seasonal_Naive | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 |
Gifteval_MAE_Results.txt ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==================================================================================
2
+ LEADERBOARD | metric=MAE agg=geometric baseline=Seasonal_Naive
3
+ datasets: 97 common (0 dropped) models: 20 uni/multi datasets: 54/43
4
+ relative < 1.000 beats the baseline; lower is better
5
+ ==================================================================================
6
+
7
+ [1] FULL RANKED (all 97 datasets)
8
+ | model | Rel_MAE | Abs_MAE |
9
+ |:------------------------------------------|----------:|----------:|
10
+ | TiRex-2 (NX-AI) | 0.673 | 28.790 |
11
+ | Chronos 2 (AWS AI Labs) | 0.679 | 29.012 |
12
+ | TiRex (NX-AI) | 0.686 | 29.334 |
13
+ | TimesFM-2.5 (Google Research) | 0.687 | 29.362 |
14
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.695 | 29.689 |
15
+ | Chronos 2 Synth (AWS AI Labs) | 0.697 | 29.777 |
16
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.698 | 29.834 |
17
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.721 | 30.811 |
18
+ | TimesFM-2.0 (Google Research) | 0.742 | 31.732 |
19
+ | Tab-PFN (PriorLabs) | 0.761 | 32.553 |
20
+ | YingLong (Alibaba) | 0.790 | 33.763 |
21
+ | Chronos-Base (AWS AI Labs) | 0.832 | 35.586 |
22
+ | MOIRAI-Base (SalesforceAI) | 0.850 | 36.349 |
23
+ | iTransformer (Tsinghua University) | 0.855 | 36.549 |
24
+ | MOIRAI-Small (SalesforceAI) | 0.905 | 38.709 |
25
+ | TimesFM (Google Research) | 0.908 | 38.795 |
26
+ | DLinear (Chinese University of Hong Kong) | 0.952 | 40.713 |
27
+ | Seasonal_Naive | 1.000 | 42.749 |
28
+ | TTM (IBM Research) | 1.004 | 42.915 |
29
+ | Lag-Llama (Morgan Stanley) | 1.188 | 50.777 |
30
+
31
+ [2] UNIVARIATE vs MULTIVARIATE
32
+ | model | Univariate | Multivariate |
33
+ |:------------------------------------------|-------------:|---------------:|
34
+ | TiRex-2 (NX-AI) | 0.655 | 0.697 |
35
+ | Chronos 2 (AWS AI Labs) | 0.679 | 0.679 |
36
+ | TiRex (NX-AI) | 0.681 | 0.693 |
37
+ | TimesFM-2.5 (Google Research) | 0.678 | 0.698 |
38
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.714 | 0.671 |
39
+ | Chronos 2 Synth (AWS AI Labs) | 0.698 | 0.695 |
40
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.711 | 0.682 |
41
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.714 | 0.729 |
42
+ | TimesFM-2.0 (Google Research) | 0.710 | 0.784 |
43
+ | Tab-PFN (PriorLabs) | 0.727 | 0.808 |
44
+ | YingLong (Alibaba) | 0.790 | 0.790 |
45
+ | Chronos-Base (AWS AI Labs) | 0.772 | 0.916 |
46
+ | MOIRAI-Base (SalesforceAI) | 0.780 | 0.948 |
47
+ | iTransformer (Tsinghua University) | 0.850 | 0.861 |
48
+ | MOIRAI-Small (SalesforceAI) | 0.875 | 0.946 |
49
+ | TimesFM (Google Research) | 0.838 | 1.002 |
50
+ | DLinear (Chinese University of Hong Kong) | 0.926 | 0.986 |
51
+ | Seasonal_Naive | 1.000 | 1.000 |
52
+ | TTM (IBM Research) | 0.981 | 1.033 |
53
+ | Lag-Llama (Morgan Stanley) | 1.221 | 1.147 |
54
+
55
+ [3] VARIATE TYPE x HORIZON (Short/Medium/Long)
56
+ | model | Uni路Short | Uni路Medium | Uni路Long | Mul路Short | Mul路Medium | Mul路Long |
57
+ |:------------------------------------------|------------:|-------------:|-----------:|------------:|-------------:|-----------:|
58
+ | TiRex-2 (NX-AI) | 0.658 | 0.651 | 0.651 | 0.626 | 0.716 | 0.804 |
59
+ | Chronos 2 (AWS AI Labs) | 0.653 | 0.737 | 0.729 | 0.616 | 0.694 | 0.774 |
60
+ | TiRex (NX-AI) | 0.657 | 0.734 | 0.731 | 0.630 | 0.712 | 0.784 |
61
+ | TimesFM-2.5 (Google Research) | 0.663 | 0.709 | 0.713 | 0.616 | 0.731 | 0.811 |
62
+ | PCFM-Small Sparse v2 CP (SCCH) | 0.697 | 0.748 | 0.749 | 0.616 | 0.674 | 0.764 |
63
+ | Chronos 2 Synth (AWS AI Labs) | 0.667 | 0.765 | 0.764 | 0.631 | 0.687 | 0.817 |
64
+ | PCFM-Small Sparse v2 CI (SCCH) | 0.694 | 0.743 | 0.748 | 0.614 | 0.692 | 0.792 |
65
+ | PCFM-Small Sparse v2 CD (SCCH) | 0.697 | 0.746 | 0.752 | 0.639 | 0.747 | 0.876 |
66
+ | TimesFM-2.0 (Google Research) | 0.668 | 0.794 | 0.812 | 0.687 | 0.830 | 0.915 |
67
+ | Tab-PFN (PriorLabs) | 0.684 | 0.808 | 0.832 | 0.728 | 0.841 | 0.914 |
68
+ | YingLong (Alibaba) | 0.758 | 0.850 | 0.869 | 0.699 | 0.837 | 0.903 |
69
+ | Chronos-Base (AWS AI Labs) | 0.716 | 0.906 | 0.887 | 0.761 | 1.041 | 1.078 |
70
+ | MOIRAI-Base (SalesforceAI) | 0.754 | 0.826 | 0.841 | 0.800 | 1.060 | 1.109 |
71
+ | iTransformer (Tsinghua University) | 0.842 | 0.863 | 0.869 | 0.842 | 0.815 | 0.942 |
72
+ | MOIRAI-Small (SalesforceAI) | 0.835 | 0.952 | 0.970 | 0.865 | 0.966 | 1.067 |
73
+ | TimesFM (Google Research) | 0.780 | 0.954 | 0.985 | 0.867 | 1.104 | 1.146 |
74
+ | DLinear (Chinese University of Hong Kong) | 0.884 | 0.998 | 1.036 | 0.927 | 0.993 | 1.080 |
75
+ | Seasonal_Naive | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 | 1.000 |
76
+ | TTM (IBM Research) | 0.964 | 1.019 | 1.015 | 0.871 | 1.163 | 1.201 |
77
+ | Lag-Llama (Morgan Stanley) | 1.223 | 1.199 | 1.236 | 1.137 | 1.134 | 1.178 |
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:746ff3eb69f9fcaa0e934f13b5afda17be7591a987eb64c671fa05b1d87b6755
3
+ size 211418848