Ornith-1.0-9B-DSpark / qualification.json
stanleyphoong's picture
Publish Ornith DSpark draft
14d7e0d verified
Raw History Blame Contribute Delete
10.1 kB
{
"qualified": true,
"checkpoint": "/workspace/checkpoints/ornith-dspark/candidates/production-candidate-sts-fast",
"gates": [
{
"name": "checkpoint_complete",
"passed": true,
"actual": {
"path": "/workspace/checkpoints/ornith-dspark/candidates/production-candidate-sts-fast",
"safetensors": 1
},
"required": "config.json and at least one safetensors file"
},
{
"name": "checkpoint_storage",
"passed": true,
"actual": 6.135368759743869,
"required": "<= 400.0 GiB"
},
{
"name": "paper_macro_acceptance",
"passed": true,
"actual": 4.605259575740357,
"required": ">= 4.5"
},
{
"name": "paper_per_task_floor",
"passed": true,
"actual": 0.8190744294356563,
"required": ">= 0.78"
},
{
"name": "frozen_candidate_repeatability",
"passed": true,
"actual": {
"repeat_count": 3,
"mean": 4.586501246454653,
"min": 4.561929853440648,
"max": 4.605259575740357
},
"required": {
"repeat_count": 3,
"minimum_accepted_length": 4.5,
"checksummed": true
}
},
{
"name": "greedy_consistency_diagnostic",
"passed": true,
"actual": {
"cases": 128,
"exact_matches": 85,
"exact_ratio": 0.6640625
},
"required": {
"min_cases": 100,
"min_exact_ratio": 0.6,
"note": "Distribution and accuracy gates establish losslessness"
}
},
{
"name": "first_token_distribution",
"passed": true,
"actual": {
"non_low_margin_mismatches": 0,
"mean_truncated_tv": 0.009450400673482224
},
"required": {
"non_low_margin_mismatches": 0,
"max_mean_truncated_tv": 0.01
}
},
{
"name": "teacher_forced_distribution",
"passed": true,
"actual": {
"non_low_margin_mismatches": 0,
"mean_truncated_tv": 0.00822962861550844
},
"required": {
"non_low_margin_mismatches": 0,
"max_mean_truncated_tv": 0.01
}
},
{
"name": "stochastic_sampling_distribution",
"passed": true,
"actual": {
"prompt_count": 8,
"min_samples_per_prompt": 1024,
"max_excess_tv": 0.0,
"top_token_mismatches": 0
},
"required": {
"min_samples_per_prompt": 1024,
"max_excess_tv": 0.02,
"top_token_mismatches": 0
}
},
{
"name": "sequential_temperature_calibration",
"passed": true,
"actual": {
"records": 1292,
"mean_ece": 0.023521986549359743,
"max_position_ece": 0.03443828972500544
},
"required": {
"min_records": 1000,
"max_mean_ece": 0.025,
"max_position_ece": 0.035
}
},
{
"name": "labeled_accuracy_non_regression",
"passed": true,
"actual": {
"samples": 1024,
"baseline_correct": 925,
"speculative_correct": 926,
"same_answers": 971,
"accuracy_difference": 0.0009765625,
"one_sided_95pct_lower_bound": -0.006387865333952388
},
"required": {
"min_samples": 100,
"one_sided_95pct_lower_bound": ">= -0.01"
}
},
{
"name": "online_acceptance",
"passed": true,
"actual": [
{
"concurrency": 1,
"acceptance_rate": 0.41404358353510895
},
{
"concurrency": 8,
"acceptance_rate": 0.3922748447204969
},
{
"concurrency": 32,
"acceptance_rate": 0.3835625159154571
}
],
"required": ">= 0.350000 at every measured load"
},
{
"name": "online_speed_non_regression",
"passed": true,
"actual": [
{
"concurrency": 1,
"speedup": 2.4609682725307374
},
{
"concurrency": 8,
"speedup": 1.6483219906948248
},
{
"concurrency": 32,
"speedup": 1.2249688238049443
}
],
"required": ">= 1.0 at every measured load"
},
{
"name": "paper_level_moderate_load_speed",
"passed": true,
"actual": 1.6483219906948248,
"required": ">= 1.5 at concurrency >= 8"
},
{
"name": "long_context_correctness",
"passed": true,
"actual": {
"contexts": [
8192,
32768,
65536,
131072,
262144
],
"all_prompt_budgets_valid": true,
"all_rows_correct": true
},
"required": {
"contexts": [
8192,
32768,
65536,
131072,
262144
],
"all_prompt_budgets_valid": true,
"all_rows_correct": true
}
},
{
"name": "long_context_performance",
"passed": true,
"actual": [
{
"context": 8192,
"route": "baseline",
"speedup": 1.130664742414886,
"acceptance_rate": null,
"baseline_memory_headroom": 0.10079642794939594,
"speculative_memory_headroom": 0.10079642794939594
},
{
"context": 8192,
"route": "baseline",
"speedup": 0.980373613644635,
"acceptance_rate": null,
"baseline_memory_headroom": 0.10079642794939594,
"speculative_memory_headroom": 0.10079642794939594
},
{
"context": 8192,
"route": "baseline",
"speedup": 0.966851611205723,
"acceptance_rate": null,
"baseline_memory_headroom": 0.10079642794939594,
"speculative_memory_headroom": 0.10079642794939594
},
{
"context": 32768,
"route": "baseline",
"speedup": 0.9521662660798582,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 32768,
"route": "baseline",
"speedup": 0.9420533202238093,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 32768,
"route": "baseline",
"speedup": 0.9484737691198221,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 65536,
"route": "baseline",
"speedup": 0.933693191425768,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 65536,
"route": "baseline",
"speedup": 0.9385807910239048,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 65536,
"route": "baseline",
"speedup": 0.9422977729157309,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 131072,
"route": "baseline",
"speedup": 0.929444115722683,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 131072,
"route": "baseline",
"speedup": 0.9422455761657648,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 131072,
"route": "baseline",
"speedup": 0.9273652817946989,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 262144,
"route": "baseline",
"speedup": 0.9386102961575811,
"acceptance_rate": null,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 262144,
"route": "baseline",
"speedup": 0.9357861169373657,
"acceptance_rate": 0.39070951320394526,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
},
{
"context": 262144,
"route": "baseline",
"speedup": 0.9298507231268106,
"acceptance_rate": 0.37947512021586094,
"baseline_memory_headroom": 0.09876973257121142,
"speculative_memory_headroom": 0.09876973257121142
}
],
"required": {
"min_speedup": 1.0,
"min_fallback_speedup": 0.9,
"min_acceptance_rate": 0.35,
"min_memory_headroom": 0.05
}
},
{
"name": "sustained_load_stability",
"passed": true,
"actual": {
"requests": 1024,
"successes": 1024,
"errors": 0,
"mismatches": 409,
"mismatch_rate": 0.3994140625,
"baseline_mismatch_rate": 0.3759765625,
"mismatch_rate_z": 1.0884718414299976,
"throughput_ratio": 0.8252932516905214,
"health_failures": 0,
"memory_headroom": 0.09105351214697764,
"memory_growth_mib": 0.0
},
"required": {
"min_requests": 500,
"zero_errors_and_health_failures": true,
"max_mismatch_rate_z": 1.6448536269514722,
"min_throughput_ratio": 0.75,
"min_memory_headroom": 0.05,
"max_memory_growth_mib": 2048.0
}
}
]
}