122 lines
6.9 KiB
JSON
122 lines
6.9 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"recorded_date": "2026-08-27",
|
|
"status": "single_seed_user_reported_test_inspected_diagnostic",
|
|
"dataset": "EasyCom",
|
|
"evaluation": {
|
|
"protocol": "EgoSpeak full timeline",
|
|
"test_sessions": [1, 2, 3],
|
|
"prediction_hz": 5,
|
|
"future_horizons_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0],
|
|
"classes": ["background", "camera_wearer_speaking", "other_people_speaking"],
|
|
"n": 23048
|
|
},
|
|
"representation": {
|
|
"checkpoint_family": "ROMA based on Qwen2.5-Omni-7B",
|
|
"thinker_frozen": true,
|
|
"streaming_mode": "open-stream-replay",
|
|
"memory_seconds": 8,
|
|
"video_fps": 2,
|
|
"state": "last_four_query_states",
|
|
"state_shape": [4, 3584],
|
|
"native_roma_one_second_streaming_recipe_reproduced": false
|
|
},
|
|
"runs": [
|
|
{
|
|
"id": "roma_last4_concat_mlp_1ep",
|
|
"layer_fusion": "concat-mlp",
|
|
"epochs": 1,
|
|
"fusion_parameters_approx": 51412480,
|
|
"reported_mean_map": 58.89219,
|
|
"mean_class_ap": {
|
|
"background": 42.17735,
|
|
"camera_wearer_speaking": 57.38537,
|
|
"other_people_speaking": 77.11384
|
|
},
|
|
"per_horizon": [
|
|
{"seconds": 0.2, "background": 52.51021, "camera_wearer_speaking": 70.83688, "other_people_speaking": 81.46725, "map": 68.27145},
|
|
{"seconds": 0.4, "background": 48.71387, "camera_wearer_speaking": 67.22686, "other_people_speaking": 79.93305, "map": 65.29126},
|
|
{"seconds": 0.6, "background": 45.13005, "camera_wearer_speaking": 62.85818, "other_people_speaking": 78.49071, "map": 62.15965},
|
|
{"seconds": 0.8, "background": 42.40835, "camera_wearer_speaking": 59.85171, "other_people_speaking": 77.60421, "map": 59.95476},
|
|
{"seconds": 1.0, "background": 41.04191, "camera_wearer_speaking": 56.82736, "other_people_speaking": 76.89293, "map": 58.25407},
|
|
{"seconds": 1.2, "background": 39.74672, "camera_wearer_speaking": 54.55271, "other_people_speaking": 76.21605, "map": 56.83849},
|
|
{"seconds": 1.4, "background": 38.80110, "camera_wearer_speaking": 52.54133, "other_people_speaking": 75.63049, "map": 55.65764},
|
|
{"seconds": 1.6, "background": 38.10108, "camera_wearer_speaking": 51.26856, "other_people_speaking": 75.38544, "map": 54.91836},
|
|
{"seconds": 1.8, "background": 37.64821, "camera_wearer_speaking": 49.94810, "other_people_speaking": 75.02545, "map": 54.20725},
|
|
{"seconds": 2.0, "background": 37.67202, "camera_wearer_speaking": 47.94197, "other_people_speaking": 74.49285, "map": 53.36895}
|
|
]
|
|
},
|
|
{
|
|
"id": "roma_last4_concat_mlp_4ep",
|
|
"layer_fusion": "concat-mlp",
|
|
"epochs": 4,
|
|
"fusion_parameters_approx": 51412480,
|
|
"reported_mean_map": 56.52425,
|
|
"mean_class_ap": {
|
|
"background": 40.19574,
|
|
"camera_wearer_speaking": 52.75190,
|
|
"other_people_speaking": 76.62513
|
|
},
|
|
"per_horizon": [
|
|
{"seconds": 0.2, "background": 52.12125, "camera_wearer_speaking": 71.51813, "other_people_speaking": 82.32518, "map": 68.65486},
|
|
{"seconds": 0.4, "background": 47.23227, "camera_wearer_speaking": 65.34609, "other_people_speaking": 80.40987, "map": 64.32941},
|
|
{"seconds": 0.6, "background": 42.97315, "camera_wearer_speaking": 59.62350, "other_people_speaking": 78.65869, "map": 60.41845},
|
|
{"seconds": 0.8, "background": 39.98155, "camera_wearer_speaking": 54.81910, "other_people_speaking": 77.25528, "map": 57.35198},
|
|
{"seconds": 1.0, "background": 38.30512, "camera_wearer_speaking": 50.99168, "other_people_speaking": 76.23874, "map": 55.17851},
|
|
{"seconds": 1.2, "background": 37.25808, "camera_wearer_speaking": 48.42603, "other_people_speaking": 75.43737, "map": 53.70716},
|
|
{"seconds": 1.4, "background": 36.84681, "camera_wearer_speaking": 46.09807, "other_people_speaking": 74.83621, "map": 52.59370},
|
|
{"seconds": 1.6, "background": 36.30368, "camera_wearer_speaking": 44.70198, "other_people_speaking": 74.50749, "map": 51.83772},
|
|
{"seconds": 1.8, "background": 35.37777, "camera_wearer_speaking": 43.20047, "other_people_speaking": 73.53734, "map": 50.70519},
|
|
{"seconds": 2.0, "background": 35.55772, "camera_wearer_speaking": 42.79393, "other_people_speaking": 73.04510, "map": 50.46558}
|
|
]
|
|
},
|
|
{
|
|
"id": "roma_last4_fixed_mean_4ep",
|
|
"layer_fusion": "fixed-mean",
|
|
"epochs": 4,
|
|
"fusion_parameters_approx": 0,
|
|
"reported_mean_map": 58.90402,
|
|
"mean_class_ap": {
|
|
"background": 42.19246,
|
|
"camera_wearer_speaking": 57.26979,
|
|
"other_people_speaking": 77.24980
|
|
},
|
|
"per_horizon": [
|
|
{"seconds": 0.2, "background": 53.55709, "camera_wearer_speaking": 72.09031, "other_people_speaking": 82.00848, "map": 69.21863},
|
|
{"seconds": 0.4, "background": 49.15182, "camera_wearer_speaking": 67.68575, "other_people_speaking": 80.18605, "map": 65.67454},
|
|
{"seconds": 0.6, "background": 45.55752, "camera_wearer_speaking": 63.19862, "other_people_speaking": 78.81368, "map": 62.52327},
|
|
{"seconds": 0.8, "background": 42.39609, "camera_wearer_speaking": 59.81524, "other_people_speaking": 77.56363, "map": 59.92499},
|
|
{"seconds": 1.0, "background": 41.02210, "camera_wearer_speaking": 56.50674, "other_people_speaking": 76.91692, "map": 58.14859},
|
|
{"seconds": 1.2, "background": 39.45131, "camera_wearer_speaking": 53.93851, "other_people_speaking": 76.46970, "map": 56.61984},
|
|
{"seconds": 1.4, "background": 38.78924, "camera_wearer_speaking": 52.30126, "other_people_speaking": 75.96435, "map": 55.68495},
|
|
{"seconds": 1.6, "background": 38.08597, "camera_wearer_speaking": 50.22392, "other_people_speaking": 75.26934, "map": 54.52641},
|
|
{"seconds": 1.8, "background": 37.22956, "camera_wearer_speaking": 49.02115, "other_people_speaking": 74.77319, "map": 53.67463},
|
|
{"seconds": 2.0, "background": 36.68386, "camera_wearer_speaking": 47.91641, "other_people_speaking": 74.53267, "map": 53.04431}
|
|
]
|
|
}
|
|
],
|
|
"comparisons_percent": {
|
|
"concat_4ep_minus_concat_1ep_horizon_0.2_map": 0.38341,
|
|
"concat_4ep_minus_concat_1ep_mean_map": -2.36794,
|
|
"fixed_mean_4ep_minus_concat_4ep_horizon_0.2_map": 0.56377,
|
|
"fixed_mean_4ep_minus_concat_4ep_mean_map": 2.37977,
|
|
"fixed_mean_4ep_minus_concat_1ep_horizon_0.2_map": 0.94718,
|
|
"fixed_mean_4ep_minus_concat_1ep_mean_map": 0.01183,
|
|
"roma_concat_4ep_minus_qwen3b_concat_4ep_horizon_0.2_map": -4.76534,
|
|
"roma_concat_4ep_minus_qwen3b_concat_4ep_mean_map": -4.34719
|
|
},
|
|
"validity": {
|
|
"single_seed": true,
|
|
"test_sessions_repeatedly_inspected": true,
|
|
"strict_causal_valid_reported": false,
|
|
"remote_metrics_json_archived": false,
|
|
"exact_training_commands_archived": false,
|
|
"train_sessions_assumed_from_project_protocol": [4, 5, 6, 7, 8, 9, 10, 11, 12]
|
|
},
|
|
"provenance": {
|
|
"source": "user-reported remote console tables",
|
|
"recorded_by": "Codex",
|
|
"values_are_percent": true
|
|
}
|
|
}
|