add new model results

static/eval_results/all_model_keywords_stats.json

@@ -1,4 +1,238 @@
 {
+    "NVLM": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.24033557047857043
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.32154059695494047
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.2937052996171993
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.22845955700594492
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2639741933075709
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.40870864071047447
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4555238095238095
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.25785191641267197
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.15679681195908274
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.0672259242345112
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.23922823287047076
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.21734036617042948
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.30313485498585124
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.34726189956094355
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.3264757655296162
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.056894830390305184
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.22868389095927066
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.2788963949121424
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.2787764976961992
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.23349712171444964
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.3215948035793096
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.18487055428231897
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.3680809151131777
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.03838410364145658
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2325581694709435
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.22773778915303383
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.08048160660797504
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.2390024647851972
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.30211261814126533
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.18857142857142856
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.24908307640275493
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3724877947012685
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.24529601154794037
+            }
+        }
+    },
     "GPT_4o_mini": {
         "skills": {
             "Object Recognition and Classification": {
@@ -167,19 +401,253 @@
                 "count": 43,
                 "num_samples": 698,
                 "tasks": [],
-                "average_score": 0.45508480503584553
+                "average_score": 0.45508480503584553
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.24651576711552803
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.3697506340557095
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.5640948591986592
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.2420320329702607
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.3458483931206892
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.43544861040322835
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5176671720617656
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.3554299482098288
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5398829253460956
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.32918280841495845
+            }
+        }
+    },
+    "Llama_3_2_11B": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.1907604552173455
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.14280015951776653
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.1960311445935766
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.22399113135844315
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.13303760019716085
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.323153603297999
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4260501253132832
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.1770852858056774
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.15366454315378308
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.06563884729522687
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.11886347847341794
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.11489351406848371
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.1693681214060816
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2123769209846321
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.2520175802062012
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.24806929522702081
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.06418655520777307
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.12349256529641485
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.16374180545556977
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.1576236804437753
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.15014439824913947
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.3003142292328822
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.19270157739425633
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.1463246409674981
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.0732004839476103
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.19579907898674231
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2123769209846321
             },
             "4-5 images": {
                 "count": 34,
                 "num_samples": 520,
                 "tasks": [],
-                "average_score": 0.24651576711552803
+                "average_score": 0.1351857051327849
             },
             "2-3 images": {
                 "count": 51,
                 "num_samples": 802,
                 "tasks": [],
-                "average_score": 0.3697506340557095
+                "average_score": 0.18586695387250338
             }
         },
         "app": {
@@ -187,113 +655,113 @@
                 "count": 72,
                 "num_samples": 1124,
                 "tasks": [],
-                "average_score": 0.5640948591986592
+                "average_score": 0.17288724679416761
             },
             "Planning": {
                 "count": 78,
                 "num_samples": 1239,
                 "tasks": [],
-                "average_score": 0.2420320329702607
+                "average_score": 0.08100042975820579
             },
             "Coding": {
                 "count": 31,
                 "num_samples": 474,
                 "tasks": [],
-                "average_score": 0.3458483931206892
+                "average_score": 0.0575426944971537
             },
             "Perception": {
                 "count": 145,
                 "num_samples": 2313,
                 "tasks": [],
-                "average_score": 0.43544861040322835
+                "average_score": 0.19853488174071646
             },
             "Metrics": {
                 "count": 20,
                 "num_samples": 309,
                 "tasks": [],
-                "average_score": 0.5176671720617656
+                "average_score": 0.254316961351997
             },
             "Science": {
                 "count": 29,
                 "num_samples": 574,
                 "tasks": [],
-                "average_score": 0.3554299482098288
+                "average_score": 0.162801811963855
             },
             "Knowledge": {
                 "count": 97,
                 "num_samples": 1605,
                 "tasks": [],
-                "average_score": 0.5398829253460956
+                "average_score": 0.28055776664538923
             },
             "Mathematics": {
                 "count": 33,
                 "num_samples": 547,
                 "tasks": [],
-                "average_score": 0.32918280841495845
+                "average_score": 0.13937853323074623
             }
         }
     },
-    "Llama_3_2_11B": {
+    "Claude_3.5_new": {
         "skills": {
             "Object Recognition and Classification": {
                 "count": 303,
                 "num_samples": 4755,
                 "tasks": [],
-                "average_score": 0.1907604552173455
+                "average_score": 0.5690042283891658
             },
             "Text Recognition (OCR)": {
                 "count": 137,
                 "num_samples": 2239,
                 "tasks": [],
-                "average_score": 0.14280015951776653
+                "average_score": 0.6220681231036606
             },
             "Language Understanding and Generation": {
                 "count": 154,
                 "num_samples": 2509,
                 "tasks": [],
-                "average_score": 0.1960311445935766
+                "average_score": 0.6077980666415158
             },
             "Scene and Event Understanding": {
                 "count": 154,
                 "num_samples": 2467,
                 "tasks": [],
-                "average_score": 0.22399113135844315
+                "average_score": 0.5511434932168607
             },
             "Mathematical and Logical Reasoning": {
                 "count": 109,
                 "num_samples": 1910,
                 "tasks": [],
-                "average_score": 0.13303760019716085
+                "average_score": 0.4885536652013625
             },
             "Commonsense and Social Reasoning": {
                 "count": 51,
                 "num_samples": 855,
                 "tasks": [],
-                "average_score": 0.323153603297999
+                "average_score": 0.590818684469149
             },
             "Ethical and Safety Reasoning": {
                 "count": 15,
                 "num_samples": 245,
                 "tasks": [],
-                "average_score": 0.4260501253132832
+                "average_score": 0.6569473684210526
             },
             "Domain-Specific Knowledge and Skills": {
                 "count": 77,
                 "num_samples": 1386,
                 "tasks": [],
-                "average_score": 0.1770852858056774
+                "average_score": 0.5486763511384175
             },
             "Spatial and Temporal Reasoning": {
                 "count": 152,
                 "num_samples": 2437,
                 "tasks": [],
-                "average_score": 0.15366454315378308
+                "average_score": 0.4315385951907387
             },
             "Planning and Decision Making": {
                 "count": 37,
                 "num_samples": 577,
                 "tasks": [],
-                "average_score": 0.06563884729522687
+                "average_score": 0.2909419331017877
             }
         },
         "input_format": {
@@ -301,43 +769,43 @@
                 "count": 93,
                 "num_samples": 1517,
                 "tasks": [],
-                "average_score": 0.11886347847341794
+                "average_score": 0.6048192628845258
             },
             "Text-Based Images and Documents": {
                 "count": 82,
                 "num_samples": 1294,
                 "tasks": [],
-                "average_score": 0.11489351406848371
+                "average_score": 0.48924295292319175
             },
             "Diagrams and Data Visualizations": {
                 "count": 101,
                 "num_samples": 1718,
                 "tasks": [],
-                "average_score": 0.1693681214060816
+                "average_score": 0.556418710368288
             },
             "Videos": {
                 "count": 43,
                 "num_samples": 698,
                 "tasks": [],
-                "average_score": 0.2123769209846321
+                "average_score": 0.4946691340754988
             },
             "Artistic and Creative Content": {
                 "count": 32,
                 "num_samples": 541,
                 "tasks": [],
-                "average_score": 0.2520175802062012
+                "average_score": 0.5558756390298104
             },
             "Photographs": {
                 "count": 143,
                 "num_samples": 2248,
                 "tasks": [],
-                "average_score": 0.24806929522702081
+                "average_score": 0.542519242638518
             },
             "3D Models and Aerial Imagery": {
                 "count": 11,
                 "num_samples": 169,
                 "tasks": [],
-                "average_score": 0.06418655520777307
+                "average_score": 0.44210335381541843
             }
         },
         "output_format": {
@@ -345,37 +813,37 @@
                 "count": 98,
                 "num_samples": 1514,
                 "tasks": [],
-                "average_score": 0.12349256529641485
+                "average_score": 0.5187252051932875
             },
             "structured_output": {
                 "count": 110,
                 "num_samples": 1714,
                 "tasks": [],
-                "average_score": 0.16374180545556977
+                "average_score": 0.5071113150600759
             },
             "exact_text": {
                 "count": 83,
                 "num_samples": 1278,
                 "tasks": [],
-                "average_score": 0.1576236804437753
+                "average_score": 0.5387340524651681
             },
             "numerical_data": {
                 "count": 49,
                 "num_samples": 862,
                 "tasks": [],
-                "average_score": 0.15014439824913947
+                "average_score": 0.4824302644151348
             },
             "open_ended_output": {
                 "count": 80,
                 "num_samples": 1454,
                 "tasks": [],
-                "average_score": 0.3003142292328822
+                "average_score": 0.6242798397166945
             },
             "multiple_choice": {
                 "count": 85,
                 "num_samples": 1363,
                 "tasks": [],
-                "average_score": 0.19270157739425633
+                "average_score": 0.5782691045270721
             }
         },
         "input_num": {
@@ -383,37 +851,37 @@
                 "count": 21,
                 "num_samples": 314,
                 "tasks": [],
-                "average_score": 0.1463246409674981
+                "average_score": 0.4630277507828528
             },
             "9-image or more": {
                 "count": 41,
                 "num_samples": 623,
                 "tasks": [],
-                "average_score": 0.0732004839476103
+                "average_score": 0.5914338446093256
             },
             "1-image": {
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.19579907898674231
+                "average_score": 0.5636254729390459
             },
             "video": {
                 "count": 43,
                 "num_samples": 698,
                 "tasks": [],
-                "average_score": 0.2123769209846321
+                "average_score": 0.4946691340754988
             },
             "4-5 images": {
                 "count": 34,
                 "num_samples": 520,
                 "tasks": [],
-                "average_score": 0.1351857051327849
+                "average_score": 0.4828123870640382
             },
             "2-3 images": {
                 "count": 51,
                 "num_samples": 802,
                 "tasks": [],
-                "average_score": 0.18586695387250338
+                "average_score": 0.48756464396063437
             }
         },
         "app": {
@@ -421,49 +889,49 @@
                 "count": 72,
                 "num_samples": 1124,
                 "tasks": [],
-                "average_score": 0.17288724679416761
+                "average_score": 0.6590137441693218
             },
             "Planning": {
                 "count": 78,
                 "num_samples": 1239,
                 "tasks": [],
-                "average_score": 0.08100042975820579
+                "average_score": 0.39901670035164916
             },
             "Coding": {
                 "count": 31,
                 "num_samples": 474,
                 "tasks": [],
-                "average_score": 0.0575426944971537
+                "average_score": 0.5166853031535193
             },
             "Perception": {
                 "count": 145,
                 "num_samples": 2313,
                 "tasks": [],
-                "average_score": 0.19853488174071646
+                "average_score": 0.5561634744977417
             },
             "Metrics": {
                 "count": 20,
                 "num_samples": 309,
                 "tasks": [],
-                "average_score": 0.254316961351997
+                "average_score": 0.6123769274172342
             },
             "Science": {
                 "count": 29,
                 "num_samples": 574,
                 "tasks": [],
-                "average_score": 0.162801811963855
+                "average_score": 0.5512015158810595
             },
             "Knowledge": {
                 "count": 97,
                 "num_samples": 1605,
                 "tasks": [],
-                "average_score": 0.28055776664538923
+                "average_score": 0.5657956645626817
             },
             "Mathematics": {
                 "count": 33,
                 "num_samples": 547,
                 "tasks": [],
-                "average_score": 0.13937853323074623
+                "average_score": 0.4763267502912362
             }
         }
     },
@@ -1181,13 +1649,13 @@
                 "count": 137,
                 "num_samples": 2239,
                 "tasks": [],
-                "average_score": 0.4989864259016192
+                "average_score": 0.49947304390648534
             },
             "Language Understanding and Generation": {
                 "count": 154,
                 "num_samples": 2509,
                 "tasks": [],
-                "average_score": 0.550842111088751
+                "average_score": 0.5512750115216515
             },
             "Scene and Event Understanding": {
                 "count": 154,
@@ -1267,7 +1735,7 @@
                 "count": 143,
                 "num_samples": 2248,
                 "tasks": [],
-                "average_score": 0.5495643443147615
+                "average_score": 0.5500305447809621
             },
             "3D Models and Aerial Imagery": {
                 "count": 11,
@@ -1281,7 +1749,7 @@
                 "count": 98,
                 "num_samples": 1514,
                 "tasks": [],
-                "average_score": 0.44828282747008336
+                "average_score": 0.44896309957892694
             },
             "structured_output": {
                 "count": 110,
@@ -1331,7 +1799,7 @@
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.5032283218366624
+                "average_score": 0.5034399620483027
             },
             "video": {
                 "count": 43,
@@ -1375,7 +1843,7 @@
                 "count": 145,
                 "num_samples": 2313,
                 "tasks": [],
-                "average_score": 0.524603412718188
+                "average_score": 0.5250631828331306
             },
             "Metrics": {
                 "count": 20,
@@ -1643,25 +2111,25 @@
                 "count": 303,
                 "num_samples": 4755,
                 "tasks": [],
-                "average_score": 0.5628292541089482
+                "average_score": 0.5630800473549525
             },
             "Text Recognition (OCR)": {
                 "count": 137,
                 "num_samples": 2239,
                 "tasks": [],
-                "average_score": 0.6173690896799526
+                "average_score": 0.6216411634729735
             },
             "Language Understanding and Generation": {
                 "count": 154,
                 "num_samples": 2509,
                 "tasks": [],
-                "average_score": 0.6122177959113034
+                "average_score": 0.616018277142757
             },
             "Scene and Event Understanding": {
                 "count": 154,
                 "num_samples": 2467,
                 "tasks": [],
-                "average_score": 0.5822888182775097
+                "average_score": 0.5823184402392676
             },
             "Mathematical and Logical Reasoning": {
                 "count": 109,
@@ -1673,7 +2141,7 @@
                 "count": 51,
                 "num_samples": 855,
                 "tasks": [],
-                "average_score": 0.6344814691282928
+                "average_score": 0.6345709158363462
             },
             "Ethical and Safety Reasoning": {
                 "count": 15,
@@ -1705,7 +2173,7 @@
                 "count": 93,
                 "num_samples": 1517,
                 "tasks": [],
-                "average_score": 0.6046575685772053
+                "average_score": 0.608083455060831
             },
             "Text-Based Images and Documents": {
                 "count": 82,
@@ -1735,7 +2203,7 @@
                 "count": 143,
                 "num_samples": 2248,
                 "tasks": [],
-                "average_score": 0.559466820210236
+                "average_score": 0.5613635226492386
             },
             "3D Models and Aerial Imagery": {
                 "count": 11,
@@ -1749,13 +2217,13 @@
                 "count": 98,
                 "num_samples": 1514,
                 "tasks": [],
-                "average_score": 0.5354190939719853
+                "average_score": 0.5388690453811203
             },
             "structured_output": {
                 "count": 110,
                 "num_samples": 1714,
                 "tasks": [],
-                "average_score": 0.4780999465727382
+                "average_score": 0.4803884979696412
             },
             "exact_text": {
                 "count": 83,
@@ -1793,13 +2261,13 @@
                 "count": 41,
                 "num_samples": 623,
                 "tasks": [],
-                "average_score": 0.5265640970967286
+                "average_score": 0.5343350103400748
             },
             "1-image": {
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.5664191419997976
+                "average_score": 0.5672657028463585
             },
             "video": {
                 "count": 43,
@@ -1817,7 +2285,7 @@
                 "count": 51,
                 "num_samples": 802,
                 "tasks": [],
-                "average_score": 0.490800991115688
+                "average_score": 0.49089043782374137
             }
         },
         "app": {
@@ -1825,7 +2293,7 @@
                 "count": 72,
                 "num_samples": 1124,
                 "tasks": [],
-                "average_score": 0.7011776751799048
+                "average_score": 0.7056027785545881
             },
             "Planning": {
                 "count": 78,
@@ -1843,7 +2311,7 @@
                 "count": 145,
                 "num_samples": 2313,
                 "tasks": [],
-                "average_score": 0.5491960044393517
+                "average_score": 0.5510350848991218
             },
             "Metrics": {
                 "count": 20,
@@ -1861,7 +2329,7 @@
                 "count": 97,
                 "num_samples": 1605,
                 "tasks": [],
-                "average_score": 0.6135384893140922
+                "average_score": 0.6135855179956459
             },
             "Mathematics": {
                 "count": 33,
@@ -2807,6 +3275,240 @@
             }
         }
     },
+    "Aria": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.3264829094772722
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.35712138797286674
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.4004806395853317
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.3783082688258977
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.27628131703993153
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.4942870225393938
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.5811228070175439
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.3279996334048362
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.2481896092177717
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.11945216302285933
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.2830308005758272
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.27833423130489043
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.32371820359400666
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.42875359425696014
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3612041984219992
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.37290568595471846
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.19554976321164697
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.3092653492193887
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.3043751656077328
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.2930015244066511
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.3092167834876797
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.4523860109667709
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.3277812604542708
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.21139455782312927
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.2711617723374526
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.3576735443060994
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.42875359425696014
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.19839956701033565
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.27267126872569447
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.38321397541649777
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.14301905320436192
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.2849545194421855
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.3779947327886569
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.39678729061309725
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.29682445889316517
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.4096377585306089
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.26194160419181234
+            }
+        }
+    },
     "Claude_3.5": {
         "skills": {
             "Object Recognition and Classification": {
@@ -2819,13 +3521,13 @@
                 "count": 137,
                 "num_samples": 2239,
                 "tasks": [],
-                "average_score": 0.6026892335040172
+                "average_score": 0.6046357055234819
             },
             "Language Understanding and Generation": {
                 "count": 154,
                 "num_samples": 2509,
                 "tasks": [],
-                "average_score": 0.5695311134746034
+                "average_score": 0.5712627152062051
             },
             "Scene and Event Understanding": {
                 "count": 154,
@@ -2905,7 +3607,7 @@
                 "count": 143,
                 "num_samples": 2248,
                 "tasks": [],
-                "average_score": 0.5078124682977725
+                "average_score": 0.5096772701625744
             },
             "3D Models and Aerial Imagery": {
                 "count": 11,
@@ -2919,7 +3621,7 @@
                 "count": 98,
                 "num_samples": 1514,
                 "tasks": [],
-                "average_score": 0.5039586533964282
+                "average_score": 0.5066797418318023
             },
             "structured_output": {
                 "count": 110,
@@ -2969,7 +3671,7 @@
                 "count": 315,
                 "num_samples": 5228,
                 "tasks": [],
-                "average_score": 0.5364554303845326
+                "average_score": 0.5373019912310933
             },
             "video": {
                 "count": 43,
@@ -3013,7 +3715,7 @@
                 "count": 145,
                 "num_samples": 2313,
                 "tasks": [],
-                "average_score": 0.5304907166726288
+                "average_score": 0.532329797132399
             },
             "Metrics": {
                 "count": 20,

static/eval_results/all_summary.json

@@ -5,16 +5,16 @@
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.5187898818829914,
-            "micro_mean_score": 0.5127977300993917
+            "macro_mean_score": 0.5203470034386184,
+            "micro_mean_score": 0.514305381949725
         },
         "core_cot": {
             "num_eval_tasks": 440,
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.5251654337401854,
-            "micro_mean_score": 0.522332974147119
+            "macro_mean_score": 0.5265059698578094,
+            "micro_mean_score": 0.5236365938368621
         },
         "open": {
             "num_eval_tasks": 65,
@@ -23,7 +23,7 @@
             "macro_mean_score": 0.6478225794744895,
             "micro_mean_score": 0.665391229578676
         },
-        "overall_score": 0.5409529871515315
+        "overall_score": 0.542120979016392
     },
     "Gemini_1.5_pro_002": {
         "core_noncot": {
@@ -39,8 +39,8 @@
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.481393687771543,
-            "micro_mean_score": 0.4756661334397647
+            "macro_mean_score": 0.48154520292305814,
+            "micro_mean_score": 0.47581906202211677
         },
         "open": {
             "num_eval_tasks": 65,
@@ -49,7 +49,7 @@
             "macro_mean_score": 0.5858190649927173,
             "micro_mean_score": 0.6104901117798793
         },
-        "overall_score": 0.4948345779089219
+        "overall_score": 0.49496659111024205
     },
     "Gemini_1.5_flash_002": {
         "core_noncot": {
@@ -91,8 +91,8 @@
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.5023557473841108,
-            "micro_mean_score": 0.4985442599850241
+            "macro_mean_score": 0.5029618079901714,
+            "micro_mean_score": 0.4991559743144323
         },
         "open": {
             "num_eval_tasks": 65,
@@ -101,7 +101,33 @@
             "macro_mean_score": 0.6373907158949892,
             "micro_mean_score": 0.6569647463456579
         },
-        "overall_score": 0.519736485905313
+        "overall_score": 0.5202645387105935
+    },
+    "Claude_3.5_new": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.4919657684484185,
+            "micro_mean_score": 0.4874520567007144
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.525918992480593,
+            "micro_mean_score": 0.5230784020211157
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.6563419761104125,
+            "micro_mean_score": 0.6724419604471196
+        },
+        "overall_score": 0.5427061091854214
     },
     "GPT_4o_mini": {
         "core_noncot": {
@@ -414,5 +440,57 @@
             "micro_mean_score": 0.35649183147033553
         },
         "overall_score": 0.138206224513898
+    },
+    "Aria": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.30485930718699694,
+            "micro_mean_score": 0.3016713629035311
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.289073788209904,
+            "micro_mean_score": 0.2859007507765791
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.5103725263180767,
+            "micro_mean_score": 0.5349957007738607
+        },
+        "overall_score": 0.3313115037088191
+    },
+    "NVLM": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.2420528895703979,
+            "micro_mean_score": 0.23838419989257642
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.21589726765847422,
+            "micro_mean_score": 0.21406043849932396
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.3478114310231307,
+            "micro_mean_score": 0.3947549441100602
+        },
+        "overall_score": 0.25566537510391796
     }
 }

utils.py

@@ -17,15 +17,18 @@ with open("./static/eval_results/all_summary.json", "r") as f:
 
 # Define model name mapping
 MODEL_NAME_MAP = {
+    "Claude_3.5_new": "Claude-3.5-Sonnet (1022)",
     "GPT_4o": "GPT-4o (0513)",
-    "Claude_3.5": "Claude-3.5-Sonnet",
+    "Claude_3.5": "Claude-3.5-Sonnet (0622)",
     "Gemini_1.5_pro_002": "Gemini-1.5-Pro-002",
     "InternVL2_76B": "InternVL2-Llama3-76B",
     "Qwen2_VL_72B": "Qwen2-VL-72B",
     "llava_onevision_72B": "Llava-OneVision-72B",
+    "NVLM": "NVLM-72B",
     "GPT_4o_mini": "GPT-4o mini",
     "Gemini_1.5_flash_002": "Gemini-1.5-Flash-002",
     "Pixtral_12B": "Pixtral 12B",
+    "Aria": "Aria-MoE-25B",
     "Qwen2_VL_7B": "Qwen2-VL-7B",
     "InternVL2_8B": "InternVL2-8B",
     "llava_onevision_7B": "Llava-OneVision-7B",
@@ -92,10 +95,6 @@ KEYWORD_NAME_MAP = {
 SUPER_GROUPS = {DIMENSION_NAME_MAP[dim]: [KEYWORD_NAME_MAP.get(k, k) for k in MODEL_DATA[next(iter(MODEL_DATA))][dim].keys()] 
                 for dim in MODEL_DATA[next(iter(MODEL_DATA))]}
 
-SUBMISSION_NAME = "test_leaderboard_submission"
-SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/cccjc/", SUBMISSION_NAME)
-CSV_DIR = "./test_leaderboard_submission/results.csv"
-
 def get_original_dimension(mapped_dimension):
     return next(k for k, v in DIMENSION_NAME_MAP.items() if v == mapped_dimension)
 
@@ -105,12 +104,12 @@ def get_original_keyword(mapped_keyword):
 # Define model groups
 MODEL_GROUPS = {
     "All": list(MODEL_DATA.keys()),
-    "Flagship Models": ['GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B'],
-    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
-    "Proprietary Flagship models": ['GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002'],
-    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
-    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B'],
-    "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM-72B'],
+    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+    "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002'],
+    "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini'],
+    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM'],
+    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
 }
 
 def get_display_model_name(model_name):