{"version":"4.27.0","mode":"blocks","app_id":11385255698202515272,"dev_mode":false,"analytics_enabled":true,"components":[{"id":1,"type":"html","props":{"value":"<h1 align=\"left\" id=\"space-title\">🏅 LLM Leaderboard for Southeast Asian Languages</h1>","show_label":true,"visible":true,"elem_classes":[],"name":"html","_selectable":false},"skip_api":false,"component_class_id":"250f2d79d505ca0e6d62e7360ebf6bb5","api_info":{"type":"string"},"example_inputs":"<p>Hello</p>"},{"id":2,"type":"markdown","props":{"value":"This leaderboard evaluates Large Language Models (LLMs) on Southeast Asian (SEA) languages through two comprehensive benchmarks - SeaExam and SeaBench [[Paper](https://aclanthology.org/2025.findings-naacl.341/)]:\n* **SeaExam** assesses world knowledge and reasoning capabilities through exam-style questions (for both base and chat version models) [[data (public)](https://huggingface.co/datasets/SeaLLMs/SeaExam)] [[eval code](https://github.com/DAMO-NLP-SG/SeaExam)]\n* **SeaBench** evaluates instruction-following abilities and multi-turn conversational skills (thus only for chat version models). [[data (public)](https://huggingface.co/datasets/SeaLLMs/SeaBench)] [[eval code](https://github.com/DAMO-NLP-SG/SeaBench?tab=readme-ov-file)]\n\nBelow are the aggregated results for SeaExam and SeaBench, shown both the public dataset (\"pub\") - which you can download via the link above - and our in-house held-out private dataset (\"prv\"). \n\nFor detailed results by language, please refer to the individual sub-tabs.\nFor more details, please refer to the \"📝 About\" tab.","show_label":true,"rtl":false,"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"visible":true,"elem_classes":["markdown-text"],"sanitize_html":true,"line_breaks":false,"header_links":false,"name":"markdown","_selectable":false},"skip_api":false,"component_class_id":"0f35515b9a3362b42bb6721c836157a0","api_info":{"type":"string"},"example_inputs":"# Hello!"},{"id":3,"type":"tabs","props":{"visible":true,"elem_classes":["tab-buttons"],"name":"tabs"},"skip_api":true,"component_class_id":"3e4202431baca71099b95e73fb9d900c"},{"id":4,"type":"tabitem","props":{"label":"🏅 Overall","visible":true,"interactive":true,"name":"tab"},"skip_api":true,"component_class_id":"18f46968e2b273dc5dbda0fe78433d7e"},{"id":5,"type":"leaderboard","props":{"row_count":[91,"fixed"],"col_count":[9,"fixed"],"headers":["R","Model","type","open?","#P(B)","SeaExam-pub","SeaExam-prv ⬇️","SeaBench-pub","SeaBench-prv"],"select_columns_config":{"default_selection":["R","Model","type","open?","#P(B)","SeaExam-pub","SeaExam-prv ⬇️","SeaBench-pub","SeaBench-prv"],"cant_deselect":["R","Model"],"allow":true,"label":"Select Columns to Display:","show_label":true,"info":null},"value":{"headers":["R","Model","type","open?","#P(B)","SeaExam-pub","SeaExam-prv ⬇️","SeaBench-pub","SeaBench-prv"],"data":[[1,"gemini-2.5-pro-preview-03-25","🔶 chat","N",null,91.2,91.2,null,null],[2,"claude-3.5-sonnet-20241022","🔶 chat","N",null,78.7,80.8,8.9,8.78],[2,"<a target=\"_blank\" href=\"https://huggingface.co/deepseek-ai/DeepSeek-V3-0324\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DeepSeek-V3-0324</a>","🔶 chat","Y",684.5,78.8,80.8,8.75,8.78],[4,"gemini-pro-1.5","🔶 chat","N",null,77.6,79.4,8.84,8.62],[5,"claude-3.7-sonnet-20250219","🔶 chat","N",null,78.7,79.1,8.84,8.77],[6,"gemini-2.0-flash","🔶 chat","N",null,77.5,78.4,8.66,8.6],[7,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-72B</a>","🟢 base","Y",72.7,75.6,77.6,null,null],[8,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Maverick-17B-128E-Instruct</a>","🔶 chat","Y",401.6,75.0,76.7,7.91,7.82],[9,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-72B-Instruct</a>","🔶 chat","Y",72.7,75.2,76.6,8.22,7.96],[10,"gemini-2.0-flash-lite","🔶 chat","N",null,74.3,76.4,8.48,8.26],[11,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>","🟢 base","Y",72.7,74.6,75.8,null,null],[12,"gpt-4.1","🔶 chat","N",null,74.8,74.6,8.92,8.9],[13,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B-Instruct</a>","🔶 chat","Y",72.7,72.3,74.5,7.46,7.35],[14,"gpt-4o-2024-08-06","🔶 chat","N",null,74.6,74.1,8.8,8.64],[15,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-20B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-20B</a>","🟢 base","Y",19.2,71.0,73.9,null,null],[16,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-27b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-27b-pt</a>","🟢 base","Y",27.4,71.4,73.3,null,null],[17,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-20B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-20B-Chat</a>","🔶 chat","Y",19.2,71.6,73.1,7.99,7.78],[18,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-14B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-14B</a>","🟢 base","Y",14.8,70.5,72.6,null,null],[19,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-14B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-14B-Instruct</a>","🔶 chat","Y",14.8,69.8,71.8,7.56,7.43],[20,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-27b-it</a>","🔶 chat","Y",27.4,68.4,71.5,8.43,8.43],[21,"gemini-flash-1.5","🔶 chat","N",null,69.4,71.1,8.39,8.18],[22,"gpt-4.1-mini","🔶 chat","N",null,70.0,71.0,8.57,8.6],[23,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-70B-Instruct</a>","🔶 chat","Y",70.6,69.7,70.1,7.12,6.77],[24,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-70B-Instruct</a>","🔶 chat","Y",70.6,66.0,68.2,7.32,7.11],[25,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Scout-17B-16E-Instruct</a>","🔶 chat","Y",108.6,63.9,66.9,7.02,6.45],[26,"<a target=\"_blank\" href=\"https://huggingface.co/AIDC-AI/Marco-LLM-SEA\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Marco-LLM-SEA</a>","🟢 base","Y",7.6,64.7,66.6,null,null],[27,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-70B</a>","🟢 base","Y",70.6,65.6,66.2,null,null],[28,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-27b-it</a>","🔶 chat","Y",27.2,65.1,65.9,8.1,7.85],[29,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-70B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-70B</a>","🟢 base","Y",70.6,64.9,65.8,null,null],[30,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-12b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-12b-pt</a>","🟢 base","Y",12.2,65.2,65.5,null,null],[31,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-7B-Instruct</a>","🔶 chat","Y",7.6,62.6,65.0,6.65,6.81],[32,"claude-3-5-haiku-20241022","🔶 chat","N",null,63.7,64.9,8.29,7.87],[33,"gpt-4o-mini-2024-07-18","🔶 chat","N",null,64.2,64.8,8.31,8.25],[34,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-7B</a>","🟢 base","Y",7.6,63.4,64.5,null,null],[35,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-8B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-8B-Chat</a>","🔶 chat","Y",8.5,62.9,64.2,7.0,7.02],[36,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-12b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-12b-it</a>","🔶 chat","Y",12.2,63.2,62.8,8.18,8.1],[37,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b-it</a>","🔶 chat","Y",9.2,62.5,62.6,7.82,7.52],[38,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-27b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-27b</a>","🟢 base","Y",27.2,61.7,62.4,null,null],[39,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-8B</a>","🟢 base","Y",8.5,60.9,62.3,null,null],[40,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/gemma2-9b-cpt-sea-lionv3-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma2-9b-cpt-sea-lionv3-instruct</a>","🔶 chat","Y",9.2,62.8,61.9,7.93,7.72],[41,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-7B-Chat</a>","🔶 chat","Y",7.6,59.5,61.6,6.57,6.33],[42,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-7B</a>","🟢 base","Y",7.6,58.1,61.5,null,null],[43,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-7B-Instruct</a>","🔶 chat","Y",7.6,57.6,59.4,6.09,5.85],[43,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>","🟢 base","Y",9.2,58.9,59.4,null,null],[45,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/gemma2-9b-cpt-sea-lionv3-base\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma2-9b-cpt-sea-lionv3-base</a>","🟢 base","Y",9.2,60.4,59.2,null,null],[46,"gpt-4.1-nano","🔶 chat","N",null,58.4,59.0,7.89,7.81],[47,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-14B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-14B-Chat</a>","🔶 chat","Y",14.2,58.2,58.6,5.32,5.24],[48,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-3B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-3B-Instruct</a>","🔶 chat","Y",3.1,55.5,58.4,5.33,5.27],[49,"claude-3-haiku","🔶 chat","N",null,59.1,57.5,7.52,7.34],[50,"<a target=\"_blank\" href=\"https://huggingface.co/THUDM/glm-4-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">glm-4-9b</a>","🟢 base","Y",9.4,54.9,57.4,null,null],[51,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-3B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-3B</a>","🟢 base","Y",3.1,55.6,57.1,null,null],[51,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-14B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-14B</a>","🟢 base","Y",14.2,56.6,57.1,null,null],[53,"<a target=\"_blank\" href=\"https://huggingface.co/THUDM/glm-4-9b-chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">glm-4-9b-chat</a>","🔶 chat","Y",9.4,53.9,54.9,6.09,5.95],[54,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLM-7B-v2.5</a>","🔶 chat","Y",8.5,54.9,54.2,5.58,5.45],[55,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sea-lionv2.1-instruct</a>","🔶 chat","Y",8.0,53.9,54.1,6.72,6.56],[56,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-8B-Instruct</a>","🔶 chat","Y",8.0,52.7,53.7,5.81,5.87],[57,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sealionv2-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sealionv2-instruct</a>","🔶 chat","Y",8.0,52.3,52.5,null,null],[58,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sealionv2-base\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sealionv2-base</a>","🟢 base","Y",8.0,51.7,52.2,null,null],[59,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-8B</a>","🟢 base","Y",8.0,50.3,51.9,null,null],[60,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-8B</a>","🟢 base","Y",8.0,49.0,50.5,null,null],[61,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-1.5B-Instruct</a>","🔶 chat","Y",1.5,47.2,49.4,null,null],[62,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-1.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-1.5B</a>","🟢 base","Y",1.5,47.3,49.2,null,null],[62,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-8B-Instruct</a>","🔶 chat","Y",8.0,49.4,49.2,null,null],[64,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-4b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-4b-pt</a>","🟢 base","Y",4.3,50.1,49.0,null,null],[64,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mixtral-8x7B-v0.1\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mixtral-8x7B-v0.1</a>","🟢 base","Y",46.7,47.9,49.0,null,null],[66,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-4b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-4b-it</a>","🔶 chat","Y",4.3,50.0,48.8,7.18,7.17],[67,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Ministral-8B-Instruct-2410\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Ministral-8B-Instruct-2410</a>","🔶 chat","Y",8.0,51.0,48.4,null,null],[68,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-7B</a>","🟢 base","Y",7.7,47.6,48.2,null,null],[69,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-7B-Chat</a>","🔶 chat","Y",7.7,48.4,47.8,4.37,4.32],[70,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mixtral-8x7B-Instruct-v0.1</a>","🔶 chat","Y",46.7,47.6,47.0,null,null],[71,"<a target=\"_blank\" href=\"https://huggingface.co/CohereForAI/aya-expanse-8b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">aya-expanse-8b</a>","🔶 chat","Y",8.0,43.8,46.7,null,null],[72,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-3B-Instruct</a>","🔶 chat","Y",3.2,45.3,46.6,null,null],[73,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-1.5B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-1.5B-Chat</a>","🔶 chat","Y",1.5,45.3,46.1,null,null],[74,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-3B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-3B</a>","🟢 base","Y",3.2,42.5,44.6,null,null],[75,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-2b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-2b-it</a>","🔶 chat","Y",2.6,45.9,44.5,null,null],[76,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-1.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-1.5B</a>","🟢 base","Y",1.5,45.0,44.4,null,null],[77,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-7B-Instruct-v0.3</a>","🔶 chat","Y",7.2,39.8,40.8,3.85,3.64],[78,"<a target=\"_blank\" href=\"https://huggingface.co/CohereForAI/aya-23-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">aya-23-8B</a>","🔶 chat","Y",8.0,40.2,40.5,4.29,4.43],[79,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-4B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-4B-Chat</a>","🔶 chat","Y",4.0,40.1,40.2,null,null],[80,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-0.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-0.5B</a>","🟢 base","Y",0.5,36.5,39.6,null,null],[81,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-2b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-2b</a>","🟢 base","Y",2.6,39.5,38.3,null,null],[82,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-1B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-1B-Chat</a>","🔶 chat","Y",1.0,33.9,37.6,null,null],[83,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-0.5B-Instruct</a>","🔶 chat","Y",0.5,36.4,37.4,null,null],[84,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-1B-Instruct</a>","🔶 chat","Y",1.2,35.1,36.8,null,null],[85,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-1.8B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-1.8B-Chat</a>","🔶 chat","Y",1.8,35.0,36.2,null,null],[86,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-4B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-4B</a>","🟢 base","Y",4.0,33.7,34.8,null,null],[87,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-1B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-1B</a>","🟢 base","Y",1.0,31.9,32.9,null,null],[88,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-1B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-1B</a>","🟢 base","Y",1.2,28.3,27.9,null,null],[89,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-0.5B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-0.5B-Chat</a>","🔶 chat","Y",0.6,26.7,26.3,null,null],[89,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-1.8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-1.8B</a>","🟢 base","Y",1.8,26.1,26.3,null,null],[91,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-0.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-0.5B</a>","🟢 base","Y",0.6,25.9,24.9,null,null]],"metadata":null},"datatype":["number","markdown","str","str","number","number","number","number","number"],"search_columns":{"primary_column":"Model","secondary_columns":[],"label":null,"placeholder":null},"filter_columns":[{"column":"type","type":"checkboxgroup","default":[["🔶 chat","🔶 chat"],["🟢 base","🟢 base"]],"choices":[["🔶 chat","🔶 chat"],["🟢 base","🟢 base"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"open?","type":"checkboxgroup","default":[["N","N"],["Y","Y"]],"choices":[["N","N"],["Y","Y"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"#P(B)","type":"slider","default":[7,9],"choices":null,"label":"Paramers(B)","info":null,"show_label":true,"min":0.5,"max":684.5}],"hide_columns":[],"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"show_label":true,"height":500,"min_width":160,"visible":true,"elem_classes":[],"wrap":false,"line_breaks":true,"column_widths":[],"name":"leaderboard","_selectable":false},"skip_api":false,"component_class_id":"dc021d732c999fcb2181404b7ae6ca17","api_info":{"properties":{"headers":{"items":{"type":"string"},"title":"Headers","type":"array"},"data":{"items":{"items":{},"type":"array"},"title":"Data","type":"array"},"metadata":{"anyOf":[{"additionalProperties":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"type":"object"},{"type":"null"}],"default":null,"title":"Metadata"}},"required":["headers","data"],"title":"DataframeData","type":"object"},"example_inputs":{"headers":["a","b"],"data":[["foo","bar"]]}},{"id":6,"type":"tabitem","props":{"label":"SeaExam","visible":true,"interactive":true,"name":"tab"},"skip_api":true,"component_class_id":"18f46968e2b273dc5dbda0fe78433d7e"},{"id":7,"type":"leaderboard","props":{"row_count":[91,"fixed"],"col_count":[13,"fixed"],"headers":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"select_columns_config":{"default_selection":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"cant_deselect":["R","Model"],"allow":true,"label":"Select Columns to Display:","show_label":true,"info":null},"value":{"headers":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"data":[[1,"gemini-2.5-pro-preview-03-25","🔶 chat","N",null,91.2,91.2,90.1,92.9,90.8,88.1,94.2,91.4],[2,"claude-3.5-sonnet-20241022","🔶 chat","N",null,78.7,80.8,76.1,80.9,79.0,83.9,76.0,82.4],[2,"<a target=\"_blank\" href=\"https://huggingface.co/deepseek-ai/DeepSeek-V3-0324\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DeepSeek-V3-0324</a>","🔶 chat","Y",684.5,78.8,80.8,76.9,77.8,81.5,83.1,73.7,85.7],[4,"gemini-pro-1.5","🔶 chat","N",null,77.6,79.4,75.3,78.0,79.4,81.9,73.9,82.3],[5,"claude-3.7-sonnet-20250219","🔶 chat","N",null,78.7,79.1,75.6,80.9,79.7,80.4,74.1,82.8],[6,"gemini-2.0-flash","🔶 chat","N",null,77.5,78.4,74.9,79.0,78.5,79.2,73.2,83.0],[7,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-72B</a>","🟢 base","Y",72.7,75.6,77.6,72.3,76.3,78.3,80.1,71.3,81.2],[8,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Maverick-17B-128E-Instruct</a>","🔶 chat","Y",401.6,75.0,76.7,75.8,75.8,73.3,78.7,71.8,79.5],[9,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-72B-Instruct</a>","🔶 chat","Y",72.7,75.2,76.6,72.5,75.2,78.0,79.9,68.8,81.1],[10,"gemini-2.0-flash-lite","🔶 chat","N",null,74.3,76.4,72.2,74.2,76.5,79.9,67.4,81.9],[11,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>","🟢 base","Y",72.7,74.6,75.8,72.9,74.4,76.5,78.2,69.7,79.5],[12,"gpt-4.1","🔶 chat","N",null,74.8,74.6,68.9,78.0,77.5,73.7,68.5,81.6],[13,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B-Instruct</a>","🔶 chat","Y",72.7,72.3,74.5,70.6,71.7,74.5,79.9,64.3,79.2],[14,"gpt-4o-2024-08-06","🔶 chat","N",null,74.6,74.1,69.7,77.0,77.2,72.0,69.5,80.8],[15,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-20B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-20B</a>","🟢 base","Y",19.2,71.0,73.9,66.8,72.0,74.3,77.7,65.3,78.9],[16,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-27b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-27b-pt</a>","🟢 base","Y",27.4,71.4,73.3,69.1,68.0,77.1,75.4,63.4,80.9],[17,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-20B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-20B-Chat</a>","🔶 chat","Y",19.2,71.6,73.1,68.5,71.6,74.8,77.2,63.6,78.5],[18,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-14B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-14B</a>","🟢 base","Y",14.8,70.5,72.6,68.9,68.7,73.8,74.4,65.0,78.3],[19,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-14B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-14B-Instruct</a>","🔶 chat","Y",14.8,69.8,71.8,67.6,68.8,73.1,75.2,62.7,77.5],[20,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-27b-it</a>","🔶 chat","Y",27.4,68.4,71.5,64.4,67.5,73.1,72.5,63.9,78.0],[21,"gemini-flash-1.5","🔶 chat","N",null,69.4,71.1,66.6,68.3,73.5,71.0,65.7,76.5],[22,"gpt-4.1-mini","🔶 chat","N",null,70.0,71.0,66.1,72.2,71.7,72.2,64.8,75.9],[23,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-70B-Instruct</a>","🔶 chat","Y",70.6,69.7,70.1,65.8,70.6,72.6,74.4,60.4,75.6],[24,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-70B-Instruct</a>","🔶 chat","Y",70.6,66.0,68.2,63.7,65.3,69.1,72.0,59.0,73.6],[25,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Scout-17B-16E-Instruct</a>","🔶 chat","Y",108.6,63.9,66.9,61.7,64.4,65.5,70.2,56.9,73.5],[26,"<a target=\"_blank\" href=\"https://huggingface.co/AIDC-AI/Marco-LLM-SEA\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Marco-LLM-SEA</a>","🟢 base","Y",7.6,64.7,66.6,61.5,63.8,68.8,68.7,58.3,72.9],[27,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-70B</a>","🟢 base","Y",70.6,65.6,66.2,62.7,65.8,68.3,67.7,59.2,71.7],[28,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-27b-it</a>","🔶 chat","Y",27.2,65.1,65.9,60.4,64.6,70.1,68.5,58.0,71.1],[29,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-70B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-70B</a>","🟢 base","Y",70.6,64.9,65.8,61.8,65.9,66.9,64.8,60.1,72.5],[30,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-12b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-12b-pt</a>","🟢 base","Y",12.2,65.2,65.5,59.9,64.7,71.0,62.8,59.0,74.7],[31,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-7B-Instruct</a>","🔶 chat","Y",7.6,62.6,65.0,59.7,61.6,66.5,67.0,57.6,70.3],[32,"claude-3-5-haiku-20241022","🔶 chat","N",null,63.7,64.9,60.6,64.4,66.1,66.5,57.6,70.7],[33,"gpt-4o-mini-2024-07-18","🔶 chat","N",null,64.2,64.8,59.9,65.7,67.0,63.3,59.2,72.0],[34,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-7B</a>","🟢 base","Y",7.6,63.4,64.5,61.3,60.9,68.0,67.0,55.9,70.7],[35,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-8B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-8B-Chat</a>","🔶 chat","Y",8.5,62.9,64.2,59.1,62.9,66.7,64.5,55.2,72.7],[36,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-12b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-12b-it</a>","🔶 chat","Y",12.2,63.2,62.8,60.2,63.2,66.1,63.3,57.6,67.7],[37,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b-it</a>","🔶 chat","Y",9.2,62.5,62.6,59.6,60.1,67.7,62.8,54.5,70.3],[38,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-27b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-27b</a>","🟢 base","Y",27.2,61.7,62.4,56.3,61.9,66.9,61.8,53.8,71.6],[39,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-8B</a>","🟢 base","Y",8.5,60.9,62.3,58.1,59.2,65.3,64.0,50.3,72.5],[40,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/gemma2-9b-cpt-sea-lionv3-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma2-9b-cpt-sea-lionv3-instruct</a>","🔶 chat","Y",9.2,62.8,61.9,59.4,60.5,68.4,62.3,53.8,69.4],[41,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-7B-Chat</a>","🔶 chat","Y",7.6,59.5,61.6,55.3,57.5,65.6,62.3,52.7,69.8],[42,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-7B</a>","🟢 base","Y",7.6,58.1,61.5,54.1,56.4,63.8,61.8,55.9,66.8],[43,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-7B-Instruct</a>","🔶 chat","Y",7.6,57.6,59.4,55.2,56.0,61.5,59.6,53.4,65.4],[43,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>","🟢 base","Y",9.2,58.9,59.4,55.8,55.0,65.8,59.8,48.5,69.8],[45,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/gemma2-9b-cpt-sea-lionv3-base\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma2-9b-cpt-sea-lionv3-base</a>","🟢 base","Y",9.2,60.4,59.2,55.9,57.4,68.0,58.8,47.8,71.0],[46,"gpt-4.1-nano","🔶 chat","N",null,58.4,59.0,56.2,57.1,61.9,59.3,51.5,66.3],[47,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-14B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-14B-Chat</a>","🔶 chat","Y",14.2,58.2,58.6,55.2,57.1,62.2,57.6,49.7,68.4],[48,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-3B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-3B-Instruct</a>","🔶 chat","Y",3.1,55.5,58.4,53.3,52.7,60.6,59.1,50.3,65.7],[49,"claude-3-haiku","🔶 chat","N",null,59.1,57.5,57.2,56.4,63.7,56.8,49.2,66.5],[50,"<a target=\"_blank\" href=\"https://huggingface.co/THUDM/glm-4-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">glm-4-9b</a>","🟢 base","Y",9.4,54.9,57.4,52.5,49.7,62.6,62.3,41.7,68.2],[51,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-3B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-3B</a>","🟢 base","Y",3.1,55.6,57.1,52.3,53.1,61.6,60.0,45.5,65.7],[51,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-14B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-14B</a>","🟢 base","Y",14.2,56.6,57.1,53.4,54.9,61.5,57.8,47.3,66.2],[53,"<a target=\"_blank\" href=\"https://huggingface.co/THUDM/glm-4-9b-chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">glm-4-9b-chat</a>","🔶 chat","Y",9.4,53.9,54.9,49.6,50.7,61.4,55.6,43.6,65.4],[54,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLM-7B-v2.5</a>","🔶 chat","Y",8.5,54.9,54.2,49.9,52.8,61.9,55.3,42.2,65.2],[55,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sea-lionv2.1-instruct</a>","🔶 chat","Y",8.0,53.9,54.1,49.2,54.0,58.6,53.6,45.0,63.8],[56,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-8B-Instruct</a>","🔶 chat","Y",8.0,52.7,53.7,49.4,51.0,57.6,56.6,42.7,61.9],[57,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sealionv2-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sealionv2-instruct</a>","🔶 chat","Y",8.0,52.3,52.5,49.4,52.4,55.2,50.6,45.0,62.0],[58,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sealionv2-base\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sealionv2-base</a>","🟢 base","Y",8.0,51.7,52.2,49.2,51.6,54.4,53.1,43.4,60.2],[59,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-8B</a>","🟢 base","Y",8.0,50.3,51.9,46.8,49.8,54.4,55.3,42.7,57.7],[60,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-8B</a>","🟢 base","Y",8.0,49.0,50.5,47.0,47.7,52.4,52.6,42.0,57.1],[61,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-1.5B-Instruct</a>","🔶 chat","Y",1.5,47.2,49.4,46.0,43.6,52.0,52.4,39.6,56.3],[62,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-1.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-1.5B</a>","🟢 base","Y",1.5,47.3,49.2,46.5,42.8,52.6,50.6,38.0,59.1],[62,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-8B-Instruct</a>","🔶 chat","Y",8.0,49.4,49.2,48.0,47.8,52.4,51.1,41.0,55.6],[64,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-4b-pt\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-4b-pt</a>","🟢 base","Y",4.3,50.1,49.0,46.3,47.3,56.6,47.6,40.8,58.7],[64,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mixtral-8x7B-v0.1\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mixtral-8x7B-v0.1</a>","🟢 base","Y",46.7,47.9,49.0,48.5,42.0,53.1,53.1,39.9,53.9],[66,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-4b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-4b-it</a>","🔶 chat","Y",4.3,50.0,48.8,46.3,49.3,54.4,52.1,41.0,53.4],[67,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Ministral-8B-Instruct-2410\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Ministral-8B-Instruct-2410</a>","🔶 chat","Y",8.0,51.0,48.4,48.0,49.6,55.4,52.1,38.2,54.8],[68,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-7B</a>","🟢 base","Y",7.7,47.6,48.2,43.7,45.8,53.2,46.2,41.7,56.7],[69,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-7B-Chat</a>","🔶 chat","Y",7.7,48.4,47.8,46.1,46.0,53.1,47.4,40.6,55.6],[70,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mixtral-8x7B-Instruct-v0.1</a>","🔶 chat","Y",46.7,47.6,47.0,48.4,42.2,52.2,51.4,38.0,51.5],[71,"<a target=\"_blank\" href=\"https://huggingface.co/CohereForAI/aya-expanse-8b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">aya-expanse-8b</a>","🔶 chat","Y",8.0,43.8,46.7,44.2,34.1,53.2,51.9,31.2,57.1],[72,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-3B-Instruct</a>","🔶 chat","Y",3.2,45.3,46.6,42.5,42.9,50.7,50.1,38.5,51.3],[73,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-1.5B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-1.5B-Chat</a>","🔶 chat","Y",1.5,45.3,46.1,43.5,44.0,48.5,50.1,36.4,51.9],[74,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-3B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-3B</a>","🟢 base","Y",3.2,42.5,44.6,40.4,39.7,47.4,45.7,37.1,51.1],[75,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-2b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-2b-it</a>","🔶 chat","Y",2.6,45.9,44.5,43.3,42.4,52.0,43.7,38.0,51.8],[76,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-1.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-1.5B</a>","🟢 base","Y",1.5,45.0,44.4,42.8,42.1,50.1,46.4,35.9,50.9],[77,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-7B-Instruct-v0.3</a>","🔶 chat","Y",7.2,39.8,40.8,42.2,34.8,42.5,44.4,30.5,47.5],[78,"<a target=\"_blank\" href=\"https://huggingface.co/CohereForAI/aya-23-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">aya-23-8B</a>","🔶 chat","Y",8.0,40.2,40.5,41.3,30.8,48.6,43.2,26.6,51.8],[79,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-4B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-4B-Chat</a>","🔶 chat","Y",4.0,40.1,40.2,39.1,36.0,45.3,41.2,34.0,45.5],[80,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-0.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-0.5B</a>","🟢 base","Y",0.5,36.5,39.6,35.8,32.6,41.1,40.2,35.0,43.6],[81,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-2b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-2b</a>","🟢 base","Y",2.6,39.5,38.3,37.0,36.6,44.9,40.9,29.1,44.8],[82,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-1B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-1B-Chat</a>","🔶 chat","Y",1.0,33.9,37.6,33.5,32.3,36.0,40.4,31.7,40.7],[83,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-0.5B-Instruct</a>","🔶 chat","Y",0.5,36.4,37.4,34.9,34.0,40.5,39.0,30.3,42.8],[84,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-1B-Instruct</a>","🔶 chat","Y",1.2,35.1,36.8,34.1,31.2,39.9,38.0,30.8,41.5],[85,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-1.8B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-1.8B-Chat</a>","🔶 chat","Y",1.8,35.0,36.2,35.2,30.7,39.2,34.0,33.6,40.9],[86,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-4B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-4B</a>","🟢 base","Y",4.0,33.7,34.8,31.4,32.8,36.9,35.0,31.7,37.8],[87,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-1B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-1B</a>","🟢 base","Y",1.0,31.9,32.9,31.1,31.7,32.9,34.7,28.0,36.1],[88,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-3.2-1B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-3.2-1B</a>","🟢 base","Y",1.2,28.3,27.9,27.4,26.2,31.2,30.5,24.9,28.3],[89,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-0.5B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-0.5B-Chat</a>","🔶 chat","Y",0.6,26.7,26.3,27.5,26.9,25.7,25.3,24.9,28.5],[89,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-1.8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-1.8B</a>","🟢 base","Y",1.8,26.1,26.3,24.8,26.2,27.4,23.3,26.8,28.9],[91,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-0.5B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-0.5B</a>","🟢 base","Y",0.6,25.9,24.9,26.4,25.2,26.0,25.6,22.6,26.5]],"metadata":null},"datatype":["number","markdown","str","str","number","number","number","number","number","number","number","number","number"],"search_columns":{"primary_column":"Model","secondary_columns":[],"label":null,"placeholder":null},"filter_columns":[{"column":"type","type":"checkboxgroup","default":[["🔶 chat","🔶 chat"],["🟢 base","🟢 base"]],"choices":[["🔶 chat","🔶 chat"],["🟢 base","🟢 base"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"open?","type":"checkboxgroup","default":[["N","N"],["Y","Y"]],"choices":[["N","N"],["Y","Y"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"#P(B)","type":"slider","default":[7,9],"choices":null,"label":null,"info":null,"show_label":true,"min":0.5,"max":684.5}],"hide_columns":[],"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"show_label":true,"height":500,"min_width":160,"visible":true,"elem_classes":[],"wrap":false,"line_breaks":true,"column_widths":[],"name":"leaderboard","_selectable":false},"skip_api":false,"component_class_id":"dc021d732c999fcb2181404b7ae6ca17","api_info":{"properties":{"headers":{"items":{"type":"string"},"title":"Headers","type":"array"},"data":{"items":{"items":{},"type":"array"},"title":"Data","type":"array"},"metadata":{"anyOf":[{"additionalProperties":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"type":"object"},{"type":"null"}],"default":null,"title":"Metadata"}},"required":["headers","data"],"title":"DataframeData","type":"object"},"example_inputs":{"headers":["a","b"],"data":[["foo","bar"]]}},{"id":8,"type":"tabitem","props":{"label":"SeaBench","visible":true,"interactive":true,"name":"tab"},"skip_api":true,"component_class_id":"18f46968e2b273dc5dbda0fe78433d7e"},{"id":9,"type":"leaderboard","props":{"row_count":[41,"fixed"],"col_count":[13,"fixed"],"headers":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"select_columns_config":{"default_selection":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"cant_deselect":["R","Model"],"allow":true,"label":"Select Columns to Display:","show_label":true,"info":null},"value":{"headers":["R","Model","type","open?","#P(B)","avg-pub","avg-prv ⬇️","id-pub","th-pub","vi-pub","id-prv","th-prv","vi-prv"],"data":[[1,"gpt-4.1","🔶 chat","N",null,8.92,8.9,8.84,8.93,8.99,8.9,8.89,8.93],[2,"claude-3.5-sonnet-20241022","🔶 chat","N",null,8.9,8.78,9.14,8.82,8.74,9.03,8.52,8.78],[2,"<a target=\"_blank\" href=\"https://huggingface.co/deepseek-ai/DeepSeek-V3-0324\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DeepSeek-V3-0324</a>","🔶 chat","Y",684.5,8.75,8.78,8.81,8.7,8.75,8.8,8.69,8.86],[4,"claude-3.7-sonnet-20250219","🔶 chat","N",null,8.84,8.77,9.05,8.78,8.7,8.87,8.67,8.78],[5,"gpt-4o-2024-08-06","🔶 chat","N",null,8.8,8.64,8.95,8.68,8.78,8.72,8.53,8.69],[6,"gemini-pro-1.5","🔶 chat","N",null,8.84,8.62,8.99,8.68,8.85,8.78,8.52,8.57],[7,"gemini-2.0-flash","🔶 chat","N",null,8.66,8.6,8.85,8.53,8.59,8.51,8.55,8.75],[7,"gpt-4.1-mini","🔶 chat","N",null,8.57,8.6,8.65,8.38,8.7,8.77,8.48,8.57],[9,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-27b-it</a>","🔶 chat","Y",27.4,8.43,8.43,8.55,8.2,8.55,8.49,8.36,8.45],[10,"gemini-2.0-flash-lite","🔶 chat","N",null,8.48,8.26,8.64,8.41,8.39,8.34,8.15,8.3],[11,"gpt-4o-mini-2024-07-18","🔶 chat","N",null,8.31,8.25,8.29,8.22,8.42,8.28,8.02,8.46],[12,"gemini-flash-1.5","🔶 chat","N",null,8.39,8.18,8.45,8.25,8.46,8.59,7.87,8.09],[13,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-12b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-12b-it</a>","🔶 chat","Y",12.2,8.18,8.1,8.36,7.86,8.32,8.15,7.96,8.19],[14,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-72B-Instruct</a>","🔶 chat","Y",72.7,8.22,7.96,8.59,7.78,8.3,8.11,7.65,8.14],[15,"claude-3-5-haiku-20241022","🔶 chat","N",null,8.29,7.87,8.77,8.01,8.1,8.37,7.36,7.89],[16,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-27b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-27b-it</a>","🔶 chat","Y",27.2,8.1,7.85,8.35,7.74,8.22,8.21,7.54,7.8],[17,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Maverick-17B-128E-Instruct</a>","🔶 chat","Y",401.6,7.91,7.82,8.27,7.65,7.83,8.21,7.51,7.76],[18,"gpt-4.1-nano","🔶 chat","N",null,7.89,7.81,7.97,7.68,8.03,8.12,7.57,7.75],[19,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-20B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-20B-Chat</a>","🔶 chat","Y",19.2,7.99,7.78,7.9,7.84,8.22,7.94,7.72,7.67],[20,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/gemma2-9b-cpt-sea-lionv3-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma2-9b-cpt-sea-lionv3-instruct</a>","🔶 chat","Y",9.2,7.93,7.72,8.08,7.55,8.17,8.03,7.52,7.61],[21,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b-it</a>","🔶 chat","Y",9.2,7.82,7.52,8.3,7.37,7.78,7.98,7.25,7.34],[22,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-14B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-14B-Instruct</a>","🔶 chat","Y",14.8,7.56,7.43,7.93,7.15,7.61,7.72,7.08,7.49],[23,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B-Instruct</a>","🔶 chat","Y",72.7,7.46,7.35,7.88,6.9,7.61,7.96,6.74,7.36],[24,"claude-3-haiku","🔶 chat","N",null,7.52,7.34,8.11,7.03,7.43,7.81,6.87,7.35],[25,"<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-3-4b-it\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-3-4b-it</a>","🔶 chat","Y",4.3,7.18,7.17,7.27,6.73,7.55,7.42,7.06,7.02],[26,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3-70B-Instruct</a>","🔶 chat","Y",70.6,7.32,7.11,7.76,6.88,7.32,7.65,6.62,7.07],[27,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor2-8B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor2-8B-Chat</a>","🔶 chat","Y",8.5,7.0,7.02,6.86,6.88,7.28,7.25,6.95,6.87],[28,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-7B-Instruct</a>","🔶 chat","Y",7.6,6.65,6.81,6.79,6.32,6.84,6.9,6.47,7.07],[29,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-70B-Instruct</a>","🔶 chat","Y",70.6,7.12,6.77,7.78,6.31,7.26,7.39,6.01,6.9],[30,"<a target=\"_blank\" href=\"https://huggingface.co/aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3-8b-cpt-sea-lionv2.1-instruct</a>","🔶 chat","Y",8.0,6.72,6.56,7.03,6.31,6.83,7.06,5.98,6.64],[31,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Llama-4-Scout-17B-16E-Instruct</a>","🔶 chat","Y",108.6,7.02,6.45,7.58,6.4,7.1,7.09,5.67,6.6],[32,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLMs-v3-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLMs-v3-7B-Chat</a>","🔶 chat","Y",7.6,6.57,6.33,6.77,6.62,6.32,6.54,6.28,6.19],[33,"<a target=\"_blank\" href=\"https://huggingface.co/THUDM/glm-4-9b-chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">glm-4-9b-chat</a>","🔶 chat","Y",9.4,6.09,5.95,6.33,5.06,6.88,6.01,5.31,6.52],[34,"<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Meta-Llama-3.1-8B-Instruct</a>","🔶 chat","Y",8.0,5.81,5.87,6.76,5.05,5.62,6.38,5.44,5.81],[35,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-7B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-7B-Instruct</a>","🔶 chat","Y",7.6,6.09,5.85,6.42,5.68,6.19,6.08,5.56,5.91],[36,"<a target=\"_blank\" href=\"https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">SeaLLM-7B-v2.5</a>","🔶 chat","Y",8.5,5.58,5.45,5.77,5.51,5.47,5.69,5.39,5.26],[37,"<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2.5-3B-Instruct\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2.5-3B-Instruct</a>","🔶 chat","Y",3.1,5.33,5.27,5.47,4.89,5.62,5.55,4.87,5.4],[38,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-14B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-14B-Chat</a>","🔶 chat","Y",14.2,5.32,5.24,5.78,4.82,5.38,5.73,4.83,5.16],[39,"<a target=\"_blank\" href=\"https://huggingface.co/CohereForAI/aya-23-8B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">aya-23-8B</a>","🔶 chat","Y",8.0,4.29,4.43,5.37,2.25,5.26,5.76,2.39,5.15],[40,"<a target=\"_blank\" href=\"https://huggingface.co/sail/Sailor-7B-Chat\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Sailor-7B-Chat</a>","🔶 chat","Y",7.7,4.37,4.32,4.7,3.98,4.45,4.5,3.88,4.58],[41,"<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-7B-Instruct-v0.3</a>","🔶 chat","Y",7.2,3.85,3.64,4.61,2.73,4.23,4.53,2.72,3.67]],"metadata":null},"datatype":["number","markdown","str","str","number","number","number","number","number","number","number","number","number"],"search_columns":{"primary_column":"Model","secondary_columns":[],"label":null,"placeholder":null},"filter_columns":[{"column":"type","type":"checkboxgroup","default":[["🔶 chat","🔶 chat"]],"choices":[["🔶 chat","🔶 chat"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"open?","type":"checkboxgroup","default":[["N","N"],["Y","Y"]],"choices":[["N","N"],["Y","Y"]],"label":null,"info":null,"show_label":true,"min":null,"max":null},{"column":"#P(B)","type":"slider","default":[7,9],"choices":null,"label":null,"info":null,"show_label":true,"min":3.1,"max":684.5}],"hide_columns":[],"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"show_label":true,"height":500,"min_width":160,"visible":true,"elem_classes":[],"wrap":false,"line_breaks":true,"column_widths":[],"name":"leaderboard","_selectable":false},"skip_api":false,"component_class_id":"dc021d732c999fcb2181404b7ae6ca17","api_info":{"properties":{"headers":{"items":{"type":"string"},"title":"Headers","type":"array"},"data":{"items":{"items":{},"type":"array"},"title":"Data","type":"array"},"metadata":{"anyOf":[{"additionalProperties":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"type":"object"},{"type":"null"}],"default":null,"title":"Metadata"}},"required":["headers","data"],"title":"DataframeData","type":"object"},"example_inputs":{"headers":["a","b"],"data":[["foo","bar"]]}},{"id":10,"type":"tabitem","props":{"label":"📝 About","visible":true,"interactive":true,"id":3,"elem_id":"llm-benchmark-tab-table","name":"tab"},"skip_api":true,"component_class_id":"18f46968e2b273dc5dbda0fe78433d7e"},{"id":11,"type":"markdown","props":{"value":"# About\nEven though large language models (LLMs) have shown impressive performance on various benchmarks for English, their performance on Southeast Asian (SEA) languages is still underexplored. This leaderboard includes two benchmarks, SeaExam and SeaBench, with public (denoted as \"pub\") and private dataset (denoted as \"prv\"), respectively. SeaExam aims to evaluate LLMs on exam-type benchmarks for SEA languages, focusing on world knowledge and reasoning abilities. SeaBench aims to evaluate LLMs on instruction-following and multi-turn conversation skills. The  three languages for evaluation are Indonesian (id), Thai (th), and Vietnamese (vi).\n\n\n## Datasets\nThe benchmark data can be found in the [SeaExam dataset](https://huggingface.co/datasets/SeaLLMs/SeaExam) and [SeaBench dataset](https://huggingface.co/datasets/SeaLLMs/SeaBench).\n- **SeaExam**: a benchmark sourced from real and official human exam questions in multiple-choice format.\n- **SeaBench**: a manually created benchmark for evaluating the model's ability to follow instructions and engage in multi-turn conversations. The questions are in open-ended format.\n\n## Evalation Criteria\n- **SeaExam**:\n    We evaluate the models with accuracy score. We have the following settings for evaluation:\n    - **few-shot**: the default setting is few-shot (3-shot). All open-source models are evaluated with 3-shot.\n    - **zero-shot**: the zero-shot setting is also available. As closed-source models has format issues with few-shot, they are evaluated with zero-shot.\n    \n_ **SeaBench**:\n    We evaluate the responses of the models with GPT-4o-2024-08-06. Each response is scored on a scale of 1-10.\n\n## Results\nHow to interpret the leaderboard?\n* Each numerical value represet the accuracy (%) for SeaExam and score for SeaBench.\n* The \"🏅 Overall\" shows the average results across the three langauges for SeaExam public dataset (SeaExam-pub), SeaExam private dataset (SeaExam-prv), SeaBench public dataset (SeaBench-pub), (SeaBench-prv). This leaderboard is ranked by SeaExam-prv. \n* SeaExam and SeaBench have the results for each langauge in both public and private dataset. The leaderboard is ranked by avg_prv, the average score across SEA languages (id, th, and vi) in private set.\n* The rank is in \"R\" column.\n* The \"#P(B)\" column shows the number of parameters of the model in billions.\n* \"open?\" column indicates whether the model is open-source or proprietary.\n\n## Reproducibility\nTo reproduce our results, use the script in [SeaExam](https://github.com/DAMO-NLP-SG/SeaExam/tree/main) and [SeaBench](https://github.com/DAMO-NLP-SG/SeaBench). The script will download the model and tokenizer, and evaluate the model on the benchmark data.","show_label":true,"rtl":false,"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"visible":true,"elem_classes":["markdown-text"],"sanitize_html":true,"line_breaks":false,"header_links":false,"name":"markdown","_selectable":false},"skip_api":false,"component_class_id":"0f35515b9a3362b42bb6721c836157a0","api_info":{"type":"string"},"example_inputs":"# Hello!"},{"id":12,"type":"row","props":{"variant":"default","visible":true,"equal_height":true,"name":"row"},"skip_api":true,"component_class_id":"b5cf3d15e2293fb5ccc071d7b8d69f64"},{"id":13,"type":"accordion","props":{"label":"📙 Citation","open":false,"visible":true,"name":"accordion"},"skip_api":true,"component_class_id":"b49c1903c3591b866d0adab5a2f6e48b"},{"id":14,"type":"textbox","props":{"value":"\n@inproceedings{liu-etal-2025-seaexam,\n    title = \"{S}ea{E}xam and {S}ea{B}ench: Benchmarking {LLM}s with Local Multilingual Questions in {S}outheast {A}sia\",\n    author = \"Liu, Chaoqun  and Zhang, Wenxuan  and Ying, Jiahao  and Aljunied, Mahani  and Luu, Anh Tuan  and  Bing, Lidong\",\n    booktitle = \"Findings of the Association for Computational Linguistics: NAACL 2025\",\n    month = apr,\n    year = \"2025\",\n    address = \"Albuquerque, New Mexico\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2025.findings-naacl.341/\",\n    pages = \"6119--6136\",\n    ISBN = \"979-8-89176-195-7\"\n}\n","lines":20,"max_lines":20,"label":"Copy the following snippet to cite these results","show_label":true,"container":true,"min_width":160,"visible":true,"elem_id":"citation-button","autofocus":false,"autoscroll":true,"elem_classes":[],"type":"text","rtl":false,"show_copy_button":true,"name":"textbox","_selectable":false},"skip_api":false,"component_class_id":"d0491bb0fbd427bdb2a4549c4d1ac869","api_info":{"type":"string"},"example_inputs":"Hello!!"},{"id":15,"type":"form","props":{"scale":0,"min_width":0,"name":"form"},"skip_api":true,"component_class_id":"43cda85c6ff7a5dd8f83d29496b014ee"},{"id":16,"type":"markdown","props":{"value":"## Contact\n\nIf you have any questions or want to include your models in the leaderboard, please contact [Chaoqun Liu](https://liuchaoqun.github.io/) and [Wenxuan Zhang](https://isakzhang.github.io/).","show_label":true,"rtl":false,"latex_delimiters":[{"left":"$$","right":"$$","display":true}],"visible":true,"elem_classes":["markdown-text"],"sanitize_html":true,"line_breaks":false,"header_links":false,"name":"markdown","_selectable":false},"skip_api":false,"component_class_id":"0f35515b9a3362b42bb6721c836157a0","api_info":{"type":"string"},"example_inputs":"# Hello!"}],"css":"\n\n.markdown-text {\n    font-size: 16px !important;\n}\n\n#models-to-add-text {\n    font-size: 18px !important;\n}\n\n#citation-button span {\n    font-size: 16px !important;\n}\n\n#citation-button textarea {\n    font-size: 16px !important;\n}\n\n#citation-button > label > button {\n    margin: 6px;\n    transform: scale(1.3);\n}\n\n#leaderboard-table {\n    margin-top: 15px\n}\n\n#leaderboard-table-lite {\n    margin-top: 15px\n}\n\n#search-bar-table-box > div:first-child {\n    background: none;\n    border: none;\n}\n \n#search-bar {\n    padding: 0px;\n}\n\n/* Hides the final AutoEvalColumn */\n#llm-benchmark-tab-table table td:last-child,\n#llm-benchmark-tab-table table th:last-child {\n    display: none;\n}\n\n/* Limit the width of the first AutoEvalColumn so that names don't expand too much */\ntable td:first-child,\ntable th:first-child {\n    max-width: 400px;\n    overflow: auto;\n    white-space: nowrap;\n}\n\n.tab-buttons button {\n    font-size: 20px;\n}\n\n#scale-logo {\n    border-style: none !important;\n    box-shadow: none;\n    display: block;\n    margin-left: auto;\n    margin-right: auto;\n    max-width: 600px;\n}\n\n#scale-logo .download {\n    display: none;\n}\n#filter_type{\n    border: 0;\n    padding-left: 0;\n    padding-top: 0;\n}\n#filter_type label {\n    display: flex;\n}\n#filter_type label > span{\n    margin-top: var(--spacing-lg);\n    margin-right: 0.5em;\n}\n#filter_type label > .wrap{\n    width: 103px;\n}\n#filter_type label > .wrap .wrap-inner{  \n    padding: 2px;\n}\n#filter_type label > .wrap .wrap-inner input{\n    width: 1px\n}\n#filter-columns-type{\n    border:0;\n    padding:0.5;\n}\n#filter-columns-size{\n    border:0;\n    padding:0.5;\n}\n#box-filter > .form{\n    border: 0\n}\n","js":null,"head":null,"title":"Gradio","space_id":"SeaLLMs/LLM_Leaderboard_for_SEA","enable_queue":true,"show_error":false,"show_api":true,"is_colab":false,"stylesheets":["https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600&display=swap","https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600&display=swap"],"theme":"default","protocol":"sse_v3","body_css":{"body_background_fill":"white","body_text_color":"#1f2937","body_background_fill_dark":"#0b0f19","body_text_color_dark":"#f3f4f6"},"fill_height":false,"layout":{"id":0,"children":[{"id":1},{"id":2},{"id":3,"children":[{"id":4,"children":[{"id":5}]},{"id":6,"children":[{"id":7}]},{"id":8,"children":[{"id":9}]},{"id":10,"children":[{"id":11}]}]},{"id":12,"children":[{"id":13,"children":[{"id":15,"children":[{"id":14}]}]}]},{"id":16}]},"dependencies":[],"root":"https://seallms-llm-leaderboard-for-sea.hf.space"}