{ "@context": "https://schema.org", "@type": "ItemList", "@id": "https://hiteshlpatel.github.io/publications/#publication-list", "name": "Hitesh Laxmichand Patel — Publications", "url": "https://hiteshlpatel.github.io/publications/", "numberOfItems": 39, "itemListElement": [ { "@type": "ListItem", "position": 1, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.emnlp-main.1748/#paper", "url": "https://aclanthology.org/2026.emnlp-main.1748/", "name": "No One Model Catches Every Harm: Benchmarking Content Moderation Across Safety Scenarios", "description": "EMNLP'26; Award Nomination.", "author": [ { "@type": "Person", "name": "Afshin Oroojlooy" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["AI safety","Content moderation","Safety evaluation","Adversarial jailbreaks"] , "abstract": "Large Language Models (LLMs) are increasingly deployed in real-world applications, yet remain vulnerable to harms ranging from adversarial jailbreaks to implicit hate. Specialized moderators and general-purpose LLMs are both used as safety layers, but there is little systematic guidance on which model to choose for which harm. We present the most comprehensive evaluation of LLM safety to date, testing 53 models on 11 datasets organized into four challenge categories, under both prompt-only (Q) and prompt-with-response (QA) settings. Large frontier models that lead on one category fall behind smaller specialized models on others, and real-world conversational safety remains unsolved across model families. These findings challenge the assumption that scale alone ensures safety and offer practitioners a structured basis for model selection." } }, { "@type": "ListItem", "position": 2, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.findings-emnlp.1402/#paper", "url": "https://aclanthology.org/2026.findings-emnlp.1402/", "name": "MLOps-Bench: Benchmarking AI Agents on Production ML Engineering in Real Repositories", "description": "EMNLP'26.", "author": [ { "@type": "Person", "name": "Quoc Co Tran" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Diwakar Mahajan" }, { "@type": "Person", "name": "Kshitij Bakliwal" }, { "@type": "Person", "name": "Avi Sil" }, { "@type": "Person", "name": "Katrin Kirchhoff" } ] , "keywords": ["AI agents","Machine learning engineering","Agent evaluation","MLOps"] , "abstract": "AI agents are increasingly expected to modify and maintain machine-learning systems in production, yet existing benchmarks primarily evaluate either competition-style ML experimentation or generic repository issue resolution. We introduce MLOps-Bench, a repository-grounded benchmark for production ML engineering. Unlike prior ML-agent benchmarks, MLOps-Bench evaluates executable lifecycle changes inside real ML repositories, spanning data pipelines, feature engineering, model training, model serving, monitoring, CI/CD and governance, and end-to-end integration. Each task is grounded in repository evidence and includes a natural-language requirement, oracle patch, plausible buggy baseline, fail-to-pass tests, pass-to-pass tests, and provenance metadata. The benchmark contains 212 validated tasks from 31 public ML repositories, with 636 fail-to-pass tests and 856 pass-to-pass tests. We evaluate five agents and find that the best system solves 27.3% of tasks while passing 67.1% of tests, showing that current agents often make partial progress but still fail production-relevant behavioral requirements." } }, { "@type": "ListItem", "position": 3, "item": { "@type": "ScholarlyArticle", "@id": "https://proceedings.mlr.press/v306/son26a.html#paper", "url": "https://proceedings.mlr.press/v306/son26a.html", "name": "Judging What We Cannot Solve: A Consequence-Based Approach for Oracle-Free Evaluation of Research-Level Math", "description": "ICML'26; Spotlight.", "author": [ { "@type": "Person", "name": "Guijin Son" }, { "@type": "Person", "name": "Donghun Yang" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Hyunwoo Ko" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Sunghee Ahn" }, { "@type": "Person", "name": "Kyong-Ha Lee" }, { "@type": "Person", "name": "Youngjae Yu" } ] , "keywords": ["Reasoning evaluation","Research-level mathematics","Oracle-free evaluation"] , "abstract": "Recent progress in reasoning models suggests that generating plausible attempts for research-level mathematics may be within reach, but verification remains a bottleneck, consuming scarce expert time. We hypothesize that a meaningful solution should contain enough method-level information that, when applied to a neighborhood of related questions, it should yield better downstream performance than incorrect solutions. Building on this idea, we propose Consequence-Based Utility , an oracle-free evaluator that scores each candidate by testing its value as an in-context exemplar in solving related yet verifiable questions. Our approach is evaluated on an original set of research-level math problems each paired with one expert-written solution and nine LLM-generated solutions. Notably, Consequence-Based Utility consistently outperforms reward models, generative reward models, and LLM judges on ranking quality. Specifically, for GPT-OSS-120B it improves Acc@1 from 67.2 to 76.3 and AUC from 71.4 to 79.6, with similarly large AUC gains on GPT-OSS-20B (69.0 to 79.2). Furthermore, compared to LLM-Judges, it also exhibits a larger solver–evaluator gap, maintaining stronger correct–wrong separation even on instances the underlying solver often fails to solve." } }, { "@type": "ListItem", "position": 4, "item": { "@type": "ScholarlyArticle", "@id": "https://openaccess.thecvf.com/content/CVPR2026/papers/Kim_World_in_a_Frame_Understanding_Culture_Mixing_as_a_New_CVPR_2026_paper.pdf#paper", "url": "https://openaccess.thecvf.com/content/CVPR2026/papers/Kim_World_in_a_Frame_Understanding_Culture_Mixing_as_a_New_CVPR_2026_paper.pdf", "name": "World in a Frame: Understanding Culture Mixing as a New Challenge for Vision-Language Models", "description": "CVPR'26.", "author": [ { "@type": "Person", "name": "Eunsu Kim" }, { "@type": "Person", "name": "Junyeong Park" }, { "@type": "Person", "name": "Na Min An" }, { "@type": "Person", "name": "Junseong Kim" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Jiho Jin" }, { "@type": "Person", "name": "Julia Kruk" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Fenal Ashokbhai Ilasariya" }, { "@type": "Person", "name": "Hyunjung Shim" }, { "@type": "Person", "name": "Alice Oh" } ] , "keywords": ["Vision-language models","Cultural evaluation","Visual question answering"] , "abstract": "In a globalized world, cultural elements from diverse origins frequently appear together within a single visual scene. We refer to these as culture mixing scenarios, yet how Large Vision-Language Models (LVLMs) perceive them remains underexplored. We investigate culture mixing as a critical challenge for LVLMs and examine how current models behave when cultural items from multiple regions appear together. To systematically analyze these behaviors, we construct CultureMix, a food Visual Question Answering (VQA) benchmark with 23k diffusion-generated, human-verified culture mixing images across four subtasks: (1) food-only, (2) food+food, (3) food+background, and (4) food+food+background. Evaluating 10 LVLMs, we find consistent failures to preserve individual cultural identities in mixed settings. Models show strong background reliance, with accuracy dropping 14% when cultural backgrounds are added to food-only baselines, and they produce inconsistent predictions for identical foods across different contexts. To address these limitations, we explore three robustness strategies. We find supervised fine-tuning using a diverse culture mixing dataset substantially improve model consistency and reduce background sensitivity. We call for increased attention to culture mixing scenarios as a critical step toward developing LVLMs capable of operating reliably in culturally diverse real-world environments." , "sameAs": "https://openaccess.thecvf.com/content/CVPR2026/html/Kim_World_in_a_Frame_Understanding_Culture_Mixing_as_a_New_CVPR_2026_paper.html" } }, { "@type": "ListItem", "position": 5, "item": { "@type": "ScholarlyArticle", "@id": "https://openreview.net/pdf?id=ABc5y3741T#paper", "url": "https://openreview.net/pdf?id=ABc5y3741T", "name": "Pushing on Multilingual Reasoning Models with Language-Mixed Chain-of-Thought", "description": "ICLR'26.", "author": [ { "@type": "Person", "name": "Guijin Son" }, { "@type": "Person", "name": "Donghun Yang" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hyunwoo Ko" }, { "@type": "Person", "name": "Chanuk Lim" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Minhyuk Kim" }, { "@type": "Person", "name": "Nikunj Drolia" }, { "@type": "Person", "name": "Dasol Choi" }, { "@type": "Person", "name": "Kyong-Ha Lee" }, { "@type": "Person", "name": "Youngjae Yu" } ] , "keywords": ["Multilingual reasoning","Chain-of-thought","Post-training","Knowledge distillation"] , "abstract": "Recent frontier models employ long-chain-of-thought reasoning to explore solution spaces in context and achieve stronger performance. While many works study distillation to build smaller yet capable models, most focus on English and little is known about language-specific reasoning. To bridge this gap, we first introduce **Language-Mixed CoT**, a reasoning schema that switches between English and a target language, using English as an anchor to excel in reasoning while minimizing translation artifacts. As a Korean case study, we curate **Yi-Sang-HQ**: 5.79M native-Korean prompts from web Q&A, exams, STEM, and code; 3.7M long reasoning traces generated from Qwen3-32B; and a targeted 260k high-yield subset. We train nine models (4B–35B) across six families (Qwen2.5, Llama-3.1, Gemma-3, etc). Our best model, **KO-REAson-35B**, achieves state-of-the-art performance, with the highest overall average score ($64.0_{\\pm2.5}$), ranking first on 5/9 benchmarks and second on the remainder. Smaller and mid-sized models also benefit substantially, with an average improvement of $+18.6$ points across the evaluated nine benchmarks. Ablations show **Language-Mixed CoT** is more effective than monolingual CoT, indicating that reasoning patterns can be engineered to improve non-English performance. We release our data-curation pipeline, evaluation system, datasets, and models to advance research on language-specific reasoning." , "sameAs": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/804b5e300c9ed4e3ea3b073f186f4adc-Abstract-Conference.html" } }, { "@type": "ListItem", "position": 6, "item": { "@type": "ScholarlyArticle", "@id": "https://openaccess.thecvf.com/content/WACV2026/papers/Rafi_Learning_from_Unknown_for_Open-Set_Test-Time_Adaptation_WACV_2026_paper.pdf#paper", "url": "https://openaccess.thecvf.com/content/WACV2026/papers/Rafi_Learning_from_Unknown_for_Open-Set_Test-Time_Adaptation_WACV_2026_paper.pdf", "name": "Learning from Unknown for Open-Set Test-Time Adaptation", "description": "WACV'26.", "author": [ { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Test-time adaptation","Open-set recognition","Distribution shift"] , "abstract": "Deep learning models often struggle to maintain performance when the training and testing data come from different distributions. Test-time adaptation (TTA) addresses this by adapting a pre-trained model to an unlabeled target domain under distribution shifts. A more challenging setting is open-set TTA (OSTTA), where the target domain may contain unknown samples outside the source classes. Existing OSTTA methods primarily detect and discard such unknowns, relying only on known samples for adaptation. In this work, we argue that unknown samples can also provide valuable cues for improving adaptation. We propose LU-OSTTA (learning from unknown for OSTTA), a simple yet effective framework that leverages both in-distribution and semantically useful out-of-distribution samples. Our approach introduces: (i) a class-conditioned dynamic energy threshold to separate OOD samples more reliably, (ii) an optimal transport-based pseudo-label refinement to mitigate noise under distribution shifts, and (iii) an adaptive prototype weighting strategy that emphasizes semantically aligned target samples while down-weighting harmful ones. Experimental results demonstrate that our LU-OSTTA consistently outperforms state-of-the-art TTA and OSTTA methods, highlighting the benefits of utilizing rather than discarding unknown samples. Our code is available at: https://github.com/takihasan/LU-OSTTA." , "sameAs": "https://openaccess.thecvf.com/content/WACV2026/html/Rafi_Learning_from_Unknown_for_Open-Set_Test-Time_Adaptation_WACV_2026_paper.html" } }, { "@type": "ListItem", "position": 7, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.emnlp-main.1758/#paper", "url": "https://aclanthology.org/2026.emnlp-main.1758/", "name": "Bias Before Disclosure: A Behavioral Audit of LLM Assistants Under User-Identity Uncertainty", "description": "EMNLP'26.", "author": [ { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Tejaswini Kumar" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Jyotika Singh" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Gouthami Vadithya" } ] , "keywords": ["LLM fairness","Demographic bias","Behavioral evaluation"] , "abstract": "LLM assistants frequently answer everyday queries that reveal nothing about the user’s demographic identity. We audit whether models nevertheless attribute demographic characteristics to the user, a behavior we call unsupported demographic commitment. Across 18 proprietary and open-weight LLMs evaluated on 212 filtered information-seeking queries, 11 of 18 models commit to a demographic label on more than 80% of queries under a forced-choice diagnostic. In the matched abstention-first setting, the eight tested models average 44% commitment, compared with 5% for human annotators. Conditional on commitment, race, SES, location, and education show the same directional skew across all six domains, while gender varies by domain. Demographic-coded patterns also appear in free-form answers when no identity inference is requested. We evaluate identity-abstention, a prompt-level intervention that reduces classifier-detected presupposition from 41% to 16%, with a measurable clarification cost on queries that can be answered without additional context." } }, { "@type": "ListItem", "position": 8, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.acl-long.1527/#paper", "url": "https://aclanthology.org/2026.acl-long.1527/", "name": "CommonLID: Re-evaluating State-of-the-Art Language Identification Performance on Web Data", "description": "ACL'26.", "author": [ { "@type": "Person", "name": "Pedro Ortiz Suarez" }, { "@type": "Person", "name": "Laurie Burchell" }, { "@type": "Person", "name": "Catherine Arnett" }, { "@type": "Person", "name": "Rafael Mosquera-Gómez" }, { "@type": "Person", "name": "Sara Hincapie-Monsalve" }, { "@type": "Person", "name": "Thom Vaughan" }, { "@type": "Person", "name": "Damian Stewart" }, { "@type": "Person", "name": "Malte Ostendorff" }, { "@type": "Person", "name": "Idris Abdulmumin" }, { "@type": "Person", "name": "Vukosi Marivate" }, { "@type": "Person", "name": "Shamsuddeen Hassan Muhammad" }, { "@type": "Person", "name": "Atnafu Lambebo Tonja" }, { "@type": "Person", "name": "Hend Al-Khalifa" }, { "@type": "Person", "name": "Nadia Ghezaiel Hammouda" }, { "@type": "Person", "name": "Verrah Otiende" }, { "@type": "Person", "name": "Tack Hwa Wong" }, { "@type": "Person", "name": "Jakhongir Saydaliev" }, { "@type": "Person", "name": "Melika Nobakhtian" }, { "@type": "Person", "name": "Muhammad Ravi Shulthan Habibi" }, { "@type": "Person", "name": "Chalamalasetti Kranti" }, { "@type": "Person", "name": "Carol Muchemi" }, { "@type": "Person", "name": "Khang Nguyen" }, { "@type": "Person", "name": "Faisal Muhammad Adam" }, { "@type": "Person", "name": "Luis Frentzen Salim" }, { "@type": "Person", "name": "Reem Alqifari" }, { "@type": "Person", "name": "Cynthia Amol" }, { "@type": "Person", "name": "Joseph Marvin Imperial" }, { "@type": "Person", "name": "Ilker Kesen" }, { "@type": "Person", "name": "Ahmad Mustafid" }, { "@type": "Person", "name": "Pavel Stepachev" }, { "@type": "Person", "name": "Leshem Choshen" }, { "@type": "Person", "name": "David Anugraha" }, { "@type": "Person", "name": "Hamada Nayel" }, { "@type": "Person", "name": "Seid Muhie Yimam" }, { "@type": "Person", "name": "Vallerie Alexandra Putra" }, { "@type": "Person", "name": "My Chiffon Nguyen" }, { "@type": "Person", "name": "Azmine Toushik Wasi" }, { "@type": "Person", "name": "Gouthami Vadithya" }, { "@type": "Person", "name": "Rob van der Goot" }, { "@type": "Person", "name": "Lanwenn ar C'horr" }, { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Andrew Yates" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Shu Okabe" }, { "@type": "Person", "name": "Fenal Ashokbhai Ilasariya" }, { "@type": "Person", "name": "Dmitry Gaynullin" }, { "@type": "Person", "name": "Genta Indra Winata" }, { "@type": "Person", "name": "Yiyuan Li" }, { "@type": "Person", "name": "Juan Pablo Martínez" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Ikhlasul Akmal Hanif" }, { "@type": "Person", "name": "Raia Abu Ahmad" }, { "@type": "Person", "name": "Esther Adenuga" }, { "@type": "Person", "name": "Filbert Aurelian Tjiaranata" }, { "@type": "Person", "name": "Weerayut Buaphet" }, { "@type": "Person", "name": "Michael Anugraha" }, { "@type": "Person", "name": "Sowmya Vajjala" }, { "@type": "Person", "name": "Benjamin Rice" }, { "@type": "Person", "name": "Azril Hafizi Amirudin" }, { "@type": "Person", "name": "Jesujoba O. Alabi" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Yassine Toughrai" }, { "@type": "Person", "name": "Bruhan Kyomuhendo" }, { "@type": "Person", "name": "Daniel Ruffinelli" }, { "@type": "Person", "name": "Akshata A" }, { "@type": "Person", "name": "Manuel Goulão" }, { "@type": "Person", "name": "Ej Zhou" }, { "@type": "Person", "name": "Ingrid Gabriela Franco Ramirez" }, { "@type": "Person", "name": "Cristina Aggazzotti" }, { "@type": "Person", "name": "Konstantin Dobler" }, { "@type": "Person", "name": "Jun Kevin" }, { "@type": "Person", "name": "Quentin Pagès" }, { "@type": "Person", "name": "Nicholas Andrews" }, { "@type": "Person", "name": "Nuhu Ibrahim" }, { "@type": "Person", "name": "Mattes Ruckdeschel" }, { "@type": "Person", "name": "Amr Keleg" }, { "@type": "Person", "name": "Mike Zhang" }, { "@type": "Person", "name": "Casper Muziri" }, { "@type": "Person", "name": "Saron Samuel" }, { "@type": "Person", "name": "Sotaro Takeshita" }, { "@type": "Person", "name": "Kun Kerdthaisong" }, { "@type": "Person", "name": "Luca Foppiano" }, { "@type": "Person", "name": "Rasul Dent" }, { "@type": "Person", "name": "Tommaso Green" }, { "@type": "Person", "name": "Ahmad Mustapha Wali" }, { "@type": "Person", "name": "Kamohelo Makaaka" }, { "@type": "Person", "name": "Vicky Feliren" }, { "@type": "Person", "name": "Inshirah Idris" }, { "@type": "Person", "name": "Hande Celikkanat" }, { "@type": "Person", "name": "Abdulhamid Abubakar" }, { "@type": "Person", "name": "Jean Maillard" }, { "@type": "Person", "name": "Benoît Sagot" }, { "@type": "Person", "name": "Thibault Clérice" }, { "@type": "Person", "name": "Kenton Murray" }, { "@type": "Person", "name": "Sarah Luger" } ] , "keywords": ["Language identification","Multilingual data","Web corpus curation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2026.acl-long.1527", "url": "https://doi.org/10.18653/v1/2026.acl-long.1527" } , "abstract": "Language identification (LID) is a fundamental step in curating multilingual corpora. However, LID models still perform poorly for many languages, especially on the noisy and heterogeneous web data often used to train multilingual language models. In this paper, we introduce CommonLID, a community-driven, human-annotated LID benchmark for the web domain, covering 109 languages. Many of the included languages have been previously under-served, making CommonLID a key resource for developing more representative high-quality text corpora. We show CommonLID’s value by using it, alongside five other common evaluation sets, to test eight popular LID models. We analyse our results to situate our contribution and to provide an overview of the state of the art. In particular, we highlight that existing evaluations overestimate LID accuracy for many languages in the web domain. We make CommonLID and the code used to create it available under an open, permissive license." } }, { "@type": "ListItem", "position": 9, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.findings-acl.129/#paper", "url": "https://aclanthology.org/2026.findings-acl.129/", "name": "RECOR: Reasoning-focused Multi-turn Conversational Retrieval Benchmark", "description": "ACL'26.", "author": [ { "@type": "Person", "name": "Mohammed Ali" }, { "@type": "Person", "name": "Abdelrahman Abdallah" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Adam Jatowt" } ] , "keywords": ["Conversational retrieval","Reasoning evaluation","Information retrieval"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2026.findings-acl.129", "url": "https://doi.org/10.18653/v1/2026.findings-acl.129" } , "abstract": "Existing benchmarks treat multi-turn conversation and reasoning-intensive retrieval separately, yet real-world information seeking requires both. To bridge this gap, we present a benchmark for reasoning-based conversational information retrieval comprising 707 conversations (2,971 turns) across eleven domains. To ensure quality, our Decomposition-and-Verification framework transforms complex queries into fact-grounded multi-turn dialogues through multi-level validation, where atomic facts are verified against sources and explicit retrieval reasoning is generated for each turn. Comprehensive evaluation reveals that combining conversation history with reasoning doubles retrieval performance (Baseline .236 → History+Reasoning .479 nDCG@10), while reasoning-specialized models substantially outperform dense encoders. Despite these gains, further analysis highlights that implicit reasoning remains challenging, particularly when logical connections are not explicitly stated in the text. [ https://github.com/RECOR-Benchmark/RECOR ]" } }, { "@type": "ListItem", "position": 10, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.emnlp-main.1344/#paper", "url": "https://aclanthology.org/2026.emnlp-main.1344/", "name": "INCLUDE-2.0: Disentangling Knowledge Gaps in Multilingual LLM Evaluation", "description": "EMNLP'26.", "author": [ { "@type": "Person", "name": "Angelika Romanou" }, { "@type": "Person", "name": "María Grandury" }, { "@type": "Person", "name": "Clara Meister" }, { "@type": "Person", "name": "Negar Foroutan" }, { "@type": "Person", "name": "Jabez Magomere" }, { "@type": "Person", "name": "Anna Sotnikova" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Akshay Kumar Dubey" }, { "@type": "Person", "name": "Alfonso Amayuelas" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Arion Das" }, { "@type": "Person", "name": "Carlos Arriaga" }, { "@type": "Person", "name": "Cheng Yu" }, { "@type": "Person", "name": "Dalia Ali" }, { "@type": "Person", "name": "David Quispe-Castillo" }, { "@type": "Person", "name": "Fenal Ashokbhai Ilasariya" }, { "@type": "Person", "name": "Francisco-Javier Rodrigo-Ginés" }, { "@type": "Person", "name": "Gonzalo Martínez" }, { "@type": "Person", "name": "Gouthami Vadithya" }, { "@type": "Person", "name": "Guido Ivetta" }, { "@type": "Person", "name": "Javier Conde" }, { "@type": "Person", "name": "Jorge Vallego" }, { "@type": "Person", "name": "Juan J. Calderón G." }, { "@type": "Person", "name": "Karan Gupta" }, { "@type": "Person", "name": "Marcos J Gomez" }, { "@type": "Person", "name": "Marvin M. Agüero-Torales" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Naira Paola Arnez Jordan" }, { "@type": "Person", "name": "Oscar M. Cumbicus-Pineda" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Reewos Erwin Talla Chumpitaz" }, { "@type": "Person", "name": "Sahil Aggarwal" }, { "@type": "Person", "name": "Sahil Raj" }, { "@type": "Person", "name": "Shruti Jalan" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Susana Zhou" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Orestis Papakyriakopoulos" }, { "@type": "Person", "name": "Scott A. Hale" }, { "@type": "Person", "name": "Shamsuddeen Hassan Muhammad" }, { "@type": "Person", "name": "Antoine Bosselut" } ] , "keywords": ["Multilingual evaluation","Factual knowledge","Regional knowledge"] , "abstract": "Current multilingual benchmarks cover only a fraction of the languages represented in modern pretraining corpora and often conflate language proficiency with the ability to answer questions grounded in specific regions and communities. We introduce INCLUDE-2.0, a large-scale benchmark for region-aware multilingual factual evaluation, which covers 91 languages from 86 countries (127 total language–country pairs), comprising over 220K curated samples drawn from native regional sources. Unlike existing benchmarks that often treat language as a proxy for region, IINCLUDE-2.0 distinguishes between questions whose answers are independent of context, vary by country, concern widely documented facts about a place, or depend on knowledge primarily expressed through local linguistic and cultural sources. Each item is annotated with its regional knowledge type and domain, and paired with an English translation, enabling controlled analysis of the relationship between language proficiency and possession of regional knowledge. Evaluating frontier and open-weight models on IINCLUDE-2.0, we find that aggregate multilingual accuracy masks substantial variation across regional knowledge types, languages, and regions. We find that frontier models perform worse on locally embedded knowledge than on widely documented facts about regions, with an average drop of ~14.5 points. We also show that for open-weight models, scale improves accuracy but does not reliably make performance invariant to whether a question is asked in English or its original language. INCLUDE-2.0 is available at https://huggingface.co/datasets/epfl-nlp/include-2.0" } }, { "@type": "ListItem", "position": 11, "item": { "@type": "ScholarlyArticle", "@id": "https://neurips.cc/virtual/2026/poster/139267#paper", "url": "https://neurips.cc/virtual/2026/poster/139267", "name": "Benchmarks Are Not Atomic: Composition-Aware LLM Evaluation using BenchHub", "description": "NeurIPS'26.", "author": [ { "@type": "Person", "name": "Eunsu Kim" }, { "@type": "Person", "name": "Haneul Yoo" }, { "@type": "Person", "name": "Guijin Son" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Alice Oh" } ] , "keywords": ["LLM evaluation","Benchmark composition","Capability measurement"] , "abstract": "LLM benchmarks are often treated as coherent measurement units, yet they are heterogeneous collections of instances spanning diverse domains, skills, formats, and contexts. As a result, aggregate benchmark scores can conflate model capability with benchmark composition, obscuring what existing benchmarks actually cover. We introduce BenchHub, a composition-aware evaluation framework that represents benchmarks as distributions over instance-level attributes. BenchHub integrates 54 benchmarks comprising 839K samples across 10 languages. It enables researchers to inspect benchmark contents, uncover reusable coverage hidden behind benchmark names, transparently compare benchmarks, and construct controllable evaluation sets for application-aligned model selection. Using BenchHub, we show that similarly motivated benchmarks can differ substantially in internal composition, existing benchmarks often contain reusable coverage beyond their stated purposes, models exhibit fine-grained category-level performance variation hidden by aggregate scores, and model rankings can shift under different reweighting and resampling configurations. Our results motivate evaluation practices that make benchmark composition explicit, inspectable, and controllable." } }, { "@type": "ListItem", "position": 12, "item": { "@type": "ScholarlyArticle", "@id": "https://neurips.cc/virtual/2026/poster/139393#paper", "url": "https://neurips.cc/virtual/2026/poster/139393", "name": "Global PIQA: Evaluating Commonsense Reasoning Across 100+ Languages and Cultures", "description": "NeurIPS'26.", "author": [ { "@type": "Person", "name": "Tyler A. Chang" }, { "@type": "Person", "name": "Catherine Arnett" }, { "@type": "Person", "name": "Abdelrahman Sadallah" }, { "@type": "Person", "name": "Abdelrahman Eldesokey" }, { "@type": "Person", "name": "Abeer Kashar" }, { "@type": "Person", "name": "Daud Abolade" }, { "@type": "Person", "name": "Abosede G Olanihun" }, { "@type": "Person", "name": "Adam Labaran M" }, { "@type": "Person", "name": "Adhikarimayum Meerajita Sharma" }, { "@type": "Person", "name": "Aditi Gupta" }, { "@type": "Person", "name": "Adril P Merin" }, { "@type": "Person", "name": "Adwoa A Bremang" }, { "@type": "Person", "name": "Afitab Iyigun" }, { "@type": "Person", "name": "Afonso Simplício" }, { "@type": "Person", "name": "Ahmed Essouaied" }, { "@type": "Person", "name": "Aicha Chorana" }, { "@type": "Person", "name": "Akhil Eppa" }, { "@type": "Person", "name": "Akintunde Oladipo" }, { "@type": "Person", "name": "Akriti Kuri" }, { "@type": "Person", "name": "Akshay Ramesh" }, { "@type": "Person", "name": "Aleksei Dorkin" }, { "@type": "Person", "name": "Alfred M Kondoro" }, { "@type": "Person", "name": "Alham Aji" }, { "@type": "Person", "name": "Ali E Çetintaş" }, { "@type": "Person", "name": "Allan Hanbury" }, { "@type": "Person", "name": "Alp Niksarli" }, { "@type": "Person", "name": "Alvaro Arroyo" }, { "@type": "Person", "name": "Amin Bajand" }, { "@type": "Person", "name": "Amol Khanna" }, { "@type": "Person", "name": "Ana Chkhaidze" }, { "@type": "Person", "name": "Ana C Condez" }, { "@type": "Person", "name": "Anamaria Roberta Hartl" }, { "@type": "Person", "name": "Andiswa Mkhonto" }, { "@type": "Person", "name": "Andrew Hoblitzell" }, { "@type": "Person", "name": "Angelos Poulis" }, { "@type": "Person", "name": "Anirban Majumder" }, { "@type": "Person", "name": "Anjali Chaudhary" }, { "@type": "Person", "name": "Anna Vacalopoulou" }, { "@type": "Person", "name": "Annika Simonsen" }, { "@type": "Person", "name": "Anton Kovalev" }, { "@type": "Person", "name": "Anupam Nayak" }, { "@type": "Person", "name": "Ashvanth.S" }, { "@type": "Person", "name": "Ayodeji Lana" }, { "@type": "Person", "name": "Ayu Purwarianti" }, { "@type": "Person", "name": "Bashar Alhafni" }, { "@type": "Person", "name": "C. Benedict C. Busole" }, { "@type": "Person", "name": "Bernard Ghanem" }, { "@type": "Person", "name": "Bharti Nathani" }, { "@type": "Person", "name": "Biljana Stojanovska Đurić" }, { "@type": "Person", "name": "Ogundipe B Funmilola" }, { "@type": "Person", "name": "Bolaotan Agbonile" }, { "@type": "Person", "name": "Bruce Torres Fischer" }, { "@type": "Person", "name": "Burak Tutar" }, { "@type": "Person", "name": "Burcu Çınar" }, { "@type": "Person", "name": "Cade Kane" }, { "@type": "Person", "name": "Can Udomcharoenchaikit" }, { "@type": "Person", "name": "Chadi Helwe" }, { "@type": "Person", "name": "Cecilia Liu" }, { "@type": "Person", "name": "Chiamaka G Nwokolo" }, { "@type": "Person", "name": "Christopher Homan" }, { "@type": "Person", "name": "Clément Sampebgo" }, { "@type": "Person", "name": "Cristina España-Bonet" }, { "@type": "Person", "name": "Cynthia Amol" }, { "@type": "Person", "name": "Daeyeop Lee" }, { "@type": "Person", "name": "Dan Saattrup Smart" }, { "@type": "Person", "name": "Dana Arad" }, { "@type": "Person", "name": "Daniil Dzenhaliou" }, { "@type": "Person", "name": "Dasol Choi" }, { "@type": "Person", "name": "David Liu" }, { "@type": "Person", "name": "David Semedo" }, { "@type": "Person", "name": "David Anugraha" }, { "@type": "Person", "name": "OLAOYE D Oluwaseun" }, { "@type": "Person", "name": "Deborah Popoola" }, { "@type": "Person", "name": "Deividas Mataciunas" }, { "@type": "Person", "name": "Dennis Asamoah Owusu" }, { "@type": "Person", "name": "Dhyuthy Krishna Kumar" }, { "@type": "Person", "name": "Diogo Tavares" }, { "@type": "Person", "name": "Diogo Glória-Silva" }, { "@type": "Person", "name": "Divyanshu Goyal" }, { "@type": "Person", "name": "DongGeon Lee" }, { "@type": "Person", "name": "Estefany Kelly Buchanan" }, { "@type": "Person", "name": "Ebele N Anajemba" }, { "@type": "Person", "name": "Ngozi G Egonu" }, { "@type": "Person", "name": "Elias Herranen" }, { "@type": "Person", "name": "Eman Nisar" }, { "@type": "Person", "name": "Emile Anand" }, { "@type": "Person", "name": "Emuobonuvie M Ajiboye" }, { "@type": "Person", "name": "Eryawan P Yulianrifat" }, { "@type": "Person", "name": "Esther Adenuga" }, { "@type": "Person", "name": "Ewa Rudnicka" }, { "@type": "Person", "name": "Faith O Itiola" }, { "@type": "Person", "name": "Fareeha Fayyaz Sheikh" }, { "@type": "Person", "name": "Fathima Thekkekara" }, { "@type": "Person", "name": "Fatima Haouari" }, { "@type": "Person", "name": "Faustin NSENGIYUMVA" }, { "@type": "Person", "name": "Fenal A Ilasariya" }, { "@type": "Person", "name": "Filbert A Tjiaranata" }, { "@type": "Person", "name": "Firas Laakom" }, { "@type": "Person", "name": "Francesca Grasso" }, { "@type": "Person", "name": "Francesco Periti" }, { "@type": "Person", "name": "Francesco Orabona" }, { "@type": "Person", "name": "Gbenga K Solomon" }, { "@type": "Person", "name": "Genta Winata" }, { "@type": "Person", "name": "Nghia G Ngo" }, { "@type": "Person", "name": "Gloria Udhedhe-oze" }, { "@type": "Person", "name": "Gonçalo Vinagre" }, { "@type": "Person", "name": "Gopi N Challagolla" }, { "@type": "Person", "name": "Gorka Urbizu-Garmendia" }, { "@type": "Person", "name": "Gouthami Vadithya" }, { "@type": "Person", "name": "Guijin Son" }, { "@type": "Person", "name": "Gyan S Mohapatra" }, { "@type": "Person", "name": "Hafeez Ullah" }, { "@type": "Person", "name": "Hafsteinn Einarsson" }, { "@type": "Person", "name": "Hai Hu" }, { "@type": "Person", "name": "Hamidreza Saffari" }, { "@type": "Person", "name": "Hamza Zaidi" }, { "@type": "Person", "name": "Haopeng Zhang" }, { "@type": "Person", "name": "Harethah Abu Shairah" }, { "@type": "Person", "name": "Hele-Andra Kuulmets" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Houda Bouamor" }, { "@type": "Person", "name": "Hwanjo Yu" }, { "@type": "Person", "name": "Iben N Debess" }, { "@type": "Person", "name": "İbrahim Ethem Deveci" }, { "@type": "Person", "name": "Ikhlasul A Hanif" }, { "@type": "Person", "name": "Ikhyun Cho" }, { "@type": "Person", "name": "Inês Vieira" }, { "@type": "Person", "name": "Inês Calvo" }, { "@type": "Person", "name": "Ismail O Daud" }, { "@type": "Person", "name": "Yusuf I Abayomi" }, { "@type": "Person", "name": "Itay Itzhak" }, { "@type": "Person", "name": "Ivan Zhelyazkov" }, { "@type": "Person", "name": "Ivan Belashkin" }, { "@type": "Person", "name": "Ivan Spada" }, { "@type": "Person", "name": "Jacob Brinton" }, { "@type": "Person", "name": "Jafar Isbarov" }, { "@type": "Person", "name": "Jaka Čibej" }, { "@type": "Person", "name": "Jan Kocon" }, { "@type": "Person", "name": "Jan Cuhel" }, { "@type": "Person", "name": "Jauza A Krito" }, { "@type": "Person", "name": "Jebish Purbey" }, { "@type": "Person", "name": "Jennifer Za" }, { "@type": "Person", "name": "Jennifer Mickel" }, { "@type": "Person", "name": "Jenny Kunz" }, { "@type": "Person", "name": "Jessica Ratovondranto" }, { "@type": "Person", "name": "Jeyarajalingam Varsha" }, { "@type": "Person", "name": "Jihae Jeong" }, { "@type": "Person", "name": "Jimena T Dávalos" }, { "@type": "Person", "name": "Jinu Lee" }, { "@type": "Person", "name": "Joao Magalhaes" }, { "@type": "Person", "name": "John S Yi" }, { "@type": "Person", "name": "Jongin Kim" }, { "@type": "Person", "name": "Joseph Chataignon" }, { "@type": "Person", "name": "Joseph Marvin Imperial" }, { "@type": "Person", "name": "Jubeerathan Thevakumar" }, { "@type": "Person", "name": "Judith Land" }, { "@type": "Person", "name": "Iuliia Alekseenko" }, { "@type": "Person", "name": "Jiang Junchen" }, { "@type": "Person", "name": "Jungwhan Kim" }, { "@type": "Person", "name": "Kairit Sirts" }, { "@type": "Person", "name": "Kamesh R" }, { "@type": "Person", "name": "Kanda Patrick Tshinu" }, { "@type": "Person", "name": "Kätriin Kukk" }, { "@type": "Person", "name": "Kaustubh Ponkshe" }, { "@type": "Person", "name": "Kavsar Huseynova" }, { "@type": "Person", "name": "Ke He" }, { "@type": "Person", "name": "Kenneth Enevoldsen" }, { "@type": "Person", "name": "Kent J Alvarez" }, { "@type": "Person", "name": "Kerem Zaman" }, { "@type": "Person", "name": "Khalil Mrini" }, { "@type": "Person", "name": "Kian Kyars" }, { "@type": "Person", "name": "Komal Gour" }, { "@type": "Person", "name": "Krishnakumar Lainitha" }, { "@type": "Person", "name": "Krister Kruusmaa" }, { "@type": "Person", "name": "Kunal Mukherjee" }, { "@type": "Person", "name": "Kusum Chouhan" }, { "@type": "Person", "name": "Laura Castro" }, { "@type": "Person", "name": "Laura M Porrino-Moscoso" }, { "@type": "Person", "name": "Lenny S Nzambi" }, { "@type": "Person", "name": "Leshem Choshen" }, { "@type": "Person", "name": "Lilja Øvrelid" }, { "@type": "Person", "name": "Lisa Alazraki" }, { "@type": "Person", "name": "Lovina Ehimen-Ugbede" }, { "@type": "Person", "name": "Luheerathan Thevakumar" }, { "@type": "Person", "name": "Luxshan Thavarasa" }, { "@type": "Person", "name": "Mamadou K KEITA" }, { "@type": "Person", "name": "Mansi Jangid" }, { "@type": "Person", "name": "Marco De Santis" }, { "@type": "Person", "name": "Marcos Garcia" }, { "@type": "Person", "name": "Marek Suppa" }, { "@type": "Person", "name": "Mariam D'Ciofalo" }, { "@type": "Person", "name": "Marii Ojastu" }, { "@type": "Person", "name": "Marium Atta" }, { "@type": "Person", "name": "Maryam Sikander" }, { "@type": "Person", "name": "Maximos Skandalis" }, { "@type": "Person", "name": "Mehak Soomro" }, { "@type": "Person", "name": "Mehmet İlteriş Bozkurt" }, { "@type": "Person", "name": "Melaku Bayu" }, { "@type": "Person", "name": "Menan Velayuthan" }, { "@type": "Person", "name": "Michael Leventhal" }, { "@type": "Person", "name": "Michał Marcińczuk" }, { "@type": "Person", "name": "Mina Almasi" }, { "@type": "Person", "name": "Mirna Potočnjak" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Mohammadamin Shafiei" }, { "@type": "Person", "name": "MOHIBA ANSARI" }, { "@type": "Person", "name": "Mridul Sharma" }, { "@type": "Person", "name": "Mrityunjaya Indoria" }, { "@type": "Person", "name": "Mughees U Rehman" }, { "@type": "Person", "name": "Muhammad Ravi Shulthan Habibi" }, { "@type": "Person", "name": "Murat B Kınay" }, { "@type": "Person", "name": "Nada Galant" }, { "@type": "Person", "name": "Naina S Rathore" }, { "@type": "Person", "name": "Narada Maugin" }, { "@type": "Person", "name": "Nathalie H Norman" }, { "@type": "Person", "name": "Nicholas Kluge Corrêa" }, { "@type": "Person", "name": "Nikola Ljubešić" }, { "@type": "Person", "name": "Nirmal Thomas" }, { "@type": "Person", "name": "Nisansa de Silva" }, { "@type": "Person", "name": "Nisheeth Joshi" }, { "@type": "Person", "name": "Nitish Ponkshe" }, { "@type": "Person", "name": "Nizar Habash" }, { "@type": "Person", "name": "Nneoma C Udeze" }, { "@type": "Person", "name": "Noel Thomas" }, { "@type": "Person", "name": "Noémi Ligeti-Nagy" }, { "@type": "Person", "name": "Odunayo Ogundepo" }, { "@type": "Person", "name": "Odunayo K Buliaminu" }, { "@type": "Person", "name": "God'spraise Okechukwu" }, { "@type": "Person", "name": "Olanrewaju Samuel" }, { "@type": "Person", "name": "Olga Snissarenko" }, { "@type": "Person", "name": "Onyinye A Chiemezie" }, { "@type": "Person", "name": "Orkun Kınay" }, { "@type": "Person", "name": "osman tursun" }, { "@type": "Person", "name": "Pablo Rodríguez" }, { "@type": "Person", "name": "Pablo Gamallo" }, { "@type": "Person", "name": "Pedro Valente" }, { "@type": "Person", "name": "Peter Rupnik" }, { "@type": "Person", "name": "Philip O. EKIUGBO" }, { "@type": "Person", "name": "Prakhar Agarwal" }, { "@type": "Person", "name": "Prokopis Prokopidis" }, { "@type": "Person", "name": "Quadri Yahya" }, { "@type": "Person", "name": "Rachele Mignone" }, { "@type": "Person", "name": "Raghav Singhal" }, { "@type": "Person", "name": "Rahul Raja" }, { "@type": "Person", "name": "Ram Mohan Rao Kadiyala" }, { "@type": "Person", "name": "Raphaël Merx" }, { "@type": "Person", "name": "Rasmus Larsen" }, { "@type": "Person", "name": "Ratnavel Rajalakshmi" }, { "@type": "Person", "name": "Rishav Ghosh" }, { "@type": "Person", "name": "Romina Oji" }, { "@type": "Person", "name": "Rui P Guerra" }, { "@type": "Person", "name": "Rushikesh Zawar" }, { "@type": "Person", "name": "Sa'ad N Bashir" }, { "@type": "Person", "name": "Saeed Alzaabi" }, { "@type": "Person", "name": "Sahil Sandeep" }, { "@type": "Person", "name": "Sai Sandeep Kantareddy" }, { "@type": "Person", "name": "Saleha Muzammil" }, { "@type": "Person", "name": "Salsabila Z Pranida" }, { "@type": "Person", "name": "Sam Buchanan" }, { "@type": "Person", "name": "Samuel Rutunda" }, { "@type": "Person", "name": "Sander Land" }, { "@type": "Person", "name": "Sarah Sulollari" }, { "@type": "Person", "name": "Sardar Ali" }, { "@type": "Person", "name": "Kengatharaiyer Sarveswaran" }, { "@type": "Person", "name": "Saulius Tautvaisas" }, { "@type": "Person", "name": "Sayambhu Sen" }, { "@type": "Person", "name": "Sayantani Banerjee" }, { "@type": "Person", "name": "Sebastien Diarra" }, { "@type": "Person", "name": "Sewoong Lee" }, { "@type": "Person", "name": "Shaan Shah" }, { "@type": "Person", "name": "Shankar Venkitachalam" }, { "@type": "Person", "name": "Sharifa Djurabaeva" }, { "@type": "Person", "name": "Sharon Ibejih" }, { "@type": "Person", "name": "Shivanya S Dutta" }, { "@type": "Person", "name": "Siddhant Gupta" }, { "@type": "Person", "name": "Silvia P Suárez" }, { "@type": "Person", "name": "Sina Ahmadi" }, { "@type": "Person", "name": "Sivasuthan Sukumar" }, { "@type": "Person", "name": "Siyuan Song" }, { "@type": "Person", "name": "Sokratis Sofianopoulos" }, { "@type": "Person", "name": "Sona E Simon" }, { "@type": "Person", "name": "Sonja Benčina" }, { "@type": "Person", "name": "Sophie Gvasalia" }, { "@type": "Person", "name": "Sphurti K More" }, { "@type": "Person", "name": "Spyridon Konstantinos Dragazis" }, { "@type": "Person", "name": "Stefan Milosavljević" }, { "@type": "Person", "name": "Stephan Kaufhold" }, { "@type": "Person", "name": "AlRashed" }, { "@type": "Person", "name": "Surangika Ranathunga" }, { "@type": "Person", "name": "Taiga Someya" }, { "@type": "Person", "name": "Taja Kuzman Pungeršek" }, { "@type": "Person", "name": "Tal Haklay" }, { "@type": "Person", "name": "Tasi'u Jibril" }, { "@type": "Person", "name": "Tatsuya Aoyama" }, { "@type": "Person", "name": "Tea Abashidze" }, { "@type": "Person", "name": "Terenz Jomar Dela Cruz" }, { "@type": "Person", "name": "Terra Blevins" }, { "@type": "Person", "name": "Themistoklis Nikas" }, { "@type": "Person", "name": "Theresa D Idoko" }, { "@type": "Person", "name": "Tilek Chubakov" }, { "@type": "Person", "name": "Tina Munda" }, { "@type": "Person", "name": "Tobiloba Moses Owoeye" }, { "@type": "Person", "name": "Tommaso Gargiani" }, { "@type": "Person", "name": "Uni Johannesen" }, { "@type": "Person", "name": "Vallerie A Putra" }, { "@type": "Person", "name": "Vanya BK" }, { "@type": "Person", "name": "Varvara Arzt" }, { "@type": "Person", "name": "Vasily Konovalov" }, { "@type": "Person", "name": "Vasudevan Nedumpozhimana" }, { "@type": "Person", "name": "Viktória Ondrejová" }, { "@type": "Person", "name": "Viktoryia Horbik" }, { "@type": "Person", "name": "Vuk Dinić" }, { "@type": "Person", "name": "Walelign Sewunetie" }, { "@type": "Person", "name": "Winston Wu" }, { "@type": "Person", "name": "Xiaojing Zhao" }, { "@type": "Person", "name": "Yacouba Diarra" }, { "@type": "Person", "name": "Yaniv Nikankin" }, { "@type": "Person", "name": "Yash Mathur" }, { "@type": "Person", "name": "Yash Bagla" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Yixi Chen" }, { "@type": "Person", "name": "Yiyuan Li" }, { "@type": "Person", "name": "Yolanda Xavier" }, { "@type": "Person", "name": "Yonatan Belinkov" }, { "@type": "Person", "name": "Zaid Alyafeai" }, { "@type": "Person", "name": "Zhengyang Shan" }, { "@type": "Person", "name": "Zhi Rui Tam" }, { "@type": "Person", "name": "Zilu Tang" }, { "@type": "Person", "name": "Zuzana Nadova" }, { "@type": "Person", "name": "Baber Abbasi" }, { "@type": "Person", "name": "Stella Biderman" }, { "@type": "Person", "name": "David Stap" }, { "@type": "Person", "name": "Duygu Ataman" }, { "@type": "Person", "name": "Fabian D Schmidt" }, { "@type": "Person", "name": "Hila Gonen" }, { "@type": "Person", "name": "Jiayi Wang" }, { "@type": "Person", "name": "David Adelani" } ] , "keywords": ["Commonsense reasoning","Multilingual evaluation","Cultural evaluation"] , "abstract": "To date, there exist almost no culturally-specific evaluation benchmarks for large language models (LLMs) that cover a large number of languages and cultures. In this paper, we present Global PIQA, a participatory commonsense reasoning benchmark for over 100 languages, constructed by hand by over 350 researchers from over 65 countries around the world. The 141 language varieties in Global PIQA cover five continents, 19 language families, and 24 writing systems. In the non-parallel split of Global PIQA, over 50% of examples reference local foods, customs, traditions, or other culturally-specific elements. In the parallel split, we translate more \"culturally agnostic\" commonsense reasoning questions into 131 language varieties, for direct cross-lingual comparisons. In both splits, all examples have been verified by native speakers of the languages. We find that state-of-the-art LLMs perform well on Global PIQA in aggregate, but they exhibit weaker performance in lower-resource languages (e.g. up to a 68% accuracy gap between languages in the parallel split). Global PIQA highlights that in many languages and cultures, everyday knowledge remains an area for improvement in LLMs, alongside more widely-discussed capabilities such as complex reasoning and expert knowledge. Beyond its uses for LLM evaluation, Global PIQA provides a glimpse into the wide diversity of cultures in which human language is embedded." } }, { "@type": "ListItem", "position": 13, "item": { "@type": "ScholarlyArticle", "@id": "https://arxiv.org/abs/2604.11490#paper", "url": "https://arxiv.org/abs/2604.11490", "name": "Anthropogenic Regional Adaptation in Multimodal Vision-Language Model", "description": "AACL'26.", "author": [ { "@type": "Person", "name": "Samuel Cahyawijaya" }, { "@type": "Person", "name": "Peerat Limkonchotiwat" }, { "@type": "Person", "name": "Tack Hwa Wong" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Manuel Antonio Rufino" }, { "@type": "Person", "name": "Carlos Rafael Catalan" }, { "@type": "Person", "name": "Muhammad Reza Qorib" }, { "@type": "Person", "name": "Vicky Feliren" }, { "@type": "Person", "name": "Holy Lovenia" }, { "@type": "Person", "name": "Aye Hninn Khine" }, { "@type": "Person", "name": "Frederikus Hudi" }, { "@type": "Person", "name": "David Anugraha" }, { "@type": "Person", "name": "Alham Fikri Aji" }, { "@type": "Person", "name": "Romrawin Chumpu" }, { "@type": "Person", "name": "Viet-Thanh Pham" }, { "@type": "Person", "name": "Minghan Wang" }, { "@type": "Person", "name": "Mohamed Fazli Imam" }, { "@type": "Person", "name": "Ruochen Zhang" }, { "@type": "Person", "name": "Joseph Marvin Imperial" }, { "@type": "Person", "name": "Khumaisa Nur'aini" }, { "@type": "Person", "name": "Do Xuan Long" }, { "@type": "Person", "name": "Musa Izzanardi Wijanarko" }, { "@type": "Person", "name": "Joel Ruben Antony Moniz" }, { "@type": "Person", "name": "Patrick Amadeus Irawan" }, { "@type": "Person", "name": "Hanif Muhammad Zhafran" }, { "@type": "Person", "name": "Isaiah Flores" }, { "@type": "Person", "name": "Salsabila Zahirah Pranida" }, { "@type": "Person", "name": "Jun Kevin" }, { "@type": "Person", "name": "Jostin Jerico Rosal" }, { "@type": "Person", "name": "Patricia Nicole Monderin" }, { "@type": "Person", "name": "Kun Kerdthaisong" }, { "@type": "Person", "name": "Ahmad Mustafid" }, { "@type": "Person", "name": "My Chiffon Nguyen" }, { "@type": "Person", "name": "Natchapon Jongwiriyanurak" }, { "@type": "Person", "name": "Siva Worajitwannakul" }, { "@type": "Person", "name": "Haochen Li" }, { "@type": "Person", "name": "Adrian Xuan Wei Lim" }, { "@type": "Person", "name": "Bin Wang" }, { "@type": "Person", "name": "Muhammad Ravi Shulthan Habibi" }, { "@type": "Person", "name": "Lynnette Hui Xian Ng" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Dun Li Chan" }, { "@type": "Person", "name": "Sherissa Caren Djuniwar" }, { "@type": "Person", "name": "Cho Chan Myei Oo" }, { "@type": "Person", "name": "Hee Ming Shan" } ] , "keywords": ["Vision-language models","Regional adaptation","Multimodal alignment","Model merging"] , "abstract": "While the field of vision-language (VL) has achieved remarkable success in integrating visual and textual information across multiple languages and domains, there is still no dedicated framework for assessing human-centric alignment in vision-language systems. We offer two contributions to address this gap. First, we introduce Anthropogenic Regional Adaptation: a novel paradigm that aims to optimize model relevance to specific regional contexts while ensuring the retention of global generalization capabilities. Second, we present a simple, but effective adaptation method named Geographical-generalization-made-easy (GG-EZ), which utilizes regional data filtering and model merging. Through comprehensive experiments on 3 VL architectures: large vision-language models, text-to-image diffusion models, and vision-language embedding models, and a case study in Southeast Asia (SEA) regional adaptation, we demonstrate the importance of Anthropogenic Regional Adaptation and the effectiveness of GG-EZ, showing 5-15% gains in cultural relevance metrics across SEA while maintaining over 98% of global performance and even occasionally surpassing it. Our findings establish Anthropogenic Regional Alignment as a foundational paradigm towards applicability of multimodal vision-language models in diverse regions and demonstrate a simple-yet-effective baseline method that optimizes regional value alignment while preserving global generalization." } }, { "@type": "ListItem", "position": 14, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.acl-long.926/#paper", "url": "https://aclanthology.org/2026.acl-long.926/", "name": "SPENCE: A Syntactic Probe for Detecting Contamination in NL2SQL Benchmarks", "description": "ACL'26.", "author": [ { "@type": "Person", "name": "Mohammadtaher Safarzadeh" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Afshin Oroojlooy" }, { "@type": "Person", "name": "Graham Horwood" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["Benchmark contamination","Text-to-SQL","LLM evaluation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2026.acl-long.926", "url": "https://doi.org/10.18653/v1/2026.acl-long.926" } , "abstract": "Large language models (LLMs) have achieved strong performance on natural language to SQL (NL2SQL) benchmarks, yet their reported accuracy may be inflated by contamination from benchmark queries or structurally similar patterns seen during training. We introduce SPENCE (Syntactic Probing and Evaluation of NL2SQL Contamination Effects), a controlled syntactic probing framework for detecting and quantifying such contamination. SPENCE systematically generates syntactic variants of test queries for four widely used NL2SQL datasets—Spider, SParC, CoSQL, and the newer BIRD benchmark. We use SPENCE to evaluate multiple high-capacity LLMs under execution-based scoring. For each model, we measure changes in execution accuracy (Delta Accuracy) across increasing levels of syntactic divergence and quantify rank sensitivity using Kendall’s tau with bootstrap confidence intervals. By aligning these robustness trends with benchmark release dates, we observe a clear temporal gradient: older benchmarks such as Spider exhibit the strongest negative values and thus the highest likelihood of training leakage, whereas the more recent BIRD dataset shows minimal sensitivity and appears largely uncontaminated. Together, these findings highlight the importance of temporally contextualized, syntactic-probing evaluation for trustworthy NL2SQL benchmarking." } }, { "@type": "ListItem", "position": 15, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2026.findings-acl.1948/#paper", "url": "https://aclanthology.org/2026.findings-acl.1948/", "name": "Do Image–Text Metrics Respect Semantic Invariances?", "description": "ACL'26.", "author": [ { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Meizhu Liu" }, { "@type": "Person", "name": "Jyotika Singh" }, { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "M. Avendi" }, { "@type": "Person", "name": "Matthew Rowe" }, { "@type": "Person", "name": "Yassi Abbasi" }, { "@type": "Person", "name": "Hansa  Meghwani" }, { "@type": "Person", "name": "Tao Sheng" }, { "@type": "Person", "name": "Sujith Ravi" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["Vision-language evaluation","Image-caption alignment","Semantic invariance"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2026.findings-acl.1948", "url": "https://doi.org/10.18653/v1/2026.findings-acl.1948" } , "abstract": "Reference-free image–to–text evaluators are now standard for scoring image–caption alignment, yet it is unclear whether they respect semantic invariances. We present an invariance probe on five popular evaluators (CLIPScore, PAC-S, UMIC, FLEUR, and a deterministic LLM judge) under semantics-preserving perturbations along three axes: spatial (flips, context-preserving repositioning, light rotations), object (scale, category), and socio-linguistic framing (cultural/economic adjectives with neutral and length-matched controls). Across curated slices of three detection datasets and three caption evaluation suites, we find consistent non-semantic sensitivities: benign spatial edits and simple phrasing changes shift scores by (≈)6–9% on average, and for systems separated by just 0.7% these shifts can cause ranking flips in upto (∼)37% of cases, particularly under spatial changes. A small human study also supports this finding and confirms that annotators generally judge perturbed pairs as equally correct, so these shifts reflect metric behavior rather than semantic change. We further propose invariance-calibrated scoring, a post-hoc adjustment that roughly halves median absolute sensitivity while retaining correlation with learned caption evaluators." } }, { "@type": "ListItem", "position": 16, "item": { "@type": "ScholarlyArticle", "@id": "https://arxiv.org/abs/2604.00018#paper", "url": "https://arxiv.org/abs/2604.00018", "name": "Think Twice Before You Write -- an Entropy-based Decoding Strategy to Enhance LLM Reasoning", "description": "Under Review.", "author": [ { "@type": "Person", "name": "Jiashu He" }, { "@type": "Person", "name": "Meizhu Liu" }, { "@type": "Person", "name": "Olaitan P Olaleye" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "M. Avendi" }, { "@type": "Person", "name": "Yassi Abbasi" }, { "@type": "Person", "name": "Matthew Rowe" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Paul Li" }, { "@type": "Person", "name": "Tao Sheng" }, { "@type": "Person", "name": "Sujith Ravi" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["LLM reasoning","Entropy-guided decoding","Adaptive inference"] , "abstract": "Decoding strategies play a central role in shaping the reasoning ability of large language models (LLMs). Traditional methods such as greedy decoding and beam search often suffer from error propagation, while sampling-based approaches introduce randomness without adequate robustness. Self-consistency improves reliability by aggregating multiple rollouts, but incurs significant computational overhead. We propose an entropy-guided decoding framework that introduces token-level adaptivity into generation. At each step, the model computes the entropy of the token distribution, identifies high-uncertainty positions, and selectively branches on these vulnerable points. A dynamic pool of partial rollouts is maintained and expanded until solutions are completed, concentrating computation where uncertainty is greatest and avoiding unnecessary exploration in confident regions. To enable efficient termination, we apply a rollout-level Entropy After (EAT) stopping criterion by performing entropy evaluation after the full reasoning trace, rather than incrementally at every step. Experiments on GSM8K, AMC2023, and their perturbed variants demonstrate that our method achieves consistently strong accuracy. Notably, on smaller LLMs, performance is comparable to GPT-5 while operating at a fraction of the cost." } }, { "@type": "ListItem", "position": 17, "item": { "@type": "ScholarlyArticle", "@id": "https://link.springer.com/chapter/10.1007/978-981-92-1947-6_48#paper", "url": "https://link.springer.com/chapter/10.1007/978-981-92-1947-6_48", "name": "A Review of Human-Centric Evaluation of Cultural Biases in Indic Languages in LLMs: Rethinking Research Directions", "description": "PAKDD'2026.", "author": [ { "@type": "Person", "name": "Azmine Toushik Wasi" }, { "@type": "Person", "name": "Omid Reza Heidari" }, { "@type": "Person", "name": "MD Mohaymen Ul Anam" }, { "@type": "Person", "name": "Abdur Rahman" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Cultural bias","Indic languages","Human-centered evaluation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1007/978-981-92-1947-6_48", "url": "https://doi.org/10.1007/978-981-92-1947-6_48" } , "abstract": "Indic languages, spoken by more than 1.3 billion people in South Asia, are often underrepresented in AI systems due to their diversity, limited resources, and evaluation challenges. This region, known for its many coexisting developed and endangered cultures, faces significant cultural bias risks as these cultures are not equally represented. These biases can significantly impact the design, development, and deployment of AI in real-life scenarios. Our survey explores cultural bias in Indic languages from a human-centered perspective, combining computational social science and NLP. We interpret different types of bias, critically review different evaluation and mitigation works, and identify research gaps and priorities to focus on to improve AI systems. Based on our review findings, we argue that we need to rethink and reprioritize our research efforts to support other Indic languages except Hindi, and focus on different evaluation approaches for cultural biases. We discuss human-centered perspectives to enhance LLMs, focusing on how they can operate more effectively and equitably within South Asia’s diverse linguistic and rich cultural landscape, thereby creating fairer and more inclusive AI systems for real-world applications." } }, { "@type": "ListItem", "position": 18, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.emnlp-main.1653/#paper", "url": "https://aclanthology.org/2025.emnlp-main.1653/", "name": "AccessEval: Benchmarking Disability Bias in Large Language Models", "description": "EMNLP'25; Best Social Impact Paper Award.", "author": [ { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["Disability bias","LLM fairness","Model evaluation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.emnlp-main.1653", "url": "https://doi.org/10.18653/v1/2025.emnlp-main.1653" } , "abstract": "Large Language Models (LLMs) are increasingly deployed across diverse domains but often exhibit disparities in how they handle real life queries. To systematically investigate these effects with various disability context, we introduce AccessEval, a large-scale benchmark evaluating total 21 close & open source LLMs across six real-world domains and nine disability types using paired Neutral and Disability-Aware Queries. We evaluated model outputs with metrics for factual accuracy, sentiment, and social perception.Our analysis reveals that responses to disability-aware queries tend to have higher factual error, more negative tone, and increased stereotyping with social perception compared to neutral queries. These effects show notable variation by domain and disability type. Disabilities affecting hearing, speech and mobility are disproportionately impacted. These disparities reveal persistent forms of ableism, highlighting the need for more comprehensive and nuanced assessment.We further argue that framing bias in terms of model performance within real-world decision making helps to better link model behaviors to the potential harms users may face. This approach guides the development of more effective and tailored fairness interventions. AccessEval, therefore, serves as a crucial tool for advancing equitable and inclusive language technologies." } }, { "@type": "ListItem", "position": 19, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.ijcnlp-long.170/#paper", "url": "https://aclanthology.org/2025.ijcnlp-long.170/", "name": "LLM-Guided Lifecycle-Aware Clustering of Multi-Turn Customer Support Conversations", "description": "AACL'25.", "author": [ { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Sanchari Chowdhuri" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["Conversational AI","Incremental clustering","Customer support"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.ijcnlp-long.170", "url": "https://doi.org/10.18653/v1/2025.ijcnlp-long.170" } , "abstract": "Clustering customer chat data is vital for cloud providers handling multi-service queries. Traditional methods struggle with overlapping concerns and create broad, static clusters that degrade over time. Re-clustering disrupts continuity, making issue tracking difficult. We propose an adaptive system that segments multi-turn chats into service-specific concerns and incrementally refines clusters as new issues arise. Cluster quality is tracked via Davies–Bouldin Index (DBI) and Silhouette Scores, with LLM-based splitting applied only to degraded clusters. Our method improves Silhouette Scores by over 100% and reduces DBI by 65.6% compared to baselines, enabling scalable, real-time analytics without full re-clustering." } }, { "@type": "ListItem", "position": 20, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.ijcnlp-long.142/#paper", "url": "https://aclanthology.org/2025.ijcnlp-long.142/", "name": "A Diagnostic Framework for Auditing Reference-Free Vision-Language Metrics", "description": "AACL'25.", "author": [ { "@type": "Person", "name": "Angeline Charles" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Tejaswini Kumar" } ] , "keywords": ["Vision-language evaluation","Reference-free metrics","Cultural evaluation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.ijcnlp-long.142", "url": "https://doi.org/10.18653/v1/2025.ijcnlp-long.142" } , "abstract": "Reference-free metrics such as CLIPScore and PAC-S are increasingly used in vision-language tasks due to their scalability and independence from human-written references. However, their reliability under linguistic, visual, and cultural variation remains underexplored. In this work, we present a systematic audit of CLIPScore and PAC-S using an eight-factor diagnostic framework applied to MS-COCO validation images. Our analysis reveals consistent failure modes across dimensions including object size, content category, syntax, named entities, spatial relations and cultural context. Both metrics penalize captions referencing African (−5.5%, −4.8%) and Arabian (−4.9%, −5.3%) cultures, favor large-object and animal-centric scenes (by 20-30%) and show limited sensitivity to spatial negation and word order. CLIPScore correlates more strongly with syntactic complexity, while PAC-S demonstrates greater robustness to verbosity and named–entity variation highlighting complementary strengths rather than superiority. These findings expose cultural and content bias, weak semantic robustness, and limited compositional understanding. We conclude with design recommendations to improve fairness, scale invariance, and semantic grounding in future reference-free evaluation metrics." } }, { "@type": "ListItem", "position": 21, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.emnlp-industry.14.pdf#paper", "url": "https://aclanthology.org/2025.emnlp-industry.14.pdf", "name": "PCRI: Measuring Context Robustness in Multimodal Models for Enterprise Applications", "description": "EMNLP'25.", "author": [ { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Paul Li" }, { "@type": "Person", "name": "Tao Sheng" }, { "@type": "Person", "name": "Sujith Ravi" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["Multimodal evaluation","Visual context","Model robustness"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.emnlp-industry.14", "url": "https://doi.org/10.18653/v1/2025.emnlp-industry.14" } , "abstract": "The reliability of Multimodal Large Language Models (MLLMs) in real-world settings is often undermined by sensitivity to irrelevant or distracting visual context, an aspect not captured by existing evaluation metrics. We introduce the Patch Context Robustness Index (PCRI), the first systematic and interpretable score for quantifying MLLM robustness to variations in visual context granularity, measuring performance changes between localized image patches and full-image input.Applying PCRI to 19 state-of-the-art MLLMs across 15 vision-language benchmarks, we find that most leading models remain brittle to background noise, with only a few, such as InternVL2-26B and Qwen2VL-72B, demonstrating consistent robustness across tasks. PCRI analysis also highlights how different model architectures handle and integrate visual context, offering actionable diagnostic insight for both researchers and practitioners.PCRI enables rigorous comparison of context robustness, supporting principled model selection and guiding the development of future architectures and training strategies for robust, real-world deployment." , "sameAs": "https://aclanthology.org/2025.emnlp-industry.14/" } }, { "@type": "ListItem", "position": 22, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.emnlp-industry.105.pdf#paper", "url": "https://aclanthology.org/2025.emnlp-industry.105.pdf", "name": "FlexDoc: Parameterized Sampling for Diverse Multilingual Synthetic Documents for Training Document Understanding Models", "description": "EMNLP'25.", "author": [ { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Puneet Mittal" }, { "@type": "Person", "name": "Ranjeet Gupta" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Praneet Pabolu" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Graham Horwood" }, { "@type": "Person", "name": "Fahad Shah" } ] , "keywords": ["Synthetic data","Multilingual documents","Document understanding"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.emnlp-industry.105", "url": "https://doi.org/10.18653/v1/2025.emnlp-industry.105" } , "abstract": "Developing document understanding models at enterprise scale requires large, diverse, and well-annotated datasets spanning a wide range of document types. However, collecting such data is prohibitively expensive due to privacy constraints, legal restrictions, and the sheer volume of manual annotation needed - costs that can scale into millions of dollars. We introduce FlexDoc, a scalable synthetic data generation framework that combines Stochastic Schemas and Parameterized Sampling to produce realistic, multilingual semi-structured documents with rich annotations. By probabilistically modeling layout patterns, visual structure, and content variability, FlexDoc enables the controlled generation of diverse document variants at scale. Experiments on Key Information Extraction (KIE) tasks demonstrate that FlexDoc-generated data improves the absolute F1 Score by up to 11% when used to augment real datasets, while reducing annotation effort by over 90% compared to traditional hard-template methods. The solution is in active deployment, where it has accelerated the development of enterprise-grade document understanding models while significantly reducing data acquisition and annotation costs." , "sameAs": "https://aclanthology.org/2025.emnlp-industry.105/" } }, { "@type": "ListItem", "position": 23, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.emnlp-industry.10/#paper", "url": "https://aclanthology.org/2025.emnlp-industry.10/", "name": "RCI: A Score for Evaluating Global and Local Reasoning in Multimodal Benchmarks", "description": "EMNLP'25.", "author": [ { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Jyotika Singh" }, { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Paul Li" }, { "@type": "Person", "name": "Tao Sheng" }, { "@type": "Person", "name": "Sujith Ravi" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["Multimodal reasoning","Benchmark evaluation","Global and local visual information"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.emnlp-industry.10", "url": "https://doi.org/10.18653/v1/2025.emnlp-industry.10" } , "abstract": "Multimodal Large Language Models (MLLMs) have achieved impressive results on vision-language benchmarks, yet it remains unclear whether these benchmarks assess genuine global reasoning or allow success via localized visual cues. Existing evaluation methods do not explicitly measure this distinction, hindering effective dataset curation and real-world focused model development.We introduce Region Comprehension Index (RCI), the first model-based score to directly quantify a dataset’s reliance on global versus local visual information. RCI systematically compares reference-model performance on image patches versus full images, revealing if tasks require holistic image understanding or can be solved with partial or localized visual cues.When applying RCI to 13 widely used multimodal benchmarks, we observed that most of them favor localized reasoning and exhibit significant spatial biases, indicating potential risks in real-world applications. RCI equips researchers & practitioners with an actionable tool for diagnosing & mitigating these biases, enabling the construction of datasets and benchmarks to foster the development of robust, enterprise-ready multimodal systems." } }, { "@type": "ListItem", "position": 24, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.emnlp-industry.9/#paper", "url": "https://aclanthology.org/2025.emnlp-industry.9/", "name": "Aligning LLMs for Multilingual Consistency in Enterprise Applications", "description": "EMNLP'25.", "author": [ { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Tao Sheng" }, { "@type": "Person", "name": "Sujith Ravi" }, { "@type": "Person", "name": "Dan Roth" } ] , "keywords": ["Multilingual alignment","Post-training","Cross-language consistency"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.emnlp-industry.9", "url": "https://doi.org/10.18653/v1/2025.emnlp-industry.9" } , "abstract": "Large language models (LLMs) remain unreliable for global enterprise applications due to substantial performance gaps between high-resource and mid/low-resource languages, driven by English-centric pretraining and internal reasoning biases. This inconsistency undermines customer experience and operational reliability in multilingual settings such as customer support, content moderation, and information retrieval. Even with advanced Retrieval-Augmented Generation (RAG) systems, we observe up to an 29% accuracy drop in non-English languages compared to English.We propose a practical, batch-wise alignment strategy for fine-tuning LLMs, leveraging semantically equivalent multilingual data in each training batch to directly align model outputs across languages. This approach improves non-English accuracy by up to 23.9% without compromising English performance, model reasoning, or retrieval quality. Our method is simple to implement, scalable, and integrates seamlessly with existing LLM training & deployment pipelines, enabling more robust and equitable multilingual AI solutions in industry." } }, { "@type": "ListItem", "position": 25, "item": { "@type": "ScholarlyArticle", "@id": "https://dl.acm.org/doi/10.1145/3746252.3760858#paper", "url": "https://dl.acm.org/doi/10.1145/3746252.3760858", "name": "Towards Robust Continual Test-Time Adaptation via Neighbor Filtration", "description": "CIKM'25.", "author": [ { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Continual test-time adaptation","Distribution shift","Pseudo-labeling"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1145/3746252.3760858", "url": "https://doi.org/10.1145/3746252.3760858" } , "abstract": "Test-Time Adaptation (TTA) aims to adapt an unseen target domain utilizing the unlabeled target data using a pre-trained source model. Continual TTA is a more challenging paradigm that deals with non-stationary environments during the test data adaptation. Most existing continual TTA methods are based on pseudo-labeling, but often (1) rely on overconfident pseudo-labels and (2) remain unstable under continual distribution shifts leading to error accumulation and catastrophic forgetting. To tackle these limitations, we propose Neighbor-Filtration based Continual Test-Time Adaptation (NF-CTTA), a reliable and memory-aware adaptation framework that addresses these challenges. NF-CTTA first calibrates pseudo-labels using class-conditional calibration error to correct over/under-confidence of the model. To further ensure reliability, we introduce an OOD Neighbor Filtration technique that selects a subset of high-confidence samples based on entropy and neighbor similarity, ensuring consistency within the semantic neighborhood. Finally, we propose a priority-guided memory buffer that retains the most informative low-entropy samples for replay, mitigating catastrophic forgetting across evolving test distributions. Extensive experiments across multiple domain shift benchmarks demonstrate that NF-CTTA achieves superior performance and stability compared to existing TTA and CTTA methods. The code is available at: https://github.com/takihasan/NF-CTTA." , "sameAs": "https://openreview.net/pdf?id=l4DUUSWug3" } }, { "@type": "ListItem", "position": 26, "item": { "@type": "ScholarlyArticle", "@id": "https://openaccess.thecvf.com/content/ICCV2025W/CV4A11y/papers/Hari_Whos_Asking_Investigating_Bias_Through_the_Lens_of_Disability-Framed_Queries_ICCVW_2025_paper.pdf#paper", "url": "https://openaccess.thecvf.com/content/ICCV2025W/CV4A11y/papers/Hari_Whos_Asking_Investigating_Bias_Through_the_Lens_of_Disability-Framed_Queries_ICCVW_2025_paper.pdf", "name": "Who's Asking? Investigating Bias Through the Lens of Disability-Framed Queries in LLMs", "description": "ICCV'25.", "author": [ { "@type": "Person", "name": "Vishnu Hari" }, { "@type": "Person", "name": "Kalpana Panda" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["Disability bias","Demographic inference","LLM fairness"] , "abstract": "Large Language Models (LLMs) routinely infer users' demographic traits from phrasing alone, which can result in biased responses, even when no explicit demographic information is provided. The role of disability cues in shaping these inferences remains largely uncharted. Thus, we present the first systematic audit of disability-conditioned demographic bias across eight state-of-the-art instruction-tuned LLMs ranging from 3B to 72B parameters. Using a balanced template corpus that pairs nine disability categories with six real-world business domains, we prompt each model to predict five demographic attributes - gender, socioeconomic status, education, cultural background, and locality - under both neutral and disability-aware conditions. Across a varied set of prompts, models deliver a definitive demographic guess in up to 97% of cases, exposing a strong tendency to make arbitrary inferences with no clear justification. Disability context heavily shifts predicted attribute distributions, and domain context can further amplify these deviations. We observe that larger models are simultaneously more sensitive to disability cues and more prone to biased reasoning, indicating that scale alone does not mitigate stereotype amplification. Our findings reveal persistent intersections between ableism and other demographic stereotypes, pinpointing critical blind spots in current alignment strategies. We release our evaluation framework and results to encourage disability-inclusive benchmarking and recommend integrating abstention calibration and counterfactual fine-tuning to curb unwarranted demographic inference. Code and data will be released on acceptance." , "sameAs": "https://openaccess.thecvf.com/content/ICCV2025W/CV4A11y/html/Hari_Whos_Asking_Investigating_Bias_Through_the_Lens_of_Disability-Framed_Queries_ICCVW_2025_paper.html" } }, { "@type": "ListItem", "position": 27, "item": { "@type": "ScholarlyArticle", "@id": "https://dl.acm.org/doi/10.1145/3715070.3749262#paper", "url": "https://dl.acm.org/doi/10.1145/3715070.3749262", "name": "LegalMind: An Intelligent Solution for Legal Document Analysis with User-Centric UI and AI-Driven Capabilities in Mobile Devices", "description": "CSCW'25.", "author": [ { "@type": "Person", "name": "Azmine Toushik Wasi" }, { "@type": "Person", "name": "Mst Rafia Islam" }, { "@type": "Person", "name": "Abdur Rahman" }, { "@type": "Person", "name": "Tawfia Yeasmin" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Legal document analysis","Mobile AI","Human-computer interaction"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1145/3715070.3749262", "url": "https://doi.org/10.1145/3715070.3749262" } , "abstract": "Navigating legal documents is challenging due to their complexity, jargon, and interconnected entities like names, dates, and provisions, leading to inefficiencies and critical oversights. With the growing reliance on mobile devices, there is an increasing demand for tools that enable efficient and accessible legal document analysis on-the-go. To address these issues, we introduce LegalMind, a mobile app that revolutionizes legal document analysis by integrating key features: Automatic Entity Mapping for quick identification of essential details, Intelligent Question Answering for context-aware responses, and Multi-document Analysis for comprehensive comparison. Designed with a user-centric UI, LegalMind streamlines information retrieval, reduces manual effort, and enhances decision-making for legal professionals. The intuitive UI allows users to easily navigate complex legal content on mobile devices. Our user study confirmed that LegalMind improves efficiency and accessibility, thus showing its ability to transform legal workflows by bridging the gap between complex legal challenges and AI-driven solutions while maintaining a seamless, accessible UI." , "sameAs": "https://scholarworks.bwise.kr/hanyang/handle/2021.sw.hanyang/211359" } }, { "@type": "ListItem", "position": 28, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.acl-long.916/#paper", "url": "https://aclanthology.org/2025.acl-long.916/", "name": "Crowdsource, Crawl, or Generate? Creating SEA-VL, a Multicultural Vision-Language Dataset for Southeast Asia", "description": "ACL'25.", "author": [ { "@type": "Person", "name": "Samuel Cahyawijaya" }, { "@type": "Person", "name": "Holy Lovenia" }, { "@type": "Person", "name": "Joel Ruben Antony Moniz" }, { "@type": "Person", "name": "Tack Hwa Wong" }, { "@type": "Person", "name": "Mohammad Rifqi Farhansyah" }, { "@type": "Person", "name": "Thant Thiri Maung" }, { "@type": "Person", "name": "Frederikus Hudi" }, { "@type": "Person", "name": "David Anugraha" }, { "@type": "Person", "name": "Muhammad Ravi Shulthan Habibi" }, { "@type": "Person", "name": "Muhammad Reza Qorib" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Joseph Marvin Imperial" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Vicky Feliren" }, { "@type": "Person", "name": "Bahrul Ilmi Nasution" }, { "@type": "Person", "name": "Manuel Antonio Rufino" }, { "@type": "Person", "name": "Genta Indra Winata" }, { "@type": "Person", "name": "Rian Adam Rajagede" }, { "@type": "Person", "name": "Carlos Rafael Catalan" }, { "@type": "Person", "name": "Mohamed Fazli Imam" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Salsabila Zahirah Pranida" }, { "@type": "Person", "name": "Kevin Pratama" }, { "@type": "Person", "name": "Yeshil Bangera" }, { "@type": "Person", "name": "Adisai Na-Thalang" }, { "@type": "Person", "name": "Patricia Nicole Monderin" }, { "@type": "Person", "name": "Yueqi Song" }, { "@type": "Person", "name": "Christian Simon" }, { "@type": "Person", "name": "Lynnette Hui Xian Ng" }, { "@type": "Person", "name": "Richardy Lobo' Sapan" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Bin Wang" }, { "@type": "Person", "name": "Supryadi" }, { "@type": "Person", "name": "Kanyakorn Veerakanjana" }, { "@type": "Person", "name": "Piyalitt Ittichaiwong" }, { "@type": "Person", "name": "Matthew Theodore Roque" }, { "@type": "Person", "name": "Karissa Vincentio" }, { "@type": "Person", "name": "Takdanai Kreangphet" }, { "@type": "Person", "name": "Phakphum Artkaew" }, { "@type": "Person", "name": "Kadek Hendrawan Palgunadi" }, { "@type": "Person", "name": "Yanzhi Yu" }, { "@type": "Person", "name": "Rochana Prih Hastuti" }, { "@type": "Person", "name": "William Nixon" }, { "@type": "Person", "name": "Mithil Bangera" }, { "@type": "Person", "name": "Adrian Xuan Wei Lim" }, { "@type": "Person", "name": "Aye Hninn Khine" }, { "@type": "Person", "name": "Hanif Muhammad Zhafran" }, { "@type": "Person", "name": "Teddy Ferdinan" }, { "@type": "Person", "name": "Audra Aurora Izzani" }, { "@type": "Person", "name": "Ayushman Singh" }, { "@type": "Person", "name": "Evan" }, { "@type": "Person", "name": "Jauza Akbar Krito" }, { "@type": "Person", "name": "Michael Anugraha" }, { "@type": "Person", "name": "Fenal Ashokbhai Ilasariya" }, { "@type": "Person", "name": "Haochen Li" }, { "@type": "Person", "name": "John Amadeo Daniswara" }, { "@type": "Person", "name": "Filbert Aurelian Tjiaranata" }, { "@type": "Person", "name": "Eryawan Presma Yulianrifat" }, { "@type": "Person", "name": "Can Udomcharoenchaikit" }, { "@type": "Person", "name": "Fadil Risdian Ansori" }, { "@type": "Person", "name": "Mahardika Krisna Ihsani" }, { "@type": "Person", "name": "Giang Nguyen" }, { "@type": "Person", "name": "Anab Maulana Barik" }, { "@type": "Person", "name": "Dan John Velasco" }, { "@type": "Person", "name": "Rifo Ahmad Genadi" }, { "@type": "Person", "name": "Saptarshi Saha" }, { "@type": "Person", "name": "Chengwei Wei" }, { "@type": "Person", "name": "Isaiah Flores" }, { "@type": "Person", "name": "Kenneth Ko Han Chen" }, { "@type": "Person", "name": "Anjela Gail Santos" }, { "@type": "Person", "name": "Wan Shen Lim" }, { "@type": "Person", "name": "Kaung Si Phyo" }, { "@type": "Person", "name": "Tim Santos" }, { "@type": "Person", "name": "Meisyarah Dwiastuti" }, { "@type": "Person", "name": "Jiayun Luo" }, { "@type": "Person", "name": "Jan Christian Blaise Cruz" }, { "@type": "Person", "name": "Ming Shan Hee" }, { "@type": "Person", "name": "Ikhlasul Akmal Hanif" }, { "@type": "Person", "name": "M.Alif Al Hakim" }, { "@type": "Person", "name": "Muhammad Rizky Sya'ban" }, { "@type": "Person", "name": "Kun Kerdthaisong" }, { "@type": "Person", "name": "Lester James V. Miranda" }, { "@type": "Person", "name": "Fajri Koto" }, { "@type": "Person", "name": "Tirana Noor Fatyanosa" }, { "@type": "Person", "name": "Alham Fikri Aji" }, { "@type": "Person", "name": "Jostin Jerico Rosal" }, { "@type": "Person", "name": "Jun Kevin" }, { "@type": "Person", "name": "Robert Wijaya" }, { "@type": "Person", "name": "Onno P. Kampman" }, { "@type": "Person", "name": "Ruochen Zhang" }, { "@type": "Person", "name": "Börje F. Karlsson" }, { "@type": "Person", "name": "Peerat Limkonchotiwat." } ] , "keywords": ["Multicultural datasets","Vision-language models","Southeast Asian languages"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.acl-long.916", "url": "https://doi.org/10.18653/v1/2025.acl-long.916" } , "abstract": "Despite Southeast Asia’s (SEA) extraordinary linguistic and cultural diversity, the region remains significantly underrepresented in vision-language (VL) research, resulting in AI models that inadequately capture SEA cultural nuances. To fill this gap, we present SEA-VL, an open-source initiative dedicated to developing culturally relevant high-quality datasets for SEA languages. By involving contributors from SEA countries, SEA-VL ensures better cultural relevance and diversity, fostering greater inclusivity of underrepresented languages and cultural depictions in VL research. Our methodology employed three approaches: community-driven crowdsourcing with SEA contributors, automated image crawling, and synthetic image generation. We evaluated each method’s effectiveness in capturing cultural relevance. We found that image crawling achieves approximately ~85% cultural relevance while being more cost- and time-efficient than crowdsourcing, whereas synthetic image generation failed to accurately reflect SEA cultural nuances and contexts. Collectively, we gathered 1.28 million SEA culturally relevant images, more than 50 times larger than other existing datasets. This work bridges the representation gap in SEA, establishes a foundation for developing culturally aware AI systems for this region, and provides a replicable framework for addressing representation gaps in other underrepresented regions." } }, { "@type": "ListItem", "position": 29, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.findings-acl.963/#paper", "url": "https://aclanthology.org/2025.findings-acl.963/", "name": "MVTamperBench: Evaluating Robustness of Vision-Language Models", "description": "ACL'25.", "author": [ { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Angeline Charles" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Tejaswini Kumar" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Karan Gupta" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Vision-language models","Adversarial robustness","Video understanding"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.findings-acl.963", "url": "https://doi.org/10.18653/v1/2025.findings-acl.963" } , "abstract": "Multimodal Large Language Models (MLLMs), are recent advancement of Vision-Language Models (VLMs) that have driven major advances in video understanding. However, their vulnerability to adversarial tampering and manipulations remains underexplored. To address this gap, we introduce MVTamperBench , a benchmark that systematically evaluates MLLM robustness against five prevalent tampering techniques: rotation, masking, substitution, repetition, and dropping; based on real-world visual tampering scenarios such as surveillance interference, social media content edits, and misinformation injection. MVTamperBench comprises ~3.4K original videos, expanded into over ~17K tampered clips covering 19 distinct video manipulation tasks. This benchmark challenges models to detect manipulations in spatial and temporal coherence. We evaluate 45 recent MLLMs from 15+ model families. We reveal substantial variability in resilience across tampering types and show that larger parameter counts do not necessarily guarantee robustness. MVTamperBench sets a new benchmark for developing tamper-resilient MLLM in safety-critical applications, including detecting clickbait, preventing harmful content distribution, and enforcing policies on media platforms. We release all code, data, and benchmark to foster open research in trustworthy video understanding." } }, { "@type": "ListItem", "position": 30, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.acl-industry.72/#paper", "url": "https://aclanthology.org/2025.acl-industry.72/", "name": "Hard Negative Mining for Domain-Specific Retrieval in Enterprise Systems", "description": "ACL'25.", "author": [ { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Srikant Panda" } ] , "keywords": ["Information retrieval","Hard negative mining","Reranking","Domain adaptation"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.acl-industry.72", "url": "https://doi.org/10.18653/v1/2025.acl-industry.72" } , "abstract": "Enterprise search systems often struggle to retrieve accurate, domain-specific information due to semantic mismatches and overlapping terminologies. These issues can degrade the performance of downstream applications such as knowledge management, customer support, and retrieval-augmented generation agents. To address this challenge, we propose a scalable hard-negative mining framework tailored specifically for domain-specific enterprise data. Our approach dynamically selects semantically challenging but contextually irrelevant documents to enhance deployed re-ranking models.Our method integrates diverse embedding models, performs dimensionality reduction, and uniquely selects hard negatives, ensuring computational efficiency and semantic precision. Evaluation on our proprietary enterprise corpus (cloud services domain) demonstrates substantial improvements of 15% in MRR@3 and 19% in MRR@10 compared to state-of-the-art baselines and other negative sampling techniques. Further validation on public domain-specific datasets (FiQA, Climate Fever, TechQA) confirms our method’s generalizability and readiness for real-world applications." } }, { "@type": "ListItem", "position": 31, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.acl-industry.51/#paper", "url": "https://aclanthology.org/2025.acl-industry.51/", "name": "SpeechWeave: Diverse Multilingual Synthetic Text & Audio Data Generation Pipeline for Training Text to Speech Models", "description": "ACL'25.", "author": [ { "@type": "Person", "name": "Karan Dua" }, { "@type": "Person", "name": "Puneet Mittal" }, { "@type": "Person", "name": "Ranjeet Gupta" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["Synthetic data","Multilingual speech","Text-to-speech"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.acl-industry.51", "url": "https://doi.org/10.18653/v1/2025.acl-industry.51" } , "abstract": "High-quality Text-to-Speech (TTS) model training requires extensive and diverse text and speech data. It is challenging to procure such data from real sources due to issues of domain specificity, licensing, and scalability. Large language models (LLMs) can certainly generate textual data, but they create repetitive text with insufficient variation in the prompt during the generation process. Another important aspect in TTS training data is text normalization. Tools for normalization might occasionally introduce anomalies or overlook valuable patterns, and thus impact data quality. Furthermore, it is also impractical to rely on voice artists for large scale speech recording in commercial TTS systems with standardized voices. To address these challenges, we propose SpeechWeave, a synthetic speech data generation pipeline that is capable of automating the generation of multilingual, domain-specific datasets for training TTS models. Our experiments reveal that our pipeline generates data that is 10–48% more diverse than the baseline across various linguistic and phonetic metrics, along with speaker-standardized speech audio while generating approximately 97% correctly normalized text. Our approach enables scalable, high-quality data generation for TTS training, improving diversity, normalization, and voice consistency in the generated datasets." } }, { "@type": "ListItem", "position": 32, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.naacl-industry.46.pdf#paper", "url": "https://aclanthology.org/2025.naacl-industry.46.pdf", "name": "SweEval: Do LLMs Really Swear? A Safety Benchmark for Testing Limits for Enterprise Use", "description": "NAACL'25.", "author": [ { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Arion Das" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Tejaswini Kumar" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["AI safety","Multilingual evaluation","Offensive language"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.naacl-industry.46", "url": "https://doi.org/10.18653/v1/2025.naacl-industry.46" } , "abstract": "Enterprise customers are increasingly adopting Large Language Models (LLMs) for critical communication tasks, such as drafting emails, crafting sales pitches, and composing casual messages. Deploying such models across different regions requires them to understand diverse cultural and linguistic contexts and generate safe and respectful responses. For enterprise applications, it is crucial to mitigate reputational risks, maintain trust, and ensure compliance by effectively identifying and handling unsafe or offensive language. To address this, we introduce SweEval, a benchmark simulating real-world scenarios with variations in tone (positive or negative) and context (formal or informal). The prompts explicitly instruct the model to include specific swear words while completing the task. This benchmark evaluates whether LLMs comply with or resist such inappropriate instructions and assesses their alignment with ethical frameworks, cultural nuances, and language comprehension capabilities. In order to advance research in building ethically aligned AI systems for enterprise use and beyond, we release the dataset and code: https://github.com/amitbcp/multilingual_profanity ." , "sameAs": "https://aclanthology.org/2025.naacl-industry.46/" } }, { "@type": "ListItem", "position": 33, "item": { "@type": "ScholarlyArticle", "@id": "https://aclanthology.org/2025.knowledgenlp-1.20.pdf#paper", "url": "https://aclanthology.org/2025.knowledgenlp-1.20.pdf", "name": "Hybrid AI for Responsive Multi-Turn Online Conversations with Novel Dynamic Routing and Feedback Adaptation", "description": "NAACL'25.", "author": [ { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hansa Meghwani" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Srikant Panda" } ] , "keywords": ["Retrieval-augmented generation","Conversational AI","Dynamic routing"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.18653/v1/2025.knowledgenlp-1.20", "url": "https://doi.org/10.18653/v1/2025.knowledgenlp-1.20" } , "abstract": "Retrieval-Augmented Generation (RAG) systems and large language model (LLM)-powered chatbots have significantly advanced conversational AI by combining generative capabilities with external knowledge retrieval. Despite their success, enterprise-scale deployments face critical challenges, including diverse user queries, high latency, hallucinations, and difficulty integrating frequently updated domain-specific knowledge. This paper introduces a novel hybrid framework that integrates RAG with intent-based canned responses, leveraging predefined high-confidence responses for efficiency while dynamically routing complex or ambiguous queries to the RAG pipeline. Our framework employs a dialogue context manager to ensure coherence in multi-turn interactions and incorporates a feedback loop to refine intents, dynamically adjust confidence thresholds, and expand response coverage over time. Experimental results demonstrate that the proposed framework achieves a balance of high accuracy (95%) and low latency (180ms), outperforming RAG and intent-based systems across diverse query types, positioning it as a scalable and adaptive solution for enterprise conversational AI applications." , "sameAs": "https://aclanthology.org/2025.knowledgenlp-1.20/" } }, { "@type": "ListItem", "position": 34, "item": { "@type": "ScholarlyArticle", "@id": "https://link.springer.com/chapter/10.1007/978-981-95-3827-0_21#paper", "url": "https://link.springer.com/chapter/10.1007/978-981-95-3827-0_21", "name": "Instance-Aware Test-Time Adaptation for Domain Generalization", "description": "DASFAA'25.", "author": [ { "@type": "Person", "name": "Taki Hasan Rafi" }, { "@type": "Person", "name": "Serbeter Karlo" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Dong-Kyu Chae" } ] , "keywords": ["Test-time adaptation","Domain generalization","Distribution shift"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1007/978-981-95-3827-0_21", "url": "https://doi.org/10.1007/978-981-95-3827-0_21" } , "abstract": "Domain generalization (DG) aims to enhance the generalization capability of models to unseen target distributions by leveraging multiple source distributions. This paper focuses on robust test-time adaptation (TTA) based DG that is mostly needed when the model experiences new unseen domains while testing. We consider two practical challenges under domain shifts: (1) existing DG methods mostly rely on domain-specific information and do not explicitly utilize class-specific information. Therefore, these approaches ignore mixed features, which are both class and domain-specific, thus resulting in the loss of useful information; (2) while existing TTA methods explicitly require a memory bank of test time samples, which is computationally complex and impractical in many applications. To overcome these limitations, we propose a new framework called IADG that utilizes class-specific information along with domain-specific information to ensure robust generalization. Our method exploits disentangled features by pulling class-relevant features to increase diversified negative pairs, facilitating flawless integration of class and domain-specific features. To leverage high-confidence samples during testing, we introduce a novel confidence-guided low-risk instance TTA approach that only considers one unlabeled sample during inference and therefore does not require a dedicated memory bank. Extensive evaluations on five public benchmarks consistently demonstrate the superior performance of our approach over the state-of-the-art. Project code can be found here: https://github.com/takihasan/IADG ." } }, { "@type": "ListItem", "position": 35, "item": { "@type": "ScholarlyArticle", "@id": "https://ieeexplore.ieee.org/document/11103625#paper", "url": "https://ieeexplore.ieee.org/document/11103625", "name": "Tokenization Matters: Improving Zero-Shot NER for Indic Languages", "description": "IEEE'25.", "author": [ { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" } ] , "keywords": ["Tokenization","Named entity recognition","Indic languages"] , "abstract": "Tokenization is a critical component of Natural Language Processing (NLP), especially for low resource languages, where subword segmentation influences vocabulary structure and downstream task accuracy. Although Byte Pair Encoding (BPE) is a standard tokenization method in multilingual language models, its suitability for Named Entity Recognition (NER) in low resource Indic languages remains underexplored due to its limitations in handling morphological complexity. In this work, we systematically compare BPE, SentencePiece, and Character Level tokenization strategies using IndicBERT for NER tasks in low resource Indic languages like Assamese, Bengali, Marathi, and Odia, as well as extremely low resource Indic languages like Santali, Manipuri, and Sindhi. We assess both intrinsic linguistic properties tokenization efficiency, out of vocabulary (OOV) rates, and morphological preservation as well as extrinsic downstream performance, including fine tuning and zero shot cross lingual transfer. Our experiments show that SentencePiece is a consistently better performing approach than BPE for NER in low resource Indic Languages, particularly in zero shot cross lingual settings, as it better preserves entity consistency. While BPE provides the most compact tokenization form, it is not capable of generalization because it misclassifies or even fails to recognize entity labels when tested on unseen languages. In contrast, SentencePiece constitutes a better linguistic structural preservation model, benefiting extremely low resource and morphologically rich Indic languages, such as Santali and Manipuri, for superior entity recognition, as well as high generalization across scripts, such as Sindhi, written in Arabic. The results point to SentencePiece as the more effective tokenization strategy for NER within multilingual and low resource Indic NLP applications." , "sameAs": "https://arxiv.org/abs/2504.16977" } }, { "@type": "ListItem", "position": 36, "item": { "@type": "ScholarlyArticle", "@id": "https://ieeexplore.ieee.org/document/11103631#paper", "url": "https://ieeexplore.ieee.org/document/11103631", "name": "Clinical QA 2.0: Multi-Task Learning for Answer Extraction and Categorization", "description": "IEEE'25.", "author": [ { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Tejaswini Kumar" } ] , "keywords": ["Clinical question answering","Multi-task learning","Answer extraction"] , "abstract": "Clinical Question Answering (CQA) plays a crucial role in medical decision-making, enabling physicians to extract relevant information from Electronic Medical Records (EMRs). While transformer-based models such as BERT, BioBERT, and ClinicalBERT have demonstrated state-of-the-art performance in CQA, existing models lack the ability to categorize extracted answers, which is critical for structured retrieval, content filtering, and medical decision support. To address this limitation, we introduce a Multi-Task Learning (MTL) framework that jointly trains CQA models for both answer extraction and medical categorization. In addition to predicting answer spans, our model classifies responses into five standardized medical categories: Diagnosis, Medication, Symptoms, Procedure, and Lab Reports. This categorization enables more structured and interpretable outputs, making clinical QA models more useful in real-world healthcare settings. We evaluate our approach on emrQA, a large-scale dataset for medical question answering. Results show that MTL improves F1-score by 2.2% compared to standard fine-tuning, while achieving 90.7% accuracy in answer categorization. These findings suggest that MTL not only enhances CQA performance but also introduces an effective mechanism for categorization and structured medical information retrieval." , "sameAs": "https://arxiv.org/abs/2502.13108" } }, { "@type": "ListItem", "position": 37, "item": { "@type": "ScholarlyArticle", "@id": "https://link.springer.com/chapter/10.1007/978-981-95-0183-0_33#paper", "url": "https://link.springer.com/chapter/10.1007/978-981-95-0183-0_33", "name": "LLM for Barcodes: Generating Diverse Synthetic Data for Identity Documents", "description": "Springer'24.", "author": [ { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Karan Gupta" }, { "@type": "Person", "name": "Priyaranjan Pattnayak" } ] , "keywords": ["Synthetic data","Barcode detection","Identity documents"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1007/978-981-95-0183-0_33", "url": "https://doi.org/10.1007/978-981-95-0183-0_33" } , "abstract": "Accurate detection and decoding of barcodes in identity documents is crucial for applications such as security, healthcare, and education, where reliable data extraction and verification are essential. However, building robust detection models is challenging due to the lack of diverse and realistic datasets, an issue often tied to privacy concerns, and the wide variety of document formats. Traditional tools like Faker rely on predefined templates, which makes them less effective in capturing the complexity of real-world identity documents. In this paper, we introduce a new approach to synthetic data generation that uses LLMs to create contextually rich and realistic data without relying on a predefined field. Using the vast knowledge LLMs have about different documents and content, our method creates data that reflect the variety found in real identity documents. This data is then encoded into barcode and overlayed on identity document templates. Our approach simplifies the process of creating a dataset, eliminating the need for extensive domain knowledge or pre-defined fields. Compared to traditional methods such as Faker, LLM-generated data demonstrates greater diversity and contextual relevance, leading to improved performance in barcode detection models. Our experiments show that using LLM-generated data improves the detection mAP@0.5 by 4.2% , making it a scalable, privacy-first solution that significantly enhance machine learning for automated document processing and identity verification." } }, { "@type": "ListItem", "position": 38, "item": { "@type": "ScholarlyArticle", "@id": "https://ieeexplore.ieee.org/abstract/document/10609903#paper", "url": "https://ieeexplore.ieee.org/abstract/document/10609903", "name": "Leveraging Machine Learning for Network Redundancy Optimization, Cost Reduction, and Latency Improvement in IoT Systems", "description": "IEEE'24.", "author": [ { "@type": "Person", "name": "Manvendra Sharma" }, { "@type": "Person", "name": "Karan Gupta" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" } ] , "keywords": ["Internet of Things","Network optimization","Machine learning"] , "abstract": "This paper introduces a novel approach for optimizing Internet of Things (IoT) network performance by employing machine learning techniques to predict network types based on measurable properties of network traffic. While traditional studies have primarily focused on either reducing latency or enhancing network redundancy, our work integrates these aspects by dynamically predicting network conditions that facilitate the selection of optimal communication pathways. This methodology not only significantly reduces latency and improves redundancy but also enhances the overall efficiency and reliability of IoT networks. By analyzing a synthesized dataset reflective of real-world IoT environments, our model achieves high accuracy in network-type prediction, demonstrating its potential for real-time network management applications. Furthermore, the study delves into error analysis and feature sensitivity, providing insights into the reliability of network predictions and their implications for network design and operation. This paper aims to serve as a cornerstone for future explorations into optimizing IoT networks, setting the stage for more resilient, efficient, and responsive IoT ecosystems." } }, { "@type": "ListItem", "position": 39, "item": { "@type": "ScholarlyArticle", "@id": "https://link.springer.com/chapter/10.1007/978-3-032-14041-8_4#paper", "url": "https://link.springer.com/chapter/10.1007/978-3-032-14041-8_4", "name": "Survey of Large Multimodal Model Datasets, Application Categories and Taxonomy", "description": "Springer'24.", "author": [ { "@type": "Person", "name": "Priyaranjan Pattnayak" }, { "@type": "Person", "name": "Hitesh Laxmichand Patel", "@id": "https://hiteshlpatel.github.io/#person" }, { "@type": "Person", "name": "Bhargava Kumar" }, { "@type": "Person", "name": "Amit Agarwal" }, { "@type": "Person", "name": "Ishan Banerjee" }, { "@type": "Person", "name": "Srikant Panda" }, { "@type": "Person", "name": "Tejaswini Kumar" } ] , "keywords": ["Multimodal models","Dataset taxonomy","Multimodal datasets"] , "identifier": { "@type": "PropertyValue", "propertyID": "DOI", "value": "10.1007/978-3-032-14041-8_4", "url": "https://doi.org/10.1007/978-3-032-14041-8_4" } , "abstract": "Multimodal learning, a rapidly evolving field in artificial intelligence, seeks to construct more versatile and robust systems by integrating and analyzing diverse types of data, including text, images, audio, and video. Inspired by the human ability to assimilate information through many senses, this method enables applications such as text-to-video conversion, visual question answering, and image captioning. Recent developments in datasets that support multimodal language models (MLLMs) are highlighted in this overview. Large-scale multimodal datasets are essential because they allow for thorough testing and training of these models. With an emphasis on their contributions to the discipline, the study examines a variety of datasets, including those for training, domain-specific tasks, and real-world applications. This survey not only catalogs multimodal datasets but also critiques prior dataset surveys that often lack comprehensive categorization or discussion on emerging dataset trends. We introduce a three-tier classification: training-specific, task-specific, and domain-specific, which offers a more functional perspective on dataset applicability. It also emphasizes how crucial benchmark datasets are for assessing models’ performance in a range of scenarios, scalability, and applicability. Since multimodal learning is always changing, overcoming these obstacles will help AI research and applications reach new heights." } } ] }