{"id":"utility-of-user-simulators","title":"Quantifying the Utility of User Simulators for Building Collaborative LLM Assistants","authors":[{"given":"Joseph","family":"Suh"},{"given":"Ayush","family":"Raj"},{"given":"Minwoo","family":"Kang"},{"given":"Serina","family":"Chang"}],"first_public":"2026-05-10","date_note":"First arXiv version.","date_source":"https://arxiv.org/abs/2605.09808","year":2026,"venue":"arXiv preprint","paper_url":"https://arxiv.org/abs/2605.09808","arxiv":"2605.09808","code_url":"https://github.com/schang-lab/utility-of-user-simulators","category":"evaluation","relationship":"uses","relationship_label":"Uses WildChat and WildBench","dataset_version":"WildChat-1M provides human utterances for user-simulator training; WildChat-4.8M and WildBench support evaluation and tests of generalization.","summary":"Tests whether learning a user simulator from real conversations helps train assistants that work better with people. The study fine-tunes simulators on WildChat user utterances, trains assistants through simulated interaction, and evaluates them on WildBench and in a study with 283 participants.","finding":"The assistant trained with the learned user simulator wins 58% of comparisons against the initial assistant in the human study; training with a role-playing model yields 51%.","evidence_url":"https://arxiv.org/html/2605.09808v1","evidence_note":"Sections 2–3 and Appendix C: WildChat preprocessing and simulator training. Section 5: human study.","bibtex_key":"suh2026utilityusersimulators","record_url":"https://yuntiandeng.com/wildchat/papers/utility-of-user-simulators/","citation_title":"Quantifying the Utility of User Simulators for Building Collaborative LLM Assistants","citation_type":"article-journal","bibtex_type":"misc","citation_url":"https://arxiv.org/abs/2605.09808","citation_year":2026,"citation_note":"arXiv preprint.","doi":"10.48550/arXiv.2605.09808"}