[ { "id": "tree-prompting", "title": "Tree Prompting: Efficient Task Adaptation without Fine-Tuning", "authors": [ { "given": "John X.", "family": "Morris" }, { "given": "Chandan", "family": "Singh" }, { "given": "Alexander M.", "family": "Rush" }, { "given": "Jianfeng", "family": "Gao" }, { "given": "Yuntian", "family": "Deng" } ], "year": 2023, "venue": "EMNLP 2023", "paper_url": "https://arxiv.org/html/2310.14034v1", "summary": "学习可重复使用的决策树，其内部判断由接收提示词的语言模型调用完成；候选提示包括少样本示例、指令和动态生成的提示。", "connection": "基于提示的决策树原始工作：通过修改树结构和提示词适配任务，同时保持语言模型参数不变。", "bibtex_type": "inproceedings", "citation_key": "treeprompting", "source_urls": [ "https://arxiv.org/html/2310.14034v1" ], "pdf_url": "https://aclanthology.org/2023.emnlp-main.384.pdf", "code_url": "https://github.com/csinva/tree-prompt", "doi": "10.18653/v1/2023.emnlp-main.384", "equal_contribution": [ "John X. Morris", "Chandan Singh" ], "arxiv_id": "2310.14034", "arxiv_first_posted": "2023-10-21", "abstract": "Prompting language models (LMs) is the main interface for applying them to new tasks. However, for smaller LMs, prompting provides low accuracy compared to gradient-based fine-tuning. Tree Prompting is an approach to prompting which builds a decision tree of prompts, linking multiple LM calls together to solve a task. At inference time, each call to the LM is determined by efficiently routing the outcome of the previous call using the tree. Experiments on classification datasets show that Tree Prompting improves accuracy over competing methods and is competitive with fine-tuning. We also show that variants of Tree Prompting allow inspection of a model’s decision-making process.", "abstract_source": "https://arxiv.org/html/2310.14034v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/tree-prompting/" }, { "id": "aug-tree", "title": "Augmenting interpretable models with large language models during training", "authors": [ { "given": "Chandan", "family": "Singh" }, { "given": "Armin", "family": "Askari" }, { "given": "Rich", "family": "Caruana" }, { "given": "Jianfeng", "family": "Gao" } ], "year": 2023, "venue": "Nature Communications 14, 7913", "paper_url": "https://www.nature.com/articles/s41467-023-43713-1", "summary": "Aug-Tree 使用语言模型生成的扩展内容，为树的划分构造可解释文本特征。", "connection": "与 Tree Prompting 遍历树时调用语言模型的方式不同，它在训练阶段完成特征扩展。", "bibtex_type": "article", "citation_key": "augtree", "source_urls": [ "https://www.nature.com/articles/s41467-023-43713-1" ], "first_public": "2022-09-23", "doi": "10.1038/s41467-023-43713-1", "container_title": "Nature Communications", "volume": "14", "issue": "1", "article_number": "7913", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/aug-tree/" }, { "id": "gptree", "title": "GPTree: Towards Explainable Decision-Making via LLM-powered Decision Trees", "authors": [ { "given": "Sichao", "family": "Xiong" }, { "given": "Yigit", "family": "Ihlamur" }, { "given": "Fuat", "family": "Alican" }, { "given": "Aaron Ontoyin", "family": "Yin" } ], "year": 2024, "venue": "arXiv:2411.08257", "paper_url": "https://arxiv.org/html/2411.08257v1", "summary": "使用语言模型辅助的划分构建决策树，并结合专家反馈修改决策路径。", "connection": "后续的大语言模型决策树系统，支持专家交互及多种划分类型。", "bibtex_type": "article", "citation_key": "gptree", "source_urls": [ "https://arxiv.org/html/2411.08257v1" ], "arxiv_id": "2411.08257", "arxiv_first_posted": "2024-11-13", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/gptree/" }, { "id": "act", "title": "ACT: Agentic Classification Tree", "authors": [ { "given": "Vincent", "family": "Grari" }, { "given": "Tim", "family": "Arni" }, { "given": "Thibault", "family": "Laugel" }, { "given": "Sylvain", "family": "Lamprier" }, { "given": "James", "family": "Zou" }, { "given": "Marcin", "family": "Detyniecki" } ], "year": 2025, "venue": "arXiv:2509.26433", "paper_url": "https://arxiv.org/html/2509.26433v4", "summary": "通过基于不纯度的划分选择与 TextGrad 优化，学习由自然语言二元问题组成的分类树。", "connection": "后续以智能体方式学习问题决策树的工作，将问题优化与树结构学习结合。", "bibtex_type": "article", "citation_key": "act", "source_urls": [ "https://arxiv.org/html/2509.26433v4" ], "arxiv_id": "2509.26433", "arxiv_first_posted": "2025-09-30", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/act/" }, { "id": "residual-ebm", "title": "Residual Energy-Based Models for Text Generation", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Anton", "family": "Bakhtin" }, { "given": "Myle", "family": "Ott" }, { "given": "Arthur", "family": "Szlam" }, { "given": "Marc’Aurelio", "family": "Ranzato" } ], "year": 2020, "venue": "ICLR 2020", "paper_url": "https://arxiv.org/html/2004.11714v1", "summary": "将固定的自回归语言模型与学到的序列级能量结合，使用 NCE 训练、基于重要性重采样的条件生成，以及似然上下界。", "connection": "原始文本生成与评估框架，将序列级残差能量与采用能量修正的扩散解码联系起来。", "bibtex_type": "inproceedings", "citation_key": "residualebm", "source_urls": [ "https://arxiv.org/html/2004.11714v1" ], "pdf_url": "https://arxiv.org/pdf/2004.11714", "code_url": "https://github.com/da03/Residual-EBM", "arxiv_id": "2004.11714", "arxiv_first_posted": "2020-04-22", "abstract": "Text generation is ubiquitous in many NLP tasks, from summarization, to dialogue and machine translation. The dominant parametric approach is based on locally normalized models which predict one word at a time. While these work remarkably well, they are plagued by exposure bias due to the greedy nature of the generation process. In this work, we investigate un-normalized energy-based models (EBMs) which operate not at the token but at the sequence level. In order to make training tractable, we first work in the residual of a pretrained locally normalized language model and second we train using noise contrastive estimation. Furthermore, since the EBM works at the sequence level, we can leverage pretrained bi-directional contextual representations, such as BERT and RoBERTa. Our experiments on two large language modeling datasets show that residual EBMs yield lower perplexity compared to locally normalized baselines. Moreover, generation via importance sampling is very efficient and of higher quality than the baseline models according to human evaluation.", "abstract_source": "https://arxiv.org/html/2004.11714v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/residual-ebm/" }, { "id": "residual-ebm-jmlr", "title": "Residual Energy-Based Models for Text", "authors": [ { "given": "Anton", "family": "Bakhtin" }, { "given": "Yuntian", "family": "Deng" }, { "given": "Sam", "family": "Gross" }, { "given": "Myle", "family": "Ott" }, { "given": "Marc’Aurelio", "family": "Ranzato" }, { "given": "Arthur", "family": "Szlam" } ], "year": 2021, "venue": "JMLR 22(40):1–41", "paper_url": "https://www.jmlr.org/papers/v22/20-326.html", "summary": "研究如何从统计上区分人类文本与模型文本，并把学到的判别器用于全局归一化生成。", "connection": "进一步讨论从判别器到生成器的联系，以及残差能量建模。", "bibtex_type": "article", "citation_key": "residualebmjmlr", "source_urls": [ "https://www.jmlr.org/papers/v22/20-326.html" ], "pdf_url": "https://www.jmlr.org/papers/volume22/20-326/20-326.pdf", "equal_contribution": [ "Anton Bakhtin", "Yuntian Deng" ], "container_title": "Journal of Machine Learning Research", "volume": "22", "issue": "40", "pages": "1–41", "abstract": "Current large-scale auto-regressive language models display impressive fluency and can generate convincing text. In this work we start by asking the question: Can the generations of these models be reliably distinguished from real text by statistical discriminators? We find experimentally that the answer is affirmative when we have access to the training data for the model, and guardedly affirmative even if we do not. This suggests that the auto-regressive models can be improved by incorporating the (globally normalized) discriminators into the generative process. We give a formalism for this using the Energy-Based Model framework, and show that it indeed improves the results of the generative models, measured both in terms of perplexity and in terms of human evaluation.", "abstract_source": "https://www.jmlr.org/papers/v22/20-326.html", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/residual-ebm-jmlr/" }, { "id": "edlm", "title": "Energy-Based Diffusion Language Models for Text Generation", "authors": [ { "given": "Minkai", "family": "Xu" }, { "given": "Tomas", "family": "Geffner" }, { "given": "Karsten", "family": "Kreis" }, { "given": "Weili", "family": "Nie" }, { "given": "Yilun", "family": "Xu" }, { "given": "Jure", "family": "Leskovec" }, { "given": "Stefano", "family": "Ermon" }, { "given": "Arash", "family": "Vahdat" } ], "year": 2025, "venue": "ICLR 2025", "paper_url": "https://arxiv.org/html/2410.21357v4", "summary": "在每一步扩散去噪分布中加入完整序列的残差能量，采用自回归模型或 NCE 参数化，并通过并行重要性采样生成文本。", "connection": "直接引用 Deng 等人（2020），将残差修正与配分函数估计用于扩散语言建模。", "bibtex_type": "inproceedings", "citation_key": "edlm", "source_urls": [ "https://arxiv.org/html/2410.21357v4" ], "arxiv_id": "2410.21357", "arxiv_first_posted": "2024-10-28", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/edlm/" }, { "id": "unified-energy", "title": "Unified Energy for Invariant and Independent Decoding in Diffusion Language Models", "authors": [ { "given": "Yuchen", "family": "Yan" }, { "given": "Minkai", "family": "Xu" }, { "given": "Zaiquan", "family": "Yang" }, { "given": "Yatao", "family": "Bian" } ], "year": 2026, "venue": "arXiv:2606.09159", "paper_url": "https://arxiv.org/html/2606.09159v1", "summary": "结合针对扩散解码中依赖关系与不变性的能量修正。", "connection": "通过统一处理解码误差，扩展当前采用能量修正的扩散研究方向。", "bibtex_type": "article", "citation_key": "unifiedenergy", "source_urls": [ "https://arxiv.org/html/2606.09159v1" ], "arxiv_id": "2606.09159", "arxiv_first_posted": "2026-06-08", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/unified-energy/" }, { "id": "neural-trf-nce", "title": "Learning Neural Trans-Dimensional Random Field Language Models with Noise-Contrastive Estimation", "authors": [ { "given": "Bin", "family": "Wang" }, { "given": "Zhijian", "family": "Ou" } ], "year": 2018, "venue": "ICASSP 2018", "paper_url": "https://arxiv.org/abs/1710.10739", "summary": "对参考分布进行指数倾斜，并用噪声对比估计训练神经随机场语言模型。", "connection": "2020 年论文明确讨论的更早残差形式与 NCE 训练方法，其评估以语音识别重评分为中心。", "bibtex_type": "inproceedings", "citation_key": "neuraltrfnce", "source_urls": [ "https://arxiv.org/abs/1710.10739" ], "arxiv_id": "1710.10739", "arxiv_first_posted": "2017-10-30", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/neural-trf-nce/" }, { "id": "cascaded2020", "title": "Cascaded Text Generation with Markov Transformers", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2020, "venue": "NeurIPS 2020", "paper_url": "https://arxiv.org/abs/2006.01112", "pdf_url": "https://papers.nips.cc/paper/2020/file/01a0683665f38d8e5e567b3b15ca98bf-Paper.pdf", "code_url": "https://github.com/harvardnlp/cascaded-generation", "summary": "用于并行文本生成的高阶 CRF 级联模型，由一个 Markov Transformer 统一参数化。", "connection": "在保持结构化推断可计算的同时增强输出依赖，与扩散解码内部的联合建模相连接。", "bibtex_type": "inproceedings", "citation_key": "deng2020cascaded", "source_urls": [ "https://arxiv.org/abs/2006.01112", "https://arxiv.org/html/2006.01112v2" ], "arxiv_id": "2006.01112", "arxiv_first_posted": "2020-06-01", "abstract": "The two dominant approaches to neural text generation are fully autoregressive models, using serial beam search decoding, and non-autoregressive models, using parallel decoding with no output dependencies. This work proposes an autoregressive model with sub-linear parallel time generation. Noting that conditional random fields with bounded context can be decoded in parallel, we propose an efficient cascaded decoding approach for generating high-quality output. To parameterize this cascade, we introduce a Markov transformer, a variant of the popular fully autoregressive model that allows us to simultaneously decode with specific autoregressive context cutoffs. This approach requires only a small modification from standard autoregressive training, while showing competitive accuracy/speed tradeoff compared to existing methods on five machine translation datasets.", "abstract_source": "https://arxiv.org/html/2006.01112v2", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/cascaded2020/" }, { "id": "lattice2021", "title": "Sequence-to-Lattice Models for Fast Translation", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2021, "venue": "Findings of EMNLP 2021", "paper_url": "https://aclanthology.org/2021.findings-emnlp.318/", "pdf_url": "https://aclanthology.org/2021.findings-emnlp.318.pdf", "code_url": "https://github.com/harvardnlp/cascaded-generation", "doi": "10.18653/v1/2021.findings-emnlp.318", "summary": "用候选查找、并行条件边打分和精确 Viterbi 解码取代深层翻译解码器。", "connection": "其“候选—边—路径”架构连接了连贯的并行草稿生成；其目标与推断方式不同于投机解码中的前缀选择。", "bibtex_type": "inproceedings", "citation_key": "deng-rush-2021-sequence", "source_urls": [ "https://aclanthology.org/2021.findings-emnlp.318/", "https://aclanthology.org/2021.findings-emnlp.318.pdf" ], "abstract": "Non-autoregressive machine translation (NAT) approaches enable fast generation by utilizing parallelizable generative processes. The remaining bottleneck in these models is their decoder layers; unfortunately unlike in autoregressive models (Kasai et al., 2020), removing decoder layers from NAT models significantly degrades accuracy. This work proposes a sequence-to-lattice model that replaces the decoder with a search lattice. Our approach first constructs a candidate lattice using efficient lookup operations, generates lattice scores from a deep encoder, and finally finds the best path using dynamic programming. Experiments on three machine translation datasets show that our method is faster than past non-autoregressive generation approaches, and more accurate than naively reducing the number of decoder layers.", "abstract_source": "https://aclanthology.org/2021.findings-emnlp.318/", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/lattice2021/" }, { "id": "sun2019", "title": "Fast Structured Decoding for Sequence Models", "authors": [ { "given": "Zhiqing", "family": "Sun" }, { "given": "Zhuohan", "family": "Li" }, { "given": "Haoqing", "family": "Wang" }, { "given": "Di", "family": "He" }, { "given": "Zi", "family": "Lin" }, { "given": "Zhi-Hong", "family": "Deng" } ], "year": 2019, "venue": "NeurIPS 2019", "paper_url": "https://proceedings.neurips.cc/paper_files/paper/2019/hash/74563ba21a90da13dacf2a73e3ddefa7-Abstract.html", "pdf_url": "https://proceedings.neurips.cc/paper_files/paper/2019/file/74563ba21a90da13dacf2a73e3ddefa7-Paper.pdf", "summary": "为非自回归序列模型加入近似一阶 CRF 推断和动态转移。", "connection": "一阶结构化解码的参考工作；Cascaded Text Generation 发展了高阶级联和共享的 Markov Transformer 势函数。", "bibtex_type": "inproceedings", "citation_key": "sun2019fast", "source_urls": [ "https://proceedings.neurips.cc/paper_files/paper/2019/hash/74563ba21a90da13dacf2a73e3ddefa7-Abstract.html", "https://arxiv.org/abs/1910.11555" ], "arxiv_id": "1910.11555", "arxiv_first_posted": "2019-10-25", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/sun2019/" }, { "id": "codd2026", "title": "Breaking the Factorization Barrier in Diffusion Language Models", "authors": [ { "given": "Ian", "family": "Li" }, { "given": "Zilei", "family": "Shao" }, { "given": "Benjie", "family": "Wang" }, { "given": "Rose", "family": "Yu" }, { "given": "Guy", "family": "Van den Broeck" }, { "given": "Anji", "family": "Liu" } ], "year": 2026, "venue": "ICML 2026", "paper_url": "https://proceedings.mlr.press/v306/li26fd.html", "summary": "CoDD 将因子化扩散预测与可进行高效推断的概率电路先验组合，建模 token 之间的联合依赖。", "connection": "与结构化生成一样，将并行神经网络打分和联合概率推断分开处理。", "bibtex_type": "inproceedings", "citation_key": "li2026factorization", "source_urls": [ "https://proceedings.mlr.press/v306/li26fd.html", "https://arxiv.org/abs/2603.00045", "https://arxiv.org/html/2603.00045v3" ], "arxiv_id": "2603.00045", "arxiv_first_posted": "2026-02-09", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/codd2026/" }, { "id": "dadlm2026", "title": "DA-DLM: Explicitly Modeling Token Dependencies in Diffusion Language Models", "authors": [ { "given": "Pengyu", "family": "Ji" }, { "given": "Zichen", "family": "Zhang" }, { "given": "Xiang", "family": "Hu" }, { "given": "Kewei", "family": "Tu" } ], "year": 2026, "venue": "EMNLP 2026", "paper_url": "https://aclanthology.org/2026.emnlp-main.2094/", "pdf_url": "https://aclanthology.org/2026.emnlp-main.2094.pdf", "doi": "10.18653/2026.emnlp-main.2094", "summary": "通过按输出位置组织的候选组和已确定 token 构成的锚点，把有向无环图依赖建模用于迭代扩散。", "connection": "把非自回归翻译中的结构化输出建模直接迁移到扩散去噪。", "bibtex_type": "inproceedings", "citation_key": "ji2026dadlm", "source_urls": [ "https://aclanthology.org/2026.emnlp-main.2094/", "https://aclanthology.org/2026.emnlp-main.2094.pdf", "https://arxiv.org/abs/2609.15070" ], "arxiv_id": "2609.15070", "arxiv_first_posted": "2026-09-14", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/dadlm2026/" }, { "id": "lilicorr2026", "title": "LiLiCorr: Lightweight Likelihood Correlation of Parallel Drafts for Speculative Decoding", "authors": [ { "given": "Matan", "family": "Rusanovsky" }, { "given": "Yoav", "family": "Miron" }, { "given": "Roy", "family": "Uziel" }, { "given": "Omer", "family": "Belhasin" }, { "given": "Hao", "family": "Guo" }, { "given": "Ran", "family": "Zilberstein" }, { "given": "Maor", "family": "Ashkenazi" }, { "given": "Michael", "family": "Elad" } ], "year": 2026, "venue": "arXiv 2026", "paper_url": "https://arxiv.org/abs/2608.20530", "summary": "对并行草稿的 top-K 候选计算相邻兼容性，再通过低开销的贪心路径选择得到草稿。", "connection": "采用“候选—边—路径”结构，并使用针对投机解码前缀接受目标的局部训练与推断。", "bibtex_type": "article", "citation_key": "rusanovsky2026lilicorr", "source_urls": [ "https://arxiv.org/abs/2608.20530", "https://arxiv.org/html/2608.20530v2" ], "arxiv_id": "2608.20530", "arxiv_first_posted": "2026-08-20", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/lilicorr2026/" }, { "id": "diffuspec2026", "title": "DiffuSpec: Unlocking Diffusion Language Models for Speculative Decoding", "authors": [ { "given": "Guanghao", "family": "Li" }, { "given": "Zhihui", "family": "Fu" }, { "given": "Min", "family": "Fang" }, { "given": "Qibin", "family": "Zhao" }, { "given": "Ming", "family": "Tang" }, { "given": "Chun", "family": "Yuan" }, { "given": "Jun", "family": "Wang" } ], "year": 2026, "venue": "Findings of ACL 2026", "paper_url": "https://aclanthology.org/2026.findings-acl.1048/", "pdf_url": "https://aclanthology.org/2026.findings-acl.1048.pdf", "doi": "10.18653/v1/2026.findings-acl.1048", "summary": "结合扩散草稿生成、token 候选格上的 Causal-consistency Path Search，以及自适应草稿长度。", "connection": "把候选格路径选择与对并行扩散提议的自回归验证相连接。", "bibtex_type": "inproceedings", "citation_key": "li2026diffuspec", "source_urls": [ "https://aclanthology.org/2026.findings-acl.1048/", "https://aclanthology.org/2026.findings-acl.1048.pdf" ], "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/diffuspec2026/" }, { "id": "latentattention2018", "title": "Latent Alignment and Variational Attention", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Yoon", "family": "Kim" }, { "given": "Justin", "family": "Chiu" }, { "given": "Demi", "family": "Guo" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2018, "venue": "NeurIPS 2018", "paper_url": "https://arxiv.org/abs/1807.03756", "pdf_url": "https://proceedings.neurips.cc/paper/2018/file/b691334ccf10d4ab144d672f7783c8a3-Paper.pdf", "code_url": "https://github.com/harvardnlp/var-attn/", "summary": "将对齐建模为隐变量，并通过变分注意力学习近似后验推断。", "connection": "对注意力建立概率分布的一般框架，超越单个确定性注意力向量。", "bibtex_type": "inproceedings", "citation_key": "deng2018latent", "source_urls": [ "https://arxiv.org/abs/1807.03756", "https://arxiv.org/html/1807.03756v2" ], "equal_contribution": [ "Yuntian Deng", "Yoon Kim" ], "arxiv_id": "1807.03756", "arxiv_first_posted": "2018-07-10", "abstract": "Neural attention has become central to many state-of-the-art models in natural language processing and related domains. Attention networks are an easy-to-train and effective method for softly simulating alignment; however, the approach does not marginalize over latent alignments in a probabilistic sense. This property makes it difficult to compare attention to other alignment approaches, to compose it with probabilistic models, and to perform posterior inference conditioned on observed data. A related latent approach, hard attention, fixes these issues, but is generally harder to train and less accurate. This work considers variational attention networks, alternatives to soft and hard attention for learning latent variable alignment models, with tighter approximation bounds based on amortized variational inference. We further propose methods for reducing the variance of gradients to make these approaches computationally feasible. Experiments show that for machine translation and visual question answering, inefficient exact latent variable models outperform standard neural attention, but these gains go away when using hard attention based training. On the other hand, variational attention retains most of the performance gain but with training speed comparable to neural attention.", "abstract_source": "https://arxiv.org/html/1807.03756v2", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/latentattention2018/" }, { "id": "showattendtell2015", "title": "Show, Attend and Tell: Neural Image Caption Generation with Visual Attention", "authors": [ { "given": "Kelvin", "family": "Xu" }, { "given": "Jimmy Lei", "family": "Ba" }, { "given": "Ryan", "family": "Kiros" }, { "given": "Kyunghyun", "family": "Cho" }, { "given": "Aaron", "family": "Courville" }, { "given": "Ruslan", "family": "Salakhutdinov" }, { "given": "Richard S.", "family": "Zemel" }, { "given": "Yoshua", "family": "Bengio" } ], "year": 2015, "venue": "ICML 2015", "paper_url": "https://proceedings.mlr.press/v37/xuc15.html", "pdf_url": "https://proceedings.mlr.press/v37/xuc15.pdf", "summary": "为图像描述生成引入确定性软视觉注意力和随机硬视觉注意力。", "connection": "随机注意力的早期参考工作，用于理解期望特征与采样选择之间的区别。", "bibtex_type": "inproceedings", "citation_key": "xu2015show", "source_urls": [ "https://proceedings.mlr.press/v37/xuc15.pdf", "https://arxiv.org/abs/1502.03044" ], "arxiv_id": "1502.03044", "arxiv_first_posted": "2015-02-10", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/showattendtell2015/" }, { "id": "structuredattention2017", "title": "Structured Attention Networks", "authors": [ { "given": "Yoon", "family": "Kim" }, { "given": "Carl", "family": "Denton" }, { "given": "Luong", "family": "Hoang" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2017, "venue": "ICLR 2017", "paper_url": "https://arxiv.org/abs/1702.00887", "summary": "使用可微图模型推断，对子序列和句法树进行注意力计算。", "connection": "建模结构化选择并计算期望表示；对完整预测进行边缘化是不同的操作。", "bibtex_type": "inproceedings", "citation_key": "kim2017structured", "source_urls": [ "https://arxiv.org/abs/1702.00887", "https://arxiv.org/html/1702.00887v3" ], "equal_contribution": [ "Yoon Kim", "Carl Denton" ], "arxiv_id": "1702.00887", "arxiv_first_posted": "2017-02-03", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/structuredattention2017/" }, { "id": "bahuleyan2018", "title": "Variational Attention for Sequence-to-Sequence Models", "authors": [ { "given": "Hareesh", "family": "Bahuleyan" }, { "given": "Lili", "family": "Mou" }, { "given": "Olga", "family": "Vechtomova" }, { "given": "Pascal", "family": "Poupart" } ], "year": 2018, "venue": "COLING 2018", "paper_url": "https://aclanthology.org/C18-1142/", "pdf_url": "https://aclanthology.org/C18-1142.pdf", "summary": "在变分编码器—解码器中，把注意力上下文向量视为高斯隐变量。", "connection": "说明“变分注意力”这一名称也用于另一个随机对象：上下文向量。", "bibtex_type": "inproceedings", "citation_key": "bahuleyan2018variational", "source_urls": [ "https://aclanthology.org/C18-1142/", "https://arxiv.org/abs/1712.08207" ], "arxiv_id": "1712.08207", "arxiv_first_posted": "2017-12-21", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/bahuleyan2018/" }, { "id": "bayesianattention2020", "title": "Bayesian Attention Modules", "authors": [ { "given": "Xinjie", "family": "Fan" }, { "given": "Shujian", "family": "Zhang" }, { "given": "Bo", "family": "Chen" }, { "given": "Mingyuan", "family": "Zhou" } ], "year": 2020, "venue": "NeurIPS 2020", "paper_url": "https://arxiv.org/abs/2010.10604", "pdf_url": "https://proceedings.neurips.cc/paper_files/paper/2020/file/bcff3f632fd16ff099a49c2f0932b47a-Paper.pdf", "summary": "由归一化的可重参数化随机变量构造注意力权重分布，并使用贝叶斯正则化学习。", "connection": "把概率注意力发展为带有数据依赖先验的模块化随机权重。", "bibtex_type": "inproceedings", "citation_key": "fan2020bayesian", "source_urls": [ "https://arxiv.org/abs/2010.10604", "https://proceedings.neurips.cc/paper_files/paper/2020/file/bcff3f632fd16ff099a49c2f0932b47a-Paper.pdf" ], "equal_contribution": [ "Xinjie Fan", "Shujian Zhang" ], "arxiv_id": "2010.10604", "arxiv_first_posted": "2020-10-20", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/bayesianattention2020/" }, { "id": "model-criticism", "title": "Model Criticism for Long-Form Text Generation", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Volodymyr", "family": "Kuleshov" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2022, "venue": "EMNLP 2022", "paper_url": "https://aclanthology.org/2022.emnlp-main.815/", "pdf_url": "https://aclanthology.org/2022.emnlp-main.815.pdf", "code_url": "https://github.com/da03/criticize_text_generation", "doi": "10.18653/v1/2022.emnlp-main.815", "summary": "通过在显式生成式检验模型定义的潜在空间中比较真实数据与生成数据，评估文本的高层结构。实验研究篇章连贯性、共指和主题性。", "connection": "诊断生成文本结构分布的原始框架，将结构评估与词元层面的流畅度分开。", "bibtex_type": "inproceedings", "citation_key": "deng-etal-2022-model", "source_urls": [ "https://aclanthology.org/2022.emnlp-main.815/", "https://arxiv.org/html/2210.08444v1" ], "arxiv_id": "2210.08444", "arxiv_first_posted": "2022-10-16", "abstract": "Language models have demonstrated the ability to generate highly fluent text; however, it remains unclear whether their output retains coherent high-level structure (e.g., story progression). Here, we propose to apply a statistical tool, model criticism in latent space, to evaluate the high-level structure of the generated text. Model criticism compares the distributions between real and generated data in a latent space obtained according to an assumptive generative process. Different generative processes identify specific failure modes of the underlying model. We perform experiments on three representative aspects of high-level discourse—coherence, coreference, and topicality—and find that transformer-based language models are able to capture topical structures but have a harder time maintaining structural coherence or modeling coreference.", "abstract_source": "https://arxiv.org/html/2210.08444v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/model-criticism/" }, { "id": "openmeva", "title": "OpenMEVA: A Benchmark for Evaluating Open-ended Story Generation Metrics", "authors": [ { "given": "Jian", "family": "Guan" }, { "given": "Zhexin", "family": "Zhang" }, { "given": "Zhuoer", "family": "Feng" }, { "given": "Zitao", "family": "Liu" }, { "given": "Wenbiao", "family": "Ding" }, { "given": "Xiaoxi", "family": "Mao" }, { "given": "Changjie", "family": "Fan" }, { "given": "Minlie", "family": "Huang" } ], "year": 2021, "venue": "ACL-IJCNLP 2021", "paper_url": "https://aclanthology.org/2021.acl-long.500/", "pdf_url": "https://aclanthology.org/2021.acl-long.500.pdf", "doi": "10.18653/v1/2021.acl-long.500", "summary": "检验自动故事评估指标的基准，涵盖与人工判断的相关性、泛化、稳健性，以及对不同连贯性错误的敏感程度。", "connection": "检验评估器能否发现已知的故事缺陷，与在潜在空间中诊断生成器输出分布的方法互补。", "bibtex_type": "inproceedings", "citation_key": "guan-etal-2021-openmeva", "source_urls": [ "https://aclanthology.org/2021.acl-long.500/" ], "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/openmeva/" }, { "id": "story-evaluation-survey", "title": "What Makes a Good Story and How Can We Measure It? A Comprehensive Survey of Story Evaluation", "authors": [ { "given": "Dingyi", "family": "Yang" }, { "given": "Qin", "family": "Jin" } ], "year": 2024, "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2408.14622", "pdf_url": "https://arxiv.org/pdf/2408.14622", "doi": "10.48550/arXiv.2408.14622", "summary": "综述故事质量标准、人工评估、自动指标和评估基准。", "connection": "将结构诊断放在连贯性、人物发展与叙事质量的更广泛评估背景中。", "bibtex_type": "article", "citation_key": "yang2024goodstory", "source_urls": [ "https://arxiv.org/abs/2408.14622", "https://arxiv.org/pdf/2408.14622" ], "arxiv_id": "2408.14622", "arxiv_first_posted": "2024-08-26", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/story-evaluation-survey/" }, { "id": "markup-diffusion", "title": "Markup-to-Image Diffusion Models with Scheduled Sampling", "authors": [ { "given": "Yuntian", "family": "Deng" }, { "given": "Noriyuki", "family": "Kojima" }, { "given": "Alexander M.", "family": "Rush" } ], "year": 2023, "venue": "ICLR 2023", "paper_url": "https://arxiv.org/abs/2210.05147", "pdf_url": "https://arxiv.org/pdf/2210.05147", "code_url": "https://github.com/da03/markup2im", "doi": "10.48550/arXiv.2210.05147", "summary": "通过条件扩散学习渲染 LaTeX、HTML 表格布局、LilyPond 乐谱和 SMILES 分子。Scheduled Sampling 让去噪器接触自己生成的输入，以处理逐步累积的错误。", "connection": "原始扩散式标记语言到图像方法，以及覆盖四个领域的渲染基准。", "bibtex_type": "inproceedings", "citation_key": "deng2023markuptoimage", "source_urls": [ "https://arxiv.org/html/2210.05147v1", "https://github.com/da03/markup2im" ], "arxiv_id": "2210.05147", "arxiv_first_posted": "2022-10-11", "abstract": "Building on recent advances in image generation, we present a fully data-driven approach to rendering markup into images. The approach is based on diffusion models, which parameterize the distribution of data using a sequence of denoising operations on top of a Gaussian noise distribution. We view the diffusion denoising process as a sequential decision making process, and show that it exhibits compounding errors similar to exposure bias issues in imitation learning problems. To mitigate these issues, we adapt the scheduled sampling algorithm to diffusion training. We conduct experiments on four markup datasets: mathematical formulas (LaTeX), table layouts (HTML), sheet music (LilyPond), and molecular images (SMILES). These experiments each verify the effectiveness of the diffusion process and the use of scheduled sampling to fix generation issues. These results also show that the markup-to-image task presents a useful controlled compositional setting for diagnosing and analyzing generative image models.", "abstract_source": "https://arxiv.org/html/2210.05147v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/markup-diffusion/" }, { "id": "fsa-cdm", "title": "Contrast-augmented Diffusion Model with Fine-grained Sequence Alignment for Markup-to-Image Generation", "authors": [ { "given": "Guojin", "family": "Zhong" }, { "given": "Jin", "family": "Yuan" }, { "given": "Pan", "family": "Wang" }, { "given": "Kailun", "family": "Yang" }, { "given": "Weili", "family": "Guan" }, { "given": "Zhiyong", "family": "Li" } ], "year": 2023, "venue": "ACM Multimedia 2023", "paper_url": "https://arxiv.org/abs/2308.01147", "pdf_url": "https://arxiv.org/pdf/2308.01147", "summary": "FSA-CDM 将细粒度跨模态序列对齐、上下文建模和对比增强的扩散目标结合，用于标记语言到图像生成。", "connection": "这项后续工作沿用 Deng、Kojima 和 Rush 提出的任务与四个数据集，研究对齐与对比反馈。", "bibtex_type": "inproceedings", "citation_key": "zhong2023contrastaugmented", "source_urls": [ "https://arxiv.org/html/2308.01147v1" ], "arxiv_id": "2308.01147", "arxiv_first_posted": "2023-08-02", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/fsa-cdm/" }, { "id": "diffusion-input-perturbation", "title": "Input Perturbation Reduces Exposure Bias in Diffusion Models", "authors": [ { "given": "Mang", "family": "Ning" }, { "given": "Enver", "family": "Sangineto" }, { "given": "Angelo", "family": "Porrello" }, { "given": "Simone", "family": "Calderara" }, { "given": "Rita", "family": "Cucchiara" } ], "year": 2023, "venue": "ICML 2023", "paper_url": "https://proceedings.mlr.press/v202/ning23a.html", "pdf_url": "https://proceedings.mlr.press/v202/ning23a/ning23a.pdf", "summary": "研究扩散模型采样过程中累积的错误，并在训练中扰动输入，以模拟推理时的预测误差。", "connection": "通过合成输入扰动处理同一种训练与采样不匹配问题；Scheduled Sampling 则在模型生成的状态上训练。", "bibtex_type": "inproceedings", "citation_key": "pmlr-v202-ning23a", "source_urls": [ "https://proceedings.mlr.press/v202/ning23a.html" ], "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/diffusion-input-perturbation/" }, { "id": "interactive-training-2025", "title": "Interactive Training: Feedback-Driven Neural Network Optimization", "authors": [ { "given": "Wentao", "family": "Zhang" }, { "given": "Yang Young", "family": "Lu" }, { "given": "Yuntian", "family": "Deng" } ], "year": 2025, "venue": "EMNLP 2025 System Demonstrations", "paper_url": "https://aclanthology.org/2025.emnlp-demos.65/", "pdf_url": "https://aclanthology.org/2025.emnlp-demos.65.pdf", "code_url": "https://github.com/yuntian-group/interactive-training", "doi": "10.18653/v1/2025.emnlp-demos.65", "summary": "一个开源框架，让人和 AI 智能体通过控制服务器干预正在进行的神经网络训练，修改优化器设置、训练数据和模型检查点。", "connection": "原始 Interactive Training 系统将实时监控与对运行中训练过程的反馈控制连接起来。", "bibtex_type": "inproceedings", "citation_key": "zhang-etal-2025-interactive", "source_urls": [ "https://aclanthology.org/2025.emnlp-demos.65/", "https://arxiv.org/html/2510.02297v1" ], "arxiv_id": "2510.02297", "arxiv_first_posted": "2025-10-02", "abstract": "Traditional neural network training typically follows fixed, predefined optimization recipes, lacking the flexibility to dynamically respond to instabilities or emerging training issues. In this paper, we introduce Interactive Training, an open-source framework that enables real-time, feedback-driven intervention during neural network training by human experts or automated AI agents. At its core, Interactive Training uses a control server to mediate communication between users or agents and the ongoing training process, allowing users to dynamically adjust optimizer hyperparameters, training data, and model checkpoints. Through three case studies, we demonstrate that Interactive Training achieves superior training stability, reduced sensitivity to initial hyperparameters, and improved adaptability to evolving user needs, paving the way toward a future training paradigm where AI agents autonomously monitor training logs, proactively resolve instabilities, and optimize training dynamics.", "abstract_source": "https://arxiv.org/html/2510.02297v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/interactive-training-2025/" }, { "id": "interactive-training-2026", "title": "Interactive Training 2: Auditable Control Plane for Live Model Training", "authors": [ { "given": "Wentao", "family": "Zhang" }, { "given": "Xuanhe", "family": "Pan" }, { "given": "Han", "family": "Zhou" }, { "given": "Yang", "family": "Lu" }, { "given": "Yuntian", "family": "Deng" } ], "year": 2026, "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2607.18314", "pdf_url": "https://arxiv.org/pdf/2607.18314", "code_url": "https://github.com/yuntian-group/interactive-training", "doi": "10.48550/arXiv.2607.18314", "summary": "可复用的协议，提供带类型的设置、结构化动作、由应用选择的控制点，以及由人、脚本和 LLM 控制器共用的有序日志。", "connection": "将 2025 年框架从依赖特定 Trainer 的命令接口，扩展为不同训练流程共用的控制协议。", "bibtex_type": "article", "citation_key": "zhang2026interactivetraining2", "source_urls": [ "https://arxiv.org/abs/2607.18314", "https://arxiv.org/html/2607.18314v1", "https://github.com/yuntian-group/interactive-training" ], "equal_contribution": [ "Wentao Zhang", "Xuanhe Pan", "Han Zhou" ], "arxiv_id": "2607.18314", "arxiv_first_posted": "2026-07-17", "publication_type": "preprint", "container_title": "arXiv", "abstract": "Experiment trackers show how training is progressing, but changing a live run still usually requires trainer-specific code. We present Interactive Training 2, an open-source control plane for steering training through a shared protocol. Training applications declare which settings and actions they expose, humans and automated controllers submit requests through the same interface, and the training loop validates and applies them at safe control points. A customized Aim workspace combines live metrics and controls with a chronological record of requests and outcomes. We demonstrate the system across five NLP and reinforcement-learning workflows. The released code and traces provide a reusable foundation for auditable human- and agent-guided training.", "abstract_source": "https://arxiv.org/html/2607.18314v1", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/interactive-training-2026/" }, { "id": "livetune", "title": "LiveTune: Dynamic Parameter Tuning for Feedback-Driven Optimization", "authors": [ { "given": "Soheil Zibakhsh", "family": "Shabgahi" }, { "given": "Nojan", "family": "Sheybani" }, { "given": "Aiden", "family": "Tabrizi" }, { "given": "Farinaz", "family": "Koushanfar" } ], "year": 2023, "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2311.17279", "pdf_url": "https://arxiv.org/pdf/2311.17279", "doi": "10.48550/arXiv.2311.17279", "summary": "LiveVariables 通过指定的系统端口开放优化参数，支持在不重启优化循环的情况下实时调整参数。", "connection": "更早的实时优化参数调节系统；Interactive Training 将人和智能体的命令接入训练，第二版提供共用的控制协议。", "bibtex_type": "article", "citation_key": "shabgahi2023livetune", "source_urls": [ "https://arxiv.org/abs/2311.17279", "https://arxiv.org/html/2311.17279v2" ], "arxiv_id": "2311.17279", "arxiv_first_posted": "2023-11-28", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/livetune/" }, { "id": "ai-training-manager", "title": "AI Training Manager: Bounded Closed-Loop Control of Adaptive Training Recipes", "authors": [ { "given": "Anjali", "family": "Rao" }, { "given": "Nikhil Kamalkumar", "family": "Advani" } ], "year": 2026, "venue": "arXiv", "paper_url": "https://arxiv.org/abs/2606.29871", "pdf_url": "https://arxiv.org/pdf/2606.29871", "doi": "10.48550/arXiv.2606.29871", "summary": "异步 LLM 监督控制器观察训练遥测数据，对正在执行的训练配置提出受约束的修改，并在刻意引入的监督学习与强化学习失败模式上实验。", "connection": "研究对正在进行的训练实施自动控制，与 Interactive Training 为人和自动化控制器提供的可复用基础设施互补。", "bibtex_type": "article", "citation_key": "rao2026aitrainingmanager", "source_urls": [ "https://arxiv.org/abs/2606.29871", "https://arxiv.org/html/2606.29871v3" ], "arxiv_id": "2606.29871", "arxiv_first_posted": "2026-06-29", "publication_type": "preprint", "container_title": "arXiv", "updated_at": "2026-11-11", "description_language": "zh-Hans", "abstract_language": "en", "record_url": "https://yuntiandeng.com/research/zh/papers/ai-training-manager/" } ]