{
  "id": "model-criticism",
  "title": "Model Criticism for Long-Form Text Generation",
  "authors": [
    {
      "given": "Yuntian",
      "family": "Deng"
    },
    {
      "given": "Volodymyr",
      "family": "Kuleshov"
    },
    {
      "given": "Alexander M.",
      "family": "Rush"
    }
  ],
  "year": 2022,
  "venue": "EMNLP 2022",
  "paper_url": "https://aclanthology.org/2022.emnlp-main.815/",
  "pdf_url": "https://aclanthology.org/2022.emnlp-main.815.pdf",
  "code_url": "https://github.com/da03/criticize_text_generation",
  "doi": "10.18653/v1/2022.emnlp-main.815",
  "summary": "A framework for evaluating high-level text structure by comparing real and generated data in latent spaces defined by explicit generative critics. Experiments study discourse coherence, coreference, and topicality.",
  "connection": "The original method behind this guide: diagnostic evaluation of structural distributions, separate from word-level fluency.",
  "bibtex_type": "inproceedings",
  "citation_key": "deng-etal-2022-model",
  "source_urls": [
    "https://aclanthology.org/2022.emnlp-main.815/",
    "https://arxiv.org/html/2210.08444v1"
  ],
  "arxiv_id": "2210.08444",
  "arxiv_first_posted": "2022-10-16",
  "abstract": "Language models have demonstrated the ability to generate highly fluent text; however, it remains unclear whether their output retains coherent high-level structure (e.g., story progression). Here, we propose to apply a statistical tool, model criticism in latent space, to evaluate the high-level structure of the generated text. Model criticism compares the distributions between real and generated data in a latent space obtained according to an assumptive generative process. Different generative processes identify specific failure modes of the underlying model. We perform experiments on three representative aspects of high-level discourse—coherence, coreference, and topicality—and find that transformer-based language models are able to capture topical structures but have a harder time maintaining structural coherence or modeling coreference.",
  "abstract_source": "https://arxiv.org/html/2210.08444v1",
  "updated_at": "2026-11-10",
  "record_url": "https://yuntiandeng.com/research/papers/model-criticism/"
}