publications | Maxime Peyrard

2024

Evaluating Language Model Agency through Negotiations

Tim R. Davidson, Veniamin Veselovsky, Martin Josifoski, and 4 more authors

ICLR, 2024

@article{davidson2024evaluating,
  title = {Evaluating Language Model Agency through Negotiations},
  author = {Davidson, Tim R. and Veselovsky, Veniamin and Josifoski, Martin and Peyrard, Maxime and Bosselut, Antoine and Kosinski, Michal and West, Robert},
  year = {2024},
  journal = {ICLR},
}

2023

A Glitch in the Matrix? Locating and Detecting Language Model Grounding with Fakepedia

Giovanni Monea, Maxime Peyrard, Martin Josifoski, and 6 more authors

2023

arXiv Bib PDF

@article{monea2023glitch,
  title = {A Glitch in the Matrix? Locating and Detecting Language Model Grounding with Fakepedia},
  author = {Monea, Giovanni and Peyrard, Maxime and Josifoski, Martin and Chaudhary, Vishrav and Eisner, Jason and Kıcıman, Emre and Palangi, Hamid and Patra, Barun and West, Robert},
  year = {2023},
  eprint = {2312.02073},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Flows: Building Blocks of Reasoning and Collaborating AI

Martin Josifoski, Lars Klein, Maxime Peyrard, and 7 more authors

2023

arXiv Bib PDF Code

@article{josifoski2023flows,
  title = {Flows: Building Blocks of Reasoning and Collaborating AI},
  author = {Josifoski, Martin and Klein, Lars and Peyrard, Maxime and Li, Yifei and Geng, Saibo and Schnitzler, Julian Paul and Yao, Yuxing and Wei, Jiheng and Paul, Debjit and West, Robert},
  year = {2023},
  eprint = {2308.01285},
  archiveprefix = {arXiv},
  primaryclass = {cs.AI},
}

Grammar-Constrained Decoding for Structured NLP Tasks without Finetuning

Saibo Geng, Martin Josifosky, Maxime Peyrard, and 1 more author

In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, 2023

arXiv Bib PDF Code

@inproceedings{geng2023grammarconstrained,
  title = {Grammar-Constrained Decoding for Structured NLP Tasks without Finetuning},
  author = {Geng, Saibo and Josifosky, Martin and Peyrard, Maxime and West, Robert},
  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
  year = {2023},
  address = {Singapore},
  publisher = {Association for Computational Linguistics},
  url = {https://aclanthology.org/2023.emnlp-main.674},
  doi = {10.18653/v1/2023.emnlp-main.674},
  pages = {10932--10952},
  eprint = {2305.13971},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Fly-Swat or Cannon? Cost-Effective Language Model Choice via Meta-Modeling

Marija Šakota, Maxime Peyrard, and Robert West

In Proceedings of The International ACM Conference on Web Search and Data Mining (WSDM), 2023

arXiv Bib PDF Code

@inproceedings{šakota2023flyswat,
  title = {Fly-Swat or Cannon? Cost-Effective Language Model Choice via Meta-Modeling},
  author = {Šakota, Marija and Peyrard, Maxime and West, Robert},
  booktitle = {Proceedings of The International ACM Conference on Web Search and Data Mining (WSDM)},
  year = {2023},
  eprint = {2308.06077},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Exploiting Asymmetry for Synthetic Training Data Generation: SynthIE and the Case of Information Extraction

Martin Josifoski, Marija Sakota, Maxime Peyrard, and 1 more author

In Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, 2023

arXiv Bib PDF Code

@inproceedings{josifoski-etal-2023-exploiting,
  title = {Exploiting Asymmetry for Synthetic Training Data Generation: {S}ynth{IE} and the Case of Information Extraction},
  author = {Josifoski, Martin and Sakota, Marija and Peyrard, Maxime and West, Robert},
  editor = {Bouamor, Houda and Pino, Juan and Bali, Kalika},
  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
  year = {2023},
  address = {Singapore},
  publisher = {Association for Computational Linguistics},
  url = {https://aclanthology.org/2023.emnlp-main.96},
  doi = {10.18653/v1/2023.emnlp-main.96},
  pages = {1555--1574},
}

REFINER: Reasoning Feedback on Intermediate Representations

Debjit Paul, Mete Ismayilzada, Maxime Peyrard, and 4 more authors

2023

arXiv Bib PDF Code

@misc{paul2023refiner,
  title = {REFINER: Reasoning Feedback on Intermediate Representations},
  author = {Paul, Debjit and Ismayilzada, Mete and Peyrard, Maxime and Borges, Beatriz and Bosselut, Antoine and West, Robert and Faltings, Boi},
  eprint = {2304.01904},
  journal = {arXiv preprint arXiv:2304.01904},
  url = {https://arxiv.org/pdf/2304.01904.pdf},
  year = {2023},
}

Descartes: Generating Short Descriptions of Wikipedia Articles

Marija Sakota, Maxime Peyrard, and Robert West

In Proceedings of The Web Conference (WWW), 2023

arXiv Bib PDF Code

@inproceedings{10.1145/3543507.3583220,
  author = {Sakota, Marija and Peyrard, Maxime and West, Robert},
  title = {Descartes: Generating Short Descriptions of Wikipedia Articles},
  year = {2023},
  booktitle = {Proceedings of The Web Conference (WWW)},
  isbn = {9781450394161},
  publisher = {Association for Computing Machinery},
  address = {New York, NY, USA},
  url = {https://doi.org/10.1145/3543507.3583220},
  doi = {10.1145/3543507.3583220},
  pages = {1446–1456},
}

Distribution Inference Risks: Identifying and Mitigating Sources of Leakage

V. Hartmann, L. Meynent, M. Peyrard, and 3 more authors

In 2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), 2023

Abs arXiv Bib PDF Code

A large body of work shows that machine learning (ML) models can leak sensitive or confidential information about their training data. Recently, leakage due to distribution inference (or property inference) attacks is gaining attention. In this attack, the goal of an adversary is to infer distributional information about the training data. So far, research on distribution inference has focused on demonstrating successful attacks, with little attention given to identifying the potential causes of the leakage and to proposing mitigations. To bridge this gap, as our main contribution, we theoretically and empirically analyze the sources of information leakage that allows an adversary to perpetrate distribution inference attacks. We identify three sources of leakage: (1) memorizing specific information about the \mathbbE[Y\vert X] (expected label given the feature values) of interest to the adversary, (2) wrong inductive bias of the model, and (3) finiteness of the training data. Next, based on our analysis, we propose principled mitigation techniques against distribution inference attacks. Specifically, we demonstrate that causal learning techniques are more resilient to a particular type of distribution inference risk termed distributional membership inference than associative learning methods. And lastly, we present a formalization of distribution inference that allows for reasoning about more general adversaries than was previously possible.
@inproceedings{10136150, author = {Hartmann, V. and Meynent, L. and Peyrard, M. and Dimitriadis, D. and Tople, S. and West, R.}, booktitle = {2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)}, title = {Distribution Inference Risks: Identifying and Mitigating Sources of Leakage}, year = {2023}, volume = {}, issn = {}, pages = {136-149}, keywords = {learning systems;training;bridges;training data;machine learning;data models;libraries}, doi = {10.1109/SaTML54575.2023.00018}, url = {https://doi.ieeecomputersociety.org/10.1109/SaTML54575.2023.00018}, }
Language Model Decoding as Likelihood–Utility Alignment

Martin Josifoski, Maxime Peyrard, Frano Rajič, and 7 more authors

In Findings of the Association for Computational Linguistics: EACL 2023, May 2023

Abs arXiv Bib PDF Code

A critical component of a successful language generation pipeline is the decoding algorithm. However, the general principles that should guide the choice of a decoding algorithm remain unclear. Previous works only compare decoding algorithms in narrow scenarios, and their findings do not generalize across tasks. We argue that the misalignment between the model’s likelihood and the task-specific notion of utility is the key factor in understanding the effectiveness of decoding algorithms. To structure the discussion, we introduce a taxonomy of misalignment mitigation strategies (MMSs), providing a unifying view of decoding as a tool for alignment. The MMS taxonomy groups decoding algorithms based on their implicit assumptions about likelihood–utility misalignment, yielding general statements about their applicability across tasks. Specifically, by analyzing the correlation between the likelihood and the utility of predictions across a diverse set of tasks, we provide empirical evidence supporting the proposed taxonomy and a set of principles to structure reasoning when choosing a decoding algorithm. Crucially, our analysis is the first to relate likelihood-based decoding algorithms with algorithms that rely on external information, such as value-guided methods and prompting, and covers the most diverse set of tasks to date. Code, data, and models are available at \urlhttps://github.com/epfl-dlab/understanding-decoding.
@inproceedings{josifoski-etal-2023-language, title = {Language Model Decoding as Likelihood{--}Utility Alignment}, author = {Josifoski, Martin and Peyrard, Maxime and Raji{\v{c}}, Frano and Wei, Jiheng and Paul, Debjit and Hartmann, Valentin and Patra, Barun and Chaudhary, Vishrav and Kiciman, Emre and Faltings, Boi}, booktitle = {Findings of the Association for Computational Linguistics: EACL 2023}, month = may, year = {2023}, address = {Dubrovnik, Croatia}, publisher = {Association for Computational Linguistics}, url = {https://aclanthology.org/2023.findings-eacl.107}, doi = {10.18653/v1/2023.findings-eacl.107}, pages = {1455--1470}, }

2022

The Glass Ceiling of Automatic Evaluation in Natural Language Generation

Pierre Colombo, Maxime Peyrard, Nathan Noiry, and 2 more authors

May 2022
Invariant Language Modeling

Maxime Peyrard, Sarvjeet Ghotra, Martin Josifoski, and 6 more authors

In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, Dec 2022

Abs

Modern pretrained language models are critical components of NLP pipelines. Yet, they suffer from spurious correlations, poor out-of-domain generalization, and biases. Inspired by recent progress in causal machine learning, in particular the invariant risk minimization (IRM) paradigm, we propose invariant language modeling, a framework for learning invariant representations that generalize better across multiple environments. In particular, we adapt a game-theoretic implementation of IRM (IRM-games) to language models, where the invariance emerges from a specific training schedule in which all the environments compete to optimize their own environment-specific loss by updating subsets of the model in a round-robin fashion. We focused on controlled experiments to precisely demonstrate the ability of our method to (i) remove structured noise, (ii) ignore specific spurious correlations without affecting global performance, and (iii) achieve better out-of-domain generalization. These benefits come with a negligible computational overhead compared to standard training, do not require changing the local loss, and can be applied to any language model. We believe this framework is promising to help mitigate spurious correlations and biases in language models.
On the Context-Free Ambiguity of Emoji

Justyna Czestochowska, Kristina Gligoric, Maxime Peyrard, and 6 more authors

In Proceedings of the Sixteenth International AAAI Conference on Web and Social Media, ICWSM 2022, Atlanta, Georgia, USA, June 6-9, 2022, Dec 2022
Predicting Is Not Understanding: Recognizing And Addressing Underspecification In Machine Learning

Damien Teney, Maxime Peyrard, and Ehsan Abbasnejad

In Computer Vision – ECCV 2022: 17th European Conference, Tel Aviv, Israel, October 23–27, 2022, Proceedings, Part XXIII, Dec 2022

Abs

Machine learning (ML) models are typically optimized for their accuracy on a given dataset. However, this predictive criterion rarely captures all desirable properties of a model, in particular how well it matches a domain expert’s understanding of a task. Underspecification [6] refers to the existence of multiple models that are indistinguishable in their in-domain accuracy, even though they differ in other desirable properties such as out-of-distribution (OOD) performance. Identifying these situations is critical for assessing the reliability of ML models. We formalize the concept of underspecification and propose a method to identify and partially address it. We train multiple models with an independence constraint that forces them to implement different functions. They discover predictive features that are otherwise ignored by standard empirical risk minimization (ERM), which we then distill into a global model with superior OOD performance. Importantly, we constrain the models to align with the data manifold to ensure that they discover meaningful features. We demonstrate the method on multiple datasets in computer vision (collages, WILDS-Camelyon17, GQA) and discuss general implications of underspecification. Most notably, in-domain performance cannot serve for OOD model selection without additional assumptions (See for the full-length version of this work).
GenIE: Generative Information Extraction

Martin Josifoski, Nicola De Cao, Maxime Peyrard, and 2 more authors

In Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Jul 2022

Abs

Structured and grounded representation of text is typically formalized by closed information extraction, the problem of extracting an exhaustive set of (subject, relation, object) triplets that are consistent with a predefined set of entities and relations from a knowledge base schema. Most existing works are pipelines prone to error accumulation, and all approaches are only applicable to unrealistically small numbers of entities and relations. We introduce GenIE (generative information extraction), the first end-to-end autoregressive formulation of closed information extraction. GenIE naturally exploits the language knowledge from the pre-trained transformer by autoregressively generating relations and entities in textual form. Thanks to a new bi-level constrained generation strategy, only triplets consistent with the predefined knowledge base schema are produced. Our experiments show that GenIE is state-of-the-art on closed information extraction, generalizes from fewer training data points than baselines, and scales to a previously unmanageable number of entities and relations. With this work, closed information extraction becomes practical in realistic scenarios, providing new opportunities for downstream tasks. Finally, this work paves the way towards a unified end-to-end approach to the core tasks of information extraction.

2021

Laughing Heads: Can Transformers Detect What Makes a Sentence Funny?

Maxime Peyrard, Beatriz Borges, Kristina Gligoric, and 1 more author

In International Joint Conference on Artificial Intelligence, Jul 2021
Better than Average: Paired Evaluation of NLP systems

Maxime Peyrard, Wei Zhao, Steffen Eger, and 1 more author

In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers), Aug 2021

Abs

Evaluation in NLP is usually done by comparing the scores of competing systems independently averaged over a common set of test instances. In this work, we question the use of averages for aggregating evaluation scores into a final number used to decide which system is best, since the average, as well as alternatives such as the median, ignores the pairing arising from the fact that systems are evaluated on the same test instances. We illustrate the importance of taking the instancelevel pairing of evaluation scores into account and demonstrate, both theoretically and empirically, the advantages of aggregation methods based on pairwise comparisons, such as the Bradley–Terry (BT) model, a mechanism based on the estimated probability that a given system scores better than another on the test set. By re-evaluating 296 real NLP evaluation setups across four tasks and 18 evaluation metrics, we show that the choice of aggregation mechanism matters and yields different conclusions as to which systems are state of the art in about 30% of the setups. To facilitate the adoption of pairwise evaluation, we release a practical tool for performing the full analysis of evaluation scores with the mean, median, BT, and two variants of BT (Elo and TrueSkill), alongside functionality for appropriate statistical testing.
Sudden Attention Shifts on Wikipedia Following COVID-19 Mobility Restrictions

Manoel Horta Ribeiro, Kristina Gligorić, Maxime Peyrard, and 3 more authors

Aug 2021

Abs

We study how the COVID-19 pandemic, alongside the severe mobility restrictions that ensued, has impacted information access on Wikipedia, the world’s largest online encyclopedia. A longitudinal analysis that combines pageview statistics for 12 Wikipedia language editions with mobility reports published by Apple and Google reveals massive shifts in the volume and nature of information seeking patterns during the pandemic. Interestingly, while we observe a transient increase in Wikipedia’s pageview volume following mobility restrictions, the nature of information sought was impacted more permanently. These changes are most pronounced for language editions associated with countries where the most severe mobility restrictions were implemented. We also find that articles belonging to different topics behaved differently; e.g., attention towards entertainment-related topics is lingering and even increasing, while the interest in health- and biology-related topics was either small or transient. Our results highlight the utility of Wikipedia for studying how the pandemic is affecting people’s needs, interests, and concerns.

2020

KLearn: Background Knowledge Inference from Summarization Data

Maxime Peyrard, and Robert West

In Findings of the Association for Computational Linguistics: EMNLP 2020, Nov 2020

Abs

The goal of text summarization is to compress documents to the relevant information while excluding background information already known to the receiver. So far, summarization researchers have given considerably more attention to relevance than to background knowledge. In contrast, this work puts background knowledge in the foreground. Building on the realization that the choices made by human summarizers and annotators contain implicit information about their background knowledge, we develop and compare techniques for inferring background knowledge from summarization data. Based on this framework, we define summary scoring functions that explicitly model background knowledge, and show that these scoring functions fit human judgments significantly better than baselines. We illustrate some of the many potential applications of our framework. First, we provide insights into human information importance priors. Second, we demonstrate that averaging the background knowledge of multiple, potentially biased annotators or corpora greatly improves summaryscoring performance. Finally, we discuss potential applications of our framework beyond summarization.
Experts and authorities receive disproportionate attention on Twitter during the COVID-19 crisis

Kristina Gligorić, Manoel Horta Ribeiro, Martin Müller, and 5 more authors

Nov 2020
On the Limitations of Cross-lingual Encoders as Exposed by Reference-Free Machine Translation Evaluation

Wei Zhao, Goran Glavaš, Maxime Peyrard, and 3 more authors

In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, Jul 2020

Abs

Evaluation of cross-lingual encoders is usually performed either via zero-shot cross-lingual transfer in supervised downstream tasks or via unsupervised cross-lingual textual similarity. In this paper, we concern ourselves with reference-free machine translation (MT) evaluation where we directly compare source texts to (sometimes low-quality) system translations, which represents a natural adversarial setup for multilingual encoders. Reference-free evaluation holds the promise of web-scale comparison of MT systems. We systematically investigate a range of metrics based on state-of-the-art cross-lingual semantic representations obtained with pretrained M-BERT and LASER. We find that they perform poorly as semantic encoders for reference-free MT evaluation and identify their two key limitations, namely, (a) a semantic mismatch between representations of mutual translations and, more prominently, (b) the inability to punish “translationese”, i.e., low-quality literal translations. We propose two partial remedies: (1) post-hoc re-alignment of the vector spaces and (2) coupling of semantic-similarity based metrics with target-side language modeling. In segment-level MT evaluation, our best metric surpasses reference-based BLEU by 5.7 correlation points.
A Ladder of Causal Distances

Maxime Peyrard, and Robert West

In International Joint Conference on Artificial Intelligence, Jul 2020

2019

MoverScore: Text Generation Evaluating with Contextualized Embeddings and Earth Mover Distance

Wei Zhao, Maxime Peyrard, Fei Liu, and 3 more authors

In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), Nov 2019

Abs

A robust evaluation metric has a profound impact on the development of text generation systems. A desirable metric compares system output against references based on their semantics rather than surface forms. In this paper we investigate strategies to encode system and reference texts to devise a metric that shows a high correlation with human judgment of text quality. We validate our new metric, namely MoverScore, on a number of text generation tasks including summarization, machine translation, image captioning, and data-to-text generation, where the outputs are produced by a variety of neural and non-neural systems. Our findings suggest that metrics combining contextualized representations with a distance measure perform the best. Such metrics also demonstrate strong generalization capability across tasks. For ease-of-use we make our metrics available as web service.
A Simple Theoretical Model of Importance for Summarization

Maxime Peyrard

In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, Jul 2019

Abs

Research on summarization has mainly been driven by empirical approaches, crafting systems to perform well on standard datasets with the notion of information Importance remaining latent. We argue that establishing theoretical models of Importance will advance our understanding of the task and help to further improve summarization systems. To this end, we propose simple but rigorous definitions of several concepts that were previously used only intuitively in summarization: Redundancy, Relevance, and Informativeness. Importance arises as a single quantity naturally unifying these concepts. Additionally, we provide intuitions to interpret the proposed quantities and experiments to demonstrate the potential of the framework to inform and guide subsequent works.
Studying Summarization Evaluation Metrics in the Appropriate Scoring Range

Maxime Peyrard

In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, Jul 2019

Abs

In summarization, automatic evaluation metrics are usually compared based on their ability to correlate with human judgments. Unfortunately, the few existing human judgment datasets have been created as by-products of the manual evaluations performed during the DUC/TAC shared tasks. However, modern systems are typically better than the best systems submitted at the time of these shared tasks. We show that, surprisingly, evaluation metrics which behave similarly on these datasets (average-scoring range) strongly disagree in the higher-scoring range in which current systems now operate. It is problematic because metrics disagree yet we can’t decide which one to trust. This is a call for collecting human judgments for high-scoring summaries as this would resolve the debate over which metrics to trust. This would also be greatly beneficial to further improve summarization systems and metrics alike.

2018

Live Blog Corpus for Summarization

Avinesh P.V.S., Maxime Peyrard, and Christian M. Meyer

In Proceedings of the Eleventh International Conference on Language Resources and Evaluation (LREC 2018), May 2018
Concatenated Power Mean Word Embeddings as Universal Cross-Lingual Sentence Representations

Andreas Rücklé, Steffen Eger, Maxime Peyrard, and 1 more author

arXiv, May 2018
Objective Function Learning to Match Human Judgements for Optimization-Based Summarization

Maxime Peyrard, and Iryna Gurevych

In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers), Jun 2018

Abs

Supervised summarization systems usually rely on supervision at the sentence or n-gram level provided by automatic metrics like ROUGE, which act as noisy proxies for human judgments. In this work, we learn a summary-level scoring function θincluding human judgments as supervision and automatically generated data as regularization. We extract summaries with a genetic algorithm using θas a fitness function. We observe strong and promising performances across datasets in both automatic and manual evaluation.

2017

A Principled Framework for Evaluating Summarizers: Comparing Models of Summary Quality against Human Judgments

Maxime Peyrard, and Judith Eckle-Kohler

In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers), Jul 2017

Abs

We present a new framework for evaluating extractive summarizers, which is based on a principled representation as optimization problem. We prove that every extractive summarizer can be decomposed into an objective function and an optimization technique. We perform a comparative analysis and evaluation of several objective functions embedded in well-known summarizers regarding their correlation with human judgments. Our comparison of these correlations across two datasets yields surprising insights into the role and performance of objective functions in the different summarizers.
Learning to Score System Summaries for Better Content Selection Evaluation.

Maxime Peyrard, Teresa Botschen, and Iryna Gurevych

In Proceedings of the Workshop on New Frontiers in Summarization, Sep 2017

Abs

The evaluation of summaries is a challenging but crucial task of the summarization field. In this work, we propose to learn an automatic scoring metric based on the human judgements available as part of classical summarization datasets like TAC-2008 and TAC-2009. Any existing automatic scoring metrics can be included as features, the model learns the combination exhibiting the best correlation with human judgments. The reliability of the new metric is tested in a further manual evaluation where we ask humans to evaluate summaries covering the whole scoring spectrum of the metric. We release the trained metric as an open-source tool.
Supervised Learning of Automatic Pyramid for Optimization-Based Multi-Document Summarization

Maxime Peyrard, and Judith Eckle-Kohler

In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Jul 2017

Abs

We present a new supervised framework that learns to estimate automatic Pyramid scores and uses them for optimization-based extractive multi-document summarization. For learning automatic Pyramid scores, we developed a method for automatic training data generation which is based on a genetic algorithm using automatic Pyramid as the fitness function. Our experimental evaluation shows that our new framework significantly outperforms strong baselines regarding automatic Pyramid, and that there is much room for improvement in comparison with the upper-bound for automatic Pyramid.
LSDSem 2017: Exploring Data Generation Methods for the Story Cloze Test

Michael Bugert, Yevgeniy Puzikov, Andreas Rücklé, and 6 more authors

In Proceedings of the 2nd Workshop on Linking Models of Lexical, Sentential and Discourse-level Semantics, Apr 2017

Abs

The Story Cloze test is a recent effort in providing a common test scenario for text understanding systems. As part of the LSDSem 2017 shared task, we present a system based on a deep learning architecture combined with a rich set of manually-crafted linguistic features. The system outperforms all known baselines for the task, suggesting that the chosen approach is promising. We additionally present two methods for generating further training data based on stories from the ROCStories corpus.

2016

The Next Step for Multi-Document Summarization: A Heterogeneous Multi-Genre Corpus Built with a Novel Construction Approach

Markus Zopf, Maxime Peyrard, and Judith Eckle-Kohler

In Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers, Dec 2016

Abs

Research in multi-document summarization has focused on newswire corpora since the early beginnings. However, the newswire genre provides genre-specific features such as sentence position which are easy to exploit in summarization systems. Such easy to exploit genre-specific features are available in other genres as well. We therefore present the new hMDS corpus for multi-document summarization, which contains heterogeneous source documents from multiple text genres, as well as summaries with different lengths. For the construction of the corpus, we developed a novel construction approach which is suited to build large and heterogeneous summarization corpora with little effort. The method reverses the usual process of writing summaries for given source documents: it combines already available summaries with appropriate source documents. In a detailed analysis, we show that our new corpus is significantly different from the homogeneous corpora commonly used, and that it is heterogeneous along several dimensions. Our experimental evaluation using well-known state-of-the-art summarization systems shows that our corpus poses new challenges in the field of multi-document summarization. Last but not least, we make our corpus publicly available to the research community at the corpus web page \urlhttps://github.com/AIPHES/hMDS.
A General Optimization Framework for Multi-Document Summarization Using Genetic Algorithms and Swarm Intelligence

Maxime Peyrard, and Judith Eckle-Kohler

In Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers, Dec 2016

Abs

Extracting summaries via integer linear programming and submodularity are popular and successful techniques in extractive multi-document summarization. However, many interesting optimization objectives are neither submodular nor factorizable into an integer linear program. We address this issue and present a general optimization framework where any function of input documents and a system summary can be plugged in. Our framework includes two kinds of summarizers – one based on genetic algorithms, the other using a swarm intelligence approach. In our experimental evaluation, we investigate the optimization of two information-theoretic summary evaluation metrics and find that our framework yields competitive results compared to several strong summarization baselines. Our comparative analysis of the genetic and swarm summarizers reveals interesting complementary properties.
Optimizing an Approximation of ROUGE - a Problem-Reduction Approach to Extractive Multi-Document Summarization

Maxime Peyrard, and Judith Eckle-Kohler

In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Aug 2016