@proceedings {644586,
	title = {exBERT: A Visual Analysis Tool to Explore Learned Representations in Transformers Models},
	year = {Submitted},
	abstract = {Large language models can produce powerful contextual representations that lead to improvements across many NLP tasks. Since these models are typically guided by a sequence of learned self attention mechanisms and may comprise undesired inductive biases, it is paramount to be able to explore what the attention has learned. While static analyses of these models lead to targeted insights, interactive tools are more dynamic and can help humans better gain an intuition for the model-internal reasoning process. We present exBERT, an interactive tool named after the popular BERT language model, that provides insights into the meaning of the contextual representations by matching a human-specified input to similar contexts in a large annotated dataset. By aggregating the annotations of the matching similar contexts, exBERT helps intuitively explain what each attention-head has learned.\&nbsp;},
	author = {Ben Hoover and Hendrik Strobelt and Sebastian Gehrmann}
}
@manuascript {644585,
	title = {Encoder-Agnostic Adaptation for Conditional Language Generation},
	year = {Submitted},
	abstract = {Large pretrained language models have changed the way researchers approach discriminative natural language understanding tasks, leading to the dominance of approaches that adapt a pretrained model for arbitrary downstream tasks. However, it is an open question how to use similar techniques for language generation. Early results in the encoder-agnostic setting have been mostly negative. In this work, we explore methods for adapting a pretrained language model to arbitrary conditional input.\&nbsp;We observe that pretrained transformer models are sensitive to large parameter changes during tuning. Therefore, we propose an adaptation that directly injects arbitrary conditioning into self attention, an approach we call pseudo self attention. Through experiments on four diverse conditional text generation tasks, we show that this encoder-agnostic technique outperforms strong baselines, produces coherent generations, and is data-efficient.},
	author = {Zachary Ziegler and Luke Melas-Kyriazi and Sebastian Gehrmann and Alexander M. Rush}
}
@conference {644584,
	title = {Generating Abstractive Summaries with Finetuned Language Models},
	booktitle = {INLG 2019 (GenChal Track)},
	year = {2019},
	abstract = {Neural abstractive document summarization is commonly approached by models that exhibit a mostly extractive behavior. This behavior is facilitated by a copy-attention which allows models to copy words from a source document.\&nbsp;\&nbsp; While models in the mostly extractive news summarization domain benefit from this inductive bias, they commonly fail to paraphrase or compress information from the source document.\&nbsp; Recent advances in transfer-learning from large pretrained language models give rise to alternative approaches that do not rely on copy-attention and instead learn to generate concise and abstractive summaries.\&nbsp;\&nbsp; In this paper, as part of the TL;DR challenge, we compare the abstractiveness of summaries from different summarization approaches and show that transfer-learning can be efficiently utilized without any changes to the model architecture.\&nbsp;\&nbsp; We demonstrate that the approach leads to a higher level of abstraction for a similar performance on the TL;DR challenge tasks, enabling true natural language compression.},
	author = {Sebastian Gehrmann and Zachary Ziegler and Alexander M. Rush}
}
@article {644583,
	title = {Visual Interactions with Deep Models through Collaborative Semantic Inference},
	journal = {IEEE Transactions on Visualization and Computer Graphics},
	year = {2019},
	abstract = {Automation of tasks can have critical consequences when humans lose agency over decision processes. Deep learning models are particularly susceptible since current black-box approaches lack explainable reasoning. We argue that both the visual interface and model structure of deep learning systems need to take into account interaction design. We propose a framework of collaborative semantic inference (CSI) for the co-design of interactions and models to enable visual collaboration between humans and algorithms. The approach exposes the intermediate reasoning process of models which allows semantic interactions with the visual metaphors of a problem, which means that a user can both understand and control parts of the model reasoning process. We demonstrate the feasibility of CSI with a co-designed case study of a document summarization system.},
	url = {https://ieeexplore.ieee.org/abstract/document/8805457},
	author = {Sebastian Gehrmann and Hendrik Strobelt and Robert Kr{\"u}ger and Pfister, Hanspeter and Alexander M. Rush}
}
@article {644582,
	title = {Evaluating an Automated Mediator for Joint Narratives in a Conflict Situation},
	journal = {Behavior \& Information Technology},
	year = {2019},
	abstract = {Joint narratives are often used in the context of reconciliation interventions for people in social conflict situations, which arise, for example, due to ethnic or religious differences. The interventions aim to encourage a change in attitudes of the participants towards each other. Typically, a human mediator is fundamental for achieving a successful intervention. In this work, we present an automated approach to support remote interactions between pairs of participants as they contribute to a shared story in their own language. A key component is an automated cognitive tutor that guides the participants through a controlled escalation/de-escalation process during the development of a joint narrative. We performed a controlled study comparing a trained human mediator to the automated mediator. The results demonstrate that an automated mediator, although simple at this stage, effectively supports interactions and helps to achieve positive outcomes comparable to those attained by the trained human mediator.},
	url = {https://www.tandfonline.com/doi/abs/10.1080/0144929X.2019.1637940?journalCode=tbit20},
	author = {Massimo Zancanaro and Oliviero Stock and Gianluca Schiavo and Alessandro Cappelletti and Sebastian Gehrmann and Daphna Canetti and Ohad Shaked and Shani Fachter and Rachel Yifat and Ravit Mimran and Patrice L. Weiss}
}
@conference {644581,
	title = {GLTR: Statistical Detection and Visualization of Generated Text},
	booktitle = {ACL 2019 (Demo)},
	year = {2019},
	author = {Sebastian Gehrmann and Hendrik Strobelt and Alexander M. Rush}
}
@proceedings {644580,
	title = {LSTMs can Perform Dynamic Counting},
	journal = {Deep Learning and Formal Languages (ACL 2019 Workshop)},
	year = {2019},
	author = {Mirac S{\"u}zg{\"u}n and Sebastian Gehrmann and Yonatan Belinkov and Shieber, Stuart}
}
@proceedings {644579,
	title = {Interactive Visual Exploration of Latent Space (IVELS) for Peptide Auto-Encoder Model Selection},
	journal = {Deep Generative Models for Highly Structured Data (ICLR 2019 Workshop)},
	year = {2019},
	author = {Tom Sercu and Sebastian Gehrmann and Hendrik Strobelt and Payel Das and Inkit Padhi and Cicero Dos Santos and Kahini Wadhawan and Vijil Chenthamarakshan}
}
@proceedings {636669,
	title = {Improving Human Text Comprehension through Semi-Markov CRF-based Neural Section Title Generation},
	journal = {North American Chapter of the Association for Computational Linguistics},
	year = {2019},
	address = {Minneapolis},
	abstract = {Titles of short sections within long documents support readers by guiding their focus towards relevant passages and by providing anchor-points that help to understand the progression of the document. The positive effects of section titles are even more pronounced when measured on readers with less developed reading abilities, for example in communities with limited labeled text resources. We, therefore, aim to develop techniques to generate section titles in low-resource environments. In particular, we present an extractive pipeline for section title generation by first selecting the most salient sentence and then applying deletion-based compression. Our compression approach is based on a Semi-Markov Conditional Random Field that leverages unsupervised word-representations such as ELMo or BERT, eliminating the need for a complex encoder-decoder architecture. The results show that this approach leads to competitive performance with sequence-to-sequence models with high resources, while strongly outperforming it with low resources. In a human-subject study across subjects with varying reading abilities, we find that our section titles improve the speed of completing comprehension tasks while retaining similar accuracy.},
	url = {https://www.aclweb.org/anthology/papers/N/N19/N19-1168/},
	author = {Sebastian Gehrmann and Steven Layne and Franck Dernoncourt}
}
@conference {644594,
	title = { Towards Controllable Generation of Diverse Natural Language},
	booktitle = {INLG 2018 (Challenge Track)},
	year = {2018},
	abstract = {In natural language generation (NLG), the task is to generate utterances from a more abstract input, such as structured data. An added challenge is to generate utterances that contain an accurate representation of the input, while reflecting the fluency and variety of human-generated text. In this paper, we report experiments with NLG models that can be used in task oriented dialogue systems. We explore the use of additional input to the model to encourage diversity and control of outputs. While our submission does not rank highly using automated metrics, qualitative investigation of generated utterances suggests the use of additional information in neural network NLG systems to be a promising research direction.},
	url = {https://www.aclweb.org/anthology/W18-6556.pdf},
	author = {Henry Elder and Sebastian Gehrmann and Alexander O{\textquoteright}Connor and Qun Lin}
}
@article {gehrmann2018bottom,
	title = {Bottom-up abstractive summarization},
	journal = {EMNLP 2018},
	year = {2018},
	abstract = {Neural network-based methods for abstractive summarization produce outputs that are more fluent than other techniques, but perform poorly at content selection. This work proposes a simple technique for addressing this issue: use a data-efficient content selector to\&nbsp;over-determine phrases in a source document that should be part of the summary. We use this selector as a bottom-up attention step to constrain the model to likely phrases. We show that this approach improves the ability to compress text, while still generating fluent summaries. This two-step process is both simpler and higher performing than other end-to-end content selection models, leading to significant improvements on ROUGE for both the\&nbsp; CNNDM and NYT corpus. Furthermore, the content selector can be trained with as little as 1,000 sentences, making it easy to transfer a trained summarizer to a new domain.},
	author = {Sebastian Gehrmann and Deng, Yuntian and Rush, Alexander M}
}
@article {gehrmann2018end,
	title = {End-to-End Content and Plan Selection for Data-to-Text Generation},
	journal = {INLG 2018},
	year = {2018},
	abstract = {Learning to generate fluent natural language from structured data with neural networks has become an common approach for NLG.\&nbsp;\&nbsp; This problem can be challenging when the form of the structured data varies between examples.\&nbsp;\&nbsp; This paper presents a survey of several extensions to sequence-to-sequence models to account for the latent content selection process, particularly variants of copy attention and coverage decoding. \&nbsp;\&nbsp; We further propose a training method based on diverse ensembling to encourage models to learn distinct sentence templates during training. An empirical evaluation of these techniques shows an increase in the quality of generated text across five automated metrics, as well as human evaluation.},
	author = {Sebastian Gehrmann and Dai, Falcon Z and Henry Elder and Rush, Alexander M}
}
@article {strobelt2018seq2seq,
	title = {Seq2Seq-Vis: A Visual Debugging Tool for Sequence-to-Sequence Models},
	journal = {IEEE Transactions on Visualization and Computer Graphics},
	year = {2018},
	abstract = {sequence-to-sequence models have proven to beaccurate and robust for many sequence prediction tasks, and havebecome the standard approach for automatic translation of text. Themodels work with a five-stage blackbox pipeline that begins with encoding asource sequence to a vector space and then decoding out to a newtarget sequence. This process is now standard, but like many deeplearning methods remains quite difficult to understand or debug. \&nbsp;Inthis work, we present a visual analysis toolthat allows interaction and "what if"-style exploration of trained sequence-to-sequence models through each stage of the translation process. The aim is to identifywhich patterns have been learned, to detect model errors, and to probe the model withcounterfactual scenario. We demonstrate the utility of our tool through several real-world \&nbsp;sequence-to-sequence use cases on large-scale models.},
	url = {https://ieeexplore.ieee.org/abstract/document/8494828/},
	author = {Hendrik Strobelt and Sebastian Gehrmann and Behrisch, Michael and Perer, Adam and Pfister, Hanspeter and Rush, Alexander M}
}
@report {606022,
	title = {End-to-End Content and Plan Selection for Natural Language Generation},
	year = {2018},
	abstract = {This paper describes our entry for the INLG 2018 E2E NLG challenge. Generating flu- ent natural language descriptions from struc- tured data is a key sub-task for conversa- tional agents. In the E2E NLG challenge, the task is to generate these utterances conditioned on multiple attributes and values. Our sys- tem utilizes several extensions to the general- purpose sequence-to-sequence (S2S) architec- ture to model the latent content selection pro- cess, particularly different variants of copy at- tention and coverage decoding. In addition, we propose a new training method based on diverse ensembling to encourage the model to learn latent plans in training. We empirically evaluate these techniques and show that the system increases the quality of generated text across five automated metrics. Out of a total of sixty submitted systems from 16 institutions, our best system ranks first-place in three of the five metrics, including ROUGE.},
	url = {http://www.macs.hw.ac.uk/InteractionLab/E2E/final_papers/E2E-HarvardNLP.pdf},
	author = {Sebastian Gehrmann and Falcon Dai and Henry Elder and Rush, Alexander}
}
@article {8017583,
	title = {LSTMVis: A Tool for Visual Analysis of Hidden State Dynamics in Recurrent Neural Networks},
	journal = {IEEE Transactions on Visualization and Computer Graphics},
	volume = {24},
	number = {1},
	year = {2018},
	month = {Jan},
	pages = {667-676},
	keywords = {black-box hidden representation, chord progressions, Computational modeling, Data models, data visualisation, hidden state dynamics, hidden state representations, learning (artificial intelligence), local state changes, long-term usage data, LSTM, LSTMVis, machine learning, machine learning community, nesting, Pattern matching, phrase structure, recurrent neural nets, recurrent neural networks, remarkably effective tool, short-term memory networks, specific hidden state properties, statistical analysis, Tools, visual analysis tool, Visualization},
	issn = {1077-2626},
	doi = {10.1109/TVCG.2017.2744158},
	author = {H. Strobelt and S. Gehrmann and H. Pfister and A. M. Rush}
}
@article {gehrmann2018comparing,
	title = {Comparing deep learning and concept extraction based methods for patient phenotyping from clinical narratives},
	journal = {PloS one},
	volume = {13},
	number = {2},
	year = {2018},
	pages = {e0192360},
	publisher = {Public Library of Science},
	author = {Sebastian Gehrmann and Franck Dernoncourt and Yeran Li and Carlson, Eric T and Wu, Joy T and Jonathan Welt and Foote Jr, John and Moseley, Edward T and Grant, David W and Tyler, Patrick D and others}
}
@article {WU2017,
	title = {Behind the Scenes: A Medical Natural Language Processing Project},
	journal = {International Journal of Medical Informatics},
	year = {2017},
	keywords = {artificial intelligence in medicine, cross-disciplinary research, machine learning, multidisciplinary teamwork, Natural Language Processing, text analytics, translational research},
	issn = {1386-5056},
	doi = {https://doi.org/10.1016/j.ijmedinf.2017.12.003},
	url = {http://www.sciencedirect.com/science/article/pii/S138650561730446X},
	author = {Joy T. Wu and Franck Dernoncourt and Sebastian Gehrmann and Tyler, Patrick D and Moseley, Edward T and Carlson, Eric T and Grant, David W and Yeran Li and Jonathan Welt and Celi, Leo Anthony}
}
@unpublished {413341,
	title = {Visual Analysis of Hidden State Dynamics in Recurrent Neural Networks},
	journal = {lstm.seas.harvard.edu},
	year = {2016},
	abstract = {Recurrent neural networks, and in particular long short-term memory networks (LSTMs), are a remarkably effective tool for sequence modeling that learn a dense black-box hidden representation of their sequential input. Researchers interested in better understanding these models have studied the changes in hidden state representations over time and noticed some interpretable patterns but also significant noise. In this work, we present LSTMVis a visual analysis tool for recurrent neural networks with a focus on understanding these hidden state dynamics. The tool allows a user to select a hypothesis input range to focus on local state changes, to match these states changes to similar patterns in a large data set, and to align these results with domain specific structural annotations. We further show several use cases of the tool for analyzing specific hidden state properties on datasets containing nesting, phrase structure, and chord progressions, and demonstrate how the tool can be used to isolate patterns for further statistical analysis.},
	url = {http://arxiv.org/abs/1606.07461},
	author = {Hendrik Strobelt and Sebastian Gehrmann and Bernd Huber and Pfister, Hanspeter and Alexander M. Rush}
}
@conference {GehrmannColWriting2015,
	title = {Deploying AI Methods to Support Collaborative Writing: a Preliminary Investigation},
	booktitle = {CHI{\textquoteright}15 Extended Abstracts on Human Factors in Computing Systems},
	year = {2015},
	publisher = {ACM},
	organization = {ACM},
	abstract = {Many documents (e.g., academic papers, government\&nbsp;reports) are typically written by multiple authors. While\&nbsp;existing tools facilitate and support such collaborative\&nbsp;efforts (e.g., Dropbox, Google Docs), these tools lack\&nbsp;intelligent information sharing mechanisms. Capabilities\&nbsp;such as "track changes" and "diff"" visualize changes to\&nbsp;authors, but do not distinguish between minor and major\&nbsp;edits and do not consider the possible effects of edits on\&nbsp;other parts of the document. Drawing collaborators{\textquoteright}\&nbsp;attention to specific edits and describing them remains\&nbsp;the responsibility of authors. This paper presents our\&nbsp;initial work toward the development of a collaborative\&nbsp;system that supports multi-author writing. We describe\&nbsp;methods for tracking paragraphs, identifying significant\&nbsp;edits, and predicting parts of the paper that are likely to\&nbsp;require changes as a result of previous edits. Preliminary\&nbsp;evaluation of these methods shows promising results.},
	url = {https://www.dropbox.com/s/ny877fw5pd4ouhp/paperFinal.pdf?dl=0},
	author = {Sebastian Gehrmann and Lauren Urke and Amir, Ofra and Grosz, Barbara J}
}
