[ { "key": "C6C7D47B", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/C6C7D47B", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/C6C7D47B", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Nangia et al.", "numChildren": 0 }, "data": { "key": "C6C7D47B", "version": 924, "itemType": "journalArticle", "title": "What Ingredients Make for an Effective Crowdsourcing Protocol for Difficult NLU Data Collection Tasks?", "creators": [ { "creatorType": "author", "firstName": "Nikita", "lastName": "Nangia" }, { "creatorType": "author", "firstName": "Saku", "lastName": "Sugawara" }, { "creatorType": "author", "firstName": "Harsh", "lastName": "Trivedi" }, { "creatorType": "author", "firstName": "Alex", "lastName": "Warstadt" }, { "creatorType": "author", "firstName": "Clara", "lastName": "Vania" }, { "creatorType": "author", "firstName": "Samuel R", "lastName": "Bowman" } ], "abstractNote": "Crowdsourcing is widely used to create data for common natural language understanding tasks. Despite the importance of these datasets for measuring and refining model understanding of language, there has been little focus on the crowdsourcing methods used for collecting the datasets. In this paper, we compare the efficacy of interventions that have been proposed in prior work as ways of improving data quality. We use multiple-choice question answering as a testbed and run a randomized trial by assigning crowdworkers to write questions under one of four different data collection protocols. We find that asking workers to write explanations for their examples is an ineffective stand-alone strategy for boosting NLU example difficulty. However, we find that training crowdworkers, and then using an iterative process of collecting data, sending feedback, and qualifying workers based on expert judgments is an effective means of collecting challenging data. But using crowdsourced, instead of expert judgments, to qualify workers and send feedback does not prove to be effective. We observe that the data from the iterative protocol with expert assessments is more challenging by several measures. Notably, the human-model gap on the unanimous agreement portion of this data is, on average, twice as large as the gap for the baseline protocol data.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "nangiaWhatIngredientsMake", "url": "https://www.mturk.com/", "accessDate": "2023-02-02", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2106.00794v1", "tags": [], "collections": [ "4X7D9IEN", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "20c44ffd-0aff-311b-ae40-ccf83c28f8b0", "mendeleyDB:remoteDocumentUUID": "20c44ffd-0aff-311b-ae40-ccf83c28f8b0" }, "dateAdded": "2023-02-03T01:17:10Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "XMGHEVV7", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/XMGHEVV7", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/XMGHEVV7", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "numChildren": 0 }, "data": { "key": "XMGHEVV7", "version": 924, "itemType": "journalArticle", "title": "AISTATS2023Paper_20221005", "creators": [], "abstractNote": "", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "AISTATS2023Paper_20221005", "url": "", "accessDate": "", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "SWD6P6ZT", "W78S7Z4P" ], "relations": { "mendeleyDB:remoteDocumentUUID": "f48d7789-e281-3357-9d23-9a58c329ace4", "mendeleyDB:documentUUID": "f48d7789-e281-3357-9d23-9a58c329ace4" }, "dateAdded": "2022-10-07T07:41:58Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "9X6PXGHF", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/9X6PXGHF", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/9X6PXGHF", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Maddela et al.", "parsedDate": "2022-04-05", "numChildren": 0 }, "data": { "key": "9X6PXGHF", "version": 924, "itemType": "journalArticle", "title": "EntSUM: A Data Set for Entity-Centric Summarization", "creators": [ { "creatorType": "author", "firstName": "Mounica", "lastName": "Maddela" }, { "creatorType": "author", "firstName": "Mayank", "lastName": "Kulkarni" }, { "creatorType": "author", "firstName": "Daniel", "lastName": "Preo" }, { "creatorType": "author", "firstName": "tiuc-Pietro", "lastName": "Bloomberg" } ], "abstractNote": "Controllable summarization aims to provide summaries that take into account\nuser-specified aspects and preferences to better assist them with their\ninformation need, as opposed to the standard summarization setup which build a\nsingle generic summary of a document. We introduce a human-annotated data set\nEntSUM for controllable summarization with a focus on named entities as the\naspects to control. We conduct an extensive quantitative analysis to motivate\nthe task of entity-centric summarization and show that existing methods for\ncontrollable summarization fail to generate entity-centric summaries. We\npropose extensions to state-of-the-art summarization approaches that achieve\nsubstantially better results on our data set. Our analysis and results show the\nchallenging nature of this task and of the proposed data set.", "publicationTitle": "", "publisher": "", "place": "", "date": "2022-04-05", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2204.02213", "citationKey": "maddelaEntSUMDataSet2022", "url": "https://arxiv.org/abs/2204.02213v1", "accessDate": "2022-04-18", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2204.02213", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "d68ce174-37de-35f0-ac40-292fbe995051", "mendeleyDB:remoteDocumentUUID": "d68ce174-37de-35f0-ac40-292fbe995051" }, "dateAdded": "2022-04-19T00:10:52Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "4BEBV5Q7", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/4BEBV5Q7", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/4BEBV5Q7", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Chen et al.", "numChildren": 0 }, "data": { "key": "4BEBV5Q7", "version": 924, "itemType": "journalArticle", "title": "Error-Sensitive Evaluation for Ordinal Target Variables", "creators": [ { "creatorType": "author", "firstName": "David Z", "lastName": "Chen" }, { "creatorType": "author", "firstName": "Maury", "lastName": "Courtland" }, { "creatorType": "author", "firstName": "Adam", "lastName": "Faulkner" }, { "creatorType": "author", "firstName": "Aysu Ezen", "lastName": "Can" } ], "abstractNote": "Product reviews and satisfaction surveys seek customer feedback in the form of ranked scales. In these settings, widely used evaluation metrics including F1 and accuracy ignore the rank in the responses (e.g., 'very likely' is closer to 'likely' than 'not at all'). In this paper, we hypothesize that the order of class values is important for evaluating classifiers on ordinal target variables and should not be disregarded. To test this hypothesis, we compared Multi-class Classification (MC) and Ordinal Regression (OR) by applying OR and MC to benchmark tasks involving ordinal target variables using the same underlying model architecture. Experimental results show that while MC outperformed OR for some datasets in accuracy and F1, OR is significantly better than MC for minimizing the error between prediction and target for all benchmarks, as revealed by error-sensitive metrics, e.g. mean-squared error (MSE) and Spearman correlation. Our findings motivate the need to establish consistent, error-sensitive met-rics for evaluating benchmarks with ordinal target variables, and we hope that it stimulates interest in exploring alternative losses for ordinal problems.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "189-199", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.26615/978-954-452-056-4_019", "citationKey": "chenErrorSensitiveEvaluationOrdinal", "url": "https://doi.org/10.26615/978-954-452-056-4_019", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "4X7D9IEN", "GIB9XLGU", "RLTFV8C7", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "31bf965f-8959-35b5-9f42-97a11d5beede", "mendeleyDB:remoteDocumentUUID": "31bf965f-8959-35b5-9f42-97a11d5beede" }, "dateAdded": "2022-04-16T00:58:56Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "Q82PVTXC", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/Q82PVTXC", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/Q82PVTXC", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Card et al.", "numChildren": 0 }, "data": { "key": "Q82PVTXC", "version": 924, "itemType": "journalArticle", "title": "With Little Power Comes Great Responsibility", "creators": [ { "creatorType": "author", "firstName": "Dallas", "lastName": "Card" }, { "creatorType": "author", "firstName": "Peter", "lastName": "Henderson" }, { "creatorType": "author", "firstName": "Urvashi", "lastName": "Khandelwal" }, { "creatorType": "author", "firstName": "Robin", "lastName": "Jia" }, { "creatorType": "author", "firstName": "Kyle", "lastName": "Mahowald" }, { "creatorType": "author", "firstName": "Dan", "lastName": "Jurafsky" } ], "abstractNote": "Despite its importance to experimental design, statistical power (the probability that, given a real effect, an experiment will reject the null hypothesis) has largely been ignored by the NLP community. Underpowered experiments make it more difficult to discern the difference between statistical noise and meaningful model improvements, and increase the chances of exaggerated findings. By meta-analyzing a set of existing NLP papers and datasets, we characterize typical power for a variety of settings and conclude that under-powered experiments are common in the NLP literature. In particular, for several tasks in the popular GLUE benchmark, small test sets mean that most attempted comparisons to state of the art models will not be adequately powered. Similarly, based on reasonable assumptions , we find that the most typical experimental design for human rating studies will be un-derpowered to detect small model differences, of the sort that are frequently studied. For machine translation, we find that typical test sets of 2000 sentences have approximately 75% power to detect differences of 1 BLEU point. To improve the situation going forward, we give an overview of best practices for power analysis in NLP and release a series of notebooks to assist with future power analyses. 1", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "9263-9274", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "cardLittlePowerComes", "url": "https://github.com/dallascard/NLP-power-analysis", "accessDate": "2022-07-25", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "4X7D9IEN", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "40771f04-6b66-328d-9389-885141a884c0", "mendeleyDB:remoteDocumentUUID": "40771f04-6b66-328d-9389-885141a884c0" }, "dateAdded": "2022-07-25T23:46:57Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "2AWM6PUE", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/2AWM6PUE", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/2AWM6PUE", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Ahuja et al.", "numChildren": 0 }, "data": { "key": "2AWM6PUE", "version": 924, "itemType": "journalArticle", "title": "ASPECTNEWS: Aspect-Oriented Summarization of News Documents", "creators": [ { "creatorType": "author", "firstName": "Ojas", "lastName": "Ahuja" }, { "creatorType": "author", "firstName": "Jiacheng", "lastName": "Xu" }, { "creatorType": "author", "firstName": "Akshay", "lastName": "Gupta" }, { "creatorType": "author", "firstName": "Kevin", "lastName": "Horecka" }, { "creatorType": "author", "firstName": "Greg", "lastName": "Durrett" } ], "abstractNote": "Generic summaries try to cover an entire document and query-based summaries try to answer document-specific questions. But real users' needs often fall in between these extremes and correspond to aspects, high-level topics discussed among similar types of documents. In this paper, we collect a dataset of realistic aspect-oriented summaries, ASPECT-NEWS, which covers different subtopics about articles in news sub-domains. We annotate data across two domains of articles, earthquakes and fraud investigations, where each article is annotated with two distinct summaries focusing on different aspects for each domain. A system producing a single generic summary cannot concisely satisfy both aspects. Our focus in evaluation is how well existing techniques can generalize to these domains without seeing in-domain training data, so we turn to techniques to construct synthetic training data that have been used in query-focused sum-marization work. We compare several training schemes that differ in how strongly keywords are used and how oracle summaries are extracted. Our evaluation shows that our final approach yields (a) focused summaries, better than those from a generic summarization system or from keyword matching; (b) a system sensitive to the choice of keywords. 1", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "ahujaASPECTNEWSAspectOrientedSummarizationa", "url": "https://github.com/oja/", "accessDate": "2022-04-16", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2110.08296v2", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "afdf7c17-0a19-38fd-838e-85b158abfe3c", "mendeleyDB:remoteDocumentUUID": "afdf7c17-0a19-38fd-838e-85b158abfe3c" }, "dateAdded": "2022-04-16T15:33:04Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "X49Z8JZH", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/X49Z8JZH", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/X49Z8JZH", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Zhang et al.", "numChildren": 0 }, "data": { "key": "X49Z8JZH", "version": 924, "itemType": "journalArticle", "title": "Fine-grained Factual Consistency Assessment for Abstractive Summarization Models", "creators": [ { "creatorType": "author", "firstName": "Sen", "lastName": "Zhang" }, { "creatorType": "author", "firstName": "Jianwei", "lastName": "Niu" }, { "creatorType": "author", "firstName": "Chuyuan", "lastName": "Wei" } ], "abstractNote": "Factual inconsistencies existed in the output of abstractive summarization models with original documents are frequently presented. Fact consistency assessment requires the reasoning capability to find subtle clues to identify whether a model-generated summary is consistent with the original document. This paper proposes a fine-grained two-stage Fact Consistency assessment framework for Summarization models (SumFC). Given a document and a summary sentence, in the first stage, SumFC selects the top-K most relevant sentences with the summary sentence from the document. In the second stage, the model performs fine-grained consistency reasoning at the sentence level, and then aggregates all sentences' consistency scores to obtain the final assessment result. We get the training data pairs by data synthesis and adopt contrastive loss of data pairs to help the model identify subtle cues. Experiment results show that SumFC has made a significant improvement over the previous state-of-the-art methods. Our experiments also indicate that SumFC distinguishes detailed differences better .", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "107-116", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "zhangFinegrainedFactualConsistency", "url": "", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "b1760121-fa9d-3d6a-a11f-53ca5f1d82eb", "mendeleyDB:remoteDocumentUUID": "b1760121-fa9d-3d6a-a11f-53ca5f1d82eb" }, "dateAdded": "2022-04-16T01:08:41Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "6KMK88I6", "version": 924, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/6KMK88I6", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/6KMK88I6", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Ahuja et al.", "numChildren": 1 }, "data": { "key": "6KMK88I6", "version": 924, "itemType": "journalArticle", "title": "ASPECTNEWS: Aspect-Oriented Summarization of News Documents", "creators": [ { "creatorType": "author", "firstName": "Ojas", "lastName": "Ahuja" }, { "creatorType": "author", "firstName": "Jiacheng", "lastName": "Xu" }, { "creatorType": "author", "firstName": "Akshay", "lastName": "Gupta" }, { "creatorType": "author", "firstName": "Kevin", "lastName": "Horecka" }, { "creatorType": "author", "firstName": "Greg", "lastName": "Durrett" } ], "abstractNote": "Generic summaries try to cover an entire document and query-based summaries try to answer document-specific questions. But real users' needs often fall in between these extremes and correspond to aspects, high-level topics discussed among similar types of documents. In this paper, we collect a dataset of realistic aspect-oriented summaries, ASPECT-NEWS, which covers different subtopics about articles in news sub-domains. We annotate data across two domains of articles, earthquakes and fraud investigations, where each article is annotated with two distinct summaries focusing on different aspects for each domain. A system producing a single generic summary cannot concisely satisfy both aspects. Our focus in evaluation is how well existing techniques can generalize to these domains without seeing in-domain training data, so we turn to techniques to construct synthetic training data that have been used in query-focused sum-marization work. We compare several training schemes that differ in how strongly keywords are used and how oracle summaries are extracted. Our evaluation shows that our final approach yields (a) focused summaries, better than those from a generic summarization system or from keyword matching; (b) a system sensitive to the choice of keywords. 1", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "ahujaASPECTNEWSAspectOrientedSummarization", "url": "https://github.com/oja/", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2110.08296v2", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "7ecff186-675d-3987-a229-0901749d0023", "mendeleyDB:remoteDocumentUUID": "7ecff186-675d-3987-a229-0901749d0023" }, "dateAdded": "2022-04-16T01:06:06Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "JZJ52F76", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/JZJ52F76", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/JZJ52F76", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Kim et al.", "numChildren": 0 }, "data": { "key": "JZJ52F76", "version": 923, "itemType": "journalArticle", "title": "Aspect-Controllable Opinion Summarization", "creators": [ { "creatorType": "author", "firstName": "Reinald", "lastName": "Kim" }, { "creatorType": "author", "firstName": "Amplayo Stefanos", "lastName": "Angelidis" }, { "creatorType": "author", "firstName": "Mirella", "lastName": "Lapata" } ], "abstractNote": "Recent work on opinion summarization produces general summaries based on a set of input reviews and the popularity of opinions expressed in them. In this paper, we propose an approach that allows the generation of cus-tomized summaries based on aspect queries (e.g., describing the location and room of a hotel). Using a review corpus, we create a synthetic training dataset of (review, summary) pairs enriched with aspect controllers which are induced by a multi-instance learning model that predicts the aspects of a document at different levels of granularity. We fine-tune a pre-trained model using our synthetic dataset and generate aspect-specific summaries by modifying the aspect controllers. Experiments on two benchmarks show that our model outperforms the previous state of the art and generates per-sonalized summaries by controlling the number of aspects discussed in them.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "kimAspectControllableOpinionSummarization", "url": "https://github.com/rktamplayo/AceSum", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2109.03171v1", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "1b6c85f0-6c5a-3883-9676-c6c6bd76d74c", "mendeleyDB:remoteDocumentUUID": "1b6c85f0-6c5a-3883-9676-c6c6bd76d74c" }, "dateAdded": "2022-04-16T00:57:50Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "JCRYDGQ9", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/JCRYDGQ9", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/JCRYDGQ9", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Chen et al.", "parsedDate": "2021-08-13", "numChildren": 0 }, "data": { "key": "JCRYDGQ9", "version": 923, "itemType": "journalArticle", "title": "A Dataset for Answering Time-Sensitive Questions", "creators": [ { "creatorType": "author", "firstName": "Wenhu", "lastName": "Chen" }, { "creatorType": "author", "firstName": "Xinyi", "lastName": "Wang" }, { "creatorType": "author", "firstName": "William Yang", "lastName": "Wang" } ], "abstractNote": "Time is an important dimension in our physical world. Lots of facts can\nevolve with respect to time. For example, the U.S. President might change every\nfour years. Therefore, it is important to consider the time dimension and\nempower the existing QA models to reason over time. However, the existing QA\ndatasets contain rather few time-sensitive questions, hence not suitable for\ndiagnosing or benchmarking the model's temporal reasoning capability. In order\nto promote research in this direction, we propose to construct a time-sensitive\nQA dataset. The dataset is constructed by 1) mining time-evolving facts from\nWikiData and aligning them to their corresponding Wikipedia page, 2) employing\ncrowd workers to verify and calibrate these noisy facts, 3) generating\nquestion-answer pairs based on the annotated time-sensitive facts. Our dataset\nposes challenges in the aspect of both temporal understanding and temporal\nreasoning. We evaluate different SoTA long-document QA systems like BigBird and\nFiD on our dataset. The best-performing model FiD can only achieve 46\\%\naccuracy, still far behind the human performance of 87\\%. We demonstrate that\nthese models are still lacking the ability to perform consistent temporal\nreasoning. Therefore, we believe that our dataset could serve as a benchmark to\ndevelop NLP models more sensitive to temporal shifts. The dataset and code are\nreleased in~\\url{https://github.com/wenhuchen/Time-Sensitive-QA}.", "publicationTitle": "", "publisher": "", "place": "", "date": "2021-08-13", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "chenDatasetAnsweringTimeSensitive2021", "url": "https://arxiv.org/abs/2108.06314v5", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2108.06314\nISBN: 2108.06314v5", "tags": [], "collections": [ "CHGL683M", "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "2f9fea43-ca95-3443-807a-6d95fa067d79", "mendeleyDB:documentUUID": "2f9fea43-ca95-3443-807a-6d95fa067d79" }, "dateAdded": "2022-04-16T00:55:28Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "NJ3QL49B", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/NJ3QL49B", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/NJ3QL49B", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Hazarika Amazon Alexa et al.", "parsedDate": "2021-06-11", "numChildren": 0 }, "data": { "key": "NJ3QL49B", "version": 923, "itemType": "journalArticle", "title": "Zero-Shot Controlled Generation with Encoder-Decoder Transformers", "creators": [ { "creatorType": "author", "firstName": "Devamanyu AI", "lastName": "Hazarika Amazon Alexa" }, { "creatorType": "author", "firstName": "Mahdi", "lastName": "Namazifar" }, { "creatorType": "author", "firstName": "Amazon AI", "lastName": "Alexa" }, { "creatorType": "author", "firstName": "Dilek", "lastName": "Hakkani-T" }, { "creatorType": "author", "firstName": "ur AI", "lastName": "Amazon Alexa" } ], "abstractNote": "Controlling neural network-based models for natural language generation (NLG)\nhas broad applications in numerous areas such as machine translation, document\nsummarization, and dialog systems. Approaches that enable such control in a\nzero-shot manner would be of great importance as, among other reasons, they\nremove the need for additional annotated data and training. In this work, we\npropose novel approaches for controlling encoder-decoder transformer-based NLG\nmodels in zero-shot. This is done by introducing three control knobs, namely,\nattention biasing, decoder mixing, and context augmentation, that are applied\nto these models at generation time. These knobs control the generation process\nby directly manipulating trained NLG models (e.g., biasing cross-attention\nlayers) to realize the desired attributes in the generated outputs. We show\nthat not only are these NLG models robust to such manipulations, but also their\nbehavior could be controlled without an impact on their generation performance.\nThese results, to the best of our knowledge, are the first of their kind.\nThrough these control knobs, we also investigate the role of transformer\ndecoder's self-attention module and show strong evidence that its primary role\nis maintaining fluency of sentences generated by these models. Based on this\nhypothesis, we show that alternative architectures for transformer decoders\ncould be viable options. We also study how this hypothesis could lead to more\nefficient ways for training encoder-decoder transformer models.", "publicationTitle": "", "publisher": "", "place": "", "date": "2021-06-11", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2106.06411", "citationKey": "hazarikaamazonalexaZeroShotControlledGeneration2021", "url": "https://arxiv.org/abs/2106.06411v3", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2106.06411", "tags": [], "collections": [ "GIB9XLGU", "P7IQVCXI", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "6a6bb370-4f33-3eaa-8395-61d0a6269707", "mendeleyDB:documentUUID": "6a6bb370-4f33-3eaa-8395-61d0a6269707" }, "dateAdded": "2022-04-16T00:52:41Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "UC8DDML8", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/UC8DDML8", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/UC8DDML8", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Hedayatnia et al.", "parsedDate": "2020-05-26", "numChildren": 0 }, "data": { "key": "UC8DDML8", "version": 923, "itemType": "journalArticle", "title": "Policy-Driven Neural Response Generation for Knowledge-Grounded Dialogue Systems", "creators": [ { "creatorType": "author", "firstName": "Behnam", "lastName": "Hedayatnia" }, { "creatorType": "author", "firstName": "Karthik", "lastName": "Gopalakrishnan" }, { "creatorType": "author", "firstName": "Seokhwan", "lastName": "Kim" }, { "creatorType": "author", "firstName": "Yang", "lastName": "Liu" }, { "creatorType": "author", "firstName": "Mihail", "lastName": "Eric" }, { "creatorType": "author", "firstName": "Dilek", "lastName": "Hakkani-Tür" } ], "abstractNote": "Open-domain dialogue systems aim to generate relevant, informative and\nengaging responses. Seq2seq neural response generation approaches do not have\nexplicit mechanisms to control the content or style of the generated response,\nand frequently result in uninformative utterances. In this paper, we propose\nusing a dialogue policy to plan the content and style of target responses in\nthe form of an action plan, which includes knowledge sentences related to the\ndialogue context, targeted dialogue acts, topic information, etc. The\nattributes within the action plan are obtained by automatically annotating the\npublicly released Topical-Chat dataset. We condition neural response generators\non the action plan which is then realized as target utterances at the turn and\nsentence levels. We also investigate different dialogue policy models to\npredict an action plan given the dialogue context. Through automated and human\nevaluation, we measure the appropriateness of the generated responses and check\nif the generation models indeed learn to realize the given action plans. We\ndemonstrate that a basic dialogue policy that operates at the sentence level\ngenerates better responses in comparison to turn level generation as well as\nbaseline models with no action plan. Additionally the basic dialogue policy has\nthe added effect of controllability.", "publicationTitle": "INLG 2020 - 13th International Conference on Natural Language Generation, Proceedings", "publisher": "Association for Computational Linguistics (ACL)", "place": "", "date": "2020-05-26", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "412-421", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2005.12529", "citationKey": "hedayatnia-PolicyDrivenNeural-2020", "url": "https://arxiv.org/abs/2005.12529v4", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2005.12529\nISBN: 9781952148545", "tags": [], "collections": [ "GIB9XLGU", "RLTFV8C7", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "a83e4fd3-1a52-3542-9181-2a23efdd22c8", "mendeleyDB:remoteDocumentUUID": "a83e4fd3-1a52-3542-9181-2a23efdd22c8" }, "dateAdded": "2022-04-16T00:50:56Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "IUXIMC9E", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/IUXIMC9E", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/IUXIMC9E", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Chowdhery et al.", "numChildren": 0 }, "data": { "key": "IUXIMC9E", "version": 923, "itemType": "journalArticle", "title": "PaLM: Scaling Language Modeling with Pathways", "creators": [ { "creatorType": "author", "firstName": "Aakanksha", "lastName": "Chowdhery" }, { "creatorType": "author", "firstName": "Sharan", "lastName": "Narang" }, { "creatorType": "author", "firstName": "Jacob", "lastName": "Devlin" }, { "creatorType": "author", "firstName": "Maarten", "lastName": "Bosma" }, { "creatorType": "author", "firstName": "Gaurav", "lastName": "Mishra" }, { "creatorType": "author", "firstName": "Adam", "lastName": "Roberts" }, { "creatorType": "author", "firstName": "Paul", "lastName": "Barham" }, { "creatorType": "author", "firstName": "Hyung", "lastName": "Won" }, { "creatorType": "author", "firstName": "Chung Charles", "lastName": "Sutton" }, { "creatorType": "author", "firstName": "Sebastian", "lastName": "Gehrmann" }, { "creatorType": "author", "firstName": "Parker", "lastName": "Schuh" }, { "creatorType": "author", "firstName": "Kensen", "lastName": "Shi" }, { "creatorType": "author", "firstName": "Sasha", "lastName": "Tsvyashchenko" }, { "creatorType": "author", "firstName": "Joshua", "lastName": "Maynez" }, { "creatorType": "author", "firstName": "Abhishek", "lastName": "Rao" }, { "creatorType": "author", "firstName": "Parker", "lastName": "Barnes" }, { "creatorType": "author", "firstName": "Yi", "lastName": "Tay" }, { "creatorType": "author", "firstName": "Noam", "lastName": "Shazeer" }, { "creatorType": "author", "firstName": "Vinodkumar", "lastName": "Prabhakaran" }, { "creatorType": "author", "firstName": "Emily", "lastName": "Reif" }, { "creatorType": "author", "firstName": "Nan", "lastName": "Du" }, { "creatorType": "author", "firstName": "Ben", "lastName": "Hutchinson" }, { "creatorType": "author", "firstName": "Reiner", "lastName": "Pope" }, { "creatorType": "author", "firstName": "James", "lastName": "Bradbury" }, { "creatorType": "author", "firstName": "Jacob", "lastName": "Austin" }, { "creatorType": "author", "firstName": "Michael Isard", "lastName": "Guy" }, { "creatorType": "author", "firstName": "Gur-Ari", "lastName": "Pengcheng" }, { "creatorType": "author", "firstName": "Yin Toju", "lastName": "Duke" }, { "creatorType": "author", "firstName": "Anselm", "lastName": "Levskaya" }, { "creatorType": "author", "firstName": "Sanjay", "lastName": "Ghemawat" }, { "creatorType": "author", "firstName": "Sunipa", "lastName": "Dev" }, { "creatorType": "author", "firstName": "Henryk", "lastName": "Michalewski" }, { "creatorType": "author", "firstName": "Xavier", "lastName": "Garcia" }, { "creatorType": "author", "firstName": "Vedant", "lastName": "Misra" }, { "creatorType": "author", "firstName": "Kevin", "lastName": "Robinson" }, { "creatorType": "author", "firstName": "Liam", "lastName": "Fedus" }, { "creatorType": "author", "firstName": "Denny", "lastName": "Zhou" }, { "creatorType": "author", "firstName": "Daphne", "lastName": "Ippolito" }, { "creatorType": "author", "firstName": "David", "lastName": "Luan" }, { "creatorType": "author", "firstName": "Hyeontaek", "lastName": "Lim" }, { "creatorType": "author", "firstName": "Barret", "lastName": "Zoph" }, { "creatorType": "author", "firstName": "Alexander", "lastName": "Spiridonov" }, { "creatorType": "author", "firstName": "Ryan", "lastName": "Sepassi" }, { "creatorType": "author", "firstName": "David", "lastName": "Dohan" }, { "creatorType": "author", "firstName": "Shivani", "lastName": "Agrawal" }, { "creatorType": "author", "firstName": "Mark", "lastName": "Omernick" }, { "creatorType": "author", "firstName": "Andrew M", "lastName": "Dai" }, { "creatorType": "author", "firstName": "Thanumalayan Sankaranarayana", "lastName": "Pillai" }, { "creatorType": "author", "firstName": "Marie", "lastName": "Pellat" }, { "creatorType": "author", "firstName": "Aitor", "lastName": "Lewkowycz" }, { "creatorType": "author", "firstName": "Erica", "lastName": "Moreira" }, { "creatorType": "author", "firstName": "Rewon", "lastName": "Child" }, { "creatorType": "author", "firstName": "Oleksandr", "lastName": "Polozov" }, { "creatorType": "author", "firstName": "Katherine", "lastName": "Lee" }, { "creatorType": "author", "firstName": "Zongwei", "lastName": "Zhou" }, { "creatorType": "author", "firstName": "Xuezhi", "lastName": "Wang" }, { "creatorType": "author", "firstName": "Brennan", "lastName": "Saeta" }, { "creatorType": "author", "firstName": "Mark", "lastName": "Diaz" }, { "creatorType": "author", "firstName": "Orhan", "lastName": "Firat" }, { "creatorType": "author", "firstName": "Michele", "lastName": "Catasta" }, { "creatorType": "author", "firstName": "Jason", "lastName": "Wei" }, { "creatorType": "author", "firstName": "Kathy", "lastName": "Meier-Hellstern" }, { "creatorType": "author", "firstName": "Douglas", "lastName": "Eck" }, { "creatorType": "author", "firstName": "Jeff", "lastName": "Dean" }, { "creatorType": "author", "firstName": "Slav", "lastName": "Petrov" }, { "creatorType": "author", "firstName": "Noah", "lastName": "Fiedel" } ], "abstractNote": "Large language models have been shown to achieve remarkable performance across a variety of natural language tasks using few-shot learning, which drastically reduces the number of task-specific training examples needed to adapt the model to a particular application. To further our understanding of the impact of scale on few-shot learning, we trained a 540-billion parameter, densely activated, Transformer language model, which we call Pathways Language Model (PaLM). We trained PaLM on 6144 TPU v4 chips using Pathways, a new ML system which enables highly efficient training across multiple TPU Pods. We demonstrate continued benefits of scaling by achieving state-of-the-art few-shot learning results on hundreds of language understanding and generation benchmarks. On a number of these tasks, PaLM 540B achieves breakthrough performance, outperforming the finetuned state-of-the-art on a suite of multi-step reasoning tasks, and outperforming average human performance on the recently released BIG-bench benchmark. A significant number of BIG-bench tasks showed discontinuous improvements from model scale, meaning that performance steeply increased as we scaled to our largest model. PaLM also has strong capabilities in multilingual tasks and source code generation, which we demonstrate on a wide array of benchmarks. We additionally provide a comprehensive analysis on bias and toxicity, and study the extent of training data memorization with respect to model scale. Finally, we discuss the ethical considerations related to large language models and discuss potential mitigation strategies.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "chowdheryPaLMScalingLanguage", "url": "", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "5UHBRRJZ", "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "f95d1e33-dfb8-33d5-89b9-b0c17d06df5e", "mendeleyDB:remoteDocumentUUID": "f95d1e33-dfb8-33d5-89b9-b0c17d06df5e" }, "dateAdded": "2022-04-16T00:50:34Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "ES7SFP3D", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/ES7SFP3D", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/ES7SFP3D", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Zhang et al.", "parsedDate": "2022-01-14", "numChildren": 0 }, "data": { "key": "ES7SFP3D", "version": 923, "itemType": "journalArticle", "title": "A Survey of Controllable Text Generation using Transformer-based Pre-trained Language Models", "creators": [ { "creatorType": "author", "firstName": "Hanqing", "lastName": "Zhang" }, { "creatorType": "author", "firstName": "Haolin", "lastName": "Song" }, { "creatorType": "author", "firstName": "Shaoyu", "lastName": "Li" }, { "creatorType": "author", "firstName": "Ming", "lastName": "Zhou" }, { "creatorType": "author", "firstName": "Dawei", "lastName": "Song" } ], "abstractNote": "Controllable Text Generation (CTG) is emerging area in the field of natural\nlanguage generation (NLG). It is regarded as crucial for the development of\nadvanced text generation technologies that are more natural and better meet the\nspecific constraints in practical applications. In recent years, methods using\nlarge-scale pre-trained language models (PLMs), in particular the widely used\ntransformer-based PLMs, have become a new paradigm of NLG, allowing generation\nof more diverse and fluent text. However, due to the lower level of\ninterpretability of deep neural networks, the controllability of these methods\nneed to be guaranteed. To this end, controllable text generation using\ntransformer-based PLMs has become a rapidly growing yet challenging new\nresearch hotspot. A diverse range of approaches have emerged in the recent 3-4\nyears, targeting different CTG tasks which may require different types of\ncontrolled constraints. In this paper, we present a systematic critical review\non the common tasks, main approaches and evaluation methods in this area.\nFinally, we discuss the challenges that the field is facing, and put forward\nvarious promising future directions. To the best of our knowledge, this is the\nfirst survey paper to summarize CTG techniques from the perspective of PLMs. We\nhope it can help researchers in related fields to quickly track the academic\nfrontier, providing them with a landscape of the area and a roadmap for future\nresearch.", "publicationTitle": "J. ACM", "publisher": "", "place": "", "date": "2022-01-14", "volume": "37", "issue": "111", "section": "", "partNumber": "", "partTitle": "", "pages": "34", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2201.05337", "citationKey": "zhangSurveyControllableText2022", "url": "https://arxiv.org/abs/2201.05337v1", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2201.05337", "tags": [ { "tag": "controllability", "type": 1 }, { "tag": "controllable text generation", "type": 1 }, { "tag": "pre-trained language models", "type": 1 }, { "tag": "systematic review", "type": 1 }, { "tag": "transformer", "type": 1 } ], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "920d98ab-f1ae-38ff-86e9-93f646abec8d", "mendeleyDB:documentUUID": "920d98ab-f1ae-38ff-86e9-93f646abec8d" }, "dateAdded": "2022-04-16T00:49:34Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "8DSQEKGY", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/8DSQEKGY", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/8DSQEKGY", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Dziri et al.", "numChildren": 0 }, "data": { "key": "8DSQEKGY", "version": 923, "itemType": "journalArticle", "title": "Neural Path Hunter: Reducing Hallucination in Dialogue Systems via Path Grounding", "creators": [ { "creatorType": "author", "firstName": "Nouha", "lastName": "Dziri" }, { "creatorType": "author", "firstName": "Andrea", "lastName": "Madotto" }, { "creatorType": "author", "firstName": "Osmar", "lastName": "Zaiane" }, { "creatorType": "author", "firstName": "Avishek", "lastName": "Joey Bose" } ], "abstractNote": "Dialogue systems powered by large pre-trained language models exhibit an innate ability to deliver fluent and natural-sounding responses. Despite their impressive performance , these models are fitful and can often generate factually incorrect statements impeding their widespread adoption. In this paper, we focus on the task of improving faithful-ness and reducing hallucination of neural dialogue systems to known facts supplied by a Knowledge Graph (KG). We propose NEU-RAL PATH HUNTER which follows a generate-then-refine strategy whereby a generated response is amended using the KG. NEURAL PATH HUNTER leverages a separate token-level fact critic to identify plausible sources of hallucination followed by a refinement stage that retrieves correct entities by crafting a query signal that is propagated over a k-hop subgraph. We empirically validate our proposed approach on the OpenDialKG dataset (Moon et al., 2019) against a suite of metrics and report a relative improvement of faithful-ness over dialogue responses by 20.35% based on FeQA (Durmus et al., 2020). The code is available at https://github.com/ nouhadziri/Neural-Path-Hunter.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "dziriNeuralPathHunter", "url": "https://github.com/", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2104.08455v2", "tags": [], "collections": [ "GIB9XLGU", "RLTFV8C7", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "33168a41-d4d3-3dce-9bbd-7f67c6fcaba3", "mendeleyDB:remoteDocumentUUID": "33168a41-d4d3-3dce-9bbd-7f67c6fcaba3" }, "dateAdded": "2022-04-16T00:58:21Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "YDLWY8RN", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/YDLWY8RN", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/YDLWY8RN", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Karpinska et al.", "numChildren": 0 }, "data": { "key": "YDLWY8RN", "version": 923, "itemType": "journalArticle", "title": "The Perils of Using Mechanical Turk to Evaluate Open-Ended Text Generation", "creators": [ { "creatorType": "author", "firstName": "Marzena", "lastName": "Karpinska" }, { "creatorType": "author", "firstName": "Nader", "lastName": "Akoury" }, { "creatorType": "author", "firstName": "Mohit", "lastName": "Iyyer" } ], "abstractNote": "Recent text generation research has increasingly focused on open-ended domains such as story and poetry generation. Because models built for such tasks are difficult to evaluate automatically, most researchers in the space justify their modeling choices by collecting crowdsourced human judgments of text quality (e.g., Likert scores of coherence or gram-maticality) from Amazon Mechanical Turk (AMT). In this paper, we first conduct a survey of 45 open-ended text generation papers and find that the vast majority of them fail to report crucial details about their AMT tasks, hindering reproducibility. We then run a series of story evaluation experiments with both AMT workers and English teachers and discover that even with strict qualification filters , AMT workers (unlike teachers) fail to distinguish between model-generated text and human-generated references. We show that AMT worker judgments improve when they are shown model-generated output alongside human-generated references, which enables the workers to better calibrate their ratings. Finally, interviews with the English teachers provide deeper insights into the challenges of the evaluation process, particularly when rating model-generated text.", "publicationTitle": "", "publisher": "Association for Computational Linguistics", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "1265-1285", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "karpinska-PerilsUsing-", "url": "https://reddit.com/r/WritingPrompts/", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "4X7D9IEN", "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "69d64b3a-cc0a-33b8-a0fa-c3d096fd6f30", "mendeleyDB:documentUUID": "69d64b3a-cc0a-33b8-a0fa-c3d096fd6f30" }, "dateAdded": "2022-04-16T00:57:03Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "RD56DNWF", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/RD56DNWF", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/RD56DNWF", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Aghajanyan Facebook et al.", "numChildren": 0 }, "data": { "key": "RD56DNWF", "version": 923, "itemType": "journalArticle", "title": "Muppet: Massive Multi-task Representations with Pre-Finetuning", "creators": [ { "creatorType": "author", "firstName": "Armen", "lastName": "Aghajanyan Facebook" }, { "creatorType": "author", "firstName": "Anchit", "lastName": "Gupta Facebook" }, { "creatorType": "author", "firstName": "Akshat", "lastName": "Shrivastava Facebook" }, { "creatorType": "author", "firstName": "Xilun", "lastName": "Chen Facebook" }, { "creatorType": "author", "firstName": "Luke", "lastName": "Zettlemoyer Facebook" }, { "creatorType": "author", "firstName": "Sonal", "lastName": "Gupta Facebook" } ], "abstractNote": "We propose pre-finetuning, an additional large-scale learning stage between language model pre-training and fine-tuning. Pre-finetuning is massively multi-task learning (around 50 datasets, over 4.8 million total labeled examples), and is designed to encourage learning of representations that generalize better to many different tasks. We show that pre-finetuning consistently improves performance for pretrained discriminators (e.g. RoBERTa) and generation models (e.g. BART) on a wide range of tasks (sentence prediction, common-sense reasoning, MRC, etc.), while also significantly improving sample efficiency during fine-tuning. We also show that large-scale multi-tasking is crucial; pre-finetuning can hurt performance when few tasks are used up until a critical point (usually above 15) after which performance improves linearly in the number of tasks.", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "5799-5811", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "aghajanyanfacebookMuppetMassiveMultitaska", "url": "", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "5UHBRRJZ", "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "ebbbd767-e865-3a1b-a1e9-663aadee0a00", "mendeleyDB:documentUUID": "ebbbd767-e865-3a1b-a1e9-663aadee0a00" }, "dateAdded": "2022-04-16T00:55:59Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "7E985K7S", "version": 923, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/7E985K7S", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/7E985K7S", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Xu et al.", "parsedDate": "2020-07-29", "numChildren": 0 }, "data": { "key": "7E985K7S", "version": 923, "itemType": "journalArticle", "title": "Fact-based Content Weighting for Evaluating Abstractive Summarisation", "creators": [ { "creatorType": "author", "firstName": "Xinnuo", "lastName": "Xu" }, { "creatorType": "author", "firstName": "Ondřej", "lastName": "Dušek" }, { "creatorType": "author", "firstName": "Jingyi", "lastName": "Li" }, { "creatorType": "author", "firstName": "Verena", "lastName": "Rieser" }, { "creatorType": "author", "firstName": "Ioannis", "lastName": "Konstas" } ], "abstractNote": "ive summarisation is notoriously hard to evaluate since standard word-overlap-based metrics are biased towards specific words in the human reference. We introduce a new evaluation metric which abstracts away from the word-level and instead is based on fact-level content weighting, i.e. relating the facts of the document to the facts of the summary. We follow the assumption that a good summary will reflect all relevant facts, i.e. the ones present in the ground truth (human-generated reference summary). We confirm this hypothesis by showing that our weightings are highly correlated to human perception and compare favourably to the recent manual highlight-based metric of Hardy et al. (2019).", "publicationTitle": "", "publisher": "Association for Computational Linguistics (ACL)", "place": "", "date": "2020-07-29", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "5071-5081", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.18653/V1/2020.ACL-MAIN.455", "citationKey": "xu-FactbasedContent-2020", "url": "https://aclanthology.org/2020.acl-main.455", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "e442da59-343e-35f3-86ee-9825155e8200", "mendeleyDB:remoteDocumentUUID": "e442da59-343e-35f3-86ee-9825155e8200" }, "dateAdded": "2022-04-16T00:49:59Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "8LGCLZJW", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/8LGCLZJW", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/8LGCLZJW", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Fabbri et al.", "parsedDate": "2021-11-11", "numChildren": 0 }, "data": { "key": "8LGCLZJW", "version": 922, "itemType": "journalArticle", "title": "AnswerSumm: A Manually-Curated Dataset and Pipeline for Answer Summarization", "creators": [ { "creatorType": "author", "firstName": "Alexander R", "lastName": "Fabbri" }, { "creatorType": "author", "firstName": "Xiaojian", "lastName": "Wu" }, { "creatorType": "author", "firstName": "Srini", "lastName": "Iyer" }, { "creatorType": "author", "firstName": "Haoran", "lastName": "Li" }, { "creatorType": "author", "firstName": "Mona", "lastName": "Diab" }, { "creatorType": "author", "firstName": "Yale", "lastName": "University" }, { "creatorType": "author", "firstName": "Facebook", "lastName": "Ai" } ], "abstractNote": "Community Question Answering (CQA) fora such as Stack Overflow and Yahoo!\nAnswers contain a rich resource of answers to a wide range of community-based\nquestions. Each question thread can receive a large number of answers with\ndifferent perspectives. One goal of answer summarization is to produce a\nsummary that reflects the range of answer perspectives. A major obstacle for\nabstractive answer summarization is the absence of a dataset to provide\nsupervision for producing such summaries. Recent works propose heuristics to\ncreate such data, but these are often noisy and do not cover all perspectives\npresent in the answers. This work introduces a novel dataset of 4,631 CQA\nthreads for answer summarization, curated by professional linguists. Our\npipeline gathers annotations for all subtasks involved in answer summarization,\nincluding the selection of answer sentences relevant to the question, grouping\nthese sentences based on perspectives, summarizing each perspective, and\nproducing an overall summary. We analyze and benchmark state-of-the-art models\non these subtasks and introduce a novel unsupervised approach for\nmulti-perspective data augmentation, that further boosts overall summarization\nperformance according to automatic evaluation. Finally, we propose\nreinforcement learning rewards to improve factual consistency and answer\ncoverage and analyze areas for improvement.", "publicationTitle": "", "publisher": "", "place": "", "date": "2021-11-11", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2111.06474", "citationKey": "fabbriAnswerSummManuallyCuratedDataset2021a", "url": "https://arxiv.org/abs/2111.06474v1", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2111.06474", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "2cd9ff31-a85a-35f5-a8d2-965845a6a5be", "mendeleyDB:remoteDocumentUUID": "2cd9ff31-a85a-35f5-a8d2-965845a6a5be" }, "dateAdded": "2022-04-16T00:47:49Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "HEDII8LL", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/HEDII8LL", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/HEDII8LL", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Fu et al.", "parsedDate": "2020-12-29", "numChildren": 0 }, "data": { "key": "HEDII8LL", "version": 922, "itemType": "journalArticle", "title": "A Theoretical Analysis of the Repetition Problem in Text Generation", "creators": [ { "creatorType": "author", "firstName": "Zihao", "lastName": "Fu" }, { "creatorType": "author", "firstName": "Wai", "lastName": "Lam" }, { "creatorType": "author", "firstName": "Anthony Man-Cho", "lastName": "So" }, { "creatorType": "author", "firstName": "Bei", "lastName": "Shi" } ], "abstractNote": "Text generation tasks, including translation, summarization, language models,\nand etc. see rapid growth during recent years. Despite the remarkable\nachievements, the repetition problem has been observed in nearly all text\ngeneration models undermining the generation performance extensively. To solve\nthe repetition problem, many methods have been proposed, but there is no\nexisting theoretical analysis to show why this problem happens and how it is\nresolved. In this paper, we propose a new framework for theoretical analysis\nfor the repetition problem. We first define the Average Repetition Probability\n(ARP) to characterize the repetition problem quantitatively. Then, we conduct\nan extensive analysis of the Markov generation model and derive several upper\nbounds of the average repetition probability with intuitive understanding. We\nshow that most of the existing methods are essentially minimizing the upper\nbounds explicitly or implicitly. Grounded on our theory, we show that the\nrepetition problem is, unfortunately, caused by the traits of our language\nitself. One major reason is attributed to the fact that there exist too many\nwords predicting the same word as the subsequent word with high probability.\nConsequently, it is easy to go back to that word and form repetitions and we\ndub it as the high inflow problem. Furthermore, we derive a concentration bound\nof the average repetition probability for a general generation model. Finally,\nbased on the theoretical upper bounds, we propose a novel rebalanced encoding\napproach to alleviate the high inflow problem. The experimental results show\nthat our theoretical framework is applicable in general generation models and\nour proposed rebalanced encoding approach alleviates the repetition problem\nsignificantly. The source code of this paper can be obtained from\nhttps://github.com/fuzihaofzh/repetition-problem-nlg.", "publicationTitle": "", "publisher": "", "place": "", "date": "2020-12-29", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2012.14660", "citationKey": "fuTheoreticalAnalysisRepetition2020", "url": "https://arxiv.org/abs/2012.14660v4", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2012.14660", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "c852b121-02c4-373a-a0c4-b5c06ddac507", "mendeleyDB:remoteDocumentUUID": "c852b121-02c4-373a-a0c4-b5c06ddac507" }, "dateAdded": "2022-04-16T00:47:02Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "G2I3I7QL", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/G2I3I7QL", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/G2I3I7QL", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Zheng et al.", "parsedDate": "2020-10-20", "numChildren": 0 }, "data": { "key": "G2I3I7QL", "version": 922, "itemType": "journalArticle", "title": "Topic-Guided Abstractive Text Summarization: a Joint Learning Approach", "creators": [ { "creatorType": "author", "firstName": "Chujie", "lastName": "Zheng" }, { "creatorType": "author", "firstName": "Kunpeng", "lastName": "Zhang" }, { "creatorType": "author", "firstName": "Harry Jiannan", "lastName": "Wang" }, { "creatorType": "author", "firstName": "Ling", "lastName": "Fan" }, { "creatorType": "author", "firstName": "Zhe", "lastName": "Wang" } ], "abstractNote": "We introduce a new approach for abstractive text summarization, Topic-Guided\nAbstractive Summarization, which calibrates long-range dependencies from\ntopic-level features with globally salient content. The idea is to incorporate\nneural topic modeling with a Transformer-based sequence-to-sequence (seq2seq)\nmodel in a joint learning framework. This design can learn and preserve the\nglobal semantics of the document, which can provide additional contextual\nguidance for capturing important ideas of the document, thereby enhancing the\ngeneration of summary. We conduct extensive experiments on two datasets and the\nresults show that our proposed model outperforms many extractive and\nabstractive systems in terms of both ROUGE measurements and human evaluation.\nOur code is available at: https://github.com/chz816/tas.", "publicationTitle": "", "publisher": "", "place": "", "date": "2020-10-20", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.2010.10323", "citationKey": "zhengTopicGuidedAbstractiveText2020b", "url": "https://arxiv.org/abs/2010.10323v2", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2010.10323", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "029f62ab-dc44-3f03-92f0-9e4a58205fe8", "mendeleyDB:remoteDocumentUUID": "029f62ab-dc44-3f03-92f0-9e4a58205fe8" }, "dateAdded": "2022-04-16T00:46:13Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "EQ9GJB72", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/EQ9GJB72", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/EQ9GJB72", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Honovich and Aharoni Jonathan Herzig G Hagai Taitelbaum G Vered Cohen G Doron Kukliansky G Thomas Scialom M Idan Szpektor G Avinatan Hassidim G Yossi Matias G", "numChildren": 0 }, "data": { "key": "EQ9GJB72", "version": 922, "itemType": "journalArticle", "title": "TRUE: Re-evaluating Factual Consistency Evaluation", "creators": [ { "creatorType": "author", "firstName": "Or T", "lastName": "Honovich" }, { "creatorType": "author", "firstName": "Roee G", "lastName": "Aharoni Jonathan Herzig G Hagai Taitelbaum G Vered Cohen G Doron Kukliansky G Thomas Scialom M Idan Szpektor G Avinatan Hassidim G Yossi Matias G" } ], "abstractNote": "Grounded text generation systems often generate text that contains factual inconsistencies, hindering their real-world applicability. Automatic factual consistency evaluation may help alleviate this limitation by accelerating evaluation cycles, filtering inconsistent outputs and augmenting training data. While attracting increasing attention, such evaluation metrics are usually developed and evaluated in silo for a single task or dataset, slowing their adoption. Moreover, previous meta-evaluation protocols focused on system-level correlations with human annotations, which leave the example-level accuracy of such metrics unclear. In this work, we introduce TRUE: a comprehensive study of factual consistency metrics on a standardized collection of existing texts from diverse tasks, manually annotated for factual consistency. Our standardization enables an example-level meta-evaluation protocol that is more actionable and interpretable than previously reported correlations, yielding clearer quality measures. Across diverse state-of-the-art metrics and 11 datasets we find that large-scale NLI and question generation-and-answering-based approaches achieve strong and complementary results. We recommend those methods as a starting point for model and metric developers, and hope TRUE will foster progress towards even better methods. 1", "publicationTitle": "", "publisher": "", "place": "", "date": "", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "honovichTRUEReevaluatingFactual", "url": "", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2204.04991v1", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "92344a30-c3a2-3e29-833f-23071e944100", "mendeleyDB:remoteDocumentUUID": "92344a30-c3a2-3e29-833f-23071e944100" }, "dateAdded": "2022-04-16T00:42:48Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "NULPTNJ7", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/NULPTNJ7", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/NULPTNJ7", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Zhu et al.", "parsedDate": "2019-11-08", "numChildren": 0 }, "data": { "key": "NULPTNJ7", "version": 922, "itemType": "journalArticle", "title": "Transforming Wikipedia into Augmented Data for Query-Focused Summarization", "creators": [ { "creatorType": "author", "firstName": "Haichao", "lastName": "Zhu" }, { "creatorType": "author", "firstName": "Li", "lastName": "Dong" }, { "creatorType": "author", "firstName": "Furu", "lastName": "Wei" }, { "creatorType": "author", "firstName": "Bing", "lastName": "Qin" }, { "creatorType": "author", "firstName": "Ting", "lastName": "Liu" } ], "abstractNote": "The manual construction of a query-focused summarization corpus is costly and\ntimeconsuming. The limited size of existing datasets renders training\ndata-driven summarization models challenging. In this paper, we use Wikipedia\nto automatically collect a large query-focused summarization dataset (named as\nWIKIREF) of more than 280,000 examples, which can serve as a means of data\naugmentation. Moreover, we develop a query-focused summarization model based on\nBERT to extract summaries from the documents. Experimental results on three DUC\nbenchmarks show that the model pre-trained on WIKIREF has already achieved\nreasonable performance. After fine-tuning on the specific datasets, the model\nwith data augmentation outperforms the state of the art on the benchmarks.", "publicationTitle": "", "publisher": "", "place": "", "date": "2019-11-08", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.1911.03324", "citationKey": "zhuTransformingWikipediaAugmented2019", "url": "https://arxiv.org/abs/1911.03324v1", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 1911.03324", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:remoteDocumentUUID": "b766e9a9-6dad-340f-9fa4-368b4b6ef057", "mendeleyDB:documentUUID": "b766e9a9-6dad-340f-9fa4-368b4b6ef057" }, "dateAdded": "2022-04-16T00:42:27Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "XFRZ8TD7", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/XFRZ8TD7", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/XFRZ8TD7", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Hu et al.", "parsedDate": "2017-03-02", "numChildren": 0 }, "data": { "key": "XFRZ8TD7", "version": 922, "itemType": "journalArticle", "title": "Toward Controlled Generation of Text", "creators": [ { "creatorType": "author", "firstName": "Zhiting", "lastName": "Hu" }, { "creatorType": "author", "firstName": "Zichao", "lastName": "Yang" }, { "creatorType": "author", "firstName": "Xiaodan", "lastName": "Liang" }, { "creatorType": "author", "firstName": "Ruslan", "lastName": "Salakhutdinov" }, { "creatorType": "author", "firstName": "Eric P.", "lastName": "Xing" } ], "abstractNote": "Generic generation and manipulation of text is challenging and has limited\nsuccess compared to recent deep generative modeling in visual domain. This\npaper aims at generating plausible natural language sentences, whose attributes\nare dynamically controlled by learning disentangled latent representations with\ndesignated semantics. We propose a new neural generative model which combines\nvariational auto-encoders and holistic attribute discriminators for effective\nimposition of semantic structures. With differentiable approximation to\ndiscrete text samples, explicit constraints on independent attribute controls,\nand efficient collaborative learning of generator and discriminators, our model\nlearns highly interpretable representations from even only word annotations,\nand produces realistic sentences with desired attributes. Quantitative\nevaluation validates the accuracy of sentence and attribute generation.", "publicationTitle": "34th International Conference on Machine Learning, ICML 2017", "publisher": "International Machine Learning Society (IMLS)", "place": "", "date": "2017-03-02", "volume": "4", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "2503-2513", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "10.48550/arxiv.1703.00955", "citationKey": "hu-ControlledGeneration-2017", "url": "https://arxiv.org/abs/1703.00955v4", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 1703.00955\nISBN: 9781510855144", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "6f53a612-8335-37ef-be32-b05043901f7d", "mendeleyDB:remoteDocumentUUID": "6f53a612-8335-37ef-be32-b05043901f7d" }, "dateAdded": "2022-04-16T00:49:24Z", "dateModified": "2026-02-17T19:30:12Z" } }, { "key": "YPE7VUVC", "version": 922, "library": { "type": "group", "id": 6028, "name": "Natural Language Processing", "links": { "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing", "type": "text/html" } } }, "links": { "self": { "href": "https://api.zotero.org/groups/6028/items/YPE7VUVC", "type": "application/json" }, "alternate": { "href": "https://www.zotero.org/groups/natural_language_processing/items/YPE7VUVC", "type": "text/html" } }, "meta": { "createdByUser": { "id": 44076, "username": "markusd", "name": "", "links": { "alternate": { "href": "https://www.zotero.org/markusd", "type": "text/html" } } }, "creatorSummary": "Kasai et al.", "parsedDate": "2022-04-11", "numChildren": 0 }, "data": { "key": "YPE7VUVC", "version": 922, "itemType": "journalArticle", "title": "Beam Decoding with Controlled Patience", "creators": [ { "creatorType": "author", "firstName": "Jungo", "lastName": "Kasai" }, { "creatorType": "author", "firstName": "Keisuke", "lastName": "Sakaguchi" }, { "creatorType": "author", "firstName": "Ronan", "lastName": "Le Bras" }, { "creatorType": "author", "firstName": "Dragomir", "lastName": "Radev" }, { "creatorType": "author", "firstName": "Yejin", "lastName": "Choi" }, { "creatorType": "author", "firstName": "Noah A", "lastName": "Smith" }, { "creatorType": "author", "firstName": "Paul G", "lastName": "Allen" } ], "abstractNote": "Text generation with beam search has proven successful in a wide range of\napplications. The commonly-used implementation of beam decoding follows a first\ncome, first served heuristic: it keeps a set of already completed sequences\nover time steps and stops when the size of this set reaches the beam size. We\nintroduce a patience factor, a simple modification to this decoding algorithm,\nthat generalizes the stopping criterion and provides flexibility to the depth\nof search. Extensive empirical results demonstrate that the patience factor\nimproves decoding performance of strong pretrained models on news text\nsummarization and machine translation over diverse language pairs, with a\nnegligible inference slowdown. Our approach only modifies one line of code and\ncan be thus readily incorporated in any implementation.", "publicationTitle": "", "publisher": "", "place": "", "date": "2022-04-11", "volume": "", "issue": "", "section": "", "partNumber": "", "partTitle": "", "pages": "", "series": "", "seriesTitle": "", "seriesText": "", "journalAbbreviation": "", "DOI": "", "citationKey": "kasaiBeamDecodingControlled2022", "url": "https://arxiv.org/abs/2204.05424v1", "accessDate": "2022-04-15", "PMID": "", "PMCID": "", "ISSN": "", "archive": "", "archiveLocation": "", "shortTitle": "", "language": "", "libraryCatalog": "", "callNumber": "", "rights": "", "extra": "arXiv: 2204.05424", "tags": [], "collections": [ "GIB9XLGU", "SWD6P6ZT" ], "relations": { "mendeleyDB:documentUUID": "b8424da8-3a9a-3e92-aaaf-85349b48a967", "mendeleyDB:remoteDocumentUUID": "b8424da8-3a9a-3e92-aaaf-85349b48a967" }, "dateAdded": "2022-04-16T00:48:39Z", "dateModified": "2026-02-17T19:30:12Z" } } ]