D5
Databases and Information Systems
2023
[1]
H. Arnaout, T.-P. Nguyen, S. Razniewski, and G. Weikum, “UnCommonSense in Action! Informative Negations for Commonsense Knowledge Bases,” in WSDM ’23, 16th ACM International Conference on Web Search and Data Mining, Singapore, 2023.
Export
BibTeX
@inproceedings{Arnaout_WSDM23, TITLE = {{UnCommonSense} in Action! {I}nformative Negations for Commonsense Knowledge Bases}, AUTHOR = {Arnaout, Hiba and Nguyen, Tuan-Phong and Razniewski, Simon and Weikum, Gerhard}, LANGUAGE = {eng}, ISBN = {978-1-4503-9407-9}, DOI = {10.1145/3539597.3573027}, PUBLISHER = {ACM}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {WSDM '23, 16th ACM International Conference on Web Search and Data Mining}, EDITOR = {Chua, Tat-Seng and Lauw, Hady and Si, Luo and Terzi, Evimaria and Tsaparas, Panayiotis}, PAGES = {1120--1123}, ADDRESS = {Singapore}, }
Endnote
%0 Conference Proceedings %A Arnaout, Hiba %A Nguyen, Tuan-Phong %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T UnCommonSense in Action! Informative Negations for Commonsense Knowledge Bases : %G eng %U http://hdl.handle.net/21.11116/0000-000C-18BC-6 %R 10.1145/3539597.3573027 %D 2023 %B 16th ACM International Conference on Web Search and Data Mining %Z date of event: 2023-02-27 - 2023-03-03 %C Singapore %B WSDM '23 %E Chua, Tat-Seng; Lauw, Hady; Si, Luo; Terzi, Evimaria; Tsaparas, Panayiotis %P 1120 - 1123 %I ACM %@ 978-1-4503-9407-9
[2]
L. Boualili and A. Yates, “A Study of Term-Topic Embeddings for Ranking,” in Advances in Information Retrieval (ECIR 2023), Dublin, Ireland, 2023.
Export
BibTeX
@inproceedings{Boualili_ECIR23, TITLE = {A Study of Term-Topic Embeddings for Ranking}, AUTHOR = {Boualili, Lila and Yates, Andrew}, LANGUAGE = {eng}, ISBN = {978-3-031-28237-9}, DOI = {10.1007/978-3-031-28238-6_25}, PUBLISHER = {Springer}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, DATE = {2023}, BOOKTITLE = {Advances in Information Retrieval (ECIR 2023)}, EDITOR = {Kamps, Jaap and Goeuriot, Lorraine and Crestani, Fabio and Maistro, Maria and Joho, Hideao and Davis, Brian and Gurrin, Cathal and Kruschwitz, Udo and Caputo, Annalina}, PAGES = {359--366}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {13981}, ADDRESS = {Dublin, Ireland}, }
Endnote
%0 Conference Proceedings %A Boualili, Lila %A Yates, Andrew %+ External Organizations Databases and Information Systems, MPI for Informatics, Max Planck Society %T A Study of Term-Topic Embeddings for Ranking : %G eng %U http://hdl.handle.net/21.11116/0000-000C-DC34-2 %R 10.1007/978-3-031-28238-6_25 %D 2023 %B 45th European Conference on IR Research %Z date of event: 2023-04-02 - 2023-04-06 %C Dublin, Ireland %B Advances in Information Retrieval %E Kamps, Jaap; Goeuriot, Lorraine; Crestani, Fabio; Maistro, Maria; Joho, Hideao; Davis, Brian; Gurrin, Cathal; Kruschwitz, Udo; Caputo, Annalina %P 359 - 366 %I Springer %@ 978-3-031-28237-9 %B Lecture Notes in Computer Science %N 13981
[3]
P. Christmann, R. Saha Roy, and G. Weikum, “CLOCQ: A Toolkit for Fast and Easy Access to Knowledge Bases,” in BTW 2023, Dresden, Germany, 2023.
Export
BibTeX
@inproceedings{Christmann_BTW2023, TITLE = {{CLOCQ}: {A} Toolkit for Fast and Easy Access to Knowledge Bases}, AUTHOR = {Christmann, Philipp and Saha Roy, Rishiraj and Weikum, Gerhard}, LANGUAGE = {eng}, ISBN = {978-3-88579-725-8}, DOI = {10.18420/BTW2023-28}, PUBLISHER = {GI}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {BTW 2023}, EDITOR = {K{\"o}nig-Ries, Birgitta and Scherzinger, Stefanie and Lehner, Wolfgang and Vossen, Gottfried}, PAGES = {579--591}, SERIES = {Lecture Notes in Informatics}, VOLUME = {P-331}, ADDRESS = {Dresden, Germany}, }
Endnote
%0 Conference Proceedings %A Christmann, Philipp %A Saha Roy, Rishiraj %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T CLOCQ: A Toolkit for Fast and Easy Access to Knowledge Bases : %G eng %U http://hdl.handle.net/21.11116/0000-000C-BF14-7 %R 10.18420/BTW2023-28 %D 2023 %B 20th Conference on Database Systems for Business, Technology and Web %Z date of event: 2023-03-06 - 2023-03-10 %C Dresden, Germany %B BTW 2023 %E König-Ries, Birgitta; Scherzinger, Stefanie; Lehner, Wolfgang; Vossen, Gottfried %P 579 - 591 %I GI %@ 978-3-88579-725-8 %B Lecture Notes in Informatics %N P-331
[4]
A. Ghazimatin, “Enhancing Explainability and Scrutability of Recommender Systems,” in BTW 2023, Dresden, Germany, 2023.
Export
BibTeX
@inproceedings{DBLP:conf/btw/Ghazimatin23, TITLE = {Enhancing Explainability and Scrutability of Recommender Systems}, AUTHOR = {Ghazimatin, Azin}, LANGUAGE = {eng}, ISBN = {978-3-88579-725-8}, DOI = {10.18420/BTW2023-32}, PUBLISHER = {GI}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {BTW 2023}, EDITOR = {K{\"o}nig-Ries, Birgitta and Scherzinger, Stefanie and Lehner, Wolfgang and Vossen, Gottfried}, PAGES = {633--640}, SERIES = {Lecture Notes in Informatics}, VOLUME = {P-331}, ADDRESS = {Dresden, Germany}, }
Endnote
%0 Conference Proceedings %A Ghazimatin, Azin %+ Databases and Information Systems, MPI for Informatics, Max Planck Society %T Enhancing Explainability and Scrutability of Recommender Systems : %G eng %U http://hdl.handle.net/21.11116/0000-000C-DC4D-7 %R 10.18420/BTW2023-32 %D 2023 %B 20th Conference on Database Systems for Business, Technology and Web %Z date of event: 2023-03-06 - 2023-03-10 %C Dresden, Germany %B BTW 2023 %E König-Ries, Birgitta; Scherzinger, Stefanie; Lehner, Wolfgang; Vossen, Gottfried %P 633 - 640 %I GI %@ 978-3-88579-725-8 %B Lecture Notes in Informatics %N P-331
[5]
S. Ghosh, S. Razniewski, and G. Weikum, “Answering Count Questions with Structured Answers from Text,” Journal of Web Semantics, vol. 76, 2023.
Export
BibTeX
@article{Ghosh23, TITLE = {Answering Count Questions with Structured Answers from Text}, AUTHOR = {Ghosh, Shrestha and Razniewski, Simon and Weikum, Gerhard}, LANGUAGE = {eng}, DOI = {10.1016/j.websem.2022.100769}, PUBLISHER = {Elsevier}, ADDRESS = {Amsterdam}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, DATE = {2023}, JOURNAL = {Journal of Web Semantics}, VOLUME = {76}, EID = {100769}, }
Endnote
%0 Journal Article %A Ghosh, Shrestha %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T Answering Count Questions with Structured Answers from Text : %G eng %U http://hdl.handle.net/21.11116/0000-000C-47CB-0 %R 10.1016/j.websem.2022.100769 %7 2022 %D 2023 %J Journal of Web Semantics %V 76 %Z sequence number: 100769 %I Elsevier %C Amsterdam
[6]
S. Ghosh, S. Razniewski, and G. Weikum, “CoQEx: Entity Counts Explained,” in WSDM ’23, 16th ACM International Conference on Web Search and Data Mining, Singapore, 2023.
Export
BibTeX
@inproceedings{Christmann_WSDM23, TITLE = {{CoQEx}: {E}ntity Counts Explained}, AUTHOR = {Ghosh, Shrestha and Razniewski, Simon and Weikum, Gerhard}, LANGUAGE = {eng}, ISBN = {978-1-4503-9407-9}, DOI = {10.1145/3539597.3573021}, PUBLISHER = {ACM}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {WSDM '23, 16th ACM International Conference on Web Search and Data Mining}, EDITOR = {Chua, Tat-Seng and Lauw, Hady and Si, Luo and Terzi, Evimaria and Tsaparas, Panayiotis}, PAGES = {1168--1171}, ADDRESS = {Singapore}, }
Endnote
%0 Conference Proceedings %A Ghosh, Shrestha %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T CoQEx: Entity Counts Explained : %G eng %U http://hdl.handle.net/21.11116/0000-000B-F41F-0 %R 10.1145/3539597.3573021 %D 2023 %B 16th ACM International Conference on Web Search and Data Mining %Z date of event: 2023-02-27 - 2023-03-03 %C Singapore %B WSDM '23 %E Chua, Tat-Seng; Lauw, Hady; Si, Luo; Terzi, Evimaria; Tsaparas, Panayiotis %P 1168 - 1171 %I ACM %@ 978-1-4503-9407-9
[7]
S. Ghosh, S. Razniewski, and G. Weikum, “Class Cardinality Comparison as a Fermi Problem,” in WWW ’23, ACM Web Conference, Austin, TX, USA. (arXiv: 2303.04532, Accepted/in press)
Abstract
Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br>
Export
BibTeX
@inproceedings{Ghosh2303.04532, TITLE = {Class Cardinality Comparison as a {F}ermi Problem}, AUTHOR = {Ghosh, Shrestha and Razniewski, Simon and Weikum, Gerhard}, URL = {https://arxiv.org/abs/2303.04532}, EPRINT = {2303.04532}, EPRINTTYPE = {arXiv}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br>}, BOOKTITLE = {WWW '23, ACM Web Conference}, ADDRESS = {Austin, TX, USA}, }
Endnote
%0 Conference Proceedings %A Ghosh, Shrestha %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T Class Cardinality Comparison as a Fermi Problem : %U http://hdl.handle.net/21.11116/0000-000C-BF05-8 %U https://arxiv.org/abs/2303.04532 %D 2023 %B ACM Web Conference %Z date of event: 2023-04-30 - 2023-05-04 %C Austin, TX, USA %X Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br> %K Computer Science, Information Retrieval, cs.IR,Computer Science, Artificial Intelligence, cs.AI %B WWW '23
[8]
T.-P. Nguyen, S. Razniewski, A. Varde, and G. Weikum, “Extracting Cultural Commonsense Knowledge at Scale,” in WWW ’23, ACM Web Conference, Austin, TX, USA. (Accepted/in press)
Export
BibTeX
@inproceedings{Nguyen_WWW23, TITLE = {Extracting Cultural Commonsense Knowledge at Scale}, AUTHOR = {Nguyen, Tuan-Phong and Razniewski, Simon and Varde, Aparna and Weikum, Gerhard}, LANGUAGE = {eng}, ISBN = {978-1-4503-9416-1}, DOI = {10.1145/3543507.3583535}, PUBLISHER = {ACM}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {WWW '23, ACM Web Conference}, ADDRESS = {Austin, TX, USA}, }
Endnote
%0 Conference Proceedings %A Nguyen, Tuan-Phong %A Razniewski, Simon %A Varde, Aparna %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society External Organizations Databases and Information Systems, MPI for Informatics, Max Planck Society %T Extracting Cultural Commonsense Knowledge at Scale : %G eng %U http://hdl.handle.net/21.11116/0000-000C-9FF7-B %R 10.1145/3543507.3583535 %D 2023 %B ACM Web Conference %Z date of event: 2023-04-30 - 2023-05-04 %C Austin, TX, USA %B WWW '23 %I ACM %@ 978-1-4503-9416-1
[9]
G. H. Torbati, G. Weikum, and A. Yates, “Search-based Recommendation : The Case for Difficult Predictions,” in WWW ’23, ACM Web Conference, Austin, TX, USA. (Accepted/in press)
Abstract
Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br>
Export
BibTeX
@inproceedings{Torbati_WWW23, TITLE = {Search-based Recommendation : {T}he Case for Difficult Predictions}, AUTHOR = {Torbati, Ghazaleh Haratinezhad and Weikum, Gerhard and Yates, Andrew}, LANGUAGE = {eng}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br>}, BOOKTITLE = {WWW '23, ACM Web Conference}, ADDRESS = {Austin, TX, USA}, }
Endnote
%0 Conference Proceedings %A Torbati, Ghazaleh Haratinezhad %A Weikum, Gerhard %A Yates, Andrew %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society %T Search-based Recommendation : The Case for Difficult Predictions : %G eng %U http://hdl.handle.net/21.11116/0000-000C-DC45-F %D 2023 %B ACM Web Conference %Z date of event: 2023-04-30 - 2023-05-04 %C Austin, TX, USA %X Questions on class cardinality comparisons are quite tricky to answer and<br>come with its own challenges. They require some kind of reasoning since web<br>documents and knowledge bases, indispensable sources of information, rarely<br>store direct answers to questions, such as, ``Are there more astronauts or<br>Physics Nobel Laureates?'' We tackle questions on class cardinality comparison<br>by tapping into three sources for absolute cardinalities as well as the<br>cardinalities of orthogonal subgroups of the classes. We propose novel<br>techniques for aggregating signals with partial coverage for more reliable<br>estimates and evaluate them on a dataset of 4005 class pairs, achieving an<br>accuracy of 83.7%.<br> %K Computer Science, Information Retrieval, cs.IR,Computer Science, Artificial Intelligence, cs.AI %B WWW '23
[10]
B. Veseli, S. Singhania, S. Razniewski, and G. Weikum, “Evaluating Language Models for Knowledge Base Completion,” in The Semantic Web (ESWC 2023), Hersonissos, Greece. (Accepted/in press)
Export
BibTeX
@inproceedings{Veseli_ESWC23, TITLE = {Evaluating Language Models for Knowledge Base Completion}, AUTHOR = {Veseli, Blerta and Singhania, Sneha and Razniewski, Simon and Weikum, Gerhard}, LANGUAGE = {eng}, PUBLISHER = {Springer}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {The Semantic Web (ESWC 2023)}, SERIES = {Lecture Notes in Computer Science}, ADDRESS = {Hersonissos, Greece}, }
Endnote
%0 Conference Proceedings %A Veseli, Blerta %A Singhania, Sneha %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society External Organizations Databases and Information Systems, MPI for Informatics, Max Planck Society %T Evaluating Language Models for Knowledge Base Completion : %G eng %U http://hdl.handle.net/21.11116/0000-000C-DC39-D %D 2023 %B The European Semantic Web Conference %Z date of event: 2023-05-28 - 2023-06-01 %C Hersonissos, Greece %B The Semantic Web %I Springer %B Lecture Notes in Computer Science
[11]
B. Veseli, S. Singhania, S. Razniewski, and G. Weikum, “Evaluating Language Models for Knowledge Base Completion,” 2023. [Online]. Available: https://arxiv.org/abs/2303.11082. (arXiv: 2303.11082)
Abstract
Structured knowledge bases (KBs) are a foundation of many intelligent<br>applications, yet are notoriously incomplete. Language models (LMs) have<br>recently been proposed for unsupervised knowledge base completion (KBC), yet,<br>despite encouraging initial results, questions regarding their suitability<br>remain open. Existing evaluations often fall short because they only evaluate<br>on popular subjects, or sample already existing facts from KBs. In this work,<br>we introduce a novel, more challenging benchmark dataset, and a methodology<br>tailored for a realistic assessment of the KBC potential of LMs. For automated<br>assessment, we curate a dataset called WD-KNOWN, which provides an unbiased<br>random sample of Wikidata, containing over 3.9 million facts. In a second step,<br>we perform a human evaluation on predictions that are not yet in the KB, as<br>only this provides real insights into the added value over existing KBs. Our<br>key finding is that biases in dataset conception of previous benchmarks lead to<br>a systematic overestimate of LM performance for KBC. However, our results also<br>reveal strong areas of LMs. We could, for example, perform a significant<br>completion of Wikidata on the relations nativeLanguage, by a factor of ~21<br>(from 260k to 5.8M) at 82% precision, usedLanguage, by a factor of ~2.1 (from<br>2.1M to 6.6M) at 82% precision, and citizenOf by a factor of ~0.3 (from 4.2M to<br>5.3M) at 90% precision. Moreover, we find that LMs possess surprisingly strong<br>generalization capabilities: even on relations where most facts were not<br>directly observed in LM training, prediction quality can be high.<br>
Export
BibTeX
@online{, TITLE = {Evaluating Language Models for Knowledge Base Completion}, AUTHOR = {Veseli, Blerta and Singhania, Sneha and Razniewski, Simon and Weikum, Gerhard}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2303.11082}, EPRINT = {2303.11082}, EPRINTTYPE = {arXiv}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Structured knowledge bases (KBs) are a foundation of many intelligent<br>applications, yet are notoriously incomplete. Language models (LMs) have<br>recently been proposed for unsupervised knowledge base completion (KBC), yet,<br>despite encouraging initial results, questions regarding their suitability<br>remain open. Existing evaluations often fall short because they only evaluate<br>on popular subjects, or sample already existing facts from KBs. In this work,<br>we introduce a novel, more challenging benchmark dataset, and a methodology<br>tailored for a realistic assessment of the KBC potential of LMs. For automated<br>assessment, we curate a dataset called WD-KNOWN, which provides an unbiased<br>random sample of Wikidata, containing over 3.9 million facts. In a second step,<br>we perform a human evaluation on predictions that are not yet in the KB, as<br>only this provides real insights into the added value over existing KBs. Our<br>key finding is that biases in dataset conception of previous benchmarks lead to<br>a systematic overestimate of LM performance for KBC. However, our results also<br>reveal strong areas of LMs. We could, for example, perform a significant<br>completion of Wikidata on the relations nativeLanguage, by a factor of ~21<br>(from 260k to 5.8M) at 82% precision, usedLanguage, by a factor of ~2.1 (from<br>2.1M to 6.6M) at 82% precision, and citizenOf by a factor of ~0.3 (from 4.2M to<br>5.3M) at 90% precision. Moreover, we find that LMs possess surprisingly strong<br>generalization capabilities: even on relations where most facts were not<br>directly observed in LM training, prediction quality can be high.<br>}, }
Endnote
%0 Report %A Veseli, Blerta %A Singhania, Sneha %A Razniewski, Simon %A Weikum, Gerhard %+ Databases and Information Systems, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society External Organizations Databases and Information Systems, MPI for Informatics, Max Planck Society %T Evaluating Language Models for Knowledge Base Completion : %G eng %U http://hdl.handle.net/21.11116/0000-000C-D3CD-F %U https://arxiv.org/abs/2303.11082 %D 2023 %X Structured knowledge bases (KBs) are a foundation of many intelligent<br>applications, yet are notoriously incomplete. Language models (LMs) have<br>recently been proposed for unsupervised knowledge base completion (KBC), yet,<br>despite encouraging initial results, questions regarding their suitability<br>remain open. Existing evaluations often fall short because they only evaluate<br>on popular subjects, or sample already existing facts from KBs. In this work,<br>we introduce a novel, more challenging benchmark dataset, and a methodology<br>tailored for a realistic assessment of the KBC potential of LMs. For automated<br>assessment, we curate a dataset called WD-KNOWN, which provides an unbiased<br>random sample of Wikidata, containing over 3.9 million facts. In a second step,<br>we perform a human evaluation on predictions that are not yet in the KB, as<br>only this provides real insights into the added value over existing KBs. Our<br>key finding is that biases in dataset conception of previous benchmarks lead to<br>a systematic overestimate of LM performance for KBC. However, our results also<br>reveal strong areas of LMs. We could, for example, perform a significant<br>completion of Wikidata on the relations nativeLanguage, by a factor of ~21<br>(from 260k to 5.8M) at 82% precision, usedLanguage, by a factor of ~2.1 (from<br>2.1M to 6.6M) at 82% precision, and citizenOf by a factor of ~0.3 (from 4.2M to<br>5.3M) at 90% precision. Moreover, we find that LMs possess surprisingly strong<br>generalization capabilities: even on relations where most facts were not<br>directly observed in LM training, prediction quality can be high.<br> %K Computer Science, Computation and Language, cs.CL,Computer Science, Artificial Intelligence, cs.AI %U https://github.com/bveseli/LMsForKBC