@online{Boehle_2301.08669,
TITLE = {Holistically Explainable Vision Transformers},
AUTHOR = {B{\"o}hle, Moritz and Fritz, Mario and Schiele, Bernt},
LANGUAGE = {eng},
URL = {https://arxiv.org/abs/2301.08669},
EPRINT = {2301.08669},
EPRINTTYPE = {arXiv},
YEAR = {2023},
MARGINALMARK = {$\bullet$},
ABSTRACT = {Transformers increasingly dominate the machine learning landscape across many<br>tasks and domains, which increases the importance for understanding their<br>outputs. While their attention modules provide partial insight into their inner<br>workings, the attention scores have been shown to be insufficient for<br>explaining the models as a whole. To address this, we propose B-cos<br>transformers, which inherently provide holistic explanations for their<br>decisions. Specifically, we formulate each model component -- such as the<br>multi-layer perceptrons, attention layers, and the tokenisation module -- to be<br>dynamic linear, which allows us to faithfully summarise the entire transformer<br>via a single linear transform. We apply our proposed design to Vision<br>Transformers (ViTs) and show that the resulting models, dubbed Bcos-ViTs, are<br>highly interpretable and perform competitively to baseline ViTs on ImageNet.<br>Code will be made available soon.<br>},
}