@online{Fischer_arXiv2110.11150,
TITLE = {Towards Strong Pruning for Lottery Tickets with Non-Zero Biases},
AUTHOR = {Fischer, Jonas and Burkholz, Rebekka},
LANGUAGE = {eng},
URL = {https://arxiv.org/abs/2110.11150},
EPRINT = {2110.11150},
EPRINTTYPE = {arXiv},
YEAR = {2021},
ABSTRACT = {The strong lottery ticket hypothesis holds the promise that pruning randomly<br>initialized deep neural networks could offer a computationally efficient<br>alternative to deep learning with stochastic gradient descent. Common parameter<br>initialization schemes and existence proofs, however, are focused on networks<br>with zero biases, thus foregoing the potential universal approximation property<br>of pruning. To fill this gap, we extend multiple initialization schemes and<br>existence proofs to non-zero biases, including explicit 'looks-linear'<br>approaches for ReLU activation functions. These do not only enable truly<br>orthogonal parameter initialization but also reduce potential pruning errors.<br>In experiments on standard benchmark data sets, we further highlight the<br>practical benefits of non-zero bias initialization schemes, and present<br>theoretically inspired extensions for state-of-the-art strong lottery ticket<br>pruning.<br>},
}