Publications

2023

Arabadzhiyska, E., Tursun, C., Seidel, H.-P., and Didyk, P. 2023. Practical Saccade Prediction for Head-Mounted Displays: Towards a Comprehensive Model. ACM Transactions on Applied Perception20, 1.

[DOI]

Links

Export

BibTeX

@article{Arabadzhiyska23, TITLE = {Practical Saccade Prediction for Head-Mounted Displays: {T}owards a Comprehensive Model}, AUTHOR = {Arabadzhiyska, Elena and Tursun, Cara and Seidel, Hans-Peter and Didyk, Piotr}, LANGUAGE = {eng}, ISSN = {1544-3558}, DOI = {10.1145/3568311}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Applied Perception}, VOLUME = {20}, NUMBER = {1}, PAGES = {1--23}, EID = {2}, }

Endnote

%0 Journal Article %A Arabadzhiyska, Elena %A Tursun, Cara %A Seidel, Hans-Peter %A Didyk, Piotr %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Practical Saccade Prediction for Head-Mounted Displays: Towards a Comprehensive Model : %G eng %U http://hdl.handle.net/21.11116/0000-000C-B76B-E %R 10.1145/3568311 %7 2023 %D 2023 %J ACM Transactions on Applied Perception %V 20 %N 1 %& 1 %P 1 - 23 %Z sequence number: 2 %I ACM %C New York, NY %@ false

Haynes, A., Reed, C.N., Nordmoen, C., and Skach, S. 2023. Being Meaningful: Weaving Soma-Reflective Technological Mediations into the Fabric of Daily Life. TEI ’23, Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction, ACM.

Links

Export

BibTeX

@inproceedings{Haynes_TEI23, TITLE = {Being Meaningful: {W}eaving Soma-Reflective Technological Mediations into the Fabric of Daily Life}, AUTHOR = {Haynes, Alice and Reed, Courtney N. and Nordmoen, Charlotte and Skach, Sophie}, LANGUAGE = {eng}, ISBN = {978-1-4503-9977-7}, DOI = {10.1145/3569009.3571844}, PUBLISHER = {ACM}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {TEI '23, Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction}, PAGES = {1--5}, EID = {68}, ADDRESS = {Warsaw, Poland}, }

Endnote

%0 Conference Proceedings %A Haynes, Alice %A Reed, Courtney N. %A Nordmoen, Charlotte %A Skach, Sophie %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Being Meaningful: Weaving Soma-Reflective Technological Mediations into the Fabric of Daily Life : %G eng %U http://hdl.handle.net/21.11116/0000-000C-BDEB-7 %R 10.1145/3569009.3571844 %D 2023 %B Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction %Z date of event: 2023-02-26 - 2023-03-01 %C Warsaw, Poland %B TEI '23 %P 1 - 5 %Z sequence number: 68 %I ACM %@ 978-1-4503-9977-7

Jambon, C., Kerbl, B., Kopanas, G., Diolatzis, S., Leimkühler, T., and Drettakis, G. NeRFshop: Interactive Editing of Neural Radiance Fields. Proceedings of the ACM on Computer Graphics and Interactive Techniques6, 1.

(Accepted/in press)

Links

Export

BibTeX

@article{JKKDLD23, TITLE = {{NeRFshop}: {I}nteractive Editing of Neural Radiance Fields}, AUTHOR = {Jambon, Cl{\'e}ment and Kerbl, Bernhard and Kopanas, Georgios and Diolatzis, Stavros and Leimk{\"u}hler, Thomas and Drettakis, George}, LANGUAGE = {eng}, ISSN = {2577-6193}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, JOURNAL = {Proceedings of the ACM on Computer Graphics and Interactive Techniques}, VOLUME = {6}, NUMBER = {1}, }

Endnote

%0 Journal Article %A Jambon, Clément %A Kerbl, Bernhard %A Kopanas, Georgios %A Diolatzis, Stavros %A Leimkühler, Thomas %A Drettakis, George %+ External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T NeRFshop: Interactive Editing of Neural Radiance Fields : %G eng %U http://hdl.handle.net/21.11116/0000-000C-BEE0-1 %D 2023 %J Proceedings of the ACM on Computer Graphics and Interactive Techniques %V 6 %N 1 %I ACM %C New York, NY %@ false %U http://www-sop.inria.fr/reves/Basilic/2023/JKKDLD23

Liao, K., Tricard, T., Piovarči, M., Seidel, H.-P., and Babaei, V. Learning Deposition Policies for Fused Multi-Material 3D Printing. IEEE International Conference on Robotics and Automation (ICRA 2023), IEEE.

(Accepted/in press)

Links

Export

BibTeX

@inproceedings{Liao_ICRA2023, TITLE = {Learning Deposition Policies for Fused Multi-Material {3D} Printing}, AUTHOR = {Liao, Kang and Tricard, Thibault and Piovar{\v c}i, Michal and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, PUBLISHER = {IEEE}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE International Conference on Robotics and Automation (ICRA 2023)}, ADDRESS = {London, UK}, }

Endnote

%0 Conference Proceedings %A Liao, Kang %A Tricard, Thibault %A Piovarči, Michal %A Seidel, Hans-Peter %A Babaei, Vahid %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Learning Deposition Policies for Fused Multi-Material 3D Printing : %G eng %U http://hdl.handle.net/21.11116/0000-000C-44C2-C %D 2023 %B IEEE International Conference on Robotics and Automation %Z date of event: 2023-05-29 - 2023-06-02 %C London, UK %B IEEE International Conference on Robotics and Automation %I IEEE

Reed, C.N., Strohmeier, P., and McPherson, A. Negotiating Experience and Communicating Information Through Abstract Metaphor. CHI ’23, CHI Conference on Human Factors in Computing Systems, ACM.

(Accepted/in press)

Links

Export

BibTeX

@inproceedings{Reed_CHI2023, TITLE = {Negotiating Experience and Communicating Information Through Abstract Metaphor}, AUTHOR = {Reed, Courtney N. and Strohmeier, Paul and McPherson, Andrew}, LANGUAGE = {eng}, DOI = {10.1145/3544548.3580700}, PUBLISHER = {ACM}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI '23, CHI Conference on Human Factors in Computing Systems}, ADDRESS = {Hamburg, Germany}, }

Endnote

%0 Conference Proceedings %A Reed, Courtney N. %A Strohmeier, Paul %A McPherson, Andrew %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Negotiating Experience and Communicating Information Through Abstract Metaphor : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A035-3 %R 10.1145/3544548.3580700 %D 2023 %B CHI Conference on Human Factors in Computing Systems %Z date of event: 2023-04-23 - 2023-04-28 %C Hamburg, Germany %B CHI '23 %I ACM

Reed, C.N. and McPherson, A.P. 2023. The Body as Sound: Unpacking Vocal Embodiment through Auditory Biofeedback. TEI ’23, Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction, ACM.

Links

Export

BibTeX

@inproceedings{Reed_TEI23, TITLE = {The Body as Sound: {U}npacking Vocal Embodiment through Auditory Biofeedback}, AUTHOR = {Reed, Courtney N. and McPherson, Andrew P.}, LANGUAGE = {eng}, ISBN = {978-1-4503-9977-7}, DOI = {10.1145/3569009.3572738}, PUBLISHER = {ACM}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {TEI '23, Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction}, PAGES = {1--15}, EID = {7}, ADDRESS = {Warsaw, Poland}, }

Endnote

%0 Conference Proceedings %A Reed, Courtney N. %A McPherson, Andrew P. %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T The Body as Sound: Unpacking Vocal Embodiment through Auditory Biofeedback : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A02B-F %R 10.1145/3569009.3572738 %D 2023 %B Seventeenth International Conference on Tangible, Embedded, and Embodied Interaction %Z date of event: 2023-02-26 - 2023-03-01 %C Warsaw, Poland %B TEI '23 %P 1 - 15 %Z sequence number: 7 %I ACM %@ 978-1-4503-9977-7

Reed, C.N. As the Luthiers Do: Designing with a Living, Growing, Changing Body-Material. CHI’ 23 Workshop - Body x Materials.

(Accepted/in press)

Links

Export

BibTeX

@inproceedings{reed2023bodyx, TITLE = {As the Luthiers Do: {D}esigning with a Living, Growing, Changing Body-Material}, AUTHOR = {Reed, Courtney N.}, LANGUAGE = {eng}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI{\textquoteright} 23 Workshop -- Body x Materials}, ADDRESS = {Hamburg, Germany}, }

Endnote

%0 Conference Proceedings %A Reed, Courtney N. %+ Computer Graphics, MPI for Informatics, Max Planck Society %T As the Luthiers Do: Designing with a Living, Growing, Changing Body-Material : %G eng %U http://hdl.handle.net/21.11116/0000-000C-BDD8-C %D 2023 %B ACM CHI Workshop on Body X Materials %Z date of event: 2023-04-23 - 2023-04-23 %C Hamburg, Germany %B CHI’ 23 Workshop - Body x Materials

Ruan, L., Bemana, M., Seidel, H.-P., Myszkowski, K., and Chen, B. 2023. Revisiting Image Deblurring with an Efficient ConvNet. https://arxiv.org/abs/2302.02234.

(arXiv: 2302.02234)

Abstract

Image deblurring aims to recover the latent sharp image from its blurry counterpart and has a wide range of applications in computer vision. The Convolution Neural Networks (CNNs) have performed well in this domain for many years, and until recently an alternative network architecture, namely Transformer, has demonstrated even stronger performance. One can attribute its superiority to the multi-head self-attention (MHSA) mechanism, which offers a larger receptive field and better input content adaptability than CNNs. However, as MHSA demands high computational costs that grow quadratically with respect to the input resolution, it becomes impractical for high-resolution image deblurring tasks. In this work, we propose a unified lightweight CNN network that features a large effective receptive field (ERF) and demonstrates comparable or even better performance than Transformers while bearing less computational costs. Our key design is an efficient CNN block dubbed LaKD, equipped with a large kernel depth-wise convolution and spatial-channel mixing structure, attaining comparable or larger ERF than Transformers but with a smaller parameter scale. Specifically, we achieve +0.17dB / +0.43dB PSNR over the state-of-the-art Restormer on defocus / motion deblurring benchmark datasets with 32% fewer parameters and 39% fewer MACs. Extensive experiments demonstrate the superior performance of our network and the effectiveness of each module. Furthermore, we propose a compact and intuitive ERFMeter metric that quantitatively characterizes ERF, and shows a high correlation to the network performance. We hope this work can inspire the research community to further explore the pros and cons of CNN and Transformer architectures beyond image deblurring tasks.

Links

Export

BibTeX

@online{ruan2023revisiting, TITLE = {Revisiting Image Deblurring with an Efficient {ConvNet}}, AUTHOR = {Ruan, Lingyan and Bemana, Mojtaba and Seidel, Hans-Peter and Myszkowski, Karol and Chen, Bin}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2302.02234}, EPRINT = {2302.02234}, EPRINTTYPE = {arXiv}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Image deblurring aims to recover the latent sharp image from its blurry counterpart and has a wide range of applications in computer vision. The Convolution Neural Networks (CNNs) have performed well in this domain for many years, and until recently an alternative network architecture, namely Transformer, has demonstrated even stronger performance. One can attribute its superiority to the multi-head self-attention (MHSA) mechanism, which offers a larger receptive field and better input content adaptability than CNNs. However, as MHSA demands high computational costs that grow quadratically with respect to the input resolution, it becomes impractical for high-resolution image deblurring tasks. In this work, we propose a unified lightweight CNN network that features a large effective receptive field (ERF) and demonstrates comparable or even better performance than Transformers while bearing less computational costs. Our key design is an efficient CNN block dubbed LaKD, equipped with a large kernel depth-wise convolution and spatial-channel mixing structure, attaining comparable or larger ERF than Transformers but with a smaller parameter scale. Specifically, we achieve +0.17dB / +0.43dB PSNR over the state-of-the-art Restormer on defocus / motion deblurring benchmark datasets with 32% fewer parameters and 39% fewer MACs. Extensive experiments demonstrate the superior performance of our network and the effectiveness of each module. Furthermore, we propose a compact and intuitive ERFMeter metric that quantitatively characterizes ERF, and shows a high correlation to the network performance. We hope this work can inspire the research community to further explore the pros and cons of CNN and Transformer architectures beyond image deblurring tasks. }, }

Endnote

%0 Report %A Ruan, Lingyan %A Bemana, Mojtaba %A Seidel, Hans-Peter %A Myszkowski, Karol %A Chen, Bin %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Revisiting Image Deblurring with an Efficient ConvNet : %G eng %U http://hdl.handle.net/21.11116/0000-000C-C7B9-3 %U https://arxiv.org/abs/2302.02234 %D 2023 %X Image deblurring aims to recover the latent sharp image from its blurry counterpart and has a wide range of applications in computer vision. The Convolution Neural Networks (CNNs) have performed well in this domain for many years, and until recently an alternative network architecture, namely Transformer, has demonstrated even stronger performance. One can attribute its superiority to the multi-head self-attention (MHSA) mechanism, which offers a larger receptive field and better input content adaptability than CNNs. However, as MHSA demands high computational costs that grow quadratically with respect to the input resolution, it becomes impractical for high-resolution image deblurring tasks. In this work, we propose a unified lightweight CNN network that features a large effective receptive field (ERF) and demonstrates comparable or even better performance than Transformers while bearing less computational costs. Our key design is an efficient CNN block dubbed LaKD, equipped with a large kernel depth-wise convolution and spatial-channel mixing structure, attaining comparable or larger ERF than Transformers but with a smaller parameter scale. Specifically, we achieve +0.17dB / +0.43dB PSNR over the state-of-the-art Restormer on defocus / motion deblurring benchmark datasets with 32% fewer parameters and 39% fewer MACs. Extensive experiments demonstrate the superior performance of our network and the effectiveness of each module. Furthermore, we propose a compact and intuitive ERFMeter metric that quantitatively characterizes ERF, and shows a high correlation to the network performance. We hope this work can inspire the research community to further explore the pros and cons of CNN and Transformer architectures beyond image deblurring tasks. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Sabnis, N., Wittchen, D., Reed, C.N., Pourjafarian, N., Steimle, J., and Strohmeier, P. Haptic Servos: Self-Contained Vibrotactile Rendering System for Creating or Augmenting Material Experiences. CHI ’23, CHI Conference on Human Factors in Computing Systems, ACM.

(Accepted/in press)

Links

Export

BibTeX

@inproceedings{Sabnis_CHI2023, TITLE = {Haptic Servos: {S}elf-Contained Vibrotactile Rendering System for Creating or Augmenting Material Experiences}, AUTHOR = {Sabnis, Nihar and Wittchen, Dennis and Reed, Courtney N. and Pourjafarian, Narjes and Steimle, J{\"u}rgen and Strohmeier, Paul}, LANGUAGE = {eng}, DOI = {10.1145/3544548.3580716}, PUBLISHER = {ACM}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI '23, CHI Conference on Human Factors in Computing Systems}, ADDRESS = {Hamburg, Germany}, }

Endnote

%0 Conference Proceedings %A Sabnis, Nihar %A Wittchen, Dennis %A Reed, Courtney N. %A Pourjafarian, Narjes %A Steimle, Jürgen %A Strohmeier, Paul %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Haptic Servos: Self-Contained Vibrotactile Rendering System for Creating or Augmenting Material Experiences : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A03C-C %R 10.1145/3544548.3580716 %D 2023 %B CHI Conference on Human Factors in Computing Systems %Z date of event: 2023-04-23 - 2023-04-28 %C Hamburg, Germany %B CHI '23 %I ACM

Sabnis, N., Wittchen, D., Vega, G., Reed, C.N., and Strohmeier, P. Tactile Symbols with Continuous and Motion-Coupled Vibration: An Exploration of using Embodied Experiences for Hermeneutic Design. CHI ’23, CHI Conference on Human Factors in Computing Systems, ACM.

(Accepted/in press)

Links

Export

BibTeX

@inproceedings{Sabnis_CHI2023B, TITLE = {Tactile Symbols with Continuous and Motion-Coupled Vibration: {A}n Exploration of using Embodied Experiences for Hermeneutic Design}, AUTHOR = {Sabnis, Nihar and Wittchen, Dennis and Vega, Gabriela and Reed, Courtney N. and Strohmeier, Paul}, LANGUAGE = {eng}, DOI = {10.1145/3544548.3581356}, PUBLISHER = {ACM}, YEAR = {2023}, PUBLREMARK = {Accepted}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI '23, CHI Conference on Human Factors in Computing Systems}, ADDRESS = {Hamburg, Germany}, }

Endnote

%0 Conference Proceedings %A Sabnis, Nihar %A Wittchen, Dennis %A Vega, Gabriela %A Reed, Courtney N. %A Strohmeier, Paul %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Tactile Symbols with Continuous and Motion-Coupled Vibration: An Exploration of using Embodied Experiences for Hermeneutic Design : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A042-4 %R 10.1145/3544548.3581356 %D 2023 %B CHI Conference on Human Factors in Computing Systems %Z date of event: 2023-04-23 - 2023-04-28 %C Hamburg, Germany %B CHI '23 %I ACM

Surace, L., Wernikowski, M., Tursun, C., Myszkowski, K., Mantiuk, R., and Didyk, P. 2023. Learning GAN-based Foveated Reconstruction to Recover Perceptually Important Image Features. ACM Transactions on Applied Perception.

Links

Export

BibTeX

@article{Surace23, TITLE = {Learning {GAN}-based Foveated Reconstruction to Recover Perceptually Important Image Features}, AUTHOR = {Surace, Luca and Wernikowski, Marek and Tursun, Cara and Myszkowski, Karol and Mantiuk, Rados{\l}aw and Didyk, Piotr}, LANGUAGE = {eng}, ISSN = {1544-3558}, DOI = {10.1145/3583072}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Applied Perception}, }

Endnote

%0 Journal Article %A Surace, Luca %A Wernikowski, Marek %A Tursun, Cara %A Myszkowski, Karol %A Mantiuk, Radosław %A Didyk, Piotr %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Learning GAN-based Foveated Reconstruction to Recover Perceptually Important Image Features : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A00D-1 %R 10.1145/3583072 %7 2023 %D 2023 %J ACM Transactions on Applied Perception %I ACM %C New York, NY %@ false

Weinrauch, A., Seidel, H.-P., Mlakar, D., Steinberger, M., and Zayer, R. 2023. A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and Reeb Graph Construction on Surfaces. Computer Graphics Forum42, 2.

Links

Export

BibTeX

@article{Weinrauch_CGF23, TITLE = {A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and {Reeb} Graph Construction on Surfaces}, AUTHOR = {Weinrauch, Alexander and Seidel, Hans-Peter and Mlakar, Daniel and Steinberger, Markus and Zayer, Rhaleb}, LANGUAGE = {eng}, ISSN = {0167-7055}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, JOURNAL = {Computer Graphics Forum}, VOLUME = {42}, NUMBER = {2}, }

Endnote

%0 Journal Article %A Weinrauch, Alexander %A Seidel, Hans-Peter %A Mlakar, Daniel %A Steinberger, Markus %A Zayer, Rhaleb %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and Reeb Graph Construction on Surfaces : %G eng %U http://hdl.handle.net/21.11116/0000-000C-B851-9 %7 2023 %D 2023 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 42 %N 2 %I Blackwell-Wiley %C Oxford %@ false

Wittchen, D., Marinez-Missir, V., Mavali, S., Sabnis, N., Reed, C.N., and Strohmeier, P. 2023. Designing Interactive Shoes for Tactile Augmented Reality. AHs ’23, Augmented Humans International Conference, ACM.

Links

Export

BibTeX

@inproceedings{Wittchen_AHs2023, TITLE = {Designing Interactive Shoes for Tactile Augmented Reality}, AUTHOR = {Wittchen, Dennis and Marinez-Missir, Valenin and Mavali, Sina and Sabnis, Nihar and Reed, Courtney N. and Strohmeier, Paul}, LANGUAGE = {eng}, ISBN = {978-1-4503-9984-5}, DOI = {10.1145/3582700.3582728}, PUBLISHER = {ACM}, YEAR = {2023}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {AHs '23, Augmented Humans International Conference}, PAGES = {1--14}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Wittchen, Dennis %A Marinez-Missir, Valenin %A Mavali, Sina %A Sabnis, Nihar %A Reed, Courtney N. %A Strohmeier, Paul %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Designing Interactive Shoes for Tactile Augmented Reality : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A04D-9 %R 10.1145/3582700.3582728 %D 2023 %B Augmented Humans International Conference %Z date of event: 2023-03-12 - 2023-03-14 %C Glasgow, UK %B AHs '23 %P 1 - 14 %I ACM %@ 978-1-4503-9984-5

2022

Ansari, N., Seidel, H.-P., and Babaei, V. 2022a. Mixed Integer Neural Inverse Design. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2022)41, 4.

Links

Export

BibTeX

@article{Ansari22, TITLE = {Mixed Integer Neural Inverse Design}, AUTHOR = {Ansari, Navid and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3528223.3530083}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {41}, NUMBER = {4}, PAGES = {1--14}, EID = {151}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2022}, }

Endnote

%0 Journal Article %A Ansari, Navid %A Seidel, Hans-Peter %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Mixed Integer Neural Inverse Design : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1678-5 %R 10.1145/3528223.3530083 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 4 %& 1 %P 1 - 14 %Z sequence number: 151 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2022 %O ACM SIGGRAPH 2022

Ansari, N., Seidel, H.-P., Vahidi Ferdowsi, N., and Babaei, V. 2022b. Autoinverse: Uncertainty Aware Inversion of Neural Networks. Advances in Neural Information Processing Systems 35 (NeurIPS 2022), Curran Associates, Inc.

Links

Export

BibTeX

@inproceedings{Ansari_Neurips22, TITLE = {Autoinverse: {U}ncertainty Aware Inversion of Neural Networks}, AUTHOR = {Ansari, Navid and Seidel, Hans-Peter and Vahidi Ferdowsi, Nima and Babaei, Vahid}, LANGUAGE = {eng}, PUBLISHER = {Curran Associates, Inc}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Advances in Neural Information Processing Systems 35 (NeurIPS 2022)}, EDITOR = {Koyejo, S. and Mohamed, S. and Agarwal, A. and Belgrave, D. and Cho, K. and Oh, A.}, PAGES = {8675--8686}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Ansari, Navid %A Seidel, Hans-Peter %A Vahidi Ferdowsi, Nima %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Autoinverse: Uncertainty Aware Inversion of Neural Networks : %G eng %U http://hdl.handle.net/21.11116/0000-000C-16F6-6 %D 2022 %B 36th Conference on Neural Information Processing Systems %Z date of event: 2022-11-28 - 2022-12-09 %C New Orleans, LA, USA %B Advances in Neural Information Processing Systems 35 %E Koyejo, S.; Mohamed, S.; Agarwal, A.; Belgrave, D.; Cho, K.; Oh, A. %P 8675 - 8686 %I Curran Associates, Inc %U https://openreview.net/pdf?id=dNyCj1AbOb

Arabadzhiyska, E., Tursun, C., Seidel, H.-P., and Didyk, P. 2022. Practical Saccade Prediction for Head-Mounted Displays: Towards a Comprehensive Model. https://arxiv.org/abs/2205.01624.

(arXiv: 2205.01624)

Abstract

Eye-tracking technology is an integral component of new display devices such as virtual and augmented reality headsets. Applications of gaze information range from new interaction techniques exploiting eye patterns to gaze-contingent digital content creation. However, system latency is still a significant issue in many of these applications because it breaks the synchronization between the current and measured gaze positions. Consequently, it may lead to unwanted visual artifacts and degradation of user experience. In this work, we focus on foveated rendering applications where the quality of an image is reduced towards the periphery for computational savings. In foveated rendering, the presence of latency leads to delayed updates to the rendered frame, making the quality degradation visible to the user. To address this issue and to combat system latency, recent work proposes to use saccade landing position prediction to extrapolate the gaze information from delayed eye-tracking samples. While the benefits of such a strategy have already been demonstrated, the solutions range from simple and efficient ones, which make several assumptions about the saccadic eye movements, to more complex and costly ones, which use machine learning techniques. Yet, it is unclear to what extent the prediction can benefit from accounting for additional factors. This paper presents a series of experiments investigating the importance of different factors for saccades prediction in common virtual and augmented reality applications. In particular, we investigate the effects of saccade orientation in 3D space and smooth pursuit eye-motion (SPEM) and how their influence compares to the variability across users. We also present a simple yet efficient correction method that adapts the existing saccade prediction methods to handle these factors without performing extensive data collection.

Links

Export

BibTeX

@online{Arabadzhiyska2205.01624, TITLE = {Practical Saccade Prediction for Head-Mounted Displays: Towards a Comprehensive Model}, AUTHOR = {Arabadzhiyska, Elena and Tursun, Cara and Seidel, Hans-Peter and Didyk, Piotr}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2205.01624}, EPRINT = {2205.01624}, EPRINTTYPE = {arXiv}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Eye-tracking technology is an integral component of new display devices such as virtual and augmented reality headsets. Applications of gaze information range from new interaction techniques exploiting eye patterns to gaze-contingent digital content creation. However, system latency is still a significant issue in many of these applications because it breaks the synchronization between the current and measured gaze positions. Consequently, it may lead to unwanted visual artifacts and degradation of user experience. In this work, we focus on foveated rendering applications where the quality of an image is reduced towards the periphery for computational savings. In foveated rendering, the presence of latency leads to delayed updates to the rendered frame, making the quality degradation visible to the user. To address this issue and to combat system latency, recent work proposes to use saccade landing position prediction to extrapolate the gaze information from delayed eye-tracking samples. While the benefits of such a strategy have already been demonstrated, the solutions range from simple and efficient ones, which make several assumptions about the saccadic eye movements, to more complex and costly ones, which use machine learning techniques. Yet, it is unclear to what extent the prediction can benefit from accounting for additional factors. This paper presents a series of experiments investigating the importance of different factors for saccades prediction in common virtual and augmented reality applications. In particular, we investigate the effects of saccade orientation in 3D space and smooth pursuit eye-motion (SPEM) and how their influence compares to the variability across users. We also present a simple yet efficient correction method that adapts the existing saccade prediction methods to handle these factors without performing extensive data collection. }, }

Endnote

%0 Report %A Arabadzhiyska, Elena %A Tursun, Cara %A Seidel, Hans-Peter %A Didyk, Piotr %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Practical Saccade Prediction for Head-Mounted Displays: Towards a Comprehensive Model : %G eng %U http://hdl.handle.net/21.11116/0000-000C-16E3-B %U https://arxiv.org/abs/2205.01624 %D 2022 %X Eye-tracking technology is an integral component of new display devices such as virtual and augmented reality headsets. Applications of gaze information range from new interaction techniques exploiting eye patterns to gaze-contingent digital content creation. However, system latency is still a significant issue in many of these applications because it breaks the synchronization between the current and measured gaze positions. Consequently, it may lead to unwanted visual artifacts and degradation of user experience. In this work, we focus on foveated rendering applications where the quality of an image is reduced towards the periphery for computational savings. In foveated rendering, the presence of latency leads to delayed updates to the rendered frame, making the quality degradation visible to the user. To address this issue and to combat system latency, recent work proposes to use saccade landing position prediction to extrapolate the gaze information from delayed eye-tracking samples. While the benefits of such a strategy have already been demonstrated, the solutions range from simple and efficient ones, which make several assumptions about the saccadic eye movements, to more complex and costly ones, which use machine learning techniques. Yet, it is unclear to what extent the prediction can benefit from accounting for additional factors. This paper presents a series of experiments investigating the importance of different factors for saccades prediction in common virtual and augmented reality applications. In particular, we investigate the effects of saccade orientation in 3D space and smooth pursuit eye-motion (SPEM) and how their influence compares to the variability across users. We also present a simple yet efficient correction method that adapts the existing saccade prediction methods to handle these factors without performing extensive data collection. %K Computer Science, Human-Computer Interaction, cs.HC,Computer Science, Graphics, cs.GR

Bemana, M., Myszkowski, K., Frisvad, J.R., Seidel, H.-P., and Ritschel, T. 2022. Eikonal Fields for Refractive Novel-View Synthesis. Proceedings SIGGRAPH 2022 Conference Papers Proceedings (ACM SIGGRAPH 2022), ACM.

Links

Export

BibTeX

@inproceedings{Bemana_SIGGRAPH22, TITLE = {Eikonal Fields for Refractive Novel-View Synthesis}, AUTHOR = {Bemana, Mojtaba and Myszkowski, Karol and Frisvad, Jeppe Revall and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, ISBN = {978-1-4503-9337-9}, DOI = {10.1145/3528233.3530706}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Proceedings SIGGRAPH 2022 Conference Papers Proceedings (ACM SIGGRAPH 2022)}, EDITOR = {Nandigjav, Munkhtsetseg and Mitra, Niloy J. and Hertzmann, Aaron}, PAGES = {1--9}, EID = {39}, ADDRESS = {Vancouver, Canada}, }

Endnote

%0 Conference Proceedings %A Bemana, Mojtaba %A Myszkowski, Karol %A Frisvad, Jeppe Revall %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Eikonal Fields for Refractive Novel-View Synthesis : %G eng %U http://hdl.handle.net/21.11116/0000-000A-BA61-7 %R 10.1145/3528233.3530706 %D 2022 %B ACM SIGGRAPH %Z date of event: 2022-08-07 - 2022-08-11 %C Vancouver, Canada %B Proceedings SIGGRAPH 2022 Conference Papers Proceedings %E Nandigjav, Munkhtsetseg; Mitra, Niloy J.; Hertzmann, Aaron %P 1 - 9 %Z sequence number: 39 %I ACM %@ 978-1-4503-9337-9

Chen, B., Piovarči, M., Wang, C., et al. 2022. Gloss Management for Consistent Reproduction of Real and Virtual Objects. Proceedings SIGGRAPH Asia 2022 (ACM SIGGRAPH Asia 2022), ACM.

Links

Export

BibTeX

@inproceedings{ChenSA22, TITLE = {Gloss Management for Consistent Reproduction of Real and Virtual Objects}, AUTHOR = {Chen, Bin and Piovar{\v c}i, Michal and Wang, Chao and Seidel, Hans-Peter and Didyk, Piotr and Myszkowski, Karol and Serrano, Ana}, LANGUAGE = {eng}, ISBN = {978-1-4503-9470-3}, DOI = {10.1145/3550469.3555406}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Proceedings SIGGRAPH Asia 2022 (ACM SIGGRAPH Asia 2022)}, EDITOR = {Jung, Soon Ki and Lee, Jehee and Bargteil, Adam}, PAGES = {1--9}, EID = {35}, }

Endnote

%0 Conference Proceedings %A Chen, Bin %A Piovarči, Michal %A Wang, Chao %A Seidel, Hans-Peter %A Didyk, Piotr %A Myszkowski, Karol %A Serrano, Ana %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Gloss Management for Consistent Reproduction of Real and Virtual Objects : %G eng %U http://hdl.handle.net/21.11116/0000-000C-167F-E %R 10.1145/3550469.3555406 %D 2022 %B Proceedings SIGGRAPH Asia 2022 %E Jung, Soon Ki; Lee, Jehee; Bargteil, Adam %P 1 - 9 %Z sequence number: 35 %I ACM %@ 978-1-4503-9470-3

Chizhov, V., Georgiev, I., Myszkowski, K., and Singh, G. 2022. Perceptual Error Optimization for Monte Carlo Rendering. ACM Transactions on Graphics41, 3.

Links

Export

BibTeX

@article{ChizhovTOG22, TITLE = {Perceptual Error Optimization for {Monte Carlo} Rendering}, AUTHOR = {Chizhov, Vassillen and Georgiev, Iliyan and Myszkowski, Karol and Singh, Gurprit}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3504002}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics}, VOLUME = {41}, NUMBER = {3}, PAGES = {1--17}, EID = {26}, }

Endnote

%0 Journal Article %A Chizhov, Vassillen %A Georgiev, Iliyan %A Myszkowski, Karol %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Perceptual Error Optimization for Monte Carlo Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-000A-BA49-3 %R 10.1145/3504002 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 3 %& 1 %P 1 - 17 %Z sequence number: 26 %I ACM %C New York, NY %@ false

Chu, M., Liu, L., Zheng, Q., et al. 2022. Physics Informed Neural Fields for Smoke Reconstruction with Sparse Data. ACM Transactions on Graphics41, 4.

Links

Export

BibTeX

@article{Chu2022, TITLE = {Physics Informed Neural Fields for Smoke Reconstruction with Sparse Data}, AUTHOR = {Chu, Mengyu and Liu, Lingjie and Zheng, Quan and Franz, Erik and Seidel, Hans-Peter and Theobalt, Christian and Zayer, Rhaleb}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3528223.3530169}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics}, VOLUME = {41}, NUMBER = {4}, PAGES = {1--14}, EID = {119}, }

Endnote

%0 Journal Article %A Chu, Mengyu %A Liu, Lingjie %A Zheng, Quan %A Franz, Erik %A Seidel, Hans-Peter %A Theobalt, Christian %A Zayer, Rhaleb %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Physics Informed Neural Fields for Smoke Reconstruction with Sparse Data : %G eng %U http://hdl.handle.net/21.11116/0000-000B-6561-6 %R 10.1145/3528223.3530169 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 4 %& 1 %P 1 - 14 %Z sequence number: 119 %I ACM %C New York, NY %@ false %U https://people.mpi-inf.mpg.de/~mchu/projects/PI-NeRF/

Çoğalan, U., Bemana, M., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2022a. Learning HDR Video Reconstruction for Dual-Exposure Sensors with Temporally-alternating Exposures. Computers and Graphics105.

Links

Export

BibTeX

@article{Cogalan2022, TITLE = {Learning {HDR} Video Reconstruction for Dual-Exposure Sensors with Temporally-alternating Exposures}, AUTHOR = {{\c C}o{\u g}alan, U{\u g}ur and Bemana, Mojtaba and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, ISSN = {0097-8493}, DOI = {10.1016/j.cag.2022.04.008}, PUBLISHER = {Elsevier}, ADDRESS = {Amsterdam}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {Computers and Graphics}, VOLUME = {105}, PAGES = {57--72}, }

Endnote

%0 Journal Article %A Çoğalan, Uğur %A Bemana, Mojtaba %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Learning HDR Video Reconstruction for Dual-Exposure Sensors with Temporally-alternating Exposures : %G eng %U http://hdl.handle.net/21.11116/0000-000A-9D95-D %R 10.1016/j.cag.2022.04.008 %7 2022 %D 2022 %J Computers and Graphics %V 105 %& 57 %P 57 - 72 %I Elsevier %C Amsterdam %@ false

Çoğalan, U., Bemana, M., Seidel, H.-P., and Myszkowski, K. 2022b. Video Frame Interpolation for High Dynamic Range Sequences Captured with Dual-exposure Sensors. https://arxiv.org/abs/2206.09485.

(arXiv: 2206.09485)

Abstract

Video frame interpolation (VFI) enables many important applications that might involve the temporal domain, such as slow motion playback, or the spatial domain, such as stop motion sequences. We are focusing on the former task, where one of the key challenges is handling high dynamic range (HDR) scenes in the presence of complex motion. To this end, we explore possible advantages of dual-exposure sensors that readily provide sharp short and blurry long exposures that are spatially registered and whose ends are temporally aligned. This way, motion blur registers temporally continuous information on the scene motion that, combined with the sharp reference, enables more precise motion sampling within a single camera shot. We demonstrate that this facilitates a more complex motion reconstruction in the VFI task, as well as HDR frame reconstruction that so far has been considered only for the originally captured frames, not in-between interpolated frames. We design a neural network trained in these tasks that clearly outperforms existing solutions. We also propose a metric for scene motion complexity that provides important insights into the performance of VFI methods at the test time.

Links

Export

BibTeX

@online{Cogalan2206.09485, TITLE = {Video Frame Interpolation for High Dynamic Range Sequences Captured with Dual-exposure Sensors}, AUTHOR = {{\c C}o{\u g}alan, U{\u g}ur and Bemana, Mojtaba and Seidel, Hans-Peter and Myszkowski, Karol}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2206.09485}, EPRINT = {2206.09485}, EPRINTTYPE = {arXiv}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Video frame interpolation (VFI) enables many important applications that might involve the temporal domain, such as slow motion playback, or the spatial domain, such as stop motion sequences. We are focusing on the former task, where one of the key challenges is handling high dynamic range (HDR) scenes in the presence of complex motion. To this end, we explore possible advantages of dual-exposure sensors that readily provide sharp short and blurry long exposures that are spatially registered and whose ends are temporally aligned. This way, motion blur registers temporally continuous information on the scene motion that, combined with the sharp reference, enables more precise motion sampling within a single camera shot. We demonstrate that this facilitates a more complex motion reconstruction in the VFI task, as well as HDR frame reconstruction that so far has been considered only for the originally captured frames, not in-between interpolated frames. We design a neural network trained in these tasks that clearly outperforms existing solutions. We also propose a metric for scene motion complexity that provides important insights into the performance of VFI methods at the test time. }, }

Endnote

%0 Report %A Çoğalan, Uğur %A Bemana, Mojtaba %A Seidel, Hans-Peter %A Myszkowski, Karol %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Video Frame Interpolation for High Dynamic Range Sequences Captured with Dual-exposure Sensors : %G eng %U http://hdl.handle.net/21.11116/0000-000C-16E8-6 %U https://arxiv.org/abs/2206.09485 %D 2022 %X Video frame interpolation (VFI) enables many important applications that might involve the temporal domain, such as slow motion playback, or the spatial domain, such as stop motion sequences. We are focusing on the former task, where one of the key challenges is handling high dynamic range (HDR) scenes in the presence of complex motion. To this end, we explore possible advantages of dual-exposure sensors that readily provide sharp short and blurry long exposures that are spatially registered and whose ends are temporally aligned. This way, motion blur registers temporally continuous information on the scene motion that, combined with the sharp reference, enables more precise motion sampling within a single camera shot. We demonstrate that this facilitates a more complex motion reconstruction in the VFI task, as well as HDR frame reconstruction that so far has been considered only for the originally captured frames, not in-between interpolated frames. We design a neural network trained in these tasks that clearly outperforms existing solutions. We also propose a metric for scene motion complexity that provides important insights into the performance of VFI methods at the test time. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Hladký, J., Stengel, M., Vining, N., Kerbl, B., Seidel, H.-P., and Steinberger, M. 2022. QuadStream: A Quad-Based Scene Streaming Architecture for Novel Viewpoint Reconstruction. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2022)41, 6.

Links

Export

BibTeX

@article{HladkySIGGRAPHAsia22, TITLE = {QuadStream: {A} Quad-Based Scene Streaming Architecture for Novel Viewpoint Reconstruction}, AUTHOR = {Hladk{\'y}, Jozef and Stengel, Michael and Vining, Nicholas and Kerbl, Bernhard and Seidel, Hans-Peter and Steinberger, Markus}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3550454.3555524}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {41}, NUMBER = {6}, PAGES = {1--13}, EID = {233}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2022}, }

Endnote

%0 Journal Article %A Hladký, Jozef %A Stengel, Michael %A Vining, Nicholas %A Kerbl, Bernhard %A Seidel, Hans-Peter %A Steinberger, Markus %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T QuadStream: A Quad-Based Scene Streaming Architecture for Novel Viewpoint Reconstruction : %G eng %U http://hdl.handle.net/21.11116/0000-000C-208B-3 %R 10.1145/3550454.3555524 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 6 %& 1 %P 1 - 13 %Z sequence number: 233 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2022 %O ACM SIGGRAPH Asia 2022 SA '22 SA 2022

Huang, X., Memari, P., Seidel, H.-P., and Singh, G. 2022. Point-Pattern Synthesis using Gabor and Random Filters. Computer Graphics Forum (Proc. Eurographics Symposium on Rendering 2022)41, 4.

Links

Export

BibTeX

@article{Huang_EGSR2022, TITLE = {Point-Pattern Synthesis using {Gabor} and Random Filters}, AUTHOR = {Huang, Xingchang and Memari, Pooran and Seidel, Hans-Peter and Singh, Gurprit}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.14596}, PUBLISHER = {Wiley-Blackwell}, ADDRESS = {Oxford}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, DATE = {2022}, JOURNAL = {Computer Graphics Forum (Proc. Eurographics Symposium on Rendering)}, VOLUME = {41}, NUMBER = {4}, PAGES = {169--179}, BOOKTITLE = {Eurographics Symposium on Rendering 2022}, EDITOR = {Ghosh, Abhijeet and Wei, Li-Yi and Wilkie, Alexander}, }

Endnote

%0 Journal Article %A Huang, Xingchang %A Memari, Pooran %A Seidel, Hans-Peter %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Point-Pattern Synthesis using Gabor and Random Filters : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1675-8 %R 10.1111/cgf.14596 %7 2022 %D 2022 %J Computer Graphics Forum %V 41 %N 4 %& 169 %P 169 - 179 %I Wiley-Blackwell %C Oxford %@ false %B Eurographics Symposium on Rendering 2022 %O Eurographics Symposium on Rendering 2022 EGSR 2022 Prague, Czech Republic & Virtual ; 4 - 6 July 2022 %U https://onlinelibrary.wiley.com/share/X44DPUPXHCYNCUKSEBEE?target=10.1111/cgf.14596

Kopanas, G., Leimkühler, T., Rainer, G., Jambon, C., and Drettakis, G. 2022. Neural Point Catacaustics for Novel-View Synthesis of Reflections. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2022)41, 6.

Links

Export

BibTeX

@article{KopanasSIGGRAPHAsia22, TITLE = {Neural Point Catacaustics for Novel-View Synthesis of Reflections}, AUTHOR = {Kopanas, Georgios and Leimk{\"u}hler, Thomas and Rainer, Gilles and Jambon, Cl{\'e}ment and Drettakis, George}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3550454.3555497}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {41}, NUMBER = {6}, PAGES = {1--15}, EID = {201}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2022}, }

Endnote

%0 Journal Article %A Kopanas, Georgios %A Leimkühler, Thomas %A Rainer, Gilles %A Jambon, Clément %A Drettakis, George %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations %T Neural Point Catacaustics for Novel-View Synthesis of Reflections : %G eng %U http://hdl.handle.net/21.11116/0000-000C-209B-1 %R 10.1145/3550454.3555497 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 6 %& 1 %P 1 - 15 %Z sequence number: 201 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2022 %O ACM SIGGRAPH Asia 2022 SA '22 SA 2022

Li, J., Chen, B., Zan, G., Qian, G., Pianetta, P., and Liu, Y. 2022. Subspace Modeling for Fast and High-sensitivity X-ray Chemical Imaging. https://arxiv.org/abs/2201.00259.

(arXiv: 2201.00259)

Abstract

Resolving morphological chemical phase transformations at the nanoscale is of vital importance to many scientific and industrial applications across various disciplines. The TXM-XANES imaging technique, by combining full field transmission X-ray microscopy (TXM) and X-ray absorption near edge structure (XANES), has been an emerging tool which operates by acquiring a series of microscopy images with multi-energy X-rays and fitting to obtain the chemical map. Its capability, however, is limited by the poor signal-to-noise ratios due to the system errors and low exposure illuminations for fast acquisition. In this work, by exploiting the intrinsic properties and subspace modeling of the TXM-XANES imaging data, we introduce a simple and robust denoising approach to improve the image quality, which enables fast and high-sensitivity chemical imaging. Extensive experiments on both synthetic and real datasets demonstrate the superior performance of the proposed method.

Links

Export

BibTeX

@online{li2022subspace, TITLE = {Subspace Modeling for Fast and High-sensitivity X-ray Chemical Imaging}, AUTHOR = {Li, Jizhou and Chen, Bin and Zan, Guibin and Qian, Guannan and Pianetta, Piero and Liu, Yijin}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2201.00259}, EPRINT = {2201.00259}, EPRINTTYPE = {arXiv}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Resolving morphological chemical phase transformations at the nanoscale is of vital importance to many scientific and industrial applications across various disciplines. The TXM-XANES imaging technique, by combining full field transmission X-ray microscopy (TXM) and X-ray absorption near edge structure (XANES), has been an emerging tool which operates by acquiring a series of microscopy images with multi-energy X-rays and fitting to obtain the chemical map. Its capability, however, is limited by the poor signal-to-noise ratios due to the system errors and low exposure illuminations for fast acquisition. In this work, by exploiting the intrinsic properties and subspace modeling of the TXM-XANES imaging data, we introduce a simple and robust denoising approach to improve the image quality, which enables fast and high-sensitivity chemical imaging. Extensive experiments on both synthetic and real datasets demonstrate the superior performance of the proposed method. }, }

Endnote

%0 Report %A Li, Jizhou %A Chen, Bin %A Zan, Guibin %A Qian, Guannan %A Pianetta, Piero %A Liu, Yijin %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T Subspace Modeling for Fast and High-sensitivity X-ray Chemical Imaging : %G eng %U http://hdl.handle.net/21.11116/0000-000C-C7BE-E %U https://arxiv.org/abs/2201.00259 %D 2022 %X Resolving morphological chemical phase transformations at the nanoscale is of vital importance to many scientific and industrial applications across various disciplines. The TXM-XANES imaging technique, by combining full field transmission X-ray microscopy (TXM) and X-ray absorption near edge structure (XANES), has been an emerging tool which operates by acquiring a series of microscopy images with multi-energy X-rays and fitting to obtain the chemical map. Its capability, however, is limited by the poor signal-to-noise ratios due to the system errors and low exposure illuminations for fast acquisition. In this work, by exploiting the intrinsic properties and subspace modeling of the TXM-XANES imaging data, we introduce a simple and robust denoising approach to improve the image quality, which enables fast and high-sensitivity chemical imaging. Extensive experiments on both synthetic and real datasets demonstrate the superior performance of the proposed method. %K eess.IV,Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Multimedia, cs.MM

Mueller, F., Semertzidis, N., Andres, J., et al. 2022. Human–Computer Integration: Towards Integrating the Human Body with the Computational Machine. Foundations and Trends in Human-Computer Interaction16, 1.

Links

Export

BibTeX

@article{Mueller22, TITLE = {Human--Computer Integration: {T}owards Integrating the Human Body with the Computational Machine}, AUTHOR = {Mueller, Florian and Semertzidis, Nathan and Andres, Josh and Weigel, Martin and Nanayakkara, Suranga and Patibanda, Rakesh and Li, Zhuying and Strohmeier, Paul and Knibbe, Jarrod and Greuter, Stefan and Obrist, Marianna and Maes, Pattie and Wang, Dakuo and Wolf, Katrin and Gerber, Liz and Marshall, Joe and Kunze, Kai and Grudin, Jonathan and Reiterer, Harald and Byrne, Richard}, LANGUAGE = {eng}, ISSN = {1551-3955}, ISBN = {978-1-63828-068-2}, DOI = {10.1561/1100000086}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {Foundations and Trends in Human-Computer Interaction}, VOLUME = {16}, NUMBER = {1}, PAGES = {1--64}, }

Endnote

%0 Journal Article %A Mueller, Florian %A Semertzidis, Nathan %A Andres, Josh %A Weigel, Martin %A Nanayakkara, Suranga %A Patibanda, Rakesh %A Li, Zhuying %A Strohmeier, Paul %A Knibbe, Jarrod %A Greuter, Stefan %A Obrist, Marianna %A Maes, Pattie %A Wang, Dakuo %A Wolf, Katrin %A Gerber, Liz %A Marshall, Joe %A Kunze, Kai %A Grudin, Jonathan %A Reiterer, Harald %A Byrne, Richard %+ External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations %T Human–Computer Integration: Towards Integrating the Human Body with the Computational Machine : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1734-0 %R 10.1561/1100000086 %@ 978-1-63828-068-2 %7 2022 %D 2022 %J Foundations and Trends in Human-Computer Interaction %O Foundations and Trends® in Human-Computer Interaction %V 16 %N 1 %& 1 %P 1 - 64 %@ false

Panetta, J., Mohammadian, H., Luci, E., and Babaei, V. 2022. Shape from Release: Inverse Design and Fabrication of Controlled Release Structures. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2022)41, 6.

Links

Export

BibTeX

@article{PanettalSIGGRAPHAsia22, TITLE = {Shape from Release: Inverse Design and Fabrication of Controlled Release Structures}, AUTHOR = {Panetta, Julian and Mohammadian, Haleh and Luci, Emiliano and Babaei, Vahid}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3550454.3555518}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {41}, NUMBER = {6}, PAGES = {1--14}, EID = {274}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2022}, }

Endnote

%0 Journal Article %A Panetta, Julian %A Mohammadian, Haleh %A Luci, Emiliano %A Babaei, Vahid %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Shape from Release: Inverse Design and Fabrication of Controlled Release Structures : %G eng %U http://hdl.handle.net/21.11116/0000-000B-5E7D-1 %R 10.1145/3550454.3555518 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 6 %& 1 %P 1 - 14 %Z sequence number: 274 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2022 %O ACM SIGGRAPH Asia 2022 SA '22 SA 2022

Piovarči, M., Foshey, M., Xu, J., et al. 2022. Closed-Loop Control of Direct Ink Writing via Reinforcement Learning. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2022)41, 4.

Links

Export

BibTeX

@article{PiovarciSIGGRAPH22, TITLE = {Closed-Loop Control of Direct Ink Writing via Reinforcement Learning}, AUTHOR = {Piovar{\v c}i, Michal and Foshey, Michael and Xu, Jie and Erps, Timmothy and Babaei, Vahid and Didyk, Piotr and Rusinkiewicz, Szymon and Matusik, Wojciech and Bickel, Bernd}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3528223.3530144}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {41}, NUMBER = {4}, PAGES = {1--10}, EID = {112}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2022}, }

Endnote

%0 Journal Article %A Piovarči, Michal %A Foshey, Michael %A Xu, Jie %A Erps, Timmothy %A Babaei, Vahid %A Didyk, Piotr %A Rusinkiewicz, Szymon %A Matusik, Wojciech %A Bickel, Bernd %+ External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T Closed-Loop Control of Direct Ink Writing via Reinforcement Learning : %G eng %U http://hdl.handle.net/21.11116/0000-000C-178D-C %R 10.1145/3528223.3530144 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 4 %& 1 %P 1 - 10 %Z sequence number: 112 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2022 %O ACM SIGGRAPH 2022

Pourjafarian, N., Koelle, M., Mjaku, F., Strohmeier, P., and Steimle, J. 2022. Print-A-Sketch: A Handheld Printer for Physical Sketching of Circuits and Sensors on Everyday Surfaces. CHI ’22, CHI Conference on Human Factors in Computing Systems, ACM.

Links

Export

BibTeX

@inproceedings{Pourjafarian_CHI2022, TITLE = {{Print-A-Sketch}: {A} Handheld Printer for Physical Sketching of Circuits and Sensors on Everyday Surfaces}, AUTHOR = {Pourjafarian, Narjes and Koelle, Marion and Mjaku, Fjolla and Strohmeier, Paul and Steimle, J{\"u}rgen}, LANGUAGE = {eng}, ISBN = {9781450391573}, DOI = {10.1145/3491102.3502074}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI '22, CHI Conference on Human Factors in Computing Systems}, PAGES = {1--17}, EID = {270}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Pourjafarian, Narjes %A Koelle, Marion %A Mjaku, Fjolla %A Strohmeier, Paul %A Steimle, Jürgen %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Print-A-Sketch: A Handheld Printer for Physical Sketching of Circuits and Sensors on Everyday Surfaces : %G eng %U http://hdl.handle.net/21.11116/0000-000A-215D-9 %R 10.1145/3491102.3502074 %D 2022 %B CHI Conference on Human Factors in Computing Systems %Z date of event: 2022-04-29 - 2022-05-05 %C New Orleans, LA, USA %B CHI '22 %P 1 - 17 %Z sequence number: 270 %I ACM %@ 9781450391573

Rao, S., Böhle, M., and Schiele, B. 2022. Towards Better Understanding Attribution Methods. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2022), IEEE.

Links

Export

BibTeX

@inproceedings{Rao_CVPR2022, TITLE = {Towards Better Understanding Attribution Methods}, AUTHOR = {Rao, Sukrut and B{\"o}hle, Moritz and Schiele, Bernt}, LANGUAGE = {eng}, ISBN = {978-1-6654-6946-3}, DOI = {10.1109/CVPR52688.2022.00998}, PUBLISHER = {IEEE}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2022)}, PAGES = {10213--10222}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Rao, Sukrut %A Böhle, Moritz %A Schiele, Bernt %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Towards Better Understanding Attribution Methods : %G eng %U http://hdl.handle.net/21.11116/0000-000A-6F91-6 %R 10.1109/CVPR52688.2022.00998 %D 2022 %B 35th IEEE/CVF Conference on Computer Vision and Pattern Recognition %Z date of event: 2022-06-19 - 2022-06-24 %C New Orleans, LA, USA %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 10213 - 10222 %I IEEE %@ 978-1-6654-6946-3

Reed, C.N., Nordmoen, C., Martelloni, A., et al. 2022a. Exploring Experiences with New Musical Instruments through Micro-phenomenology. NIME 2022, International Conference on New Interfaces for Musical Expression, PubPub.

Links

Export

BibTeX

@inproceedings{Reed2022Exploring, TITLE = {Exploring Experiences with New Musical Instruments through Micro-phenomenology}, AUTHOR = {Reed, Courtney N. and Nordmoen, Charlotte and Martelloni, Andrea and Lepri, Giacomo and Robson, Nicole and Zayas-Garin, Eevee and Cotton, Kelsey and Mice, Lia and McPherson, Andrew}, LANGUAGE = {eng}, DOI = {10.21428/92fbeb44.b304e4b1}, PUBLISHER = {PubPub}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {NIME 2022, International Conference on New Interfaces for Musical Expression}, ADDRESS = {Auckland, New Zealand}, }

Endnote

%0 Conference Proceedings %A Reed, Courtney N. %A Nordmoen, Charlotte %A Martelloni, Andrea %A Lepri, Giacomo %A Robson, Nicole %A Zayas-Garin, Eevee %A Cotton, Kelsey %A Mice, Lia %A McPherson, Andrew %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations %T Exploring Experiences with New Musical Instruments through Micro-phenomenology : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A023-7 %R 10.21428/92fbeb44.b304e4b1 %D 2022 %B International Conference on New Interfaces for Musical Expression %Z date of event: 2022-06-28 - 2022-07-01 %C Auckland, New Zealand %B NIME 2022 %I PubPub

Reed, C.N., Skach, S., Strohmeier, P., and McPherson, A.P. 2022b. Singing Knit: Soft Knit Biosensing for Augmenting Vocal Performances. AHs ’22, Augmented Humans International Conference, ACM.

Links

Export

BibTeX

@inproceedings{Reed_AHs2022, TITLE = {Singing Knit: {S}oft Knit Biosensing for Augmenting Vocal Performances}, AUTHOR = {Reed, Courtney N. and Skach, Sophie and Strohmeier, Paul and McPherson, Andrew P.}, LANGUAGE = {eng}, DOI = {10.1145/3519391.3519412}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {AHs '22, Augmented Humans International Conference}, PAGES = {170--183}, ADDRESS = {Munich, Germany (Hybrid)}, }

Endnote

%0 Conference Proceedings %A Reed, Courtney N. %A Skach, Sophie %A Strohmeier, Paul %A McPherson, Andrew P. %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Singing Knit: Soft Knit Biosensing for Augmenting Vocal Performances : %G eng %U http://hdl.handle.net/21.11116/0000-000A-2178-A %R 10.1145/3519391.3519412 %D 2022 %B Augmented Humans International Conference %Z date of event: 2022-03-13 - 2022-03-15 %C Munich, Germany (Hybrid) %B AHs '22 %P 170 - 183 %I ACM

Ruan, L., Chen, B., Li, J., and Lam, M. 2022. Learning to Deblur using Light Field Generated and Real Defocus Images. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2022), IEEE.

Abstract

Although considerable progress has been made in semantic scene understanding under clear weather, it is still a tough problem under adverse weather conditions, such as dense fog, due to the uncertainty caused by imperfect observations. Besides, difficulties in collecting and labeling foggy images hinder the progress of this field. Considering the success in semantic scene understanding under clear weather, we think it is reasonable to transfer knowledge learned from clear images to the foggy domain. As such, the problem becomes to bridge the domain gap between clear images and foggy images. Unlike previous methods that mainly focus on closing the domain gap caused by fog -- defogging the foggy images or fogging the clear images, we propose to alleviate the domain gap by considering fog influence and style variation simultaneously. The motivation is based on our finding that the style-related gap and the fog-related gap can be divided and closed respectively, by adding an intermediate domain. Thus, we propose a new pipeline to cumulatively adapt style, fog and the dual-factor (style and fog). Specifically, we devise a unified framework to disentangle the style factor and the fog factor separately, and then the dual-factor from images in different domains. Furthermore, we collaborate the disentanglement of three factors with a novel cumulative loss to thoroughly disentangle these three factors. Our method achieves the state-of-the-art performance on three benchmarks and shows generalization ability in rainy and snowy scenes.

Links

Export

BibTeX

@inproceedings{Ruan_CVPR2022, TITLE = {Learning to Deblur using Light Field Generated and Real Defocus Images}, AUTHOR = {Ruan, Lingyan and Chen, Bin and Li, Jizhou and Lam, Miuling}, LANGUAGE = {eng}, ISBN = {978-1-6654-6946-3}, DOI = {10.1109/CVPR52688.2022.01582}, PUBLISHER = {IEEE}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Although considerable progress has been made in semantic scene understanding under clear weather, it is still a tough problem under adverse weather conditions, such as dense fog, due to the uncertainty caused by imperfect observations. Besides, difficulties in collecting and labeling foggy images hinder the progress of this field. Considering the success in semantic scene understanding under clear weather, we think it is reasonable to transfer knowledge learned from clear images to the foggy domain. As such, the problem becomes to bridge the domain gap between clear images and foggy images. Unlike previous methods that mainly focus on closing the domain gap caused by fog -- defogging the foggy images or fogging the clear images, we propose to alleviate the domain gap by considering fog influence and style variation simultaneously. The motivation is based on our finding that the style-related gap and the fog-related gap can be divided and closed respectively, by adding an intermediate domain. Thus, we propose a new pipeline to cumulatively adapt style, fog and the dual-factor (style and fog). Specifically, we devise a unified framework to disentangle the style factor and the fog factor separately, and then the dual-factor from images in different domains. Furthermore, we collaborate the disentanglement of three factors with a novel cumulative loss to thoroughly disentangle these three factors. Our method achieves the state-of-the-art performance on three benchmarks and shows generalization ability in rainy and snowy scenes. }, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2022)}, PAGES = {16283--16292}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Ruan, Lingyan %A Chen, Bin %A Li, Jizhou %A Lam, Miuling %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Learning to Deblur using Light Field Generated and Real Defocus Images : %G eng %U http://hdl.handle.net/21.11116/0000-000C-4354-A %R 10.1109/CVPR52688.2022.01582 %D 2022 %B 35th IEEE/CVF Conference on Computer Vision and Pattern Recognition %Z date of event: 2022-06-19 - 2022-06-24 %C New Orleans, LA, USA %X Although considerable progress has been made in semantic scene understanding under clear weather, it is still a tough problem under adverse weather conditions, such as dense fog, due to the uncertainty caused by imperfect observations. Besides, difficulties in collecting and labeling foggy images hinder the progress of this field. Considering the success in semantic scene understanding under clear weather, we think it is reasonable to transfer knowledge learned from clear images to the foggy domain. As such, the problem becomes to bridge the domain gap between clear images and foggy images. Unlike previous methods that mainly focus on closing the domain gap caused by fog -- defogging the foggy images or fogging the clear images, we propose to alleviate the domain gap by considering fog influence and style variation simultaneously. The motivation is based on our finding that the style-related gap and the fog-related gap can be divided and closed respectively, by adding an intermediate domain. Thus, we propose a new pipeline to cumulatively adapt style, fog and the dual-factor (style and fog). Specifically, we devise a unified framework to disentangle the style factor and the fog factor separately, and then the dual-factor from images in different domains. Furthermore, we collaborate the disentanglement of three factors with a novel cumulative loss to thoroughly disentangle these three factors. Our method achieves the state-of-the-art performance on three benchmarks and shows generalization ability in rainy and snowy scenes. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 16283 - 16292 %I IEEE %@ 978-1-6654-6946-3

Salaün, C., Georgiev, I., Seidel, H.-P., and Singh, G. 2022a. Scalable Multi-Class Sampling via Filtered Sliced Optimal Transport. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2022)41, 6.

Links

Export

BibTeX

@article{SalauenSIGGRAPHAsia22, TITLE = {Scalable Multi-Class Sampling via Filtered Sliced Optimal Transport}, AUTHOR = {Sala{\"u}n, Corentin and Georgiev, Iliyan and Seidel, Hans-Peter and Singh, Gurprit}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3550454.3555484}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {41}, NUMBER = {6}, PAGES = {1--14}, EID = {261}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2022}, }

Endnote

%0 Journal Article %A Salaün, Corentin %A Georgiev, Iliyan %A Seidel, Hans-Peter %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Scalable Multi-Class Sampling via Filtered Sliced Optimal Transport : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1716-2 %R 10.1145/3550454.3555484 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 6 %& 1 %P 1 - 14 %Z sequence number: 261 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2022 %O ACM SIGGRAPH Asia 2022 SA '22 SA 2022

Salaün, C., Gruson, A., Hua, B.-S., Hachisuka, T., and Singh, G. 2022b. Regression-based Monte Carlo integration. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2022)41, 4.

Links

Export

BibTeX

@article{Salauen_SIGGRAPH22, TITLE = {Regression-based {Monte Carlo} integration}, AUTHOR = {Sala{\"u}n, Corentin and Gruson, Adrien and Hua, Binh-Son and Hachisuka, Toshiya and Singh, Gurprit}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3528223.3530095}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {41}, NUMBER = {4}, PAGES = {1--14}, EID = {79}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2022}, }

Endnote

%0 Journal Article %A Salaün, Corentin %A Gruson, Adrien %A Hua, Binh-Son %A Hachisuka, Toshiya %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Regression-based Monte Carlo integration : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1793-4 %R 10.1145/3528223.3530095 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 4 %& 1 %P 1 - 14 %Z sequence number: 79 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2022 %O ACM SIGGRAPH 2022

Schneider, O., Fruchard, B., Wittchen, D., et al. 2022. Sustainable Haptic Design: Improving Collaboration, Sharing, and Reuse in Haptic Design Research. CHI ’22, CHI Conference on Human Factors in Computing Systems, ACM.

Links

Export

BibTeX

@inproceedings{Schneider_CHIEA22, TITLE = {Sustainable Haptic Design: {I}mproving Collaboration, Sharing, and Reuse in Haptic Design Research}, AUTHOR = {Schneider, Oliver and Fruchard, Bruno and Wittchen, Dennis and Joshi, Bibhushan Raj and Freitag, Georg and Degraen, Donald and Strohmeier, Paul}, LANGUAGE = {eng}, ISBN = {9781450391566}, DOI = {10.1145/3491101.3503734}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {CHI '22, CHI Conference on Human Factors in Computing Systems}, EDITOR = {Barbosa, Simone and Lampe, Cliff and Appert, Caroline and Shamma, David A.}, PAGES = {1--5}, EID = {79}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Schneider, Oliver %A Fruchard, Bruno %A Wittchen, Dennis %A Joshi, Bibhushan Raj %A Freitag, Georg %A Degraen, Donald %A Strohmeier, Paul %+ External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Sustainable Haptic Design: Improving Collaboration, Sharing, and Reuse in Haptic Design Research : %G eng %U http://hdl.handle.net/21.11116/0000-000C-1748-A %R 10.1145/3491101.3503734 %D 2022 %B CHI Conference on Human Factors in Computing Systems %Z date of event: 2022-04-30 - 2022-05-05 %C New Orleans, LA, USA %B CHI '22 %E Barbosa, Simone; Lampe, Cliff; Appert, Caroline; Shamma, David A. %P 1 - 5 %Z sequence number: 79 %I ACM %@ 9781450391566

Shen, Z., Lin, C., Liao, K., Nie, L., Zheng, Z., and Zhao, Y. 2022. PanoFormer: Panorama Transformer for Indoor 360° Depth Estimation. Computer Vision -- ECCV 2022, Springer.

Links

Export

BibTeX

@inproceedings{Shen_ECCV2022, TITLE = {{PanoFormer}: {P}anorama Transformer for Indoor 360$^{\circ}$ Depth Estimation}, AUTHOR = {Shen, Zhijie and Lin, Chunyu and Liao, Kang and Nie, Lang and Zheng, Zishuo and Zhao, Yao}, LANGUAGE = {eng}, ISBN = {978-3-031-19768-0}, DOI = {10.1007/978-3-031-19769-7_12}, PUBLISHER = {Springer}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, DATE = {2022}, BOOKTITLE = {Computer Vision -- ECCV 2022}, EDITOR = {Avidan, Shai and Brostow, Gabriel and Ciss{\'e}, Moustapha and Farinella, Giovanni Maria and Hassner, Tal}, PAGES = {195--211}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {13661}, ADDRESS = {Tel Aviv, Israel}, }

Endnote

%0 Conference Proceedings %A Shen, Zhijie %A Lin, Chunyu %A Liao, Kang %A Nie, Lang %A Zheng, Zishuo %A Zhao, Yao %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations %T PanoFormer: Panorama Transformer for Indoor 360° Depth Estimation : %G eng %U http://hdl.handle.net/21.11116/0000-000C-9585-5 %R 10.1007/978-3-031-19769-7_12 %D 2022 %B 17th European Conference on Computer Vision %Z date of event: 2022-10-23 - 2022-10-27 %C Tel Aviv, Israel %B Computer Vision -- ECCV 2022 %E Avidan, Shai; Brostow, Gabriel; Cissé, Moustapha; Farinella, Giovanni Maria; Hassner, Tal %P 195 - 211 %I Springer %@ 978-3-031-19768-0 %B Lecture Notes in Computer Science %N 13661 %U https://rdcu.be/c5Ays

Shimada, S., Golyanik, V., Li, Z., Pérez, P., Xu, W., and Theobalt, C. 2022. HULC: 3D HUman Motion Capture with Pose Manifold SampLing and Dense Contact Guidance. Computer Vision -- ECCV 2022, Springer.

Links

Export

BibTeX

@inproceedings{Shimada_ECCV2022, TITLE = {{HULC}: {3D} {HU}man Motion Capture with Pose Manifold Samp{Li}ng and Dense {C}ontact Guidance}, AUTHOR = {Shimada, Soshi and Golyanik, Vladislav and Li, Zhi and P{\'e}rez, Patrick and Xu, Weipeng and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-031-20046-5}, DOI = {10.1007/978-3-031-20047-2_30}, PUBLISHER = {Springer}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, DATE = {2022}, BOOKTITLE = {Computer Vision -- ECCV 2022}, EDITOR = {Avidan, Shai and Brostow, Gabriel and Ciss{\'e}, Moustapha and Farinella, Giovanni Maria and Hassner, Tal}, PAGES = {516--533}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {13682}, ADDRESS = {Tel Aviv, Israel}, }

Endnote

%0 Conference Proceedings %A Shimada, Soshi %A Golyanik, Vladislav %A Li, Zhi %A Pérez, Patrick %A Xu, Weipeng %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T HULC: 3D HUman Motion Capture with Pose Manifold SampLing and Dense Contact Guidance : %G eng %U http://hdl.handle.net/21.11116/0000-000B-7918-3 %R 10.1007/978-3-031-20047-2_30 %D 2022 %B 17th European Conference on Computer Vision %Z date of event: 2022-10-23 - 2022-10-27 %C Tel Aviv, Israel %B Computer Vision -- ECCV 2022 %E Avidan, Shai; Brostow, Gabriel; Cissé, Moustapha; Farinella, Giovanni Maria; Hassner, Tal %P 516 - 533 %I Springer %@ 978-3-031-20046-5 %B Lecture Notes in Computer Science %N 13682 %U https://rdcu.be/c0aoZ

Strohmeier, P., Mottelson, A., Pohl, H., et al. 2022. Body-based user interfaces. In: The Routledge Handbook of Bodily Awareness. Routledge, London.

Links

Export

BibTeX

@incollection{strohmeier2022body, TITLE = {Body-based user interfaces}, AUTHOR = {Strohmeier, Paul and Mottelson, Aske and Pohl, Henning and McIntosh, Jess and Knibbe, Jarrod and Bergstr{\"o}m, Joanna and Jansen, Yvonne and Hornb{\ae}k, Kasper}, LANGUAGE = {eng}, ISBN = {9780429321542}, DOI = {10.4324/9780429321542}, PUBLISHER = {Routledge}, ADDRESS = {London}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, DATE = {2022}, BOOKTITLE = {The Routledge Handbook of Bodily Awareness}, EDITOR = {Alsmith, Adrian J. T. and Longo, Matthew R.}, PAGES = {478--502}, }

Endnote

%0 Book Section %A Strohmeier, Paul %A Mottelson, Aske %A Pohl, Henning %A McIntosh, Jess %A Knibbe, Jarrod %A Bergström, Joanna %A Jansen, Yvonne %A Hornbæk, Kasper %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations %T Body-based user interfaces : %G eng %U http://hdl.handle.net/21.11116/0000-000C-A013-9 %D 2022 %B The Routledge Handbook of Bodily Awareness %E Alsmith, Adrian J. T.; Longo, Matthew R. %P 478 - 502 %I Routledge %C London %@ 9780429321542

Wang, C., Serrano, A., Pan, X., et al. 2022a. GlowGAN: Unsupervised Learning of HDR Images from LDR Images in the Wild. https://arxiv.org/abs/2211.12352.

(arXiv: 2211.12352)

Abstract

Most in-the-wild images are stored in Low Dynamic Range (LDR) form, serving as a partial observation of the High Dynamic Range (HDR) visual world. Despite limited dynamic range, these LDR images are often captured with different exposures, implicitly containing information about the underlying HDR image distribution. Inspired by this intuition, in this work we present, to the best of our knowledge, the first method for learning a generative model of HDR images from in-the-wild LDR image collections in a fully unsupervised manner. The key idea is to train a generative adversarial network (GAN) to generate HDR images which, when projected to LDR under various exposures, are indistinguishable from real LDR images. The projection from HDR to LDR is achieved via a camera model that captures the stochasticity in exposure and camera response function. Experiments show that our method GlowGAN can synthesize photorealistic HDR images in many challenging cases such as landscapes, lightning, or windows, where previous supervised generative models produce overexposed images. We further demonstrate the new application of unsupervised inverse tone mapping (ITM) enabled by GlowGAN. Our ITM method does not need HDR images or paired multi-exposure images for training, yet it reconstructs more plausible information for overexposed regions than state-of-the-art supervised learning models trained on such data.

Links

Export

BibTeX

@online{Wang2211.12352, TITLE = {{GlowGAN}: Unsupervised Learning of {HDR} Images from {LDR} Images in the Wild}, AUTHOR = {Wang, Chao and Serrano, Ana and Pan, X. and Chen, Bin and Seidel, Hans-Peter and Theobalt, Christian and Myszkowski, Karol and Leimk{\"u}hler, Thomas}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2211.12352}, EPRINT = {2211.12352}, EPRINTTYPE = {arXiv}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Most in-the-wild images are stored in Low Dynamic Range (LDR) form, serving as a partial observation of the High Dynamic Range (HDR) visual world. Despite limited dynamic range, these LDR images are often captured with different exposures, implicitly containing information about the underlying HDR image distribution. Inspired by this intuition, in this work we present, to the best of our knowledge, the first method for learning a generative model of HDR images from in-the-wild LDR image collections in a fully unsupervised manner. The key idea is to train a generative adversarial network (GAN) to generate HDR images which, when projected to LDR under various exposures, are indistinguishable from real LDR images. The projection from HDR to LDR is achieved via a camera model that captures the stochasticity in exposure and camera response function. Experiments show that our method GlowGAN can synthesize photorealistic HDR images in many challenging cases such as landscapes, lightning, or windows, where previous supervised generative models produce overexposed images. We further demonstrate the new application of unsupervised inverse tone mapping (ITM) enabled by GlowGAN. Our ITM method does not need HDR images or paired multi-exposure images for training, yet it reconstructs more plausible information for overexposed regions than state-of-the-art supervised learning models trained on such data. }, }

Endnote

%0 Report %A Wang, Chao %A Serrano, Ana %A Pan, X. %A Chen, Bin %A Seidel, Hans-Peter %A Theobalt, Christian %A Myszkowski, Karol %A Leimkühler, Thomas %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T GlowGAN: Unsupervised Learning of HDR Images from LDR Images in the Wild : %G eng %U http://hdl.handle.net/21.11116/0000-000B-9D08-C %U https://arxiv.org/abs/2211.12352 %D 2022 %X Most in-the-wild images are stored in Low Dynamic Range (LDR) form, serving as a partial observation of the High Dynamic Range (HDR) visual world. Despite limited dynamic range, these LDR images are often captured with different exposures, implicitly containing information about the underlying HDR image distribution. Inspired by this intuition, in this work we present, to the best of our knowledge, the first method for learning a generative model of HDR images from in-the-wild LDR image collections in a fully unsupervised manner. The key idea is to train a generative adversarial network (GAN) to generate HDR images which, when projected to LDR under various exposures, are indistinguishable from real LDR images. The projection from HDR to LDR is achieved via a camera model that captures the stochasticity in exposure and camera response function. Experiments show that our method GlowGAN can synthesize photorealistic HDR images in many challenging cases such as landscapes, lightning, or windows, where previous supervised generative models produce overexposed images. We further demonstrate the new application of unsupervised inverse tone mapping (ITM) enabled by GlowGAN. Our ITM method does not need HDR images or paired multi-exposure images for training, yet it reconstructs more plausible information for overexposed regions than state-of-the-art supervised learning models trained on such data. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,eess.IV

Wang, C., Chen, B., Seidel, H.-P., Myszkowski, K., and Serrano, A. 2022b. Learning a self-supervised tone mapping operator via feature contrast masking loss. Computer Graphics Forum (Proc. EUROGRAPHICS 2022)41, 2.

Links

Export

BibTeX

@article{Wang2022, TITLE = {Learning a self-supervised tone mapping operator via feature contrast masking loss}, AUTHOR = {Wang, Chao and Chen, Bin and Seidel, Hans-Peter and Myszkowski, Karol and Serrano, Ana}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.14459}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {41}, NUMBER = {2}, PAGES = {71--84}, BOOKTITLE = {The European Association for Computer Graphics 43rdAnnual Conference (EUROGRAPHICS 2022)}, EDITOR = {Caine, Rapha{\"e}lle and Kim, Min H.}, }

Endnote

%0 Journal Article %A Wang, Chao %A Chen, Bin %A Seidel, Hans-Peter %A Myszkowski, Karol %A Serrano, Ana %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Learning a self-supervised tone mapping operator via feature contrast masking loss : %G eng %U http://hdl.handle.net/21.11116/0000-000A-BA09-B %R 10.1111/cgf.14459 %7 2022 %D 2022 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 41 %N 2 %& 71 %P 71 - 84 %I Blackwell-Wiley %C Oxford %@ false %B The European Association for Computer Graphics 43rdAnnual Conference %O EUROGRAPHICS 2022 EG 2022 Reims, France, April 25 - 29, 2022

Wang, Y., Chen, H., Fan, Y., et al. 2022c. USB: A Unified Semi-supervised Learning Benchmark for Classification. Advances in Neural Information Processing Systems 35 (NeurIPS 2022), Curran Associates, Inc.

Links

Export

BibTeX

@inproceedings{Wang_Neurips22, TITLE = {{USB}: {A} Unified Semi-supervised Learning Benchmark for Classification}, AUTHOR = {Wang, Yidong and Chen, Hao and Fan, Yue and Sun, Wang and Tao, Ran and Hou, Wenxin and Wang, Renjie and Yang, Linyi and Zhou, Zhi and Guo, Lan-Zhe and Qi, Heli and Wu, Zhen and Li, Yu-Feng and Nakamura, Satoshi and Ye, Wei and Savvides, Marios and Raj, Bhiksha and Shinozaki, Takahiro and Schiele, Bernt and Wang, Jindong and Xie, Xing and Zhang, Yue}, LANGUAGE = {eng}, PUBLISHER = {Curran Associates, Inc.}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Advances in Neural Information Processing Systems 35 (NeurIPS 2022)}, EDITOR = {Koyejo, S. and Mohamed, S. and Agarwal, A. and Belgrave, D. and Cho, K. and Oh, A.}, PAGES = {3938--3961}, ADDRESS = {New Orleans, LA, USA}, }

Endnote

%0 Conference Proceedings %A Wang, Yidong %A Chen, Hao %A Fan, Yue %A Sun, Wang %A Tao, Ran %A Hou, Wenxin %A Wang, Renjie %A Yang, Linyi %A Zhou, Zhi %A Guo, Lan-Zhe %A Qi, Heli %A Wu, Zhen %A Li, Yu-Feng %A Nakamura, Satoshi %A Ye, Wei %A Savvides, Marios %A Raj, Bhiksha %A Shinozaki, Takahiro %A Schiele, Bernt %A Wang, Jindong %A Xie, Xing %A Zhang, Yue %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T USB: A Unified Semi-supervised Learning Benchmark for Classification : %G eng %U http://hdl.handle.net/21.11116/0000-000C-184A-7 %D 2022 %B 36th Conference on Neural Information Processing Systems %Z date of event: 2022-11-28 - 2022-12-09 %C New Orleans, LA, USA %B Advances in Neural Information Processing Systems 35 %E Koyejo, S.; Mohamed, S.; Agarwal, A.; Belgrave, D.; Cho, K.; Oh, A. %P 3938 - 3961 %I Curran Associates, Inc. %U https://openreview.net/pdf?id=QeuwINa96C

Wittchen, D., Spiel, K., Fruchard, B., et al. 2022. TactJam: An End-to-End Prototyping Suite for Collaborative Design of On-Body Vibrotactile Feedback. TEI ’22, Sixteenth International Conference on Tangible, Embedded, and Embodied Interaction, ACM.

Links

Export

BibTeX

@inproceedings{Wittchen_TEI22, TITLE = {{TactJam}: {A}n End-to-End Prototyping Suite for Collaborative Design of On-Body Vibrotactile Feedback}, AUTHOR = {Wittchen, Dennis and Spiel, Katta and Fruchard, Bruno and Degraen, Donald and Schneider, Oliver and Freitag, Georg and Strohmeier, Paul}, LANGUAGE = {eng}, ISBN = {978-1-4503-9147-4}, DOI = {10.1145/3490149.3501307}, PUBLISHER = {ACM}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {TEI '22, Sixteenth International Conference on Tangible, Embedded, and Embodied Interaction}, PAGES = {1--13}, EID = {1}, ADDRESS = {Daejeon, Republic of Korea (Online)}, }

Endnote

%0 Conference Proceedings %A Wittchen, Dennis %A Spiel, Katta %A Fruchard, Bruno %A Degraen, Donald %A Schneider, Oliver %A Freitag, Georg %A Strohmeier, Paul %+ External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T TactJam: An End-to-End Prototyping Suite for Collaborative Design of On-Body Vibrotactile Feedback : %G eng %U http://hdl.handle.net/21.11116/0000-000A-20B9-1 %R 10.1145/3490149.3501307 %D 2022 %B Sixteenth International Conference on Tangible, Embedded, and Embodied Interaction %Z date of event: 2022-02-13 - 2022-02-16 %C Daejeon, Republic of Korea (Online) %B TEI '22 %P 1 - 13 %Z sequence number: 1 %I ACM %@ 978-1-4503-9147-4

Wolski, K., Trutoiu, L., Dong, Z., Shen, Z., Mackenzie, K., and Chapiro, A. 2022a. Geo-Metric: A Perceptual Dataset of Distortions on Faces. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2022)41, 6.

Links

Export

BibTeX

@article{WolskiSIGGRAPHAsia22, TITLE = {Geo-Metric: {A} Perceptual Dataset of Distortions on Faces}, AUTHOR = {Wolski, Krzysztof and Trutoiu, Laura and Dong, Zhao and Shen, Zhengyang and Mackenzie, Kevin and Chapiro, Alexandre}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3550454.3555475}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {41}, NUMBER = {6}, PAGES = {1--13}, EID = {215}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2022}, }

Endnote

%0 Journal Article %A Wolski, Krzysztof %A Trutoiu, Laura %A Dong, Zhao %A Shen, Zhengyang %A Mackenzie, Kevin %A Chapiro, Alexandre %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations %T Geo-Metric: A Perceptual Dataset of Distortions on Faces : %G eng %U http://hdl.handle.net/21.11116/0000-000C-209F-D %R 10.1145/3550454.3555475 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 6 %& 1 %P 1 - 13 %Z sequence number: 215 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2022 %O ACM SIGGRAPH Asia 2022 SA '22 SA 2022

Wolski, K., Zhong, F., Myszkowski, K., and Mantiuk, R.K. 2022b. Dark Stereo: Improving Depth Perception Under Low Luminance. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2022)41, 4.

Links

Export

BibTeX

@article{Wolski_SIGGRAPH22, TITLE = {Dark Stereo: {I}mproving Depth Perception Under Low Luminance}, AUTHOR = {Wolski, Krzysztof and Zhong, Fangcheng and Myszkowski, Karol and Mantiuk, Rafa{\l} K.}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3528223.3530136}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2022}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {41}, NUMBER = {4}, PAGES = {1--12}, EID = {146}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2022}, }

Endnote

%0 Journal Article %A Wolski, Krzysztof %A Zhong, Fangcheng %A Myszkowski, Karol %A Mantiuk, Rafał K. %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Dark Stereo: Improving Depth Perception Under Low Luminance : %G eng %U http://hdl.handle.net/21.11116/0000-000A-BA6D-B %R 10.1145/3528223.3530136 %7 2022 %D 2022 %J ACM Transactions on Graphics %V 41 %N 4 %& 1 %P 1 - 12 %Z sequence number: 146 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2022 %O ACM SIGGRAPH 2022

2021

Ali, J., Lahoti, P., and Gummadi, K.P. 2021. Accounting for Model Uncertainty in Algorithmic Discrimination. AIES ’21, Fourth AAAI/ACM Conference on Artificial Intelligence, Ethics and Society, ACM.

Links

Export

BibTeX

@inproceedings{Ali_AIES2021, TITLE = {Accounting for Model Uncertainty in Algorithmic Discrimination}, AUTHOR = {Ali, Junaid and Lahoti, Preethi and Gummadi, Krishna P.}, LANGUAGE = {eng}, ISBN = {978-1-4503-8473-5}, DOI = {10.1145/3461702.3462630}, PUBLISHER = {ACM}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {AIES '21, Fourth AAAI/ACM Conference on Artificial Intelligence, Ethics and Society}, EDITOR = {Fourcade, Marion and Kuipers, Benjamin and Lazar, Seth and Mulligan, Deirdre}, PAGES = {336--345}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Ali, Junaid %A Lahoti, Preethi %A Gummadi, Krishna P. %+ Computer Graphics, MPI for Informatics, Max Planck Society Databases and Information Systems, MPI for Informatics, Max Planck Society External Organizations %T Accounting for Model Uncertainty in Algorithmic Discrimination : %G eng %U http://hdl.handle.net/21.11116/0000-0008-72E3-7 %R 10.1145/3461702.3462630 %D 2021 %B Fourth AAAI/ACM Conference on Artificial Intelligence, Ethics and Society %Z date of event: 2021-05-19 - 2021-05-21 %C Virtual Conference %B AIES '21 %E Fourcade, Marion; Kuipers, Benjamin; Lazar, Seth; Mulligan, Deirdre %P 336 - 345 %I ACM %@ 978-1-4503-8473-5

Ansari, N., Seidel, H.-P., and Babaei, V. 2021. Mixed Integer Neural Inverse Design. https://arxiv.org/abs/2109.12888.

(arXiv: 2109.12888)

Abstract

In computational design and fabrication, neural networks are becoming important surrogates for bulky forward simulations. A long-standing, intertwined question is that of inverse design: how to compute a design that satisfies a desired target performance? Here, we show that the piecewise linear property, very common in everyday neural networks, allows for an inverse design formulation based on mixed-integer linear programming. Our mixed-integer inverse design uncovers globally optimal or near optimal solutions in a principled manner. Furthermore, our method significantly facilitates emerging, but challenging, combinatorial inverse design tasks, such as material selection. For problems where finding the optimal solution is not desirable or tractable, we develop an efficient yet near-optimal hybrid optimization. Eventually, our method is able to find solutions provably robust to possible fabrication perturbations among multiple designs with similar performances.

Links

Export

BibTeX

@online{Ansari_2109.12888, TITLE = {Mixed Integer Neural Inverse Design}, AUTHOR = {Ansari, Navid and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2109.12888}, EPRINT = {2109.12888}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {In computational design and fabrication, neural networks are becoming important surrogates for bulky forward simulations. A long-standing, intertwined question is that of inverse design: how to compute a design that satisfies a desired target performance? Here, we show that the piecewise linear property, very common in everyday neural networks, allows for an inverse design formulation based on mixed-integer linear programming. Our mixed-integer inverse design uncovers globally optimal or near optimal solutions in a principled manner. Furthermore, our method significantly facilitates emerging, but challenging, combinatorial inverse design tasks, such as material selection. For problems where finding the optimal solution is not desirable or tractable, we develop an efficient yet near-optimal hybrid optimization. Eventually, our method is able to find solutions provably robust to possible fabrication perturbations among multiple designs with similar performances. }, }

Endnote

%0 Report %A Ansari, Navid %A Seidel, Hans-Peter %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Mixed Integer Neural Inverse Design : %G eng %U http://hdl.handle.net/21.11116/0000-0009-7104-3 %U https://arxiv.org/abs/2109.12888 %D 2021 %X In computational design and fabrication, neural networks are becoming important surrogates for bulky forward simulations. A long-standing, intertwined question is that of inverse design: how to compute a design that satisfies a desired target performance? Here, we show that the piecewise linear property, very common in everyday neural networks, allows for an inverse design formulation based on mixed-integer linear programming. Our mixed-integer inverse design uncovers globally optimal or near optimal solutions in a principled manner. Furthermore, our method significantly facilitates emerging, but challenging, combinatorial inverse design tasks, such as material selection. For problems where finding the optimal solution is not desirable or tractable, we develop an efficient yet near-optimal hybrid optimization. Eventually, our method is able to find solutions provably robust to possible fabrication perturbations among multiple designs with similar performances. %K Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Chen, B., Wang, C., Piovarči, M., et al. 2021. The Effect of Geometry and Illumination on Appearance Perception of Different Material Categories. The Visual Computer37.

Links

Export

BibTeX

@article{Chen2021b, TITLE = {The Effect of Geometry and Illumination on Appearance Perception of Different Material Categories}, AUTHOR = {Chen, Bin and Wang, Chao and Piovar{\v c}i, Michal and Seidel, Hans-Peter and Didyk, Piotr and Myszkowski, Karol and Serrano, Ana}, LANGUAGE = {eng}, ISSN = {0178-2789}, DOI = {10.1007/s00371-021-02227-x}, PUBLISHER = {Springer}, ADDRESS = {Berlin}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {The Visual Computer}, VOLUME = {37}, PAGES = {2975--2987}, }

Endnote

%0 Journal Article %A Chen, Bin %A Wang, Chao %A Piovarči, Michal %A Seidel, Hans-Peter %A Didyk, Piotr %A Myszkowski, Karol %A Serrano, Ana %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T The Effect of Geometry and Illumination on Appearance Perception of Different Material Categories : %G eng %U http://hdl.handle.net/21.11116/0000-0008-F05C-2 %R 10.1007/s00371-021-02227-x %7 2021 %D 2021 %J The Visual Computer %V 37 %& 2975 %P 2975 - 2987 %I Springer %C Berlin %@ false

Chu, M., Thuerey, N., Seidel, H.-P., Theobalt, C., and Zayer, R. 2021. Learning Meaningful Controls for Fluids. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2021)40, 4.

Links

Export

BibTeX

@article{Chu2021, TITLE = {Learning Meaningful Controls for Fluids}, AUTHOR = {Chu, Mengyu and Thuerey, Nils and Seidel, Hans-Peter and Theobalt, Christian and Zayer, Rhaleb}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3450626.3459845}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {40}, NUMBER = {4}, PAGES = {1--13}, EID = {100}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2021}, }

Endnote

%0 Journal Article %A Chu, Mengyu %A Thuerey, Nils %A Seidel, Hans-Peter %A Theobalt, Christian %A Zayer, Rhaleb %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Learning Meaningful Controls for Fluids : %G eng %U http://hdl.handle.net/21.11116/0000-0009-4B91-F %R 10.1145/3450626.3459845 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 4 %& 1 %P 1 - 13 %Z sequence number: 100 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2021 %O ACM SIGGRAPH 2021

Delanoy, J., Serrano, A., Masia, B., and Gutierrez, D. 2021. Perception of Material Appearance: A Comparison between Painted and Rendered Images. Journal of Vision21, 5.

Links

Export

BibTeX

@article{Delanoy2021, TITLE = {Perception of Material Appearance: {A} Comparison between Painted and Rendered Images}, AUTHOR = {Delanoy, Johanna and Serrano, Ana and Masia, Belen and Gutierrez, Diego}, LANGUAGE = {eng}, ISSN = {1534-7362}, DOI = {10.1167/jov.21.5.16}, PUBLISHER = {Scholar One, Inc.}, ADDRESS = {Charlottesville, VA}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {Journal of Vision}, VOLUME = {21}, NUMBER = {5}, EID = {16}, }

Endnote

%0 Journal Article %A Delanoy, Johanna %A Serrano, Ana %A Masia, Belen %A Gutierrez, Diego %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Perception of Material Appearance: A Comparison between Painted and Rendered Images : %G eng %U http://hdl.handle.net/21.11116/0000-0008-A6CC-7 %R 10.1167/jov.21.5.16 %2 PMC8131993 %7 2021 %D 2021 %J Journal of Vision %V 21 %N 5 %Z sequence number: 16 %I Scholar One, Inc. %C Charlottesville, VA %@ false

Doosti, N., Panetta, J., and Babaei, V. 2021. Topology Optimization via Frequency Tuning of Neural Design Representations. Proceedings SCF 2021, ACM.

Links

Export

BibTeX

@inproceedings{Doosti21, TITLE = {Topology Optimization via Frequency Tuning of Neural Design Representations}, AUTHOR = {Doosti, Nikan and Panetta, Julian and Babaei, Vahid}, LANGUAGE = {eng}, ISBN = {978-1-4503-9090-3}, DOI = {10.1145/3485114.3485124}, PUBLISHER = {ACM}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Proceedings SCF 2021}, EDITOR = {Whiting, Emily and Hart, John and Sung, Cynthia and McCann, James and Peek, Nadya}, PAGES = {1--9}, EID = {1}, ADDRESS = {Virtual Event}, }

Endnote

%0 Conference Proceedings %A Doosti, Nikan %A Panetta, Julian %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Topology Optimization via Frequency Tuning of Neural Design Representations : %G eng %U http://hdl.handle.net/21.11116/0000-000C-C159-6 %R 10.1145/3485114.3485124 %D 2021 %B ACM Symposium on Computational Fabrication %Z date of event: 2021-10-28 - 2021-10-29 %C Virtual Event %B Proceedings SCF 2021 %E Whiting, Emily; Hart, John; Sung, Cynthia; McCann, James; Peek, Nadya %P 1 - 9 %Z sequence number: 1 %I ACM %@ 978-1-4503-9090-3

Elek, O., Zhang, R., Sumin, D., et al. 2021. Robust and Practical Measurement of Volume Transport Parameters in Solid Photo-polymer Materials for 3D Printing. Optics Express29, 5.

Links

Export

BibTeX

@article{Elek2021, TITLE = {Robust and Practical Measurement of Volume Transport Parameters in Solid Photo-polymer Materials for {3D} Printing}, AUTHOR = {Elek, Oskar and Zhang, Ran and Sumin, Denis and Myszkowski, Karol and Bickel, Bernd and Wilkie, Alexander and Krivanek, Jaroslav and Weyrich, Tim}, LANGUAGE = {eng}, ISSN = {1094-4087}, DOI = {10.1364/OE.406095}, PUBLISHER = {Optical Society of America}, ADDRESS = {Washington, DC}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {Optics Express}, VOLUME = {29}, NUMBER = {5}, PAGES = {7568--7588}, }

Endnote

%0 Journal Article %A Elek, Oskar %A Zhang, Ran %A Sumin, Denis %A Myszkowski, Karol %A Bickel, Bernd %A Wilkie, Alexander %A Krivanek, Jaroslav %A Weyrich, Tim %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T Robust and Practical Measurement of Volume Transport Parameters in Solid Photo-polymer Materials for 3D Printing : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E013-6 %R 10.1364/OE.406095 %7 2021 %D 2021 %J Optics Express %O Opt. Express %V 29 %N 5 %& 7568 %P 7568 - 7588 %I Optical Society of America %C Washington, DC %@ false

Fox, G., Liu, W., Kim, H., Seidel, H.-P., Elgharib, M., and Theobalt, C. 2021. VideoForensicsHQ: Detecting High-quality Manipulated Face Videos. IEEE International Conference on Multimedia and Expo (ICME 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Fox_ICME2021, TITLE = {{Video\-Foren\-sics\-HQ}: {D}etecting High-quality Manipulated Face Videos}, AUTHOR = {Fox, Gereon and Liu, Wentao and Kim, Hyeongwoo and Seidel, Hans-Peter and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-3864-3}, DOI = {10.1109/ICME51207.2021.9428101}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE International Conference on Multimedia and Expo (ICME 2021)}, PAGES = {1--6}, ADDRESS = {Shenzhen, China (Virtual)}, }

Endnote

%0 Conference Proceedings %A Fox, Gereon %A Liu, Wentao %A Kim, Hyeongwoo %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T VideoForensicsHQ: Detecting High-quality Manipulated Face Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0008-88DF-4 %R 10.1109/ICME51207.2021.9428101 %D 2021 %B 22nd IEEE International Conference on Multimedia and Expo %Z date of event: 2021-07-05 - 2021-07-07 %C Shenzhen, China (Virtual) %B IEEE International Conference on Multimedia and Expo %P 1 - 6 %I IEEE %@ 978-1-6654-3864-3 %U http://gvv.mpi-inf.mpg.de/projects/VForensicsHQ/

Habermann, M., Xu, W., Zollhöfer, M., Pons-Moll, G., and Theobalt, C. 2021. A Deeper Look into DeepCap. IEEE Transactions on Pattern Analysis and Machine Intelligence45, 4.

Abstract

Human performance capture is a highly important computer vision problem with many applications in movie production and virtual/augmented reality. Many previous performance capture approaches either required expensive multi-view setups or did not recover dense space-time coherent geometry with frame-to-frame correspondences. We propose a novel deep learning approach for monocular dense human performance capture. Our method is trained in a weakly supervised manner based on multi-view supervision completely removing the need for training data with 3D ground truth annotations. The network architecture is based on two separate networks that disentangle the task into a pose estimation and a non-rigid surface deformation step. Extensive qualitative and quantitative evaluations show that our approach outperforms the state of the art in terms of quality and robustness. This work is an extended version of DeepCap where we provide more detailed explanations, comparisons and results as well as applications.

Links

Export

BibTeX

@article{Habermann2111.10563, TITLE = {A Deeper Look into {DeepCap}}, AUTHOR = {Habermann, Marc and Xu, Weipeng and Zollh{\"o}fer, Michael and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0162-8828}, DOI = {10.1109/TPAMI.2021.3093553}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Human performance capture is a highly important computer vision problem with many applications in movie production and virtual/augmented reality. Many previous performance capture approaches either required expensive multi-view setups or did not recover dense space-time coherent geometry with frame-to-frame correspondences. We propose a novel deep learning approach for monocular dense human performance capture. Our method is trained in a weakly supervised manner based on multi-view supervision completely removing the need for training data with 3D ground truth annotations. The network architecture is based on two separate networks that disentangle the task into a pose estimation and a non-rigid surface deformation step. Extensive qualitative and quantitative evaluations show that our approach outperforms the state of the art in terms of quality and robustness. This work is an extended version of DeepCap where we provide more detailed explanations, comparisons and results as well as applications. }, JOURNAL = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, VOLUME = {45}, NUMBER = {4}, PAGES = {4009--4002}, }

Endnote

%0 Journal Article %A Habermann, Marc %A Xu, Weipeng %A Zollhöfer, Michael %A Pons-Moll, Gerard %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T A Deeper Look into DeepCap : (Invited Paper) %G eng %U http://hdl.handle.net/21.11116/0000-0009-8C33-0 %R 10.1109/TPAMI.2021.3093553 %7 2021 %D 2021 %X Human performance capture is a highly important computer vision problem with many applications in movie production and virtual/augmented reality. Many previous performance capture approaches either required expensive multi-view setups or did not recover dense space-time coherent geometry with frame-to-frame correspondences. We propose a novel deep learning approach for monocular dense human performance capture. Our method is trained in a weakly supervised manner based on multi-view supervision completely removing the need for training data with 3D ground truth annotations. The network architecture is based on two separate networks that disentangle the task into a pose estimation and a non-rigid surface deformation step. Extensive qualitative and quantitative evaluations show that our approach outperforms the state of the art in terms of quality and robustness. This work is an extended version of DeepCap where we provide more detailed explanations, comparisons and results as well as applications. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV %J IEEE Transactions on Pattern Analysis and Machine Intelligence %O IEEE Trans. Pattern Anal. Mach. Intell. %V 45 %N 4 %& 4009 %P 4009 - 4002 %I IEEE %C Piscataway, NJ %@ false

Habermann, M. 2021. Real-time human performance capture and synthesis. nbn:de:bsz:291--ds-349617.

Abstract

Most of the images one finds in the media, such as on the Internet or in textbooks and magazines, contain humans as the main point of attention. Thus, there is an inherent necessity for industry, society, and private persons to be able to thoroughly analyze and synthesize the human-related content in these images. One aspect of this analysis and subject of this thesis is to infer the 3D pose and surface deformation, using only visual information, which is also known as human performance capture. Human performance capture enables the tracking of virtual characters from real-world observations, and this is key for visual effects, games, VR, and AR, to name just a few application areas. However, traditional capture methods usually rely on expensive multi-view (marker-based) systems that are prohibitively expensive for the vast majority of people, or they use depth sensors, which are still not as common as single color cameras. Recently, some approaches have attempted to solve the task by assuming only a single RGB image is given. Nonetheless, they can either not track the dense deforming geometry of the human, such as the clothing layers, or they are far from real time, which is indispensable for many applications. To overcome these shortcomings, this thesis proposes two monocular human performance capture methods, which for the first time allow the real-time capture of the dense deforming geometry as well as an unseen 3D accuracy for pose and surface deformations. At the technical core, this work introduces novel GPU-based and data-parallel optimization strategies in conjunction with other algorithmic design choices that are all geared towards real-time performance at high accuracy. Moreover, this thesis presents a new weakly supervised multiview training strategy combined with a fully differentiable character representation that shows superior 3D accuracy. However, there is more to human-related Computer Vision than only the analysis of people in images. It is equally important to synthesize new images of humans in unseen poses and also from camera viewpoints that have not been observed in the real world. Such tools are essential for the movie industry because they, for example, allow the synthesis of photo-realistic virtual worlds with real-looking humans or of contents that are too dangerous for actors to perform on set. But also video conferencing and telepresence applications can benefit from photo-real 3D characters, as they can enhance the immersive experience of these applications. Here, the traditional Computer Graphics pipeline for rendering photo-realistic images involves many tedious and time-consuming steps that require expert knowledge and are far from real time. Traditional rendering involves character rigging and skinning, the modeling of the surface appearance properties, and physically based ray tracing. Recent learning-based methods attempt to simplify the traditional rendering pipeline and instead learn the rendering function from data resulting in methods that are easier accessible to non-experts. However, most of them model the synthesis task entirely in image space such that 3D consistency cannot be achieved, and/or they fail to model motion- and view-dependent appearance effects. To this end, this thesis presents a method and ongoing work on character synthesis, which allow the synthesis of controllable photoreal characters that achieve motion- and view-dependent appearance effects as well as 3D consistency and which run in real time. This is technically achieved by a novel coarse-to-fine geometric character representation for efficient synthesis, which can be solely supervised on multi-view imagery. Furthermore, this work shows how such a geometric representation can be combined with an implicit surface representation to boost synthesis and geometric quality.

Links

Export

BibTeX

@phdthesis{Habermannphd2021, TITLE = {Real-time human performance capture and synthesis}, AUTHOR = {Habermann, Marc}, LANGUAGE = {eng}, URL = {nbn:de:bsz:291--ds-349617}, DOI = {10.22028/D291-34961}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, ABSTRACT = {Most of the images one finds in the media, such as on the Internet or in textbooks and magazines, contain humans as the main point of attention. Thus, there is an inherent necessity for industry, society, and private persons to be able to thoroughly analyze and synthesize the human-related content in these images. One aspect of this analysis and subject of this thesis is to infer the 3D pose and surface deformation, using only visual information, which is also known as human performance capture. Human performance capture enables the tracking of virtual characters from real-world observations, and this is key for visual effects, games, VR, and AR, to name just a few application areas. However, traditional capture methods usually rely on expensive multi-view (marker-based) systems that are prohibitively expensive for the vast majority of people, or they use depth sensors, which are still not as common as single color cameras. Recently, some approaches have attempted to solve the task by assuming only a single RGB image is given. Nonetheless, they can either not track the dense deforming geometry of the human, such as the clothing layers, or they are far from real time, which is indispensable for many applications. To overcome these shortcomings, this thesis proposes two monocular human performance capture methods, which for the first time allow the real-time capture of the dense deforming geometry as well as an unseen 3D accuracy for pose and surface deformations. At the technical core, this work introduces novel GPU-based and data-parallel optimization strategies in conjunction with other algorithmic design choices that are all geared towards real-time performance at high accuracy. Moreover, this thesis presents a new weakly supervised multiview training strategy combined with a fully differentiable character representation that shows superior 3D accuracy. However, there is more to human-related Computer Vision than only the analysis of people in images. It is equally important to synthesize new images of humans in unseen poses and also from camera viewpoints that have not been observed in the real world. Such tools are essential for the movie industry because they, for example, allow the synthesis of photo-realistic virtual worlds with real-looking humans or of contents that are too dangerous for actors to perform on set. But also video conferencing and telepresence applications can benefit from photo-real 3D characters, as they can enhance the immersive experience of these applications. Here, the traditional Computer Graphics pipeline for rendering photo-realistic images involves many tedious and time-consuming steps that require expert knowledge and are far from real time. Traditional rendering involves character rigging and skinning, the modeling of the surface appearance properties, and physically based ray tracing. Recent learning-based methods attempt to simplify the traditional rendering pipeline and instead learn the rendering function from data resulting in methods that are easier accessible to non-experts. However, most of them model the synthesis task entirely in image space such that 3D consistency cannot be achieved, and/or they fail to model motion- and view-dependent appearance effects. To this end, this thesis presents a method and ongoing work on character synthesis, which allow the synthesis of controllable photoreal characters that achieve motion- and view-dependent appearance effects as well as 3D consistency and which run in real time. This is technically achieved by a novel coarse-to-fine geometric character representation for efficient synthesis, which can be solely supervised on multi-view imagery. Furthermore, this work shows how such a geometric representation can be combined with an implicit surface representation to boost synthesis and geometric quality.}, }

Endnote

%0 Thesis %A Habermann, Marc %Y Theobalt, Christian %A referee: Seidel, Hans-Peter %A referee: Hilton, Adrian %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Real-time human performance capture and synthesis : %G eng %U http://hdl.handle.net/21.11116/0000-0009-7D87-3 %R 10.22028/D291-34961 %U nbn:de:bsz:291--ds-349617 %F OTHER: hdl:20.500.11880/31986 %I Universität des Saarlandes %C Saarbrücken %D 2021 %P 153 p. %V phd %9 phd %X Most of the images one finds in the media, such as on the Internet or in textbooks and magazines, contain humans as the main point of attention. Thus, there is an inherent necessity for industry, society, and private persons to be able to thoroughly analyze and synthesize the human-related content in these images. One aspect of this analysis and subject of this thesis is to infer the 3D pose and surface deformation, using only visual information, which is also known as human performance capture. Human performance capture enables the tracking of virtual characters from real-world observations, and this is key for visual effects, games, VR, and AR, to name just a few application areas. However, traditional capture methods usually rely on expensive multi-view (marker-based) systems that are prohibitively expensive for the vast majority of people, or they use depth sensors, which are still not as common as single color cameras. Recently, some approaches have attempted to solve the task by assuming only a single RGB image is given. Nonetheless, they can either not track the dense deforming geometry of the human, such as the clothing layers, or they are far from real time, which is indispensable for many applications. To overcome these shortcomings, this thesis proposes two monocular human performance capture methods, which for the first time allow the real-time capture of the dense deforming geometry as well as an unseen 3D accuracy for pose and surface deformations. At the technical core, this work introduces novel GPU-based and data-parallel optimization strategies in conjunction with other algorithmic design choices that are all geared towards real-time performance at high accuracy. Moreover, this thesis presents a new weakly supervised multiview training strategy combined with a fully differentiable character representation that shows superior 3D accuracy. However, there is more to human-related Computer Vision than only the analysis of people in images. It is equally important to synthesize new images of humans in unseen poses and also from camera viewpoints that have not been observed in the real world. Such tools are essential for the movie industry because they, for example, allow the synthesis of photo-realistic virtual worlds with real-looking humans or of contents that are too dangerous for actors to perform on set. But also video conferencing and telepresence applications can benefit from photo-real 3D characters, as they can enhance the immersive experience of these applications. Here, the traditional Computer Graphics pipeline for rendering photo-realistic images involves many tedious and time-consuming steps that require expert knowledge and are far from real time. Traditional rendering involves character rigging and skinning, the modeling of the surface appearance properties, and physically based ray tracing. Recent learning-based methods attempt to simplify the traditional rendering pipeline and instead learn the rendering function from data resulting in methods that are easier accessible to non-experts. However, most of them model the synthesis task entirely in image space such that 3D consistency cannot be achieved, and/or they fail to model motion- and view-dependent appearance effects. To this end, this thesis presents a method and ongoing work on character synthesis, which allow the synthesis of controllable photoreal characters that achieve motion- and view-dependent appearance effects as well as 3D consistency and which run in real time. This is technically achieved by a novel coarse-to-fine geometric character representation for efficient synthesis, which can be solely supervised on multi-view imagery. Furthermore, this work shows how such a geometric representation can be combined with an implicit surface representation to boost synthesis and geometric quality. %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/31986

Habibie, I., Xu, W., Mehta, D., et al. 2021a. Learning Speech-driven 3D Conversational Gestures from Video. https://arxiv.org/abs/2102.06837.

(arXiv: 2102.06837)

Abstract

We propose the first approach to automatically and jointly synthesize both the synchronous 3D conversational body and hand gestures, as well as 3D face and head animations, of a virtual character from speech input. Our algorithm uses a CNN architecture that leverages the inherent correlation between facial expression and hand gestures. Synthesis of conversational body gestures is a multi-modal problem since many similar gestures can plausibly accompany the same input speech. To synthesize plausible body gestures in this setting, we train a Generative Adversarial Network (GAN) based model that measures the plausibility of the generated sequences of 3D body motion when paired with the input audio features. We also contribute a new way to create a large corpus of more than 33 hours of annotated body, hand, and face data from in-the-wild videos of talking people. To this end, we apply state-of-the-art monocular approaches for 3D body and hand pose estimation as well as dense 3D face performance capture to the video corpus. In this way, we can train on orders of magnitude more data than previous algorithms that resort to complex in-studio motion capture solutions, and thereby train more expressive synthesis algorithms. Our experiments and user study show the state-of-the-art quality of our speech-synthesized full 3D character animations.

Links

Export

BibTeX

@online{Habibie_2102.06837, TITLE = {Learning Speech-driven {3D} Conversational Gestures from Video}, AUTHOR = {Habibie, Ikhsanul and Xu, Weipeng and Mehta, Dushyant and Liu, Lingjie and Seidel, Hans-Peter and Pons-Moll, Gerard and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2102.06837}, EPRINT = {2102.06837}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {We propose the first approach to automatically and jointly synthesize both the synchronous 3D conversational body and hand gestures, as well as 3D face and head animations, of a virtual character from speech input. Our algorithm uses a CNN architecture that leverages the inherent correlation between facial expression and hand gestures. Synthesis of conversational body gestures is a multi-modal problem since many similar gestures can plausibly accompany the same input speech. To synthesize plausible body gestures in this setting, we train a Generative Adversarial Network (GAN) based model that measures the plausibility of the generated sequences of 3D body motion when paired with the input audio features. We also contribute a new way to create a large corpus of more than 33 hours of annotated body, hand, and face data from in-the-wild videos of talking people. To this end, we apply state-of-the-art monocular approaches for 3D body and hand pose estimation as well as dense 3D face performance capture to the video corpus. In this way, we can train on orders of magnitude more data than previous algorithms that resort to complex in-studio motion capture solutions, and thereby train more expressive synthesis algorithms. Our experiments and user study show the state-of-the-art quality of our speech-synthesized full 3D character animations. }, }

Endnote

%0 Report %A Habibie, Ikhsanul %A Xu, Weipeng %A Mehta, Dushyant %A Liu, Lingjie %A Seidel, Hans-Peter %A Pons-Moll, Gerard %A Elgharib, Mohamed %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Learning Speech-driven 3D Conversational Gestures from Video : %G eng %U http://hdl.handle.net/21.11116/0000-0009-70C7-8 %U https://arxiv.org/abs/2102.06837 %D 2021 %X We propose the first approach to automatically and jointly synthesize both the synchronous 3D conversational body and hand gestures, as well as 3D face and head animations, of a virtual character from speech input. Our algorithm uses a CNN architecture that leverages the inherent correlation between facial expression and hand gestures. Synthesis of conversational body gestures is a multi-modal problem since many similar gestures can plausibly accompany the same input speech. To synthesize plausible body gestures in this setting, we train a Generative Adversarial Network (GAN) based model that measures the plausibility of the generated sequences of 3D body motion when paired with the input audio features. We also contribute a new way to create a large corpus of more than 33 hours of annotated body, hand, and face data from in-the-wild videos of talking people. To this end, we apply state-of-the-art monocular approaches for 3D body and hand pose estimation as well as dense 3D face performance capture to the video corpus. In this way, we can train on orders of magnitude more data than previous algorithms that resort to complex in-studio motion capture solutions, and thereby train more expressive synthesis algorithms. Our experiments and user study show the state-of-the-art quality of our speech-synthesized full 3D character animations. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Habibie, I., Xu, W., Mehta, D., et al. 2021b. Learning Speech-driven 3D Conversational Gestures from Video. Proceedings of the 21st ACM International Conference on Intelligent Virtual Agents (IVA 2021), ACM.

Links

Export

BibTeX

@inproceedings{Habibie_IVA2021, TITLE = {Learning Speech-driven {3D} Conversational Gestures from Video}, AUTHOR = {Habibie, Ikhsanul and Xu, Weipeng and Mehta, Dushyant and Liu, Lingjie and Seidel, Hans-Peter and Pons-Moll, Gerard and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {9781450386197}, DOI = {10.1145/3472306.3478335}, PUBLISHER = {ACM}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Proceedings of the 21st ACM International Conference on Intelligent Virtual Agents (IVA 2021)}, PAGES = {101--108}, ADDRESS = {Virtual Event, Japan}, }

Endnote

%0 Conference Proceedings %A Habibie, Ikhsanul %A Xu, Weipeng %A Mehta, Dushyant %A Liu, Lingjie %A Seidel, Hans-Peter %A Pons-Moll, Gerard %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Learning Speech-driven 3D Conversational Gestures from Video : %G eng %U http://hdl.handle.net/21.11116/0000-0009-4D19-6 %R 10.1145/3472306.3478335 %D 2021 %B 21st ACM International Conference on Intelligent Virtual Agents %Z date of event: 2021-09-14 - 2021-09-17 %C Virtual Event, Japan %B Proceedings of the 21st ACM International Conference on Intelligent Virtual Agents %P 101 - 108 %I ACM %@ 9781450386197

Hladký, J., Seidel, H.-P., and Steinberger, M. 2021. SnakeBinning: Efficient Temporally Coherent Triangle Packing for Shading Streaming. Computer Graphics Forum (Proc. EUROGRAPHICS 2021)40, 2.

Links

Export

BibTeX

@article{10.1111:cgf.142648, TITLE = {{SnakeBinning}: {E}fficient Temporally Coherent Triangle Packing for Shading Streaming}, AUTHOR = {Hladk{\'y}, Jozef and Seidel, Hans-Peter and Steinberger, Markus}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.142648}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {40}, NUMBER = {2}, PAGES = {475--488}, BOOKTITLE = {42nd Annual Conference of the European Association for Computer Graphics (EUROGRAPHICS 2021)}, EDITOR = {Mitra, Niloy and Viola, Ivan}, }

Endnote

%0 Journal Article %A Hladký, Jozef %A Seidel, Hans-Peter %A Steinberger, Markus %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T SnakeBinning: Efficient Temporally Coherent Triangle Packing for Shading Streaming : %G eng %U http://hdl.handle.net/21.11116/0000-0008-7AFD-3 %R 10.1111/cgf.142648 %7 2021 %D 2021 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 40 %N 2 %& 475 %P 475 - 488 %I Blackwell-Wiley %C Oxford %@ false %B 42nd Annual Conference of the European Association for Computer Graphics %O EUROGRAPHICS 2021 EG 2021

Jiang, C., Tang, C., Seidel, H.-P., Chen, R., and Wonka, P. 2021. Computational Design of Lightweight Trusses. Computer-Aided Design141.

Links

Export

BibTeX

@article{Jiang2021, TITLE = {Computational Design of Lightweight Trusses}, AUTHOR = {Jiang, Caigui and Tang, Chengcheng and Seidel, Hans-Peter and Chen, Renjie and Wonka, Peter}, ISSN = {0010-4485}, DOI = {10.1016/j.cad.2021.103076}, PUBLISHER = {Elsevier}, ADDRESS = {Amsterdam}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {Computer-Aided Design}, VOLUME = {141}, EID = {103076}, }

Endnote

%0 Journal Article %A Jiang, Caigui %A Tang, Chengcheng %A Seidel, Hans-Peter %A Chen, Renjie %A Wonka, Peter %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Computational Design of Lightweight Trusses : %U http://hdl.handle.net/21.11116/0000-0009-70C2-D %R 10.1016/j.cad.2021.103076 %7 2021 %D 2021 %J Computer-Aided Design %V 141 %Z sequence number: 103076 %I Elsevier %C Amsterdam %@ false

Jindal, A., Wolski, K., Mantiuk, R.K., and Myszkowski, K. 2021. Perceptual Model for Adaptive Local Shading and Refresh Rate. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2021)40, 6.

Links

Export

BibTeX

@article{JindalSIGGRAPHAsia21, TITLE = {Perceptual Model for Adaptive Local Shading and Refresh Rate}, AUTHOR = {Jindal, Akshay and Wolski, Krzysztof and Mantiuk, Rafa{\l} K. and Myszkowski, Karol}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3478513.3480514}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {40}, NUMBER = {6}, PAGES = {1--18}, EID = {281}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2021}, }

Endnote

%0 Journal Article %A Jindal, Akshay %A Wolski, Krzysztof %A Mantiuk, Rafał K. %A Myszkowski, Karol %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Perceptual Model for Adaptive Local Shading and Refresh Rate : %G eng %U http://hdl.handle.net/21.11116/0000-0009-9B45-B %R 10.1145/3478513.3480514 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 6 %& 1 %P 1 - 18 %Z sequence number: 281 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2021 %O ACM SIGGRAPH Asia 2021 SA '21 SA 2021

Kappel, M., Golyanik, V., Elgharib, M., et al. 2021. High-Fidelity Neural Human Motion Transfer from Monocular Video Computer Vision and Pattern Recognition. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Kappel_CVPR2021, TITLE = {High-Fidelity Neural Human Motion Transfer from Monocular Video Computer Vision and Pattern Recognition}, AUTHOR = {Kappel, Moritz and Golyanik, Vladislav and Elgharib, Mohamed and Henningson, Jann-Ole and Seidel, Hans-Peter and Castillo, Susana and Theobalt, Christian and Magnor, Marcus A.}, LANGUAGE = {eng}, ISBN = {978-1-6654-4509-2}, DOI = {10.1109/CVPR46437.2021.00159}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021)}, PAGES = {1541--1550}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Kappel, Moritz %A Golyanik, Vladislav %A Elgharib, Mohamed %A Henningson, Jann-Ole %A Seidel, Hans-Peter %A Castillo, Susana %A Theobalt, Christian %A Magnor, Marcus A. %+ External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations %T High-Fidelity Neural Human Motion Transfer from Monocular Video Computer Vision and Pattern Recognition : %G eng %U http://hdl.handle.net/21.11116/0000-0008-8947-E %R 10.1109/CVPR46437.2021.00159 %D 2021 %B 34th IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2021-06-19 - 2021-06-25 %C Virtual Conference %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 1541 - 1550 %I IEEE %@ 978-1-6654-4509-2 %U https://gvv.mpi-inf.mpg.de/projects/NHMT/

Knibbe, J., Freire, R., Koelle, M., and Strohmeier, P. 2021. Skill-Sleeves: Designing Electrode Garments for Wearability. TEI ’21, Fifteenth International Conference on Tangible, Embedded, and Embodied Interaction, ACM.

Links

Export

BibTeX

@inproceedings{Knibbe_TEI21, TITLE = {Skill-Sleeves: {D}esigning Electrode Garments for Wearability}, AUTHOR = {Knibbe, Jarrod and Freire, Rachel and Koelle, Marion and Strohmeier, Paul}, LANGUAGE = {eng}, ISBN = {978-1-4503-8213-7}, DOI = {10.1145/3430524.3440652}, PUBLISHER = {ACM}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {TEI '21, Fifteenth International Conference on Tangible, Embedded, and Embodied Interaction}, PAGES = {1--16}, EID = {33}, ADDRESS = {Salzburg, Austria}, }

Endnote

%0 Conference Proceedings %A Knibbe, Jarrod %A Freire, Rachel %A Koelle, Marion %A Strohmeier, Paul %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Skill-Sleeves: Designing Electrode Garments for Wearability : %G eng %U http://hdl.handle.net/21.11116/0000-000C-BDF0-0 %R 10.1145/3430524.3440652 %D 2021 %B Fifteenth International Conference on Tangible, Embedded, and Embodied Interaction %Z date of event: 2021-02-14 - 2021-02-17 %C Salzburg, Austria %B TEI '21 %P 1 - 16 %Z sequence number: 33 %I ACM %@ 978-1-4503-8213-7

Lagunas, M., Serrano, A., Gutierrez, D., and Masia, B. 2021. The Joint Role of Geometry and Illumination on Material Recognition. Journal of Vision21, 2.

Links

Export

BibTeX

@article{Lagunas2021_MatRecog, TITLE = {The Joint Role of Geometry and Illumination on Material Recognition}, AUTHOR = {Lagunas, Manuel and Serrano, Ana and Gutierrez, Diego and Masia, Belen}, LANGUAGE = {eng}, ISSN = {1534-7362}, DOI = {10.1167/jov.21.2.2}, PUBLISHER = {Scholar One, Inc.}, ADDRESS = {Charlottesville, VA}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {Journal of Vision}, VOLUME = {21}, NUMBER = {2}, PAGES = {1--18}, }

Endnote

%0 Journal Article %A Lagunas, Manuel %A Serrano, Ana %A Gutierrez, Diego %A Masia, Belen %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T The Joint Role of Geometry and Illumination on Material Recognition : %G eng %U http://hdl.handle.net/21.11116/0000-0007-EAF9-9 %R 10.1167/jov.21.2.2 %7 2021 %D 2021 %J Journal of Vision %V 21 %N 2 %& 1 %P 1 - 18 %I Scholar One, Inc. %C Charlottesville, VA %@ false

Leimkühler, T. and Drettakis, G. 2021. FreeStyleGAN: Free-view Editable Portrait Rendering with the Camera Manifold. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2021)40, 6.

Links

Export

BibTeX

@article{Leimkuehler_SIGGRAPHAsia21, TITLE = {{FreeStyleGAN}: {F}ree-view editable portrait rendering with the camera manifold}, AUTHOR = {Leimk{\"u}hler, Thomas and Drettakis, George}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3478513.3480538}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {40}, NUMBER = {6}, PAGES = {1--15}, EID = {224}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2021}, }

Endnote

%0 Journal Article %A Leimkühler, Thomas %A Drettakis, George %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T FreeStyleGAN: Free-view Editable Portrait Rendering with the Camera Manifold : %G eng %U http://hdl.handle.net/21.11116/0000-000C-AFF0-0 %R 10.1145/3478513.3480538 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 6 %& 1 %P 1 - 15 %Z sequence number: 224 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2021 %O ACM SIGGRAPH Asia 2021 SA '21 SA 2021

Liu, L., Xu, W., Habermann, M., et al. 2021. Neural Human Video Rendering by Learning Dynamic Textures and Rendering-to-Video Translation. IEEE Transactions on Visualization and Computer Graphics27, 10.

Links

Export

BibTeX

@article{liu2020NeuralHumanRendering, TITLE = {Neural Human Video Rendering by Learning Dynamic Textures and Rendering-to-Video Translation}, AUTHOR = {Liu, Lingjie and Xu, Weipeng and Habermann, Marc and Zollh{\"o}fer, Michael and Bernard, Florian and Kim, Hyeongwoo and Wang, Wenping and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {1077-2626}, DOI = {10.1109/TVCG.2020.2996594}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {IEEE Transactions on Visualization and Computer Graphics}, VOLUME = {27}, NUMBER = {10}, PAGES = {4009--4022}, }

Endnote

%0 Journal Article %A Liu, Lingjie %A Xu, Weipeng %A Habermann, Marc %A Zollhöfer, Michael %A Bernard, Florian %A Kim, Hyeongwoo %A Wang, Wenping %A Theobalt, Christian %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Human Video Rendering by Learning Dynamic Textures and Rendering-to-Video Translation : %G eng %U http://hdl.handle.net/21.11116/0000-0008-0369-F %R 10.1109/TVCG.2020.2996594 %7 2020 %D 2021 %J IEEE Transactions on Visualization and Computer Graphics %V 27 %N 10 %& 4009 %P 4009 - 4022 %I IEEE %C Piscataway, NJ %@ false

Li, Y., Habermann, M., Thomaszewski,, B., Coros, S., Beeler, T., and Theobalt, C. 2021. Deep Physics-aware Inference of Cloth Deformation for Monocular Human Performance Capture. 2021 International Conference on 3D Vision, IEEE.

Links

Export

BibTeX

@inproceedings{Li_3DV21, TITLE = {Deep Physics-aware Inference of Cloth Deformation for Monocular Human Performance Capture}, AUTHOR = {Li, Yue and Habermann, Marc and Thomaszewski,, Bernhard and Coros, Stelian and Beeler, Thabo and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-2688-6}, DOI = {10.1109/3DV53792.2021.00047}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, BOOKTITLE = {2021 International Conference on 3D Vision}, PAGES = {373--384}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Li, Yue %A Habermann, Marc %A Thomaszewski,, Bernhard %A Coros, Stelian %A Beeler, Thabo %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Deep Physics-aware Inference of Cloth Deformation for Monocular Human Performance Capture : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E9D0-7 %R 10.1109/3DV53792.2021.00047 %D 2021 %B International Conference on 3D Vision %Z date of event: 2021-12-01 - 2021-12-03 %C Virtual Conference %B 2021 International Conference on 3D Vision %P 373 - 384 %I IEEE %@ 978-1-6654-2688-6

Mallikarjun B R, Tewari, A., Seidel, H.-P., Elgharib, M., and Theobalt, C. 2021a. Learning Complete 3D Morphable Face Models from Images and Videos. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Mallikarjun_CVPR2021b, TITLE = {Learning Complete {3D} Morphable Face Models from Images and Videos}, AUTHOR = {Mallikarjun B R and Tewari, Ayush and Seidel, Hans-Peter and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-4509-2}, DOI = {10.1109/CVPR46437.2021.00337}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021)}, PAGES = {3360--3370}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Mallikarjun B R, %A Tewari, Ayush %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Learning Complete 3D Morphable Face Models from Images and Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0008-8926-3 %R 10.1109/CVPR46437.2021.00337 %D 2021 %B 34th IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2021-06-19 - 2021-06-25 %C Virtual Conference %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 3360 - 3370 %I IEEE %@ 978-1-6654-4509-2 %U https://gvv.mpi-inf.mpg.de/projects/LeMoMo/

Mallikarjun B R, Tewari, A., Oh, T.-H., et al. 2021b. Monocular Reconstruction of Neural Face Reflectance Fields. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Mallikarjun_CVPR2021, TITLE = {Monocular Reconstruction of Neural Face Reflectance Fields}, AUTHOR = {Mallikarjun B R and Tewari, Ayush and Oh, Tae-Hyun and Weyrich, Tim and Bickel, Bernd and Seidel, Hans-Peter and Pfister, Hanspeter and Matusik, Wojciech and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-4509-2}, DOI = {10.1109/CVPR46437.2021.00476}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021)}, PAGES = {4789--4798}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Mallikarjun B R, %A Tewari, Ayush %A Oh, Tae-Hyun %A Weyrich, Tim %A Bickel, Bernd %A Seidel, Hans-Peter %A Pfister, Hanspeter %A Matusik, Wojciech %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Monocular Reconstruction of Neural Face Reflectance Fields : %G eng %U http://hdl.handle.net/21.11116/0000-0008-88FB-4 %R 10.1109/CVPR46437.2021.00476 %D 2021 %B 34th IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2021-06-19 - 2021-06-25 %C Virtual Conference %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 4789 - 4798 %I IEEE %@ 978-1-6654-4509-2 %U https://gvv.mpi-inf.mpg.de/projects/FaceReflectanceFields/

Mallikarjun B R, Tewari, A., Dib, A., et al. 2021c. PhotoApp: Photorealistic Appearance Editing of Head Portraits. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2021)40, 4.

Links

Export

BibTeX

@article{MallikarjunBR2021, TITLE = {{PhotoApp}: {P}hotorealistic Appearance Editing of Head Portraits}, AUTHOR = {Mallikarjun B R and Tewari, Ayush and Dib, Abdallah and Weyrich, Tim and Bickel, Bernd and Seidel, Hans-Peter and Pfister, Hanspeter and Matusik, Wojciech and Chevallier, Louis and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3450626.3459765}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {40}, NUMBER = {4}, PAGES = {1--16}, EID = {44}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2021}, }

Endnote

%0 Journal Article %A Mallikarjun B R, %A Tewari, Ayush %A Dib, Abdallah %A Weyrich, Tim %A Bickel, Bernd %A Seidel, Hans-Peter %A Pfister, Hanspeter %A Matusik, Wojciech %A Chevallier, Louis %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T PhotoApp: Photorealistic Appearance Editing of Head Portraits : %G eng %U http://hdl.handle.net/21.11116/0000-0009-2A9B-A %R 10.1145/3450626.3459765 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 4 %& 1 %P 1 - 16 %Z sequence number: 44 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2021 %O ACM SIGGRAPH 2021

Martin, D., Malpica, S., Gutierrez, D., Masia, B., and Serrano, A. 2021. Multimodality in VR: A Survey. https://arxiv.org/abs/2101.07906.

(arXiv: 2101.07906)

Abstract

Virtual reality has the potential to change the way we create and consume content in our everyday life. Entertainment, training, design and manufacturing, communication, or advertising are all applications that already benefit from this new medium reaching consumer level. VR is inherently different from traditional media: it offers a more immersive experience, and has the ability to elicit a sense of presence through the place and plausibility illusions. It also gives the user unprecedented capabilities to explore their environment, in contrast with traditional media. In VR, like in the real world, users integrate the multimodal sensory information they receive to create a unified perception of the virtual world. Therefore, the sensory cues that are available in a virtual environment can be leveraged to enhance the final experience. This may include increasing realism, or the sense of presence; predicting or guiding the attention of the user through the experience; or increasing their performance if the experience involves the completion of certain tasks. In this state-of-the-art report, we survey the body of work addressing multimodality in virtual reality, its role and benefits in the final user experience. The works here reviewed thus encompass several fields of research, including computer graphics, human computer interaction, or psychology and perception. Additionally, we give an overview of different applications that leverage multimodal input in areas such as medicine, training and education, or entertainment; we include works in which the integration of multiple sensory information yields significant improvements, demonstrating how multimodality can play a fundamental role in the way VR systems are designed, and VR experiences created and consumed.

Links

Export

BibTeX

@online{Martin2021_VRsurvey, TITLE = {Multimodality in {VR}: {A} Survey}, AUTHOR = {Martin, Daniel and Malpica, Sandra and Gutierrez, Diego and Masia, Belen and Serrano, Ana}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2101.07906}, EPRINT = {2101.07906}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Virtual reality has the potential to change the way we create and consume content in our everyday life. Entertainment, training, design and manufacturing, communication, or advertising are all applications that already benefit from this new medium reaching consumer level. VR is inherently different from traditional media: it offers a more immersive experience, and has the ability to elicit a sense of presence through the place and plausibility illusions. It also gives the user unprecedented capabilities to explore their environment, in contrast with traditional media. In VR, like in the real world, users integrate the multimodal sensory information they receive to create a unified perception of the virtual world. Therefore, the sensory cues that are available in a virtual environment can be leveraged to enhance the final experience. This may include increasing realism, or the sense of presence; predicting or guiding the attention of the user through the experience; or increasing their performance if the experience involves the completion of certain tasks. In this state-of-the-art report, we survey the body of work addressing multimodality in virtual reality, its role and benefits in the final user experience. The works here reviewed thus encompass several fields of research, including computer graphics, human computer interaction, or psychology and perception. Additionally, we give an overview of different applications that leverage multimodal input in areas such as medicine, training and education, or entertainment; we include works in which the integration of multiple sensory information yields significant improvements, demonstrating how multimodality can play a fundamental role in the way VR systems are designed, and VR experiences created and consumed. }, }

Endnote

%0 Report %A Martin, Daniel %A Malpica, Sandra %A Gutierrez, Diego %A Masia, Belen %A Serrano, Ana %+ External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Multimodality in VR: A Survey : %G eng %U http://hdl.handle.net/21.11116/0000-0007-EB00-0 %U https://arxiv.org/abs/2101.07906 %D 2021 %X Virtual reality has the potential to change the way we create and consume content in our everyday life. Entertainment, training, design and manufacturing, communication, or advertising are all applications that already benefit from this new medium reaching consumer level. VR is inherently different from traditional media: it offers a more immersive experience, and has the ability to elicit a sense of presence through the place and plausibility illusions. It also gives the user unprecedented capabilities to explore their environment, in contrast with traditional media. In VR, like in the real world, users integrate the multimodal sensory information they receive to create a unified perception of the virtual world. Therefore, the sensory cues that are available in a virtual environment can be leveraged to enhance the final experience. This may include increasing realism, or the sense of presence; predicting or guiding the attention of the user through the experience; or increasing their performance if the experience involves the completion of certain tasks. In this state-of-the-art report, we survey the body of work addressing multimodality in virtual reality, its role and benefits in the final user experience. The works here reviewed thus encompass several fields of research, including computer graphics, human computer interaction, or psychology and perception. Additionally, we give an overview of different applications that leverage multimodal input in areas such as medicine, training and education, or entertainment; we include works in which the integration of multiple sensory information yields significant improvements, demonstrating how multimodality can play a fundamental role in the way VR systems are designed, and VR experiences created and consumed. %K Computer Science, Human-Computer Interaction, cs.HC,Computer Science, Graphics, cs.GR

Masia, B., Camon, J., Gutierrez,, D., and Serrano, A. 2021. Influence of Directional Sound Cues on Users’ Exploration Across 360° Movie Cuts. IEEE Computer Graphics and Applications41, 4.

Links

Export

BibTeX

@article{Masia2021, TITLE = {Influence of Directional Sound Cues on Users' Exploration Across 360{\textdegree} Movie Cuts}, AUTHOR = {Masia, Belen and Camon, Javier and Gutierrez,, Diego and Serrano, Ana}, LANGUAGE = {eng}, ISSN = {0272-1716}, DOI = {10.1109/MCG.2021.3064688}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {IEEE Computer Graphics and Applications}, VOLUME = {41}, NUMBER = {4}, PAGES = {64--75}, }

Endnote

%0 Journal Article %A Masia, Belen %A Camon, Javier %A Gutierrez,, Diego %A Serrano, Ana %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Influence of Directional Sound Cues on Users' Exploration Across 360° Movie Cuts : %G eng %U http://hdl.handle.net/21.11116/0000-0008-F077-3 %R 10.1109/MCG.2021.3064688 %7 2021 %D 2021 %J IEEE Computer Graphics and Applications %V 41 %N 4 %& 64 %P 64 - 75 %I IEEE %C Piscataway, NJ %@ false

Meka, A., Shafiei, M., Zollhöfer, M., Richardt, C., and Theobalt, C. 2021. Real-time Global Illumination Decomposition of Videos. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2021)40, 4.

Links

Export

BibTeX

@article{Meka:2021, TITLE = {Real-time Global Illumination Decomposition of Videos}, AUTHOR = {Meka, Abhimitra and Shafiei, Mohammad and Zollh{\"o}fer, Michael and Richardt, Christian and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3374753}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {40}, NUMBER = {4}, PAGES = {1--16}, EID = {22}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2021}, }

Endnote

%0 Journal Article %A Meka, Abhimitra %A Shafiei, Mohammad %A Zollhöfer, Michael %A Richardt, Christian %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Real-time Global Illumination Decomposition of Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0007-EE07-6 %R 10.1145/3374753 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 4 %& 1 %P 1 - 16 %Z sequence number: 22 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2021 %O ACM SIGGRAPH 2021 %U http://gvv.mpi-inf.mpg.de/projects/LiveIlluminationDecomposition/

Nehvi, J., Golyanik, V., Mueller, F., Seidel, H.-P., Elgharib, M., and Theobalt, C. 2021. Differentiable Event Stream Simulator for Non-Rigid 3D Tracking. Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPR 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Nehvi_CVPR2021Workshop, TITLE = {Differentiable Event Stream Simulator for Non-Rigid {3D} Tracking}, AUTHOR = {Nehvi, Jalees and Golyanik, Vladislav and Mueller, Franziska and Seidel, Hans-Peter and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-4899-4}, DOI = {10.1109/CVPRW53098.2021.00143}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops (CVPR 2021)}, PAGES = {1302--1311}, ADDRESS = {Virtual Workshop}, }

Endnote

%0 Conference Proceedings %A Nehvi, Jalees %A Golyanik, Vladislav %A Mueller, Franziska %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Differentiable Event Stream Simulator for Non-Rigid 3D Tracking : %G eng %U http://hdl.handle.net/21.11116/0000-0008-8957-C %R 10.1109/CVPRW53098.2021.00143 %D 2021 %B Third International Workshop on Event-Based Vision %Z date of event: 2021-06-19 - 2021-06-19 %C Virtual Workshop %B Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition Workshops %P 1302 - 1311 %I IEEE %@ 978-1-6654-4899-4 %U https://gvv.mpi-inf.mpg.de/projects/Event-based_Non-rigid_3D_Tracking/

Rao, S., Stutz, D., and Schiele, B. 2021. Adversarial Training Against Location-Optimized Adversarial Patches. Computer Vision -- ECCV Workshops 2020, Springer.

Links

Export

BibTeX

@inproceedings{DBLP:conf/eccv/RaoSS20, TITLE = {Adversarial Training Against Location-Optimized Adversarial Patches}, AUTHOR = {Rao, Sukrut and Stutz, David and Schiele, Bernt}, LANGUAGE = {eng}, ISBN = {978-3-030-68237-8}, DOI = {10.1007/978-3-030-68238-5_32}, PUBLISHER = {Springer}, YEAR = {2020}, MARGINALMARK = {$\bullet$}, DATE = {2021}, BOOKTITLE = {Computer Vision -- ECCV Workshops 2020}, EDITOR = {Bartoli, Adrian and Fusiello, Andrea}, PAGES = {429--448}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12539}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Rao, Sukrut %A Stutz, David %A Schiele, Bernt %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Adversarial Training Against Location-Optimized Adversarial Patches : %G eng %U http://hdl.handle.net/21.11116/0000-0008-1662-1 %R 10.1007/978-3-030-68238-5_32 %D 2021 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV Workshops 2020 %E Bartoli, Adrian; Fusiello, Andrea %P 429 - 448 %I Springer %@ 978-3-030-68237-8 %B Lecture Notes in Computer Science %N 12539

Rittig, T., Sumin, D., Babaei, V., et al. 2021. Neural Acceleration of Scattering-Aware Color 3D Printing. Computer Graphics Forum (Proc. EUROGRAPHICS 2021)40, 2.

Links

Export

BibTeX

@article{rittig2021neural, TITLE = {Neural Acceleration of Scattering-Aware Color {3D} Printing}, AUTHOR = {Rittig, Tobias and Sumin, Denis and Babaei, Vahid and Didyk, Piotr and Voloboy, Alexei and Wilkie, Alexander and Bickel, Bernd and Myszkowski, Karol and Weyrich, Tim and Krivanek, Jaroslav}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.142626}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, EDITOR = {Mitra, Niloy and Violoa, Ivan}, VOLUME = {40}, NUMBER = {2}, PAGES = {205--219}, BOOKTITLE = {42nd Annual Conference of the European Association for Computer Graphics (EUROGRAPHICS 2021)}, }

Endnote

%0 Journal Article %A Rittig, Tobias %A Sumin, Denis %A Babaei, Vahid %A Didyk, Piotr %A Voloboy, Alexei %A Wilkie, Alexander %A Bickel, Bernd %A Myszkowski, Karol %A Weyrich, Tim %A Krivanek, Jaroslav %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Neural Acceleration of Scattering-Aware Color 3D Printing : %G eng %U http://hdl.handle.net/21.11116/0000-0007-F073-8 %R 10.1111/cgf.142626 %7 2021 %D 2021 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 40 %N 2 %& 205 %P 205 - 219 %I Blackwell-Wiley %C Oxford %@ false %B 42nd Annual Conference of the European Association for Computer Graphics %O EUROGRAPHICS 2021 EG 2021

Ruan, L., Chen, B., Li, J., and Lam, M.-L. 2021. AIFNet: All-in-Focus Image Restoration Network Using a Light Field-Based Dataset. IEEE Transactions on Computational Imaging7.

Links

Export

BibTeX

@article{Ruan2021, TITLE = {{AIFNet}: {A}ll-in-Focus Image Restoration Network Using a Light Field-Based Dataset}, AUTHOR = {Ruan, Lingyan and Chen, Bin and Li, Jizhou and Lam, Miu-Ling}, LANGUAGE = {eng}, ISSN = {2573-0436}, DOI = {10.1109/TCI.2021.3092891}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {IEEE Transactions on Computational Imaging}, VOLUME = {7}, PAGES = {675--688}, }

Endnote

%0 Journal Article %A Ruan, Lingyan %A Chen, Bin %A Li, Jizhou %A Lam, Miu-Ling %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T AIFNet: All-in-Focus Image Restoration Network Using a Light Field-Based Dataset : %G eng %U http://hdl.handle.net/21.11116/0000-0009-4795-F %R 10.1109/TCI.2021.3092891 %7 2021 %D 2021 %J IEEE Transactions on Computational Imaging %V 7 %& 675 %P 675 - 688 %I IEEE %C Piscataway, NJ %@ false

Rudnev, V., Golyanik, V., Wang, J., et al. 2021. EventHands: Real-Time Neural 3D Hand Pose Estimation from an Event Stream. ICCV 2021, IEEE.

Links

Export

BibTeX

@inproceedings{Rudnev_2021_ICCV, TITLE = {{EventHands}: {R}eal-Time Neural {3D} Hand Pose Estimation from an Event Stream}, AUTHOR = {Rudnev, Viktor and Golyanik, Vladislav and Wang, Jiayi and Seidel, Hans-Peter and Mueller, Franziska and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-2812-5}, DOI = {10.1109/ICCV48922.2021.01216}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {ICCV 2021}, PAGES = {12365--12375}, ADDRESS = {Virtual Event}, }

Endnote

%0 Conference Proceedings %A Rudnev, Viktor %A Golyanik, Vladislav %A Wang, Jiayi %A Seidel, Hans-Peter %A Mueller, Franziska %A Elgharib, Mohamed %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T EventHands: Real-Time Neural 3D Hand Pose Estimation from an Event Stream : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B709-1 %R 10.1109/ICCV48922.2021.01216 %D 2021 %B IEEE/CVF International Conference on Computer Vision %Z date of event: 2021-10-10 - 2021-10-17 %C Virtual Event %B ICCV 2021 %P 12365 - 12375 %I IEEE %@ 978-1-6654-2812-5

Sarkar, K., Mehta, D., Xu, W., Golyanik, V., and Theobalt, C. 2021. Neural Re-Rendering of Humans from a Single Image. https://arxiv.org/abs/2101.04104.

(arXiv: 2101.04104)

Abstract

Human re-rendering from a single image is a starkly under-constrained problem, and state-of-the-art algorithms often exhibit undesired artefacts, such as over-smoothing, unrealistic distortions of the body parts and garments, or implausible changes of the texture. To address these challenges, we propose a new method for neural re-rendering of a human under a novel user-defined pose and viewpoint, given one input image. Our algorithm represents body pose and shape as a parametric mesh which can be reconstructed from a single image and easily reposed. Instead of a colour-based UV texture map, our approach further employs a learned high-dimensional UV feature map to encode appearance. This rich implicit representation captures detailed appearance variation across poses, viewpoints, person identities and clothing styles better than learned colour texture maps. The body model with the rendered feature maps is fed through a neural image-translation network that creates the final rendered colour image. The above components are combined in an end-to-end-trained neural network architecture that takes as input a source person image, and images of the parametric body model in the source pose and desired target pose. Experimental evaluation demonstrates that our approach produces higher quality single image re-rendering results than existing methods.

Links

Export

BibTeX

@online{Sarkar_arXiv2101.04104, TITLE = {Neural Re-Rendering of Humans from a Single Image}, AUTHOR = {Sarkar, Kripasindhu and Mehta, Dushyant and Xu, Weipeng and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2101.04104}, EPRINT = {2101.04104}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Human re-rendering from a single image is a starkly under-constrained problem, and state-of-the-art algorithms often exhibit undesired artefacts, such as over-smoothing, unrealistic distortions of the body parts and garments, or implausible changes of the texture. To address these challenges, we propose a new method for neural re-rendering of a human under a novel user-defined pose and viewpoint, given one input image. Our algorithm represents body pose and shape as a parametric mesh which can be reconstructed from a single image and easily reposed. Instead of a colour-based UV texture map, our approach further employs a learned high-dimensional UV feature map to encode appearance. This rich implicit representation captures detailed appearance variation across poses, viewpoints, person identities and clothing styles better than learned colour texture maps. The body model with the rendered feature maps is fed through a neural image-translation network that creates the final rendered colour image. The above components are combined in an end-to-end-trained neural network architecture that takes as input a source person image, and images of the parametric body model in the source pose and desired target pose. Experimental evaluation demonstrates that our approach produces higher quality single image re-rendering results than existing methods. }, }

Endnote

%0 Report %A Sarkar, Kripasindhu %A Mehta, Dushyant %A Xu, Weipeng %A Golyanik, Vladislav %A Theobalt, Christian %+ Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Neural Re-Rendering of Humans from a Single Image : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CF05-B %U https://arxiv.org/abs/2101.04104 %D 2021 %X Human re-rendering from a single image is a starkly under-constrained problem, and state-of-the-art algorithms often exhibit undesired artefacts, such as over-smoothing, unrealistic distortions of the body parts and garments, or implausible changes of the texture. To address these challenges, we propose a new method for neural re-rendering of a human under a novel user-defined pose and viewpoint, given one input image. Our algorithm represents body pose and shape as a parametric mesh which can be reconstructed from a single image and easily reposed. Instead of a colour-based UV texture map, our approach further employs a learned high-dimensional UV feature map to encode appearance. This rich implicit representation captures detailed appearance variation across poses, viewpoints, person identities and clothing styles better than learned colour texture maps. The body model with the rendered feature maps is fed through a neural image-translation network that creates the final rendered colour image. The above components are combined in an end-to-end-trained neural network architecture that takes as input a source person image, and images of the parametric body model in the source pose and desired target pose. Experimental evaluation demonstrates that our approach produces higher quality single image re-rendering results than existing methods. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Serrano, A., Chen, B., Wang, C., et al. 2021. The Effect of Shape and Illumination on Material Perception. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2021)40, 4.

Links

Export

BibTeX

@article{SIGG2021_Materials, TITLE = {The Effect of Shape and Illumination on Material Perception}, AUTHOR = {Serrano, Ana and Chen, Bin and Wang, Chao and Piovar{\v c}i, Michal and Seidel, Hans-Peter and Didyk, Piotr and Myszkowski, Karol}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3450626.3459813}, PUBLISHER = {Association for Computing Machinery}, ADDRESS = {New York, NY}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {40}, NUMBER = {4}, PAGES = {1--16}, EID = {125}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2021}, }

Endnote

%0 Journal Article %A Serrano, Ana %A Chen, Bin %A Wang, Chao %A Piovarči, Michal %A Seidel, Hans-Peter %A Didyk, Piotr %A Myszkowski, Karol %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T The Effect of Shape and Illumination on Material Perception : Model and Applications %G eng %U http://hdl.handle.net/21.11116/0000-0009-0565-0 %R 10.1145/3450626.3459813 %7 2021 %D 2021 %J ACM Transactions on Graphics %V 40 %N 4 %& 1 %P 1 - 16 %Z sequence number: 125 %I Association for Computing Machinery %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2021 %O ACM SIGGRAPH 2021

Surace, L., Wernikowski, M., Tursun, O.T., Myszkowski, K., Mantiuk, R., and Didyk, P. 2021. Learning Foveated Reconstruction to Preserve Perceived Image Statistics. https://arxiv.org/abs/2108.03499.

(arXiv: 2108.03499)

Abstract

Foveated image reconstruction recovers full image from a sparse set of samples distributed according to the human visual system's retinal sensitivity that rapidly drops with eccentricity. Recently, the use of Generative Adversarial Networks was shown to be a promising solution for such a task as they can successfully hallucinate missing image information. Like for other supervised learning approaches, also for this one, the definition of the loss function and training strategy heavily influences the output quality. In this work, we pose the question of how to efficiently guide the training of foveated reconstruction techniques such that they are fully aware of the human visual system's capabilities and limitations, and therefore, reconstruct visually important image features. Due to the nature of GAN-based solutions, we concentrate on the human's sensitivity to hallucination for different input sample densities. We present new psychophysical experiments, a dataset, and a procedure for training foveated image reconstruction. The strategy provides flexibility to the generator network by penalizing only perceptually important deviations in the output. As a result, the method aims to preserve perceived image statistics rather than natural image statistics. We evaluate our strategy and compare it to alternative solutions using a newly trained objective metric and user experiments.

Links

Export

BibTeX

@online{Surace2108.03499, TITLE = {Learning Foveated Reconstruction to Preserve Perceived Image Statistics}, AUTHOR = {Surace, Luca and Wernikowski, Marek and Tursun, Okan Tarhan and Myszkowski, Karol and Mantiuk, Rados{\l}aw and Didyk, Piotr}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2108.03499}, EPRINT = {2108.03499}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {Foveated image reconstruction recovers full image from a sparse set of samples distributed according to the human visual system's retinal sensitivity that rapidly drops with eccentricity. Recently, the use of Generative Adversarial Networks was shown to be a promising solution for such a task as they can successfully hallucinate missing image information. Like for other supervised learning approaches, also for this one, the definition of the loss function and training strategy heavily influences the output quality. In this work, we pose the question of how to efficiently guide the training of foveated reconstruction techniques such that they are fully aware of the human visual system's capabilities and limitations, and therefore, reconstruct visually important image features. Due to the nature of GAN-based solutions, we concentrate on the human's sensitivity to hallucination for different input sample densities. We present new psychophysical experiments, a dataset, and a procedure for training foveated image reconstruction. The strategy provides flexibility to the generator network by penalizing only perceptually important deviations in the output. As a result, the method aims to preserve perceived image statistics rather than natural image statistics. We evaluate our strategy and compare it to alternative solutions using a newly trained objective metric and user experiments. }, }

Endnote

%0 Report %A Surace, Luca %A Wernikowski, Marek %A Tursun, Okan Tarhan %A Myszkowski, Karol %A Mantiuk, Radosław %A Didyk, Piotr %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Learning Foveated Reconstruction to Preserve Perceived Image Statistics : %G eng %U http://hdl.handle.net/21.11116/0000-0009-73D9-1 %U https://arxiv.org/abs/2108.03499 %D 2021 %X Foveated image reconstruction recovers full image from a sparse set of samples distributed according to the human visual system's retinal sensitivity that rapidly drops with eccentricity. Recently, the use of Generative Adversarial Networks was shown to be a promising solution for such a task as they can successfully hallucinate missing image information. Like for other supervised learning approaches, also for this one, the definition of the loss function and training strategy heavily influences the output quality. In this work, we pose the question of how to efficiently guide the training of foveated reconstruction techniques such that they are fully aware of the human visual system's capabilities and limitations, and therefore, reconstruct visually important image features. Due to the nature of GAN-based solutions, we concentrate on the human's sensitivity to hallucination for different input sample densities. We present new psychophysical experiments, a dataset, and a procedure for training foveated image reconstruction. The strategy provides flexibility to the generator network by penalizing only perceptually important deviations in the output. As a result, the method aims to preserve perceived image statistics rather than natural image statistics. We evaluate our strategy and compare it to alternative solutions using a newly trained objective metric and user experiments. %K Computer Science, Graphics, cs.GR,Computer Science, Computer Vision and Pattern Recognition, cs.CV

Tewari, A. 2021. Self-supervised reconstruction and synthesis of faces. nbn:de:bsz:291--ds-345982.

Links

Export

BibTeX

@phdthesis{Tewariphd2021, TITLE = {Self-supervised reconstruction and synthesis of faces}, AUTHOR = {Tewari, Ayush}, LANGUAGE = {eng}, URL = {nbn:de:bsz:291--ds-345982}, DOI = {10.22028/D291-34598}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, }

Endnote

%0 Thesis %A Tewari, Ayush %Y Theobalt, Christian %A referee: Zollhöfer, Michael %A referee: Wonka, Peter %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Self-supervised reconstruction and synthesis of faces : %G eng %U http://hdl.handle.net/21.11116/0000-0009-9CD2-A %R 10.22028/D291-34598 %U nbn:de:bsz:291--ds-345982 %F OTHER: hdl:20.500.11880/31754 %I Universität des Saarlandes %C Saarbrücken %D 2021 %P 173 p. %V phd %9 phd %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/31754

Van Onzenoodt, C., Singh, G., Ropinski, T., and Ritschel, T. 2021a. Blue Noise Plots. https://arxiv.org/abs/2102.04072.

(arXiv: 2102.04072)

Abstract

We propose Blue Noise Plots, two-dimensional dot plots that depict data points of univariate data sets. While often one-dimensional strip plots are used to depict such data, one of their main problems is visual clutter which results from overlap. To reduce this overlap, jitter plots were introduced, whereby an additional, non-encoding plot dimension is introduced, along which the data point representing dots are randomly perturbed. Unfortunately, this randomness can suggest non-existent clusters, and often leads to visually unappealing plots, in which overlap might still occur. To overcome these shortcomings, we introduce BlueNoise Plots where random jitter along the non-encoding plot dimension is replaced by optimizing all dots to keep a minimum distance in 2D i. e., Blue Noise. We evaluate the effectiveness as well as the aesthetics of Blue Noise Plots through both, a quantitative and a qualitative user study.

Links

Export

BibTeX

@online{Onzenoodt_2102.04072, TITLE = {Blue Noise Plots}, AUTHOR = {van Onzenoodt, Christian and Singh, Gurprit and Ropinski, Timo and Ritschel, Tobias}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2102.04072}, EPRINT = {2102.04072}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {We propose Blue Noise Plots, two-dimensional dot plots that depict data points of univariate data sets. While often one-dimensional strip plots are used to depict such data, one of their main problems is visual clutter which results from overlap. To reduce this overlap, jitter plots were introduced, whereby an additional, non-encoding plot dimension is introduced, along which the data point representing dots are randomly perturbed. Unfortunately, this randomness can suggest non-existent clusters, and often leads to visually unappealing plots, in which overlap might still occur. To overcome these shortcomings, we introduce BlueNoise Plots where random jitter along the non-encoding plot dimension is replaced by optimizing all dots to keep a minimum distance in 2D i. e., Blue Noise. We evaluate the effectiveness as well as the aesthetics of Blue Noise Plots through both, a quantitative and a qualitative user study. }, }

Endnote

%0 Report %A van Onzenoodt, Christian %A Singh, Gurprit %A Ropinski, Timo %A Ritschel, Tobias %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Blue Noise Plots : %G eng %U http://hdl.handle.net/21.11116/0000-0008-01ED-C %U https://arxiv.org/abs/2102.04072 %D 2021 %X We propose Blue Noise Plots, two-dimensional dot plots that depict data points of univariate data sets. While often one-dimensional strip plots are used to depict such data, one of their main problems is visual clutter which results from overlap. To reduce this overlap, jitter plots were introduced, whereby an additional, non-encoding plot dimension is introduced, along which the data point representing dots are randomly perturbed. Unfortunately, this randomness can suggest non-existent clusters, and often leads to visually unappealing plots, in which overlap might still occur. To overcome these shortcomings, we introduce BlueNoise Plots where random jitter along the non-encoding plot dimension is replaced by optimizing all dots to keep a minimum distance in 2D i. e., Blue Noise. We evaluate the effectiveness as well as the aesthetics of Blue Noise Plots through both, a quantitative and a qualitative user study. %K Computer Science, Graphics, cs.GR

Van Onzenoodt, C., Singh, G., Ropinski, T., and Ritschel, T. 2021b. Blue Noise Plots. Computer Graphics Forum (Proc. EUROGRAPHICS 2021)40, 2.

Links

Export

BibTeX

@article{onzenoodt2021blue, TITLE = {Blue Noise Plots}, AUTHOR = {van Onzenoodt, Christian and Singh, Gurprit and Ropinski, Timo and Ritschel, Tobias}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.142644}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, DATE = {2021}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {40}, NUMBER = {2}, PAGES = {425--433}, BOOKTITLE = {42nd Annual Conference of the European Association for Computer Graphics (EUROGRAPHICS 2021)}, EDITOR = {Mitra, Niloy and Viola, Ivan}, }

Endnote

%0 Journal Article %A van Onzenoodt, Christian %A Singh, Gurprit %A Ropinski, Timo %A Ritschel, Tobias %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Blue Noise Plots : %G eng %U http://hdl.handle.net/21.11116/0000-0008-010F-7 %R 10.1111/cgf.142644 %7 2021 %D 2021 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 40 %N 2 %& 425 %P 425 - 433 %I Blackwell-Wiley %C Oxford %@ false %B 42nd Annual Conference of the European Association for Computer Graphics %O EUROGRAPHICS 2021 EG 2021

Wang, C., Chen, B., Seidel, H.-P., Myszkowski, K., and Serrano, A. 2021. Learning a self-supervised tone mapping operator via feature contrast masking loss. https://arxiv.org/abs/2110.09866.

(arXiv: 2110.09866)

Abstract

High Dynamic Range (HDR) content is becoming ubiquitous due to the rapid development of capture technologies. Nevertheless, the dynamic range of common display devices is still limited, therefore tone mapping (TM) remains a key challenge for image visualization. Recent work has demonstrated that neural networks can achieve remarkable performance in this task when compared to traditional methods, however, the quality of the results of these learning-based methods is limited by the training data. Most existing works use as training set a curated selection of best-performing results from existing traditional tone mapping operators (often guided by a quality metric), therefore, the quality of newly generated results is fundamentally limited by the performance of such operators. This quality might be even further limited by the pool of HDR content that is used for training. In this work we propose a learning-based self-supervised tone mapping operator that is trained at test time specifically for each HDR image and does not need any data labeling. The key novelty of our approach is a carefully designed loss function built upon fundamental knowledge on contrast perception that allows for directly comparing the content in the HDR and tone mapped images. We achieve this goal by reformulating classic VGG feature maps into feature contrast maps that normalize local feature differences by their average magnitude in a local neighborhood, allowing our loss to account for contrast masking effects. We perform extensive ablation studies and exploration of parameters and demonstrate that our solution outperforms existing approaches with a single set of fixed parameters, as confirmed by both objective and subjective metrics.

Links

Export

BibTeX

@online{Wang_2110.09866, TITLE = {Learning a self-supervised tone mapping operator via feature contrast masking loss}, AUTHOR = {Wang, Chao and Chen, Bin and Seidel, Hans-Peter and Myszkowski, Karol and Serrano, Ana}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2110.09866}, EPRINT = {2110.09866}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {High Dynamic Range (HDR) content is becoming ubiquitous due to the rapid development of capture technologies. Nevertheless, the dynamic range of common display devices is still limited, therefore tone mapping (TM) remains a key challenge for image visualization. Recent work has demonstrated that neural networks can achieve remarkable performance in this task when compared to traditional methods, however, the quality of the results of these learning-based methods is limited by the training data. Most existing works use as training set a curated selection of best-performing results from existing traditional tone mapping operators (often guided by a quality metric), therefore, the quality of newly generated results is fundamentally limited by the performance of such operators. This quality might be even further limited by the pool of HDR content that is used for training. In this work we propose a learning-based self-supervised tone mapping operator that is trained at test time specifically for each HDR image and does not need any data labeling. The key novelty of our approach is a carefully designed loss function built upon fundamental knowledge on contrast perception that allows for directly comparing the content in the HDR and tone mapped images. We achieve this goal by reformulating classic VGG feature maps into feature contrast maps that normalize local feature differences by their average magnitude in a local neighborhood, allowing our loss to account for contrast masking effects. We perform extensive ablation studies and exploration of parameters and demonstrate that our solution outperforms existing approaches with a single set of fixed parameters, as confirmed by both objective and subjective metrics. }, }

Endnote

%0 Report %A Wang, Chao %A Chen, Bin %A Seidel, Hans-Peter %A Myszkowski, Karol %A Serrano, Ana %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Learning a self-supervised tone mapping operator via feature contrast masking loss : %G eng %U http://hdl.handle.net/21.11116/0000-0009-710E-9 %U https://arxiv.org/abs/2110.09866 %D 2021 %X High Dynamic Range (HDR) content is becoming ubiquitous due to the rapid development of capture technologies. Nevertheless, the dynamic range of common display devices is still limited, therefore tone mapping (TM) remains a key challenge for image visualization. Recent work has demonstrated that neural networks can achieve remarkable performance in this task when compared to traditional methods, however, the quality of the results of these learning-based methods is limited by the training data. Most existing works use as training set a curated selection of best-performing results from existing traditional tone mapping operators (often guided by a quality metric), therefore, the quality of newly generated results is fundamentally limited by the performance of such operators. This quality might be even further limited by the pool of HDR content that is used for training. In this work we propose a learning-based self-supervised tone mapping operator that is trained at test time specifically for each HDR image and does not need any data labeling. The key novelty of our approach is a carefully designed loss function built upon fundamental knowledge on contrast perception that allows for directly comparing the content in the HDR and tone mapped images. We achieve this goal by reformulating classic VGG feature maps into feature contrast maps that normalize local feature differences by their average magnitude in a local neighborhood, allowing our loss to account for contrast masking effects. We perform extensive ablation studies and exploration of parameters and demonstrate that our solution outperforms existing approaches with a single set of fixed parameters, as confirmed by both objective and subjective metrics. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,eess.IV

Weinrauch, A., Seidel, H.-P., Mlakar, D., Steinberger, M., and Zayer, R. 2021. A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and Reeb Graph Construction on Surfaces. https://arxiv.org/abs/2105.13168.

(arXiv: 2105.13168)

Abstract

The humble loop shrinking property played a central role in the inception of modern topology but it has been eclipsed by more abstract algebraic formalism. This is particularly true in the context of detecting relevant non-contractible loops on surfaces where elaborate homological and/or graph theoretical constructs are favored in algorithmic solutions. In this work, we devise a variational analogy to the loop shrinking property and show that it yields a simple, intuitive, yet powerful solution allowing a streamlined treatment of the problem of handle and tunnel loop detection. Our formalization tracks the evolution of a diffusion front randomly initiated on a single location on the surface. Capitalizing on a diffuse interface representation combined with a set of rules for concurrent front interactions, we develop a dynamic data structure for tracking the evolution on the surface encoded as a sparse matrix which serves for performing both diffusion numerics and loop detection and acts as the workhorse of our fully parallel implementation. The substantiated results suggest our approach outperforms state of the art and robustly copes with highly detailed geometric models. As a byproduct, our approach can be used to construct Reeb graphs by diffusion thus avoiding commonly encountered issues when using Morse functions.

Links

Export

BibTeX

@online{Weinrauch_2105.13168, TITLE = {A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and {Reeb} Graph Construction on Surfaces}, AUTHOR = {Weinrauch, Alexander and Seidel, Hans-Peter and Mlakar, Daniel and Steinberger, Markus and Zayer, Rhaleb}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2105.13168}, EPRINT = {2105.13168}, EPRINTTYPE = {arXiv}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, ABSTRACT = {The humble loop shrinking property played a central role in the inception of modern topology but it has been eclipsed by more abstract algebraic formalism. This is particularly true in the context of detecting relevant non-contractible loops on surfaces where elaborate homological and/or graph theoretical constructs are favored in algorithmic solutions. In this work, we devise a variational analogy to the loop shrinking property and show that it yields a simple, intuitive, yet powerful solution allowing a streamlined treatment of the problem of handle and tunnel loop detection. Our formalization tracks the evolution of a diffusion front randomly initiated on a single location on the surface. Capitalizing on a diffuse interface representation combined with a set of rules for concurrent front interactions, we develop a dynamic data structure for tracking the evolution on the surface encoded as a sparse matrix which serves for performing both diffusion numerics and loop detection and acts as the workhorse of our fully parallel implementation. The substantiated results suggest our approach outperforms state of the art and robustly copes with highly detailed geometric models. As a byproduct, our approach can be used to construct Reeb graphs by diffusion thus avoiding commonly encountered issues when using Morse functions. }, }

Endnote

%0 Report %A Weinrauch, Alexander %A Seidel, Hans-Peter %A Mlakar, Daniel %A Steinberger, Markus %A Zayer, Rhaleb %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T A Variational Loop Shrinking Analogy for Handle and Tunnel Detection and Reeb Graph Construction on Surfaces : %G eng %U http://hdl.handle.net/21.11116/0000-0009-70EE-D %U https://arxiv.org/abs/2105.13168 %D 2021 %X The humble loop shrinking property played a central role in the inception of modern topology but it has been eclipsed by more abstract algebraic formalism. This is particularly true in the context of detecting relevant non-contractible loops on surfaces where elaborate homological and/or graph theoretical constructs are favored in algorithmic solutions. In this work, we devise a variational analogy to the loop shrinking property and show that it yields a simple, intuitive, yet powerful solution allowing a streamlined treatment of the problem of handle and tunnel loop detection. Our formalization tracks the evolution of a diffusion front randomly initiated on a single location on the surface. Capitalizing on a diffuse interface representation combined with a set of rules for concurrent front interactions, we develop a dynamic data structure for tracking the evolution on the surface encoded as a sparse matrix which serves for performing both diffusion numerics and loop detection and acts as the workhorse of our fully parallel implementation. The substantiated results suggest our approach outperforms state of the art and robustly copes with highly detailed geometric models. As a byproduct, our approach can be used to construct Reeb graphs by diffusion thus avoiding commonly encountered issues when using Morse functions. %K Computer Science, Graphics, cs.GR,Computer Science, Computational Geometry, cs.CG,Mathematics, Algebraic Topology, math.AT

Yenamandra, T., Tewari, A., Bernard, F., et al. 2021. i3DMM: Deep Implicit 3D Morphable Model of Human Heads. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021), IEEE.

Links

Export

BibTeX

@inproceedings{Yenamandra_CVPR2021, TITLE = {{i3DMM}: {D}eep Implicit {3D} Morphable Model of Human Heads}, AUTHOR = {Yenamandra, Tarun and Tewari, Ayush and Bernard, Florian and Seidel, Hans-Peter and Elgharib, Mohamed and Cremers, Daniel and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-6654-4509-2}, DOI = {10.1109/CVPR46437.2021.01261}, PUBLISHER = {IEEE}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2021)}, PAGES = {12798--12808}, ADDRESS = {Virtual Conference}, }

Endnote

%0 Conference Proceedings %A Yenamandra, Tarun %A Tewari, Ayush %A Bernard, Florian %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Cremers, Daniel %A Theobalt, Christian %+ External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T i3DMM: Deep Implicit 3D Morphable Model of Human Heads : %G eng %U http://hdl.handle.net/21.11116/0000-0008-8966-B %R 10.1109/CVPR46437.2021.01261 %D 2021 %B 34th IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2021-06-19 - 2021-06-25 %C Virtual Conference %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 12798 - 12808 %I IEEE %@ 978-1-6654-4509-2 %U https://gvv.mpi-inf.mpg.de/projects/i3DMM/

Zheng, Q., Singh, G., and Seidel, H.-P. 2021. Neural Relightable Participating Media Rendering. Advances in Neural Information Processing Systems 34 (NeurIPS 2021), Curran Associates, Inc.

Links

Export

BibTeX

@inproceedings{Zheng_Neurips2021, TITLE = {Neural Relightable Participating Media Rendering}, AUTHOR = {Zheng, Quan and Singh, Gurprit and Seidel, Hans-Peter}, LANGUAGE = {eng}, ISBN = {9781713845393}, PUBLISHER = {Curran Associates, Inc.}, YEAR = {2021}, MARGINALMARK = {$\bullet$}, BOOKTITLE = {Advances in Neural Information Processing Systems 34 (NeurIPS 2021)}, EDITOR = {Ranzato, M. and Beygelzimer, A. and Liang, P. S. and Vaughan, J. W. and Dauphin, Y.}, PAGES = {15203--15215}, ADDRESS = {Virtual}, }

Endnote

%0 Conference Proceedings %A Zheng, Quan %A Singh, Gurprit %A Seidel, Hans-Peter %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Relightable Participating Media Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0009-7117-E %D 2021 %B 35th Conference on Neural Information Processing Systems %Z date of event: 2021-12-06 - 2021-12-14 %C Virtual %B Advances in Neural Information Processing Systems 34 %E Ranzato, M.; Beygelzimer, A.; Liang, P. S.; Vaughan, J. W.; Dauphin, Y. %P 15203 - 15215 %I Curran Associates, Inc. %@ 9781713845393

2020

Ali, S.A., Kahraman, K., Theobalt, C., Stricker, D., and Golyanik, V. 2020. Fast Gravitational Approach for Rigid Point Set Registration with Ordinary Differential Equations. https://arxiv.org/abs/2009.14005.

(arXiv: 2009.14005)

Abstract

This article introduces a new physics-based method for rigid point set alignment called Fast Gravitational Approach (FGA). In FGA, the source and target point sets are interpreted as rigid particle swarms with masses interacting in a globally multiply-linked manner while moving in a simulated gravitational force field. The optimal alignment is obtained by explicit modeling of forces acting on the particles as well as their velocities and displacements with second-order ordinary differential equations of motion. Additional alignment cues (point-based or geometric features, and other boundary conditions) can be integrated into FGA through particle masses. We propose a smooth-particle mass function for point mass initialization, which improves robustness to noise and structural discontinuities. To avoid prohibitive quadratic complexity of all-to-all point interactions, we adapt a Barnes-Hut tree for accelerated force computation and achieve quasilinear computational complexity. We show that the new method class has characteristics not found in previous alignment methods such as efficient handling of partial overlaps, inhomogeneous point sampling densities, and coping with large point clouds with reduced runtime compared to the state of the art. Experiments show that our method performs on par with or outperforms all compared competing non-deep-learning-based and general-purpose techniques (which do not assume the availability of training data and a scene prior) in resolving transformations for LiDAR data and gains state-of-the-art accuracy and speed when coping with different types of data disturbances.

Links

Export

BibTeX

@online{Ali_2009.14005, TITLE = {Fast Gravitational Approach for Rigid Point Set Registration with Ordinary Differential Equations}, AUTHOR = {Ali, Sk Aziz and Kahraman, Kerem and Theobalt, Christian and Stricker, Didier and Golyanik, Vladislav}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2009.14005}, EPRINT = {2009.14005}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {This article introduces a new physics-based method for rigid point set alignment called Fast Gravitational Approach (FGA). In FGA, the source and target point sets are interpreted as rigid particle swarms with masses interacting in a globally multiply-linked manner while moving in a simulated gravitational force field. The optimal alignment is obtained by explicit modeling of forces acting on the particles as well as their velocities and displacements with second-order ordinary differential equations of motion. Additional alignment cues (point-based or geometric features, and other boundary conditions) can be integrated into FGA through particle masses. We propose a smooth-particle mass function for point mass initialization, which improves robustness to noise and structural discontinuities. To avoid prohibitive quadratic complexity of all-to-all point interactions, we adapt a Barnes-Hut tree for accelerated force computation and achieve quasilinear computational complexity. We show that the new method class has characteristics not found in previous alignment methods such as efficient handling of partial overlaps, inhomogeneous point sampling densities, and coping with large point clouds with reduced runtime compared to the state of the art. Experiments show that our method performs on par with or outperforms all compared competing non-deep-learning-based and general-purpose techniques (which do not assume the availability of training data and a scene prior) in resolving transformations for LiDAR data and gains state-of-the-art accuracy and speed when coping with different types of data disturbances. }, }

Endnote

%0 Report %A Ali, Sk Aziz %A Kahraman, Kerem %A Theobalt, Christian %A Stricker, Didier %A Golyanik, Vladislav %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Fast Gravitational Approach for Rigid Point Set Registration with Ordinary Differential Equations : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8FA-A %U https://arxiv.org/abs/2009.14005 %D 2020 %X This article introduces a new physics-based method for rigid point set alignment called Fast Gravitational Approach (FGA). In FGA, the source and target point sets are interpreted as rigid particle swarms with masses interacting in a globally multiply-linked manner while moving in a simulated gravitational force field. The optimal alignment is obtained by explicit modeling of forces acting on the particles as well as their velocities and displacements with second-order ordinary differential equations of motion. Additional alignment cues (point-based or geometric features, and other boundary conditions) can be integrated into FGA through particle masses. We propose a smooth-particle mass function for point mass initialization, which improves robustness to noise and structural discontinuities. To avoid prohibitive quadratic complexity of all-to-all point interactions, we adapt a Barnes-Hut tree for accelerated force computation and achieve quasilinear computational complexity. We show that the new method class has characteristics not found in previous alignment methods such as efficient handling of partial overlaps, inhomogeneous point sampling densities, and coping with large point clouds with reduced runtime compared to the state of the art. Experiments show that our method performs on par with or outperforms all compared competing non-deep-learning-based and general-purpose techniques (which do not assume the availability of training data and a scene prior) in resolving transformations for LiDAR data and gains state-of-the-art accuracy and speed when coping with different types of data disturbances. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,eess.IV

Ansari, N., Alizadeh-Mousavi, O., Seidel, H.-P., and Babaei, V. 2020. Mixed Integer Ink Selection for Spectral Reproduction. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Ansari_ToG2020, TITLE = {Mixed Integer Ink Selection for Spectral Reproduction}, AUTHOR = {Ansari, Navid and Alizadeh-Mousavi, Omid and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417761}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {255}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Ansari, Navid %A Alizadeh-Mousavi, Omid %A Seidel, Hans-Peter %A Babaei, Vahid %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Mixed Integer Ink Selection for Spectral Reproduction : %G eng %U http://hdl.handle.net/21.11116/0000-0007-9B23-3 %R 10.1145/3414685.3417761 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 255 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Bemana, M., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2020a. X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Bemana2020, TITLE = {X-{F}ields: {I}mplicit Neural View-, Light- and Time-Image Interpolation}, AUTHOR = {Bemana, Mojtaba and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417827}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {257}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Bemana, Mojtaba %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation : %G eng %U http://hdl.handle.net/21.11116/0000-0006-FBF0-0 %R 10.1145/3414685.3417827 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 257 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Bemana, M., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2020b. X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation. https://arxiv.org/abs/2010.00450.

(arXiv: 2010.00450)

Abstract

We suggest to represent an X-Field -a set of 2D images taken across different view, time or illumination conditions, i.e., video, light field, reflectance fields or combinations thereof-by learning a neural network (NN) to map their view, time or light coordinates to 2D images. Executing this NN at new coordinates results in joint view, time or light interpolation. The key idea to make this workable is a NN that already knows the "basic tricks" of graphics (lighting, 3D projection, occlusion) in a hard-coded and differentiable form. The NN represents the input to that rendering as an implicit map, that for any view, time, or light coordinate and for any pixel can quantify how it will move if view, time or light coordinates change (Jacobian of pixel position with respect to view, time, illumination, etc.). Our X-Field representation is trained for one scene within minutes, leading to a compact set of trainable parameters and hence real-time navigation in view, time and illumination.

Links

Export

BibTeX

@online{Bemana_arXiv2010.00450, TITLE = {X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation}, AUTHOR = {Bemana, Mojtaba and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2010.00450}, EPRINT = {2010.00450}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We suggest to represent an X-Field -a set of 2D images taken across different view, time or illumination conditions, i.e., video, light field, reflectance fields or combinations thereof-by learning a neural network (NN) to map their view, time or light coordinates to 2D images. Executing this NN at new coordinates results in joint view, time or light interpolation. The key idea to make this workable is a NN that already knows the "basic tricks" of graphics (lighting, 3D projection, occlusion) in a hard-coded and differentiable form. The NN represents the input to that rendering as an implicit map, that for any view, time, or light coordinate and for any pixel can quantify how it will move if view, time or light coordinates change (Jacobian of pixel position with respect to view, time, illumination, etc.). Our X-Field representation is trained for one scene within minutes, leading to a compact set of trainable parameters and hence real-time navigation in view, time and illumination. }, }

Endnote

%0 Report %A Bemana, Mojtaba %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T X-Fields: Implicit Neural View-, Light- and Time-Image Interpolation : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B6EC-2 %U https://arxiv.org/abs/2010.00450 %D 2020 %X We suggest to represent an X-Field -a set of 2D images taken across different view, time or illumination conditions, i.e., video, light field, reflectance fields or combinations thereof-by learning a neural network (NN) to map their view, time or light coordinates to 2D images. Executing this NN at new coordinates results in joint view, time or light interpolation. The key idea to make this workable is a NN that already knows the "basic tricks" of graphics (lighting, 3D projection, occlusion) in a hard-coded and differentiable form. The NN represents the input to that rendering as an implicit map, that for any view, time, or light coordinate and for any pixel can quantify how it will move if view, time or light coordinates change (Jacobian of pixel position with respect to view, time, illumination, etc.). Our X-Field representation is trained for one scene within minutes, leading to a compact set of trainable parameters and hence real-time navigation in view, time and illumination. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Bernard, F., Suri, Z.K., and Theobalt, C. 2020a. MINA: Convex Mixed-Integer Programming for Non-Rigid Shape Alignment. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Bernard_CVPR2020, TITLE = {{MINA}: {C}onvex Mixed-Integer Programming for Non-Rigid Shape Alignment}, AUTHOR = {Bernard, Florian and Suri, Zeeshan Khan and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.01384}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {13823--13832}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Bernard, Florian %A Suri, Zeeshan Khan %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T MINA: Convex Mixed-Integer Programming for Non-Rigid Shape Alignment : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D059-A %R 10.1109/CVPR42600.2020.01384 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 13823 - 13832 %I IEEE %@ 978-1-7281-7168-5

Bernard, F., Suri, Z.K., and Theobalt, C. 2020b. MINA: Convex Mixed-Integer Programming for Non-Rigid Shape Alignment. https://arxiv.org/abs/2002.12623.

(arXiv: 2002.12623)

Abstract

We present a convex mixed-integer programming formulation for non-rigid shape matching. To this end, we propose a novel shape deformation model based on an efficient low-dimensional discrete model, so that finding a globally optimal solution is tractable in (most) practical cases. Our approach combines several favourable properties: it is independent of the initialisation, it is much more efficient to solve to global optimality compared to analogous quadratic assignment problem formulations, and it is highly flexible in terms of the variants of matching problems it can handle. Experimentally we demonstrate that our approach outperforms existing methods for sparse shape matching, that it can be used for initialising dense shape matching methods, and we showcase its flexibility on several examples.

Links

Export

BibTeX

@online{Bernard_arXiv2002.12623, TITLE = {MINA: {C}onvex Mixed-Integer Programming for Non-Rigid Shape Alignment}, AUTHOR = {Bernard, Florian and Suri, Zeeshan Khan and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2002.12623}, EPRINT = {2002.12623}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present a convex mixed-integer programming formulation for non-rigid shape matching. To this end, we propose a novel shape deformation model based on an efficient low-dimensional discrete model, so that finding a globally optimal solution is tractable in (most) practical cases. Our approach combines several favourable properties: it is independent of the initialisation, it is much more efficient to solve to global optimality compared to analogous quadratic assignment problem formulations, and it is highly flexible in terms of the variants of matching problems it can handle. Experimentally we demonstrate that our approach outperforms existing methods for sparse shape matching, that it can be used for initialising dense shape matching methods, and we showcase its flexibility on several examples. }, }

Endnote

%0 Report %A Bernard, Florian %A Suri, Zeeshan Khan %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T MINA: Convex Mixed-Integer Programming for Non-Rigid Shape Alignment : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E00C-F %U https://arxiv.org/abs/2002.12623 %D 2020 %X We present a convex mixed-integer programming formulation for non-rigid shape matching. To this end, we propose a novel shape deformation model based on an efficient low-dimensional discrete model, so that finding a globally optimal solution is tractable in (most) practical cases. Our approach combines several favourable properties: it is independent of the initialisation, it is much more efficient to solve to global optimality compared to analogous quadratic assignment problem formulations, and it is highly flexible in terms of the variants of matching problems it can handle. Experimentally we demonstrate that our approach outperforms existing methods for sparse shape matching, that it can be used for initialising dense shape matching methods, and we showcase its flexibility on several examples. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG,Mathematics, Optimization and Control, math.OC

Bhatnagar, B.L., Sminchisescu, C., Theobalt, C., and Pons-Moll, G. 2020a. LoopReg: Self-supervised Learning of Implicit Surface Correspondences, Pose and Shape for 3D Human Mesh Registration. https://arxiv.org/abs/2010.12447.

(arXiv: 2010.12447)

Abstract

We address the problem of fitting 3D human models to 3D scans of dressed humans. Classical methods optimize both the data-to-model correspondences and the human model parameters (pose and shape), but are reliable only when initialized close to the solution. Some methods initialize the optimization based on fully supervised correspondence predictors, which is not differentiable end-to-end, and can only process a single scan at a time. Our main contribution is LoopReg, an end-to-end learning framework to register a corpus of scans to a common 3D human model. The key idea is to create a self-supervised loop. A backward map, parameterized by a Neural Network, predicts the correspondence from every scan point to the surface of the human model. A forward map, parameterized by a human model, transforms the corresponding points back to the scan based on the model parameters (pose and shape), thus closing the loop. Formulating this closed loop is not straightforward because it is not trivial to force the output of the NN to be on the surface of the human model - outside this surface the human model is not even defined. To this end, we propose two key innovations. First, we define the canonical surface implicitly as the zero level set of a distance field in R3, which in contrast to morecommon UV parameterizations, does not require cutting the surface, does not have discontinuities, and does not induce distortion. Second, we diffuse the human model to the 3D domain R3. This allows to map the NN predictions forward,even when they slightly deviate from the zero level set. Results demonstrate that we can train LoopRegmainly self-supervised - following a supervised warm-start, the model becomes increasingly more accurate as additional unlabelled raw scans are processed. Our code and pre-trained models can be downloaded for research.

Links

Export

BibTeX

@online{Bhatnagar_2010.12447, TITLE = {{LoopReg}: {S}elf-supervised Learning of Implicit Surface Correspondences, Pose and Shape for {3D} Human Mesh Registration}, AUTHOR = {Bhatnagar, Bharat Lal and Sminchisescu, Cristian and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2010.12447}, EPRINT = {2010.12447}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We address the problem of fitting 3D human models to 3D scans of dressed humans. Classical methods optimize both the data-to-model correspondences and the human model parameters (pose and shape), but are reliable only when initialized close to the solution. Some methods initialize the optimization based on fully supervised correspondence predictors, which is not differentiable end-to-end, and can only process a single scan at a time. Our main contribution is LoopReg, an end-to-end learning framework to register a corpus of scans to a common 3D human model. The key idea is to create a self-supervised loop. A backward map, parameterized by a Neural Network, predicts the correspondence from every scan point to the surface of the human model. A forward map, parameterized by a human model, transforms the corresponding points back to the scan based on the model parameters (pose and shape), thus closing the loop. Formulating this closed loop is not straightforward because it is not trivial to force the output of the NN to be on the surface of the human model -- outside this surface the human model is not even defined. To this end, we propose two key innovations. First, we define the canonical surface implicitly as the zero level set of a distance field in R3, which in contrast to morecommon UV parameterizations, does not require cutting the surface, does not have discontinuities, and does not induce distortion. Second, we diffuse the human model to the 3D domain R3. This allows to map the NN predictions forward,even when they slightly deviate from the zero level set. Results demonstrate that we can train LoopRegmainly self-supervised -- following a supervised warm-start, the model becomes increasingly more accurate as additional unlabelled raw scans are processed. Our code and pre-trained models can be downloaded for research. }, }

Endnote

%0 Report %A Bhatnagar, Bharat Lal %A Sminchisescu, Cristian %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T LoopReg: Self-supervised Learning of Implicit Surface Correspondences, Pose and Shape for 3D Human Mesh Registration : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E91C-4 %U https://arxiv.org/abs/2010.12447 %D 2020 %X We address the problem of fitting 3D human models to 3D scans of dressed humans. Classical methods optimize both the data-to-model correspondences and the human model parameters (pose and shape), but are reliable only when initialized close to the solution. Some methods initialize the optimization based on fully supervised correspondence predictors, which is not differentiable end-to-end, and can only process a single scan at a time. Our main contribution is LoopReg, an end-to-end learning framework to register a corpus of scans to a common 3D human model. The key idea is to create a self-supervised loop. A backward map, parameterized by a Neural Network, predicts the correspondence from every scan point to the surface of the human model. A forward map, parameterized by a human model, transforms the corresponding points back to the scan based on the model parameters (pose and shape), thus closing the loop. Formulating this closed loop is not straightforward because it is not trivial to force the output of the NN to be on the surface of the human model - outside this surface the human model is not even defined. To this end, we propose two key innovations. First, we define the canonical surface implicitly as the zero level set of a distance field in R3, which in contrast to morecommon UV parameterizations, does not require cutting the surface, does not have discontinuities, and does not induce distortion. Second, we diffuse the human model to the 3D domain R3. This allows to map the NN predictions forward,even when they slightly deviate from the zero level set. Results demonstrate that we can train LoopRegmainly self-supervised - following a supervised warm-start, the model becomes increasingly more accurate as additional unlabelled raw scans are processed. Our code and pre-trained models can be downloaded for research. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Bhatnagar, B.L., Sminchisescu, C., Theobalt, C., and Pons-Moll, G. 2020b. LoopReg: Self-supervised Learning of Implicit Surface Correspondences, Pose and Shape for 3D Human Mesh Registration. Advances in Neural Information Processing Systems 33 (NeurIPS 2020), Curran Associates, Inc.

Links

Export

BibTeX

@inproceedings{bhatnagar2020loopreg, TITLE = {{LoopReg}: Self-supervised Learning of Implicit Surface Correspondences, Pose and Shape for {3D} Human Mesh Registration}, AUTHOR = {Bhatnagar, Bharat Lal and Sminchisescu, Cristian and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, ISBN = {9781713829546}, PUBLISHER = {Curran Associates, Inc.}, YEAR = {2020}, BOOKTITLE = {Advances in Neural Information Processing Systems 33 (NeurIPS 2020)}, EDITOR = {Larochelle, H. and Ranzato, M. and Hadsell, R. and Balcan, M. F. and Lin, H.}, PAGES = {12909--12922}, ADDRESS = {Virtual Event}, }

Endnote

%0 Conference Proceedings %A Bhatnagar, Bharat Lal %A Sminchisescu, Cristian %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T LoopReg: Self-supervised Learning of Implicit Surface Correspondences, Pose and Shape for 3D Human Mesh Registration : %G eng %U http://hdl.handle.net/21.11116/0000-0007-6FD1-1 %D 2020 %B 34th Conference on Neural Information Processing Systems %Z date of event: 2020-12-06 - 2020-12-12 %C Virtual Event %B Advances in Neural Information Processing Systems 33 %E Larochelle, H.; Ranzato, M.; Hadsell, R.; Balcan, M. F.; Lin, H. %P 12909 - 12922 %I Curran Associates, Inc. %@ 9781713829546 %U https://papers.nips.cc/paper/2020/file/970af30e481057c48f87e101b61e6994-Paper.pdf

Bhatnagar, B.L., Sminchisescu, C., Theobalt, C., and Pons-Moll, G. 2020c. Combining Implicit Function Learning and Parametric Models for 3D Human Reconstruction. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{bhatnagar2020ipnet, TITLE = {Combining Implicit Function Learning and Parametric Models for {3D} Human Reconstruction}, AUTHOR = {Bhatnagar, Bharat Lal and Sminchisescu, Cristian and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, ISBN = {978-3-030-58535-8}, DOI = {10.1007/978-3-030-58536-5_19}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {311--329}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12347}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Bhatnagar, Bharat Lal %A Sminchisescu, Cristian %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Combining Implicit Function Learning and Parametric Models for 3D Human Reconstruction : %G eng %U http://hdl.handle.net/21.11116/0000-0006-E89E-3 %R 10.1007/978-3-030-58536-5_19 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 311 - 329 %I Springer %@ 978-3-030-58535-8 %B Lecture Notes in Computer Science %N 12347

Bhatnagar, B.L., Sminchisescu, C., Theobalt, C., and Pons-Moll, G. 2020d. Combining Implicit Function Learning and Parametric Models for 3D Human Reconstruction. https://arxiv.org/abs/2007.11432.

(arXiv: 2007.11432)

Abstract

Implicit functions represented as deep learning approximations are powerful for reconstructing 3D surfaces. However, they can only produce static surfaces that are not controllable, which provides limited ability to modify the resulting model by editing its pose or shape parameters. Nevertheless, such features are essential in building flexible models for both computer graphics and computer vision. In this work, we present methodology that combines detail-rich implicit functions and parametric representations in order to reconstruct 3D models of people that remain controllable and accurate even in the presence of clothing. Given sparse 3D point clouds sampled on the surface of a dressed person, we use an Implicit Part Network (IP-Net)to jointly predict the outer 3D surface of the dressed person, the and inner body surface, and the semantic correspondences to a parametric body model. We subsequently use correspondences to fit the body model to our inner surface and then non-rigidly deform it (under a parametric body + displacement model) to the outer surface in order to capture garment, face and hair detail. In quantitative and qualitative experiments with both full body data and hand scans we show that the proposed methodology generalizes, and is effective even given incomplete point clouds collected from single-view depth images. Our models and code can be downloaded from http://virtualhumans.mpi-inf.mpg.de/ipnet.

Links

Export

BibTeX

@online{Bhatnagar_2007.11432, TITLE = {Combining Implicit Function Learning and Parametric Models for {3D} Human Reconstruction}, AUTHOR = {Bhatnagar, Bharat Lal and Sminchisescu, Cristian and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2007.11432}, EPRINT = {2007.11432}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Implicit functions represented as deep learning approximations are powerful for reconstructing 3D surfaces. However, they can only produce static surfaces that are not controllable, which provides limited ability to modify the resulting model by editing its pose or shape parameters. Nevertheless, such features are essential in building flexible models for both computer graphics and computer vision. In this work, we present methodology that combines detail-rich implicit functions and parametric representations in order to reconstruct 3D models of people that remain controllable and accurate even in the presence of clothing. Given sparse 3D point clouds sampled on the surface of a dressed person, we use an Implicit Part Network (IP-Net)to jointly predict the outer 3D surface of the dressed person, the and inner body surface, and the semantic correspondences to a parametric body model. We subsequently use correspondences to fit the body model to our inner surface and then non-rigidly deform it (under a parametric body + displacement model) to the outer surface in order to capture garment, face and hair detail. In quantitative and qualitative experiments with both full body data and hand scans we show that the proposed methodology generalizes, and is effective even given incomplete point clouds collected from single-view depth images. Our models and code can be downloaded from http://virtualhumans.mpi-inf.mpg.de/ipnet. }, }

Endnote

%0 Report %A Bhatnagar, Bharat Lal %A Sminchisescu, Cristian %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Combining Implicit Function Learning and Parametric Models for 3D Human Reconstruction : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8A0-E %U https://arxiv.org/abs/2007.11432 %D 2020 %X Implicit functions represented as deep learning approximations are powerful for reconstructing 3D surfaces. However, they can only produce static surfaces that are not controllable, which provides limited ability to modify the resulting model by editing its pose or shape parameters. Nevertheless, such features are essential in building flexible models for both computer graphics and computer vision. In this work, we present methodology that combines detail-rich implicit functions and parametric representations in order to reconstruct 3D models of people that remain controllable and accurate even in the presence of clothing. Given sparse 3D point clouds sampled on the surface of a dressed person, we use an Implicit Part Network (IP-Net)to jointly predict the outer 3D surface of the dressed person, the and inner body surface, and the semantic correspondences to a parametric body model. We subsequently use correspondences to fit the body model to our inner surface and then non-rigidly deform it (under a parametric body + displacement model) to the outer surface in order to capture garment, face and hair detail. In quantitative and qualitative experiments with both full body data and hand scans we show that the proposed methodology generalizes, and is effective even given incomplete point clouds collected from single-view depth images. Our models and code can be downloaded from http://virtualhumans.mpi-inf.mpg.de/ipnet. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Božič, A., Zollhöfer, M., Theobalt, C., and Nießner, M. 2020. DeepDeform: Learning Non-Rigid RGB-D Reconstruction With Semi-Supervised Data. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Bozic_CVPR2020, TITLE = {{DeepDeform}: {L}earning Non-Rigid {RGB}-{D} Reconstruction With Semi-Supervised Data}, AUTHOR = {Bo{\v z}i{\v c}, Alja{\v z} and Zollh{\"o}fer, Michael and Theobalt, Christian and Nie{\ss}ner, Matthias}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00703}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {7000--7010}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Božič, Aljaž %A Zollhöfer, Michael %A Theobalt, Christian %A Nießner, Matthias %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T DeepDeform: Learning Non-Rigid RGB-D Reconstruction With Semi-Supervised Data : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CF63-1 %R 10.1109/CVPR42600.2020.00703 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 7000 - 7010 %I IEEE %@ 978-1-7281-7168-5

Chizhov, V., Georgiev, I., Myszkowski, K., and Singh, G. 2020. Perceptual Error Optimization for Monte Carlo Rendering. https://arxiv.org/abs/2012.02344.

(arXiv: 2012.02344)

Abstract

Realistic image synthesis involves computing high-dimensional light transport integrals which in practice are numerically estimated using Monte Carlo integration. The error of this estimation manifests itself in the image as visually displeasing aliasing or noise. To ameliorate this, we develop a theoretical framework for optimizing screen-space error distribution. Our model is flexible and works for arbitrary target error power spectra. We focus on perceptual error optimization by leveraging models of the human visual system's (HVS) point spread function (PSF) from halftoning literature. This results in a specific optimization problem whose solution distributes the error as visually pleasing blue noise in image space. We develop a set of algorithms that provide a trade-off between quality and speed, showing substantial improvements over prior state of the art. We perform evaluations using both quantitative and perceptual error metrics to support our analysis, and provide extensive supplemental material to help evaluate the perceptual improvements achieved by our methods.

Links

Export

BibTeX

@online{Chizhov_arXiv2012.02344, TITLE = {Perceptual Error Optimization for {Monte Carlo} Rendering}, AUTHOR = {Chizhov, Vassillen and Georgiev, Iliyan and Myszkowski, Karol and Singh, Gurprit}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.02344}, EPRINT = {2012.02344}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Realistic image synthesis involves computing high-dimensional light transport integrals which in practice are numerically estimated using Monte Carlo integration. The error of this estimation manifests itself in the image as visually displeasing aliasing or noise. To ameliorate this, we develop a theoretical framework for optimizing screen-space error distribution. Our model is flexible and works for arbitrary target error power spectra. We focus on perceptual error optimization by leveraging models of the human visual system's (HVS) point spread function (PSF) from halftoning literature. This results in a specific optimization problem whose solution distributes the error as visually pleasing blue noise in image space. We develop a set of algorithms that provide a trade-off between quality and speed, showing substantial improvements over prior state of the art. We perform evaluations using both quantitative and perceptual error metrics to support our analysis, and provide extensive supplemental material to help evaluate the perceptual improvements achieved by our methods. }, }

Endnote

%0 Report %A Chizhov, Vassillen %A Georgiev, Iliyan %A Myszkowski, Karol %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Perceptual Error Optimization for Monte Carlo Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CEB7-3 %U https://arxiv.org/abs/2012.02344 %D 2020 %X Realistic image synthesis involves computing high-dimensional light transport integrals which in practice are numerically estimated using Monte Carlo integration. The error of this estimation manifests itself in the image as visually displeasing aliasing or noise. To ameliorate this, we develop a theoretical framework for optimizing screen-space error distribution. Our model is flexible and works for arbitrary target error power spectra. We focus on perceptual error optimization by leveraging models of the human visual system's (HVS) point spread function (PSF) from halftoning literature. This results in a specific optimization problem whose solution distributes the error as visually pleasing blue noise in image space. We develop a set of algorithms that provide a trade-off between quality and speed, showing substantial improvements over prior state of the art. We perform evaluations using both quantitative and perceptual error metrics to support our analysis, and provide extensive supplemental material to help evaluate the perceptual improvements achieved by our methods. %K Computer Science, Graphics, cs.GR

Çoğalan, U. and Akyüz, A.O. 2020. Deep Joint Deinterlacing and Denoising for Single Shot Dual-ISO HDR Reconstruction. IEEE Transactions on Image Processing29.

Links

Export

BibTeX

@article{Cogalan2020, TITLE = {Deep Joint Deinterlacing and Denoising for Single Shot Dual-{ISO HDR} Reconstruction}, AUTHOR = {{\c C}o{\u g}alan, U{\u g}ur and Aky{\"u}z, Ahmet O{\u g}uz}, LANGUAGE = {eng}, ISSN = {1057-7149}, DOI = {10.1109/TIP.2020.3004014}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2020}, DATE = {2020}, JOURNAL = {IEEE Transactions on Image Processing}, VOLUME = {29}, PAGES = {7511--7524}, }

Endnote

%0 Journal Article %A Çoğalan, Uğur %A Akyüz , Ahmet Oğuz %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Deep Joint Deinterlacing and Denoising for Single Shot Dual-ISO HDR Reconstruction : %G eng %U http://hdl.handle.net/21.11116/0000-0006-DCA7-6 %R 10.1109/TIP.2020.3004014 %7 2020 %D 2020 %J IEEE Transactions on Image Processing %V 29 %& 7511 %P 7511 - 7524 %I IEEE %C Piscataway, NJ %@ false

Çoğalan, U., Bemana, M., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2020. HDR Denoising and Deblurring by Learning Spatio-temporal Distortion Models. https://arxiv.org/abs/2012.12009.

(arXiv: 2012.12009)

Abstract

We seek to reconstruct sharp and noise-free high-dynamic range (HDR) video from a dual-exposure sensor that records different low-dynamic range (LDR) information in different pixel columns: Odd columns provide low-exposure, sharp, but noisy information; even columns complement this with less noisy, high-exposure, but motion-blurred data. Previous LDR work learns to deblur and denoise (DISTORTED->CLEAN) supervised by pairs of CLEAN and DISTORTED images. Regrettably, capturing DISTORTED sensor readings is time-consuming; as well, there is a lack of CLEAN HDR videos. We suggest a method to overcome those two limitations. First, we learn a different function instead: CLEAN->DISTORTED, which generates samples containing correlated pixel noise, and row and column noise, as well as motion blur from a low number of CLEAN sensor readings. Second, as there is not enough CLEAN HDR video available, we devise a method to learn from LDR video in-stead. Our approach compares favorably to several strong baselines, and can boost existing methods when they are re-trained on our data. Combined with spatial and temporal super-resolution, it enables applications such as re-lighting with low noise or blur.

Links

Export

BibTeX

@online{Cogalan_arXiv2012.12009, TITLE = {{HDR} Denoising and Deblurring by Learning Spatio-temporal Distortion Model}, AUTHOR = {{\c C}o{\u g}alan, U{\u g}ur and Bemana, Mojtaba and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.12009}, EPRINT = {2012.12009}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We seek to reconstruct sharp and noise-free high-dynamic range (HDR) video from a dual-exposure sensor that records different low-dynamic range (LDR) information in different pixel columns: Odd columns provide low-exposure, sharp, but noisy information; even columns complement this with less noisy, high-exposure, but motion-blurred data. Previous LDR work learns to deblur and denoise (DISTORTED->CLEAN) supervised by pairs of CLEAN and DISTORTED images. Regrettably, capturing DISTORTED sensor readings is time-consuming; as well, there is a lack of CLEAN HDR videos. We suggest a method to overcome those two limitations. First, we learn a different function instead: CLEAN->DISTORTED, which generates samples containing correlated pixel noise, and row and column noise, as well as motion blur from a low number of CLEAN sensor readings. Second, as there is not enough CLEAN HDR video available, we devise a method to learn from LDR video in-stead. Our approach compares favorably to several strong baselines, and can boost existing methods when they are re-trained on our data. Combined with spatial and temporal super-resolution, it enables applications such as re-lighting with low noise or blur. }, }

Endnote

%0 Report %A Çoğalan, Uğur %A Bemana, Mojtaba %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T HDR Denoising and Deblurring by Learning Spatio-temporal Distortion Models : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B721-5 %U https://arxiv.org/abs/2012.12009 %D 2020 %X We seek to reconstruct sharp and noise-free high-dynamic range (HDR) video from a dual-exposure sensor that records different low-dynamic range (LDR) information in different pixel columns: Odd columns provide low-exposure, sharp, but noisy information; even columns complement this with less noisy, high-exposure, but motion-blurred data. Previous LDR work learns to deblur and denoise (DISTORTED->CLEAN) supervised by pairs of CLEAN and DISTORTED images. Regrettably, capturing DISTORTED sensor readings is time-consuming; as well, there is a lack of CLEAN HDR videos. We suggest a method to overcome those two limitations. First, we learn a different function instead: CLEAN->DISTORTED, which generates samples containing correlated pixel noise, and row and column noise, as well as motion blur from a low number of CLEAN sensor readings. Second, as there is not enough CLEAN HDR video available, we devise a method to learn from LDR video in-stead. Our approach compares favorably to several strong baselines, and can boost existing methods when they are re-trained on our data. Combined with spatial and temporal super-resolution, it enables applications such as re-lighting with low noise or blur. %K eess.IV,Computer Science, Computer Vision and Pattern Recognition, cs.CV

Cucerca, S., Didyk, P., Seidel, H.-P., and Babaei, V. 2020. Computational Image Marking on Metals via Laser Induced Heating. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2020)39, 4.

Links

Export

BibTeX

@article{Cucerca_SIGGRAPH2020, TITLE = {Computational Image Marking on Metals via Laser Induced Heating}, AUTHOR = {Cucerca, Sebastian and Didyk, Piotr and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3386569.3392423}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {39}, NUMBER = {4}, EID = {70}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2020}, }

Endnote

%0 Journal Article %A Cucerca, Sebastian %A Didyk, Piotr %A Seidel, Hans-Peter %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Computational Image Marking on Metals via Laser Induced Heating : %G eng %U http://hdl.handle.net/21.11116/0000-0007-9664-F %R 10.1145/3386569.3392423 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 4 %Z sequence number: 70 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2020 %O ACM SIGGRAPH 2020 Virtual Conference ; 2020, 17-28 August

Dunn, D., Tursun, O., Yu, H., Didyk, P., Myszkowski, K., and Fuchs, H. 2020. Stimulating the Human Visual System Beyond Real World Performance in Future Augmented Reality Displays. IEEE International Symposium on Mixed and Augmented Reality (ISMAR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Dunn2020, TITLE = {Stimulating the Human Visual System Beyond Real World Performance in Future Augmented Reality Displays}, AUTHOR = {Dunn, David and Tursun, Okan and Yu, Hyeonseung and Didyk, Piotr and Myszkowski, Karol and Fuchs, Henry}, LANGUAGE = {eng}, ISBN = {978-1-7281-8508-8}, DOI = {10.1109/ISMAR50242.2020.00029}, PUBLISHER = {IEEE}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {IEEE International Symposium on Mixed and Augmented Reality (ISMAR 2020)}, PAGES = {90--100}, ADDRESS = {Recife/Porto de Galinhas, Brazil (Virtual Conference)}, }

Endnote

%0 Conference Proceedings %A Dunn, David %A Tursun, Okan %A Yu, Hyeonseung %A Didyk, Piotr %A Myszkowski, Karol %A Fuchs, Henry %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Stimulating the Human Visual System Beyond Real World Performance in Future Augmented Reality Displays : %G eng %U http://hdl.handle.net/21.11116/0000-0006-FBDF-5 %R 10.1109/ISMAR50242.2020.00029 %D 2020 %B International Symposium on Mixed and Augmented Reality %Z date of event: 2020-11-09 - 2020-11-13 %C Recife/Porto de Galinhas, Brazil (Virtual Conference) %B IEEE International Symposium on Mixed and Augmented Reality %P 90 - 100 %I IEEE %@ 978-1-7281-8508-8

Egger, B., Smith, W.A.P., Tewari, A., et al. 2020. 3D Morphable Face Models -Past, Present and Future. ACM Transactions on Graphics39, 5.

Links

Export

BibTeX

@article{Egger_TOG2020, TITLE = {{3D} Morphable Face Models -- Past, Present and Future}, AUTHOR = {Egger, Bernhard and Smith, William A. P. and Tewari, Ayush and Wuhrer, Stefanie and Zollh{\"o}fer, Michael and Beeler, Thabo and Bernard, Florian and Bolkart, Timo and Kortylewski, Adam and Romdhani, Sami and Theobalt, Christian and Blanz, Volker and Vetter, Thomas}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3395208}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics}, VOLUME = {39}, NUMBER = {5}, EID = {157}, }

Endnote

%0 Journal Article %A Egger, Bernhard %A Smith, William A. P. %A Tewari, Ayush %A Wuhrer, Stefanie %A Zollhöfer, Michael %A Beeler, Thabo %A Bernard, Florian %A Bolkart, Timo %A Kortylewski, Adam %A Romdhani, Sami %A Theobalt, Christian %A Blanz, Volker %A Vetter, Thomas %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T 3D Morphable Face Models -Past, Present and Future : %G eng %U http://hdl.handle.net/21.11116/0000-0007-1CF5-6 %R 10.1145/3395208 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 5 %Z sequence number: 157 %I ACM %C New York, NY %@ false

Elgharib, M., Mendiratta, M., Thies, J., et al. 2020. Egocentric Videoconferencing. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Elgharib_ToG2020, TITLE = {Egocentric Videoconferencing}, AUTHOR = {Elgharib, Mohamed and Mendiratta, Mohit and Thies, Justus and Nie{\ss}ner, Matthias and Seidel, Hans-Peter and Tewari, Ayush and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417808}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {268}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Elgharib, Mohamed %A Mendiratta, Mohit %A Thies, Justus %A Nießner, Matthias %A Seidel, Hans-Peter %A Tewari, Ayush %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Egocentric Videoconferencing : %G eng %U http://hdl.handle.net/21.11116/0000-0007-9B36-E %R 10.1145/3414685.3417808 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 268 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Fox, G., Liu, W., Kim, H., Seidel, H.-P., Elgharib, M., and Theobalt, C. 2020. VideoForensicsHQ: Detecting High-quality Manipulated Face Videos. https://arxiv.org/abs/2005.10360.

(arXiv: 2005.10360)

Abstract

New approaches to synthesize and manipulate face videos at very high quality have paved the way for new applications in computer animation, virtual and augmented reality, or face video analysis. However, there are concerns that they may be used in a malicious way, e.g. to manipulate videos of public figures, politicians or reporters, to spread false information. The research community therefore developed techniques for automated detection of modified imagery, and assembled benchmark datasets showing manipulatons by state-of-the-art techniques. In this paper, we contribute to this initiative in two ways: First, we present a new audio-visual benchmark dataset. It shows some of the highest quality visual manipulations available today. Human observers find them significantly harder to identify as forged than videos from other benchmarks. Furthermore we propose new family of deep-learning-based fake detectors, demonstrating that existing detectors are not well-suited for detecting fakes of a quality as high as presented in our dataset. Our detectors examine spatial and temporal features. This allows them to outperform existing approaches both in terms of high detection accuracy and generalization to unseen fake generation methods and unseen identities.

Links

Export

BibTeX

@online{Fox_2005.10360, TITLE = {{Video\-Foren\-sics\-HQ}: {D}etecting High-quality Manipulated Face Videos}, AUTHOR = {Fox, Gereon and Liu, Wentao and Kim, Hyeongwoo and Seidel, Hans-Peter and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2005.10360}, EPRINT = {2005.10360}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {New approaches to synthesize and manipulate face videos at very high quality have paved the way for new applications in computer animation, virtual and augmented reality, or face video analysis. However, there are concerns that they may be used in a malicious way, e.g. to manipulate videos of public figures, politicians or reporters, to spread false information. The research community therefore developed techniques for automated detection of modified imagery, and assembled benchmark datasets showing manipulatons by state-of-the-art techniques. In this paper, we contribute to this initiative in two ways: First, we present a new audio-visual benchmark dataset. It shows some of the highest quality visual manipulations available today. Human observers find them significantly harder to identify as forged than videos from other benchmarks. Furthermore we propose new family of deep-learning-based fake detectors, demonstrating that existing detectors are not well-suited for detecting fakes of a quality as high as presented in our dataset. Our detectors examine spatial and temporal features. This allows them to outperform existing approaches both in terms of high detection accuracy and generalization to unseen fake generation methods and unseen identities. }, }

Endnote

%0 Report %A Fox, Gereon %A Liu, Wentao %A Kim, Hyeongwoo %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T VideoForensicsHQ: Detecting High-quality Manipulated Face Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B109-7 %U https://arxiv.org/abs/2005.10360 %D 2020 %X New approaches to synthesize and manipulate face videos at very high quality have paved the way for new applications in computer animation, virtual and augmented reality, or face video analysis. However, there are concerns that they may be used in a malicious way, e.g. to manipulate videos of public figures, politicians or reporters, to spread false information. The research community therefore developed techniques for automated detection of modified imagery, and assembled benchmark datasets showing manipulatons by state-of-the-art techniques. In this paper, we contribute to this initiative in two ways: First, we present a new audio-visual benchmark dataset. It shows some of the highest quality visual manipulations available today. Human observers find them significantly harder to identify as forged than videos from other benchmarks. Furthermore we propose new family of deep-learning-based fake detectors, demonstrating that existing detectors are not well-suited for detecting fakes of a quality as high as presented in our dataset. Our detectors examine spatial and temporal features. This allows them to outperform existing approaches both in terms of high detection accuracy and generalization to unseen fake generation methods and unseen identities. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Golyanik, V., Shimada, S., and Theobalt, C. 2020a. Fast Simultaneous Gravitational Alignment of Multiple Point Sets. International Conference on 3D Vision, IEEE.

Links

Export

BibTeX

@inproceedings{Golyanik_MBGA2020, TITLE = {Fast Simultaneous Gravitational Alignment of Multiple Point Sets}, AUTHOR = {Golyanik, Vladislav and Shimada, Soshi and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-8128-8}, DOI = {10.1109/3DV50981.2020.00019}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {International Conference on 3D Vision}, PAGES = {91--100}, ADDRESS = {Fukuoka, Japan (Virtual Event)}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Shimada, Soshi %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Fast Simultaneous Gravitational Alignment of Multiple Point Sets : %G eng %U http://hdl.handle.net/21.11116/0000-0008-0358-2 %R 10.1109/3DV50981.2020.00019 %D 2020 %B International Conference on 3D Vision %Z date of event: 2020-11-25 - 2020-11-28 %C Fukuoka, Japan (Virtual Event) %B International Conference on 3D Vision %P 91 - 100 %I IEEE %@ 978-1-7281-8128-8

Golyanik, V. and Theobalt, C. 2020. A Quantum Computational Approach to Correspondence Problems on Point Sets. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Golyanik_CVPR2020, TITLE = {A Quantum Computational Approach to Correspondence Problems on Point Sets}, AUTHOR = {Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00920}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {9179--9188}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T A Quantum Computational Approach to Correspondence Problems on Point Sets : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D053-0 %R 10.1109/CVPR42600.2020.00920 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 9179 - 9188 %I IEEE %@ 978-1-7281-7168-5

Golyanik, V., Jonas, A., Stricker, D., and Theobalt, C. 2020b. Intrinsic Dynamic Shape Prior for Dense Non-Rigid Structure from Motion. International Conference on 3D Vision, IEEE.

Links

Export

BibTeX

@inproceedings{Golyanik2020DSPR, TITLE = {Intrinsic Dynamic Shape Prior for Dense Non-Rigid Structure from Motion}, AUTHOR = {Golyanik, Vladislav and Jonas, Andr{\'e} and Stricker, Didier and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-8128-8}, DOI = {10.1109/3DV50981.2020.00079}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {International Conference on 3D Vision}, PAGES = {692--701}, ADDRESS = {Fukuoka, Japan (Virtual Event)}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Jonas, André %A Stricker, Didier %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Intrinsic Dynamic Shape Prior for Dense Non-Rigid Structure from Motion : %G eng %U http://hdl.handle.net/21.11116/0000-0008-0364-4 %R 10.1109/3DV50981.2020.00079 %D 2020 %B International Conference on 3D Vision %Z date of event: 2020-11-25 - 2020-11-28 %C Fukuoka, Japan (Virtual Event) %B International Conference on 3D Vision %P 692 - 701 %I IEEE %@ 978-1-7281-8128-8

Günther, F., Jiang, C., and Pottmann, H. 2020. Smooth Polyhedral Surfaces. Advances in Mathematics363.

(arXiv: 1703.05318)

Abstract

Polyhedral surfaces are fundamental objects in architectural geometry and industrial design. Whereas closeness of a given mesh to a smooth reference surface and its suitability for numerical simulations were already studied extensively, the aim of our work is to find and to discuss suitable assessments of smoothness of polyhedral surfaces that only take the geometry of the polyhedral surface itself into account. Motivated by analogies to classical differential geometry, we propose a theory of smoothness of polyhedral surfaces including suitable notions of normal vectors, tangent planes, asymptotic directions, and parabolic curves that are invariant under projective transformations. It is remarkable that seemingly mild conditions significantly limit the shapes of faces of a smooth polyhedral surface. Besides being of theoretical interest, we believe that smoothness of polyhedral surfaces is of interest in the architectural context, where vertices and edges of polyhedral surfaces are highly visible.

Links

Export

BibTeX

@article{Guenther2020, TITLE = {Smooth Polyhedral Surfaces}, AUTHOR = {G{\"u}nther, Felix and Jiang, Caigui and Pottmann, Helmut}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1703.05318}, DOI = {10.1016/j.aim.2020.107004}, EPRINT = {1703.05318}, EPRINTTYPE = {arXiv}, PUBLISHER = {Elsevier}, YEAR = {2020}, DATE = {2020}, ABSTRACT = {Polyhedral surfaces are fundamental objects in architectural geometry and industrial design. Whereas closeness of a given mesh to a smooth reference surface and its suitability for numerical simulations were already studied extensively, the aim of our work is to find and to discuss suitable assessments of smoothness of polyhedral surfaces that only take the geometry of the polyhedral surface itself into account. Motivated by analogies to classical differential geometry, we propose a theory of smoothness of polyhedral surfaces including suitable notions of normal vectors, tangent planes, asymptotic directions, and parabolic curves that are invariant under projective transformations. It is remarkable that seemingly mild conditions significantly limit the shapes of faces of a smooth polyhedral surface. Besides being of theoretical interest, we believe that smoothness of polyhedral surfaces is of interest in the architectural context, where vertices and edges of polyhedral surfaces are highly visible. }, JOURNAL = {Advances in Mathematics}, VOLUME = {363}, EID = {107004}, }

Endnote

%0 Journal Article %A Günther, Felix %A Jiang, Caigui %A Pottmann, Helmut %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Smooth Polyhedral Surfaces : %G eng %U http://hdl.handle.net/21.11116/0000-0006-9760-3 %U http://arxiv.org/abs/1703.05318 %R 10.1016/j.aim.2020.107004 %D 2020 %* Review method: peer-reviewed %X Polyhedral surfaces are fundamental objects in architectural geometry and industrial design. Whereas closeness of a given mesh to a smooth reference surface and its suitability for numerical simulations were already studied extensively, the aim of our work is to find and to discuss suitable assessments of smoothness of polyhedral surfaces that only take the geometry of the polyhedral surface itself into account. Motivated by analogies to classical differential geometry, we propose a theory of smoothness of polyhedral surfaces including suitable notions of normal vectors, tangent planes, asymptotic directions, and parabolic curves that are invariant under projective transformations. It is remarkable that seemingly mild conditions significantly limit the shapes of faces of a smooth polyhedral surface. Besides being of theoretical interest, we believe that smoothness of polyhedral surfaces is of interest in the architectural context, where vertices and edges of polyhedral surfaces are highly visible. %K Mathematics, Metric Geometry, Mathematics, Differential Geometry %J Advances in Mathematics %O Adv. Math. %V 363 %Z sequence number: 107004 %I Elsevier

Habermann, M., Xu, W., Zollhöfer, M., Pons-Moll, G., and Theobalt, C. 2020a. DeepCap: Monocular Human Performance Capture Using Weak Supervision. https://arxiv.org/abs/2003.08325.

(arXiv: 2003.08325)

Abstract

Links

Export

BibTeX

@online{Habermann2003.08325, TITLE = {{DeepCap}: {M}onocular Human Performance Capture Using Weak Supervision}, AUTHOR = {Habermann, Marc and Xu, Weipeng and Zollh{\"o}fer, Michael and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2003.08325}, EPRINT = {2003.08325}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Human performance capture is a highly important computer vision problem with many applications in movie production and virtual/augmented reality. Many previous performance capture approaches either required expensive multi-view setups or did not recover dense space-time coherent geometry with frame-to-frame correspondences. We propose a novel deep learning approach for monocular dense human performance capture. Our method is trained in a weakly supervised manner based on multi-view supervision completely removing the need for training data with 3D ground truth annotations. The network architecture is based on two separate networks that disentangle the task into a pose estimation and a non-rigid surface deformation step. Extensive qualitative and quantitative evaluations show that our approach outperforms the state of the art in terms of quality and robustness. }, }

Endnote

%0 Report %A Habermann, Marc %A Xu, Weipeng %A Zollhöfer, Michael %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T DeepCap: Monocular Human Performance Capture Using Weak Supervision : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E010-9 %U https://arxiv.org/abs/2003.08325 %D 2020 %X Human performance capture is a highly important computer vision problem with many applications in movie production and virtual/augmented reality. Many previous performance capture approaches either required expensive multi-view setups or did not recover dense space-time coherent geometry with frame-to-frame correspondences. We propose a novel deep learning approach for monocular dense human performance capture. Our method is trained in a weakly supervised manner based on multi-view supervision completely removing the need for training data with 3D ground truth annotations. The network architecture is based on two separate networks that disentangle the task into a pose estimation and a non-rigid surface deformation step. Extensive qualitative and quantitative evaluations show that our approach outperforms the state of the art in terms of quality and robustness. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Habermann, M., Xu, W., Zollhöfer, M., Pons-Moll, G., and Theobalt, C. 2020b. DeepCap: Monocular Human Performance Capture Using Weak Supervision. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{deepcap2020, TITLE = {{DeepCap}: {M}onocular Human Performance Capture Using Weak Supervision}, AUTHOR = {Habermann, Marc and Xu, Weipeng and Zollh{\"o}fer, Michael and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00510}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {5051--5062}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Habermann, Marc %A Xu, Weipeng %A Zollhöfer, Michael %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T DeepCap: Monocular Human Performance Capture Using Weak Supervision : %G eng %U http://hdl.handle.net/21.11116/0000-0006-A895-4 %R 10.1109/CVPR42600.2020.00510 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 5051 - 5062 %I IEEE %@ 978-1-7281-7168-5

Huang, L., Gao, C., Zhou, Y., et al. 2020. Universal Physical Camouﬂage Attacks on Object Detectors. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Huang_CVPR2020, TITLE = {Universal Physical Camou{fl}age Attacks on Object Detectors}, AUTHOR = {Huang, Lifeng and Gao, Chengying and Zhou, Yuyin and Xie, Cihang and Yuille, Alan and Zou, Changqing and Liu, Ning}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00080}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {717--726}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Huang, Lifeng %A Gao, Chengying %A Zhou, Yuyin %A Xie, Cihang %A Yuille, Alan %A Zou, Changqing %A Liu, Ning %+ External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Universal Physical Camouﬂage Attacks on Object Detectors : %G eng %U http://hdl.handle.net/21.11116/0000-0006-09F0-1 %R 10.1109/CVPR42600.2020.00080 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 717 - 726 %I IEEE %@ 978-1-7281-7168-5

Kappel, M., Golyanik, V., Elgharib, M., et al. 2020. High-Fidelity Neural Human Motion Transfer from Monocular Video. https://arxiv.org/abs/2012.10974.

(arXiv: 2012.10974)

Abstract

Video-based human motion transfer creates video animations of humans following a source motion. Current methods show remarkable results for tightly-clad subjects. However, the lack of temporally consistent handling of plausible clothing dynamics, including fine and high-frequency details, significantly limits the attainable visual quality. We address these limitations for the first time in the literature and present a new framework which performs high-fidelity and temporally-consistent human motion transfer with natural pose-dependent non-rigid deformations, for several types of loose garments. In contrast to the previous techniques, we perform image generation in three subsequent stages, synthesizing human shape, structure, and appearance. Given a monocular RGB video of an actor, we train a stack of recurrent deep neural networks that generate these intermediate representations from 2D poses and their temporal derivatives. Splitting the difficult motion transfer problem into subtasks that are aware of the temporal motion context helps us to synthesize results with plausible dynamics and pose-dependent detail. It also allows artistic control of results by manipulation of individual framework stages. In the experimental results, we significantly outperform the state-of-the-art in terms of video realism. Our code and data will be made publicly available.

Links

Export

BibTeX

@online{Kappel_arXiv2012.10974, TITLE = {High-Fidelity Neural Human Motion Transfer from Monocular Video}, AUTHOR = {Kappel, Moritz and Golyanik, Vladislav and Elgharib, Mohamed and Henningson, Jann-Ole and Seidel, Hans-Peter and Castillo, Susana and Theobalt, Christian and Magnor, Marcus A.}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.10974}, EPRINT = {2012.10974}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Video-based human motion transfer creates video animations of humans following a source motion. Current methods show remarkable results for tightly-clad subjects. However, the lack of temporally consistent handling of plausible clothing dynamics, including fine and high-frequency details, significantly limits the attainable visual quality. We address these limitations for the first time in the literature and present a new framework which performs high-fidelity and temporally-consistent human motion transfer with natural pose-dependent non-rigid deformations, for several types of loose garments. In contrast to the previous techniques, we perform image generation in three subsequent stages, synthesizing human shape, structure, and appearance. Given a monocular RGB video of an actor, we train a stack of recurrent deep neural networks that generate these intermediate representations from 2D poses and their temporal derivatives. Splitting the difficult motion transfer problem into subtasks that are aware of the temporal motion context helps us to synthesize results with plausible dynamics and pose-dependent detail. It also allows artistic control of results by manipulation of individual framework stages. In the experimental results, we significantly outperform the state-of-the-art in terms of video realism. Our code and data will be made publicly available. }, }

Endnote

%0 Report %A Kappel, Moritz %A Golyanik, Vladislav %A Elgharib, Mohamed %A Henningson, Jann-Ole %A Seidel, Hans-Peter %A Castillo, Susana %A Theobalt, Christian %A Magnor, Marcus A. %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T High-Fidelity Neural Human Motion Transfer from Monocular Video : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B715-3 %U https://arxiv.org/abs/2012.10974 %D 2020 %X Video-based human motion transfer creates video animations of humans following a source motion. Current methods show remarkable results for tightly-clad subjects. However, the lack of temporally consistent handling of plausible clothing dynamics, including fine and high-frequency details, significantly limits the attainable visual quality. We address these limitations for the first time in the literature and present a new framework which performs high-fidelity and temporally-consistent human motion transfer with natural pose-dependent non-rigid deformations, for several types of loose garments. In contrast to the previous techniques, we perform image generation in three subsequent stages, synthesizing human shape, structure, and appearance. Given a monocular RGB video of an actor, we train a stack of recurrent deep neural networks that generate these intermediate representations from 2D poses and their temporal derivatives. Splitting the difficult motion transfer problem into subtasks that are aware of the temporal motion context helps us to synthesize results with plausible dynamics and pose-dependent detail. It also allows artistic control of results by manipulation of individual framework stages. In the experimental results, we significantly outperform the state-of-the-art in terms of video realism. Our code and data will be made publicly available. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Liu, L., Gu, J., Lin, K.Z., Chua, T.-S., and Theobalt, C. 2020a. Neural Sparse Voxel Fields. Advances in Neural Information Processing Systems 33 (NeurIPS 2020), Curran Associates, Inc.

Links

Export

BibTeX

@inproceedings{LiuNeural20, TITLE = {Neural Sparse Voxel Fields}, AUTHOR = {Liu, Lingjie and Gu, Jiatao and Lin, Kyaw Zaw and Chua, Tat-Seng and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {9781713829546}, PUBLISHER = {Curran Associates, Inc.}, YEAR = {2020}, BOOKTITLE = {Advances in Neural Information Processing Systems 33 (NeurIPS 2020)}, EDITOR = {Larochelle, H. and Ranzato, M. and Hadsell, R. and Balcan, M. F. and Lin, H.}, PAGES = {15651--15663}, ADDRESS = {Virtual Event}, }

Endnote

%0 Conference Proceedings %A Liu, Lingjie %A Gu, Jiatao %A Lin, Kyaw Zaw %A Chua, Tat-Seng %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Sparse Voxel Fields : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D437-C %D 2020 %B 34th Conference on Neural Information Processing Systems %Z date of event: 2020-12-06 - 2020-12-12 %C Virtual Event %B Advances in Neural Information Processing Systems 33 %E Larochelle, H.; Ranzato, M.; Hadsell, R.; Balcan, M. F.; Lin, H. %P 15651 - 15663 %I Curran Associates, Inc. %@ 9781713829546 %U https://proceedings.neurips.cc/paper/2020/file/b4b758962f17808746e9bb832a6fa4b8-Paper.pdf

Liu, L., Xu, W., Habermann, M., et al. 2020b. Learning Dynamic Textures for Neural Rendering of Human Actors. IEEE Transactions on Visualization and Computer Graphics27, 10.

Links

Export

BibTeX

@article{Liu2020, TITLE = {Learning Dynamic Textures for Neural Rendering of Human Actors}, AUTHOR = {Liu, Lingjie and Xu, Weipeng and Habermann, Marc and Zollh{\"o}fer, Michael and Bernard, Florian and Kim, Hyeongwoo and Wang, Wenping and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {1077-2626}, DOI = {10.1109/TVCG.2020.2996594}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2020}, JOURNAL = {IEEE Transactions on Visualization and Computer Graphics}, VOLUME = {27}, NUMBER = {10}, PAGES = {4009--4022}, }

Endnote

%0 Journal Article %A Liu, Lingjie %A Xu, Weipeng %A Habermann, Marc %A Zollhöfer, Michael %A Bernard, Florian %A Kim, Hyeongwoo %A Wang, Wenping %A Theobalt, Christian %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Visual Computing and Artificial Intelligence, MPI for Informatics, Max Planck Society %T Learning Dynamic Textures for Neural Rendering of Human Actors : %G eng %U http://hdl.handle.net/21.11116/0000-0009-4C96-9 %R 10.1109/TVCG.2020.2996594 %7 2020 %D 2020 %J IEEE Transactions on Visualization and Computer Graphics %V 27 %N 10 %& 4009 %P 4009 - 4022 %I IEEE %C Piscataway, NJ %@ false

Liu, L., Gu, J., Lin, K.Z., Chua, T.-S., and Theobalt, C. 2020c. Neural Sparse Voxel Fields. https://arxiv.org/abs/2007.11571.

(arXiv: 2007.11571)

Abstract

Photo-realistic free-viewpoint rendering of real-world scenes using classical computer graphics techniques is challenging, because it requires the difficult step of capturing detailed appearance and geometry models. Recent studies have demonstrated promising results by learning scene representations that implicitly encode both geometry and appearance without 3D supervision. However, existing approaches in practice often show blurry renderings caused by the limited network capacity or the difficulty in finding accurate intersections of camera rays with the scene geometry. Synthesizing high-resolution imagery from these representations often requires time-consuming optical ray marching. In this work, we introduce Neural Sparse Voxel Fields (NSVF), a new neural scene representation for fast and high-quality free-viewpoint rendering. NSVF defines a set of voxel-bounded implicit fields organized in a sparse voxel octree to model local properties in each cell. We progressively learn the underlying voxel structures with a differentiable ray-marching operation from only a set of posed RGB images. With the sparse voxel octree structure, rendering novel views can be accelerated by skipping the voxels containing no relevant scene content. Our method is typically over 10 times faster than the state-of-the-art (namely, NeRF(Mildenhall et al., 2020)) at inference time while achieving higher quality results. Furthermore, by utilizing an explicit sparse voxel representation, our method can easily be applied to scene editing and scene composition. We also demonstrate several challenging tasks, including multi-scene learning, free-viewpoint rendering of a moving human, and large-scale scene rendering. Code and data are available at our website: https://github.com/facebookresearch/NSVF.

Links

Export

BibTeX

@online{Liu_2007.11571, TITLE = {Neural Sparse Voxel Fields}, AUTHOR = {Liu, Lingjie and Gu, Jiatao and Lin, Kyaw Zaw and Chua, Tat-Seng and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2007.11571}, EPRINT = {2007.11571}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Photo-realistic free-viewpoint rendering of real-world scenes using classical computer graphics techniques is challenging, because it requires the difficult step of capturing detailed appearance and geometry models. Recent studies have demonstrated promising results by learning scene representations that implicitly encode both geometry and appearance without 3D supervision. However, existing approaches in practice often show blurry renderings caused by the limited network capacity or the difficulty in finding accurate intersections of camera rays with the scene geometry. Synthesizing high-resolution imagery from these representations often requires time-consuming optical ray marching. In this work, we introduce Neural Sparse Voxel Fields (NSVF), a new neural scene representation for fast and high-quality free-viewpoint rendering. NSVF defines a set of voxel-bounded implicit fields organized in a sparse voxel octree to model local properties in each cell. We progressively learn the underlying voxel structures with a differentiable ray-marching operation from only a set of posed RGB images. With the sparse voxel octree structure, rendering novel views can be accelerated by skipping the voxels containing no relevant scene content. Our method is typically over 10 times faster than the state-of-the-art (namely, NeRF(Mildenhall et al., 2020)) at inference time while achieving higher quality results. Furthermore, by utilizing an explicit sparse voxel representation, our method can easily be applied to scene editing and scene composition. We also demonstrate several challenging tasks, including multi-scene learning, free-viewpoint rendering of a moving human, and large-scale scene rendering. Code and data are available at our website: https://github.com/facebookresearch/NSVF. }, }

Endnote

%0 Report %A Liu, Lingjie %A Gu, Jiatao %A Lin, Kyaw Zaw %A Chua, Tat-Seng %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Sparse Voxel Fields : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8B2-A %U https://arxiv.org/abs/2007.11571 %D 2020 %X Photo-realistic free-viewpoint rendering of real-world scenes using classical computer graphics techniques is challenging, because it requires the difficult step of capturing detailed appearance and geometry models. Recent studies have demonstrated promising results by learning scene representations that implicitly encode both geometry and appearance without 3D supervision. However, existing approaches in practice often show blurry renderings caused by the limited network capacity or the difficulty in finding accurate intersections of camera rays with the scene geometry. Synthesizing high-resolution imagery from these representations often requires time-consuming optical ray marching. In this work, we introduce Neural Sparse Voxel Fields (NSVF), a new neural scene representation for fast and high-quality free-viewpoint rendering. NSVF defines a set of voxel-bounded implicit fields organized in a sparse voxel octree to model local properties in each cell. We progressively learn the underlying voxel structures with a differentiable ray-marching operation from only a set of posed RGB images. With the sparse voxel octree structure, rendering novel views can be accelerated by skipping the voxels containing no relevant scene content. Our method is typically over 10 times faster than the state-of-the-art (namely, NeRF(Mildenhall et al., 2020)) at inference time while achieving higher quality results. Furthermore, by utilizing an explicit sparse voxel representation, our method can easily be applied to scene editing and scene composition. We also demonstrate several challenging tasks, including multi-scene learning, free-viewpoint rendering of a moving human, and large-scale scene rendering. Code and data are available at our website: https://github.com/facebookresearch/NSVF. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Long, X., Liu, L., Li, W., Theobalt, C., and Wang, W. 2020a. Multi-view Depth Estimation using Epipolar Spatio-Temporal Networks. https://arxiv.org/abs/2011.13118.

(arXiv: 2011.13118)

Abstract

We present a novel method for multi-view depth estimation from a single video, which is a critical task in various applications, such as perception, reconstruction and robot navigation. Although previous learning-based methods have demonstrated compelling results, most works estimate depth maps of individual video frames independently, without taking into consideration the strong geometric and temporal coherence among the frames. Moreover, current state-of-the-art (SOTA) models mostly adopt a fully 3D convolution network for cost regularization and therefore require high computational cost, thus limiting their deployment in real-world applications. Our method achieves temporally coherent depth estimation results by using a novel Epipolar Spatio-Temporal (EST) transformer to explicitly associate geometric and temporal correlation with multiple estimated depth maps. Furthermore, to reduce the computational cost, inspired by recent Mixture-of-Experts models, we design a compact hybrid network consisting of a 2D context-aware network and a 3D matching network which learn 2D context information and 3D disparity cues separately. Extensive experiments demonstrate that our method achieves higher accuracy in depth estimation and significant speedup than the SOTA methods.

Links

Export

BibTeX

@online{Long_2011.13118, TITLE = {Multi-view Depth Estimation using Epipolar Spatio-Temporal Networks}, AUTHOR = {Long, Xiaoxiao and Liu, Lingjie and Li, Wei and Theobalt, Christian and Wang, Wenping}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2011.13118}, EPRINT = {2011.13118}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present a novel method for multi-view depth estimation from a single video, which is a critical task in various applications, such as perception, reconstruction and robot navigation. Although previous learning-based methods have demonstrated compelling results, most works estimate depth maps of individual video frames independently, without taking into consideration the strong geometric and temporal coherence among the frames. Moreover, current state-of-the-art (SOTA) models mostly adopt a fully 3D convolution network for cost regularization and therefore require high computational cost, thus limiting their deployment in real-world applications. Our method achieves temporally coherent depth estimation results by using a novel Epipolar Spatio-Temporal (EST) transformer to explicitly associate geometric and temporal correlation with multiple estimated depth maps. Furthermore, to reduce the computational cost, inspired by recent Mixture-of-Experts models, we design a compact hybrid network consisting of a 2D context-aware network and a 3D matching network which learn 2D context information and 3D disparity cues separately. Extensive experiments demonstrate that our method achieves higher accuracy in depth estimation and significant speedup than the SOTA methods. }, }

Endnote

%0 Report %A Long, Xiaoxiao %A Liu, Lingjie %A Li, Wei %A Theobalt, Christian %A Wang, Wenping %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Multi-view Depth Estimation using Epipolar Spatio-Temporal Networks : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E9EA-B %U https://arxiv.org/abs/2011.13118 %D 2020 %X We present a novel method for multi-view depth estimation from a single video, which is a critical task in various applications, such as perception, reconstruction and robot navigation. Although previous learning-based methods have demonstrated compelling results, most works estimate depth maps of individual video frames independently, without taking into consideration the strong geometric and temporal coherence among the frames. Moreover, current state-of-the-art (SOTA) models mostly adopt a fully 3D convolution network for cost regularization and therefore require high computational cost, thus limiting their deployment in real-world applications. Our method achieves temporally coherent depth estimation results by using a novel Epipolar Spatio-Temporal (EST) transformer to explicitly associate geometric and temporal correlation with multiple estimated depth maps. Furthermore, to reduce the computational cost, inspired by recent Mixture-of-Experts models, we design a compact hybrid network consisting of a 2D context-aware network and a 3D matching network which learn 2D context information and 3D disparity cues separately. Extensive experiments demonstrate that our method achieves higher accuracy in depth estimation and significant speedup than the SOTA methods. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Long, X., Liu, L., Theobalt, C., and Wang, W. 2020b. Occlusion-Aware Depth Estimation with Adaptive Normal Constraints. ECCV 2020. Lecture Notes in Computer Science, vol 12354. Springer, Cham. https://arxiv.org/abs/2004.00845.

(arXiv: 2004.00845)

Abstract

We present a new learning-based method for multi-frame depth estimation from a color video, which is a fundamental problem in scene understanding, robot navigation or handheld 3D reconstruction. While recent learning-based methods estimate depth at high accuracy, 3D point clouds exported from their depth maps often fail to preserve important geometric feature (e.g., corners, edges, planes) of man-made scenes. Widely-used pixel-wise depth errors do not specifically penalize inconsistency on these features. These inaccuracies are particularly severe when subsequent depth reconstructions are accumulated in an attempt to scan a full environment with man-made objects with this kind of features. Our depth estimation algorithm therefore introduces a Combined Normal Map (CNM) constraint, which is designed to better preserve high-curvature features and global planar regions. In order to further improve the depth estimation accuracy, we introduce a new occlusion-aware strategy that aggregates initial depth predictions from multiple adjacent views into one final depth map and one occlusion probability map for the current reference view. Our method outperforms the state-of-the-art in terms of depth estimation accuracy, and preserves essential geometric features of man-made indoor scenes much better than other algorithms.

Links

Export

BibTeX

@online{Long2004.00845, TITLE = {Occlusion-Aware Depth Estimation with Adaptive Normal Constraints}, AUTHOR = {Long, Xiaoxiao and Liu, Lingjie and Theobalt, Christian and Wang, Wenping}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2004.00845}, EPRINT = {2004.00845}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present a new learning-based method for multi-frame depth estimation from a color video, which is a fundamental problem in scene understanding, robot navigation or handheld 3D reconstruction. While recent learning-based methods estimate depth at high accuracy, 3D point clouds exported from their depth maps often fail to preserve important geometric feature (e.g., corners, edges, planes) of man-made scenes. Widely-used pixel-wise depth errors do not specifically penalize inconsistency on these features. These inaccuracies are particularly severe when subsequent depth reconstructions are accumulated in an attempt to scan a full environment with man-made objects with this kind of features. Our depth estimation algorithm therefore introduces a Combined Normal Map (CNM) constraint, which is designed to better preserve high-curvature features and global planar regions. In order to further improve the depth estimation accuracy, we introduce a new occlusion-aware strategy that aggregates initial depth predictions from multiple adjacent views into one final depth map and one occlusion probability map for the current reference view. Our method outperforms the state-of-the-art in terms of depth estimation accuracy, and preserves essential geometric features of man-made indoor scenes much better than other algorithms. }, JOURNAL = {ECCV 2020. Lecture Notes in Computer Science, vol 12354. Springer, Cham}, }

Endnote

%0 Report %A Long, Xiaoxiao %A Liu, Lingjie %A Theobalt, Christian %A Wang, Wenping %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Occlusion-Aware Depth Estimation with Adaptive Normal Constraints : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E0E9-5 %U https://arxiv.org/abs/2004.00845 %D 2020 %X We present a new learning-based method for multi-frame depth estimation from a color video, which is a fundamental problem in scene understanding, robot navigation or handheld 3D reconstruction. While recent learning-based methods estimate depth at high accuracy, 3D point clouds exported from their depth maps often fail to preserve important geometric feature (e.g., corners, edges, planes) of man-made scenes. Widely-used pixel-wise depth errors do not specifically penalize inconsistency on these features. These inaccuracies are particularly severe when subsequent depth reconstructions are accumulated in an attempt to scan a full environment with man-made objects with this kind of features. Our depth estimation algorithm therefore introduces a Combined Normal Map (CNM) constraint, which is designed to better preserve high-curvature features and global planar regions. In order to further improve the depth estimation accuracy, we introduce a new occlusion-aware strategy that aggregates initial depth predictions from multiple adjacent views into one final depth map and one occlusion probability map for the current reference view. Our method outperforms the state-of-the-art in terms of depth estimation accuracy, and preserves essential geometric features of man-made indoor scenes much better than other algorithms. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV %J ECCV 2020. Lecture Notes in Computer Science, vol 12354. Springer, Cham

Long, X., Liu, L., Theobalt, C., and Wang, W. 2020c. Occlusion-Aware Depth Estimation with Adaptive Normal Constraints. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Long_ECCV20, TITLE = {Occlusion-Aware Depth Estimation with Adaptive Normal Constraints}, AUTHOR = {Long, Xiaoxiao and Liu, Lingjie and Theobalt, Christian and Wang, Wenping}, LANGUAGE = {eng}, ISBN = {978-3-030-58544-0}, DOI = {10.1007/978-3-030-58545-7_37}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {640--657}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12354}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Long, Xiaoxiao %A Liu, Lingjie %A Theobalt, Christian %A Wang, Wenping %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Occlusion-Aware Depth Estimation with Adaptive Normal Constraints : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D42B-A %R 10.1007/978-3-030-58545-7_37 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 640 - 657 %I Springer %@ 978-3-030-58544-0 %B Lecture Notes in Computer Science %N 12354

Malik, J., Abdelaziz, I., Elhayek, A., et al. 2020a. HandVoxNet: Deep Voxel-Based Network for 3D Hand Shape and Pose Estimation from a Single Depth Map. https://arxiv.org/abs/2004.01588.

(arXiv: 2004.01588)

Abstract

3D hand shape and pose estimation from a single depth map is a new and challenging computer vision problem with many applications. The state-of-the-art methods directly regress 3D hand meshes from 2D depth images via 2D convolutional neural networks, which leads to artefacts in the estimations due to perspective distortions in the images. In contrast, we propose a novel architecture with 3D convolutions trained in a weakly-supervised manner. The input to our method is a 3D voxelized depth map, and we rely on two hand shape representations. The first one is the 3D voxelized grid of the shape which is accurate but does not preserve the mesh topology and the number of mesh vertices. The second representation is the 3D hand surface which is less accurate but does not suffer from the limitations of the first representation. We combine the advantages of these two representations by registering the hand surface to the voxelized hand shape. In the extensive experiments, the proposed approach improves over the state of the art by 47.8% on the SynHand5M dataset. Moreover, our augmentation policy for voxelized depth maps further enhances the accuracy of 3D hand pose estimation on real data. Our method produces visually more reasonable and realistic hand shapes on NYU and BigHand2.2M datasets compared to the existing approaches.

Links

Export

BibTeX

@online{Malik2004.01588, TITLE = {{HandVoxNet}: {D}eep Voxel-Based Network for {3D} Hand Shape and Pose Estimation from a Single Depth Map}, AUTHOR = {Malik, Jameel and Abdelaziz, Ibrahim and Elhayek, Ahmed and Shimada, Soshi and Ali, Sk Aziz and Golyanik, Vladislav and Theobalt, Christian and Stricker, Didier}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2004.01588}, EPRINT = {2004.01588}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {3D hand shape and pose estimation from a single depth map is a new and challenging computer vision problem with many applications. The state-of-the-art methods directly regress 3D hand meshes from 2D depth images via 2D convolutional neural networks, which leads to artefacts in the estimations due to perspective distortions in the images. In contrast, we propose a novel architecture with 3D convolutions trained in a weakly-supervised manner. The input to our method is a 3D voxelized depth map, and we rely on two hand shape representations. The first one is the 3D voxelized grid of the shape which is accurate but does not preserve the mesh topology and the number of mesh vertices. The second representation is the 3D hand surface which is less accurate but does not suffer from the limitations of the first representation. We combine the advantages of these two representations by registering the hand surface to the voxelized hand shape. In the extensive experiments, the proposed approach improves over the state of the art by 47.8% on the SynHand5M dataset. Moreover, our augmentation policy for voxelized depth maps further enhances the accuracy of 3D hand pose estimation on real data. Our method produces visually more reasonable and realistic hand shapes on NYU and BigHand2.2M datasets compared to the existing approaches. }, }

Endnote

%0 Report %A Malik, Jameel %A Abdelaziz, Ibrahim %A Elhayek, Ahmed %A Shimada, Soshi %A Ali, Sk Aziz %A Golyanik, Vladislav %A Theobalt, Christian %A Stricker, Didier %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T HandVoxNet: Deep Voxel-Based Network for 3D Hand Shape and Pose Estimation from a Single Depth Map : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E0FF-D %U https://arxiv.org/abs/2004.01588 %D 2020 %X 3D hand shape and pose estimation from a single depth map is a new and challenging computer vision problem with many applications. The state-of-the-art methods directly regress 3D hand meshes from 2D depth images via 2D convolutional neural networks, which leads to artefacts in the estimations due to perspective distortions in the images. In contrast, we propose a novel architecture with 3D convolutions trained in a weakly-supervised manner. The input to our method is a 3D voxelized depth map, and we rely on two hand shape representations. The first one is the 3D voxelized grid of the shape which is accurate but does not preserve the mesh topology and the number of mesh vertices. The second representation is the 3D hand surface which is less accurate but does not suffer from the limitations of the first representation. We combine the advantages of these two representations by registering the hand surface to the voxelized hand shape. In the extensive experiments, the proposed approach improves over the state of the art by 47.8% on the SynHand5M dataset. Moreover, our augmentation policy for voxelized depth maps further enhances the accuracy of 3D hand pose estimation on real data. Our method produces visually more reasonable and realistic hand shapes on NYU and BigHand2.2M datasets compared to the existing approaches. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Malik, J., Abdelaziz, I., Elhayek, A., et al. 2020b. HandVoxNet: Deep Voxel-Based Network for 3D Hand Shape and Pose Estimation From a Single Depth Map. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Malik_CVPR2020, TITLE = {{HandVoxNet}: {D}eep Voxel-Based Network for {3D} Hand Shape and Pose Estimation From a Single Depth Map}, AUTHOR = {Malik, Jameel and Abdelaziz, Ibrahim and Elhayek, Ahmed and Shimada, Soshi and Ali, Sk Aziz and Golyanik, Vladislav and Theobalt, Christian and Stricker, Didier}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00714}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {7111--7120}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Malik, Jameel %A Abdelaziz, Ibrahim %A Elhayek, Ahmed %A Shimada, Soshi %A Ali, Sk Aziz %A Golyanik, Vladislav %A Theobalt, Christian %A Stricker, Didier %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T HandVoxNet: Deep Voxel-Based Network for 3D Hand Shape and Pose Estimation From a Single Depth Map : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CFCA-D %R 10.1109/CVPR42600.2020.00714 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 7111 - 7120 %I IEEE %@ 978-1-7281-7168-5

Mallikarjun B R, Tewari, A., Seidel, H.-P., Elgharib, M., and Theobalt, C. 2020a. Learning Complete 3D Morphable Face Models from Images and Videos. https://arxiv.org/abs/2010.01679.

(arXiv: 2010.01679)

Abstract

Most 3D face reconstruction methods rely on 3D morphable models, which disentangle the space of facial deformations into identity geometry, expressions and skin reflectance. These models are typically learned from a limited number of 3D scans and thus do not generalize well across different identities and expressions. We present the first approach to learn complete 3D models of face identity geometry, albedo and expression just from images and videos. The virtually endless collection of such data, in combination with our self-supervised learning-based approach allows for learning face models that generalize beyond the span of existing approaches. Our network design and loss functions ensure a disentangled parameterization of not only identity and albedo, but also, for the first time, an expression basis. Our method also allows for in-the-wild monocular reconstruction at test time. We show that our learned models better generalize and lead to higher quality image-based reconstructions than existing approaches.

Links

Export

BibTeX

@online{Mallikarjun_arXiv2010.01679, TITLE = {Learning Complete {3D} Morphable Face Models from Images and Videos}, AUTHOR = {Mallikarjun B R and Tewari, Ayush and Seidel, Hans-Peter and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2010.01679}, EPRINT = {2010.01679}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Most 3D face reconstruction methods rely on 3D morphable models, which disentangle the space of facial deformations into identity geometry, expressions and skin reflectance. These models are typically learned from a limited number of 3D scans and thus do not generalize well across different identities and expressions. We present the first approach to learn complete 3D models of face identity geometry, albedo and expression just from images and videos. The virtually endless collection of such data, in combination with our self-supervised learning-based approach allows for learning face models that generalize beyond the span of existing approaches. Our network design and loss functions ensure a disentangled parameterization of not only identity and albedo, but also, for the first time, an expression basis. Our method also allows for in-the-wild monocular reconstruction at test time. We show that our learned models better generalize and lead to higher quality image-based reconstructions than existing approaches. }, }

Endnote

%0 Report %A Mallikarjun B R, %A Tewari, Ayush %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Learning Complete 3D Morphable Face Models from Images and Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B6FB-1 %U https://arxiv.org/abs/2010.01679 %D 2020 %X Most 3D face reconstruction methods rely on 3D morphable models, which disentangle the space of facial deformations into identity geometry, expressions and skin reflectance. These models are typically learned from a limited number of 3D scans and thus do not generalize well across different identities and expressions. We present the first approach to learn complete 3D models of face identity geometry, albedo and expression just from images and videos. The virtually endless collection of such data, in combination with our self-supervised learning-based approach allows for learning face models that generalize beyond the span of existing approaches. Our network design and loss functions ensure a disentangled parameterization of not only identity and albedo, but also, for the first time, an expression basis. Our method also allows for in-the-wild monocular reconstruction at test time. We show that our learned models better generalize and lead to higher quality image-based reconstructions than existing approaches. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Artificial Intelligence, cs.AI,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG,Computer Science, Multimedia, cs.MM

Mallikarjun B R, Tewari, A., Oh, T.-H., et al. 2020b. Monocular Reconstruction of Neural Face Reflectance Fields. https://arxiv.org/abs/2008.10247.

(arXiv: 2008.10247)

Abstract

The reflectance field of a face describes the reflectance properties responsible for complex lighting effects including diffuse, specular, inter-reflection and self shadowing. Most existing methods for estimating the face reflectance from a monocular image assume faces to be diffuse with very few approaches adding a specular component. This still leaves out important perceptual aspects of reflectance as higher-order global illumination effects and self-shadowing are not modeled. We present a new neural representation for face reflectance where we can estimate all components of the reflectance responsible for the final appearance from a single monocular image. Instead of modeling each component of the reflectance separately using parametric models, our neural representation allows us to generate a basis set of faces in a geometric deformation-invariant space, parameterized by the input light direction, viewpoint and face geometry. We learn to reconstruct this reflectance field of a face just from a monocular image, which can be used to render the face from any viewpoint in any light condition. Our method is trained on a light-stage training dataset, which captures 300 people illuminated with 150 light conditions from 8 viewpoints. We show that our method outperforms existing monocular reflectance reconstruction methods, in terms of photorealism due to better capturing of physical premitives, such as sub-surface scattering, specularities, self-shadows and other higher-order effects.

Links

Export

BibTeX

@online{Mallikarjun_2008.10247, TITLE = {Monocular Reconstruction of Neural Face Reflectance Fields}, AUTHOR = {Mallikarjun B R and Tewari, Ayush and Oh, Tae-Hyun and Weyrich, Tim and Bickel, Bernd and Seidel, Hans-Peter and Pfister, Hanspeter and Matusik, Wojciech and Elgharib, Mohamed and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2008.10247}, EPRINT = {2008.10247}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {The reflectance field of a face describes the reflectance properties responsible for complex lighting effects including diffuse, specular, inter-reflection and self shadowing. Most existing methods for estimating the face reflectance from a monocular image assume faces to be diffuse with very few approaches adding a specular component. This still leaves out important perceptual aspects of reflectance as higher-order global illumination effects and self-shadowing are not modeled. We present a new neural representation for face reflectance where we can estimate all components of the reflectance responsible for the final appearance from a single monocular image. Instead of modeling each component of the reflectance separately using parametric models, our neural representation allows us to generate a basis set of faces in a geometric deformation-invariant space, parameterized by the input light direction, viewpoint and face geometry. We learn to reconstruct this reflectance field of a face just from a monocular image, which can be used to render the face from any viewpoint in any light condition. Our method is trained on a light-stage training dataset, which captures 300 people illuminated with 150 light conditions from 8 viewpoints. We show that our method outperforms existing monocular reflectance reconstruction methods, in terms of photorealism due to better capturing of physical premitives, such as sub-surface scattering, specularities, self-shadows and other higher-order effects. }, }

Endnote

%0 Report %A Mallikarjun B R, %A Tewari, Ayush %A Oh, Tae-Hyun %A Weyrich, Tim %A Bickel, Bernd %A Seidel, Hans-Peter %A Pfister, Hanspeter %A Matusik, Wojciech %A Elgharib, Mohamed %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Monocular Reconstruction of Neural Face Reflectance Fields : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B110-E %U https://arxiv.org/abs/2008.10247 %D 2020 %X The reflectance field of a face describes the reflectance properties responsible for complex lighting effects including diffuse, specular, inter-reflection and self shadowing. Most existing methods for estimating the face reflectance from a monocular image assume faces to be diffuse with very few approaches adding a specular component. This still leaves out important perceptual aspects of reflectance as higher-order global illumination effects and self-shadowing are not modeled. We present a new neural representation for face reflectance where we can estimate all components of the reflectance responsible for the final appearance from a single monocular image. Instead of modeling each component of the reflectance separately using parametric models, our neural representation allows us to generate a basis set of faces in a geometric deformation-invariant space, parameterized by the input light direction, viewpoint and face geometry. We learn to reconstruct this reflectance field of a face just from a monocular image, which can be used to render the face from any viewpoint in any light condition. Our method is trained on a light-stage training dataset, which captures 300 people illuminated with 150 light conditions from 8 viewpoints. We show that our method outperforms existing monocular reflectance reconstruction methods, in terms of photorealism due to better capturing of physical premitives, such as sub-surface scattering, specularities, self-shadows and other higher-order effects. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Mehta, D., Sotnychenko, O., Mueller, F., et al. 2020. XNect: Real-time Multi-person 3D Human Pose Estimation with a Single RGB Camera. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2020)39, 4.

Links

Export

BibTeX

@article{Mehta_TOG2020, TITLE = {{XNect}: {R}eal-time Multi-person {3D} Human Pose Estimation with a Single {RGB} Camera}, AUTHOR = {Mehta, Dushyant and Sotnychenko, Oleksandr and Mueller, Franziska and Xu, Weipeng and Elgharib, Mohamed and Fua, Pascal and Seidel, Hans-Peter and Rhodin, Helge and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3386569.3392410}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {39}, NUMBER = {4}, EID = {82}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2020}, }

Endnote

%0 Journal Article %A Mehta, Dushyant %A Sotnychenko, Oleksandr %A Mueller, Franziska %A Xu, Weipeng %A Elgharib, Mohamed %A Fua, Pascal %A Seidel, Hans-Peter %A Rhodin, Helge %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T XNect: Real-time Multi-person 3D Human Pose Estimation with a Single RGB Camera : %G eng %U http://hdl.handle.net/21.11116/0000-0007-832D-3 %R 10.1145/3386569.3392410 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 4 %Z sequence number: 82 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2020 %O ACM SIGGRAPH 2020 Virtual Conference ; 2020, 17-28 August

Meka, A., Pandey, R., Häne, C., et al. 2020. Deep Relightable Textures Volumetric Performance Capture with Neural Rendering. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Meka_ToG2020, TITLE = {Deep Relightable Textures Volumetric Performance Capture with Neural Rendering}, AUTHOR = {Meka, Abhimitra and Pandey, Rohit and H{\"a}ne, Christian and Orts-Escolano, Sergio and Barnum, Peter and David-Son, Philip and Erickson, Daniel and Zhang, Yinda and Taylor, Jonathan and Bouaziz, Sofien and Legendre, Chloe and Ma, Wan-Chun and Overbeck, Ryan and Beeler, Thabo and Debevec, Paul and Izadi, Shahram and Theobalt, Christian and Rhemann, Christoph and Fanello, Sean}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417814}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {259}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Meka, Abhimitra %A Pandey, Rohit %A Häne, Christian %A Orts-Escolano, Sergio %A Barnum, Peter %A David-Son, Philip %A Erickson, Daniel %A Zhang, Yinda %A Taylor, Jonathan %A Bouaziz, Sofien %A Legendre, Chloe %A Ma, Wan-Chun %A Overbeck, Ryan %A Beeler, Thabo %A Debevec, Paul %A Izadi, Shahram %A Theobalt, Christian %A Rhemann, Christoph %A Fanello, Sean %+ External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Deep Relightable Textures Volumetric Performance Capture with Neural Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0007-A6FA-4 %R 10.1145/3414685.3417814 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 259 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020 %U https://dl.acm.org/doi/pdf/10.1145/3414685.3417814

Meka, A. 2020. Live inverse rendering. urn:nbn:de:bsz:291--ds-302066.

Links

Export

BibTeX

@phdthesis{Meka_2019, TITLE = {Live inverse rendering}, AUTHOR = {Meka, Abhimitra}, LANGUAGE = {eng}, URL = {urn:nbn:de:bsz:291--ds-302066}, DOI = {http://dx.doi.org/10.22028/D291-30206}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2020}, DATE = {2020}, }

Endnote

%0 Thesis %A Meka, Abhimitra %Y Theobalt, Christian %A referee: Drettakis, George %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Live inverse rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0007-715A-5 %R http://dx.doi.org/10.22028/D291-30206 %U urn:nbn:de:bsz:291--ds-302066 %F OTHER: hdl:20.500.11880/28721 %I Universität des Saarlandes %C Saarbrücken %D 2020 %P 189 p. %V phd %9 phd %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/28721

Meng, X., Zheng, Q., Varshney, A., Singh, G., and Zwicker, M. 2020. Real-time Monte Carlo Denoising with the Neural Bilateral Grid. Rendering 2020 - DL-only Track (Eurographics Symposium on Rendering 2020), The Eurographics Association.

Links

Export

BibTeX

@inproceedings{Meng_EGRendering20, TITLE = {Real-time {Monte Carlo} Denoising with the Neural Bilateral Grid}, AUTHOR = {Meng, Xiaoxu and Zheng, Quan and Varshney, Amitabh and Singh, Gurprit and Zwicker, Matthias}, LANGUAGE = {eng}, ISBN = {978-3-03868-117-5}, URL = {https://diglib.eg.org:443/handle/10.2312/sr20201133}, DOI = {10.2312/sr.20201133}, PUBLISHER = {The Eurographics Association}, YEAR = {2020}, BOOKTITLE = {Rendering 2020 -- DL-only Track (Eurographics Symposium on Rendering 2020)}, EDITOR = {Dachsbacher, Carsten and Pharr, Matt}, PAGES = {1--12}, ADDRESS = {London, UK}, }

Endnote

%0 Conference Proceedings %A Meng, Xiaoxu %A Zheng, Quan %A Varshney, Amitabh %A Singh, Gurprit %A Zwicker, Matthias %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Real-time Monte Carlo Denoising with the Neural Bilateral Grid : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CEC2-6 %R 10.2312/sr.20201133 %U https://diglib.eg.org:443/handle/10.2312/sr20201133 %D 2020 %B 31st Eurographics Symposium on Rendering %Z date of event: 2020-06-29 - 2020-07-02 %C London, UK %B Rendering 2020 - DL-only Track %E Dachsbacher, Carsten; Pharr, Matt %P 1 - 12 %I The Eurographics Association %@ 978-3-03868-117-5

Mlakar, D., Winter, M., Stadlbauer, P., Seidel, H.-P., Steinberger, M., and Zayer, R. 2020. Subdivision-Specialized Linear Algebra Kernels for Static and Dynamic Mesh Connectivity on the GPU. Computer Graphics Forum (Proc. EUROGRAPHICS 2020)39, 2.

Links

Export

BibTeX

@article{Mlakar_EG2020, TITLE = {Subdivision-Specialized Linear Algebra Kernels for Static and Dynamic Mesh Connectivity on the {GPU}}, AUTHOR = {Mlakar, Daniel and Winter, M. and Stadlbauer, Pascal and Seidel, Hans-Peter and Steinberger, Markus and Zayer, Rhaleb}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.13934}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {39}, NUMBER = {2}, PAGES = {335--349}, BOOKTITLE = {The European Association for Computer Graphics 41st Annual Conference (EUROGRAPHICS 2020)}, EDITOR = {Panozzo, Daniele and Assarsson, Ulf}, }

Endnote

%0 Journal Article %A Mlakar, Daniel %A Winter, M. %A Stadlbauer, Pascal %A Seidel, Hans-Peter %A Steinberger, Markus %A Zayer, Rhaleb %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Subdivision-Specialized Linear Algebra Kernels for Static and Dynamic Mesh Connectivity on the GPU : %G eng %U http://hdl.handle.net/21.11116/0000-0006-DB80-2 %R 10.1111/cgf.13934 %7 2020 %D 2020 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 39 %N 2 %& 335 %P 335 - 349 %I Blackwell-Wiley %C Oxford %@ false %B The European Association for Computer Graphics 41st Annual Conference %O EUROGRAPHICS 2020 EG 2020 The European Association for Computer Graphics 41st Annual Conference ; Norrköping, Sweden, May 25 – 29, 2020

Mueller, F. 2020. Real-time 3D Hand Reconstruction in Challenging Scenes from a Single Color or Depth Camera. urn:nbn:de:bsz:291--ds-328467.

Links

Export

BibTeX

@phdthesis{MuellerFDiss_2020, TITLE = {Real-time 3{D} Hand Reconstruction in Challenging Scenes from a Single Color or Depth Camera}, AUTHOR = {Mueller, Franziska}, LANGUAGE = {eng}, URL = {urn:nbn:de:bsz:291--ds-328467}, DOI = {10.22028/D291-32846}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2020}, DATE = {2020}, }

Endnote

%0 Thesis %A Mueller, Franziska %Y Theobalt, Christian %A referee: Seidel, Hans-Peter %A referee: Izadi, Shahram %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Real-time 3D Hand Reconstruction in Challenging Scenes from a Single Color or Depth Camera : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D8C7-5 %R 10.22028/D291-32846 %F OTHER: hdl:20.500.11880/30313 %U urn:nbn:de:bsz:291--ds-328467 %I Universität des Saarlandes %C Saarbrücken %D 2020 %P 155 p. %V phd %9 phd %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/30313

Piovarči, M., Foshey, M., Babaei, V., Rusinkiewicz, S., Matusik, W., and Didyk, P. 2020. Towards Spatially Varying Gloss Reproduction for 3D Printing. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Piovarci_ToG2020, TITLE = {Towards Spatially Varying Gloss Reproduction for {3D} Printing}, AUTHOR = {Piovar{\v c}i, Michal and Foshey, Michael and Babaei, Vahid and Rusinkiewicz, Szymon and Matusik, Wojciech and Didyk, Piotr}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417850}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {206}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Piovarči, Michal %A Foshey, Michael %A Babaei, Vahid %A Rusinkiewicz, Szymon %A Matusik, Wojciech %A Didyk, Piotr %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations %T Towards Spatially Varying Gloss Reproduction for 3D Printing : %G eng %U http://hdl.handle.net/21.11116/0000-0007-A6FE-0 %R 10.1145/3414685.3417850 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 206 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Qian, N., Wang, J., Mueller, F., Bernard, F., Golyanik, V., and Theobalt, C. 2020a. HTML: A Parametric Hand Texture Model for 3D Hand Reconstruction and Personalization. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Qian_ECCV20, TITLE = {{HTML}: {A} Parametric Hand Texture Model for {3D} Hand Reconstruction and Personalization}, AUTHOR = {Qian, Neng and Wang, Jiayi and Mueller, Franziska and Bernard, Florian and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-58621-8}, DOI = {10.1007/978-3-030-58621-8_4}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {54--71}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12356}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Qian, Neng %A Wang, Jiayi %A Mueller, Franziska %A Bernard, Florian %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T HTML: A Parametric Hand Texture Model for 3D Hand Reconstruction and Personalization : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D062-F %R 10.1007/978-3-030-58621-8_4 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 54 - 71 %I Springer %@ 978-3-030-58621-8 %B Lecture Notes in Computer Science %N 12356

Qian, N., Wang, J., Mueller, F., Bernard, F., Golyanik, V., and Theobalt, C. 2020b. Parametric Hand Texture Model for 3D Hand Reconstruction and Personalization. Max-Planck-Institut für Informatik, Saarbrücken.

[PuRe]

[BibTeX]

[fulltext version]

Abstract

3D hand reconstruction from image data is a widely-studied problem in com- puter vision and graphics, and has a particularly high relevance for virtual and augmented reality. Although several 3D hand reconstruction approaches leverage hand models as a strong prior to resolve ambiguities and achieve a more robust reconstruction, most existing models account only for the hand shape and poses and do not model the texture. To ﬁll this gap, in this work we present the ﬁrst parametric texture model of human hands. Our model spans several dimensions of hand appearance variability (e.g., related to gen- der, ethnicity, or age) and only requires a commodity camera for data acqui- sition. Experimentally, we demonstrate that our appearance model can be used to tackle a range of challenging problems such as 3D hand reconstruc- tion from a single monocular image. Furthermore, our appearance model can be used to deﬁne a neural rendering layer that enables training with a self-supervised photometric loss. We make our model publicly available.

Links

[PuRe]

[BibTeX]

[fulltext version]

Export

BibTeX

@techreport{Qian_report2020, TITLE = {Parametric Hand Texture Model for {3D} Hand Reconstruction and Personalization}, AUTHOR = {Qian, Neng and Wang, Jiayi and Mueller, Franziska and Bernard, Florian and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0946-011X}, NUMBER = {MPI-I-2020-4-001}, INSTITUTION = {Max-Planck-Institut f{\"u}r Informatik}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2020}, ABSTRACT = {3D hand reconstruction from image data is a widely-studied problem in com- puter vision and graphics, and has a particularly high relevance for virtual and augmented reality. Although several 3D hand reconstruction approaches leverage hand models as a strong prior to resolve ambiguities and achieve a more robust reconstruction, most existing models account only for the hand shape and poses and do not model the texture. To {fi}ll this gap, in this work we present the {fi}rst parametric texture model of human hands. Our model spans several dimensions of hand appearance variability (e.g., related to gen- der, ethnicity, or age) and only requires a commodity camera for data acqui- sition. Experimentally, we demonstrate that our appearance model can be used to tackle a range of challenging problems such as 3D hand reconstruc- tion from a single monocular image. Furthermore, our appearance model can be used to de{fi}ne a neural rendering layer that enables training with a self-supervised photometric loss. We make our model publicly available.}, TYPE = {Research Report}, }

Endnote

%0 Report %A Qian, Neng %A Wang, Jiayi %A Mueller, Franziska %A Bernard, Florian %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Parametric Hand Texture Model for 3D Hand Reconstruction and Personalization : %G eng %U http://hdl.handle.net/21.11116/0000-0006-9128-9 %Y Max-Planck-Institut für Informatik %C Saarbrücken %D 2020 %P 37 p. %X 3D hand reconstruction from image data is a widely-studied problem in com- puter vision and graphics, and has a particularly high relevance for virtual and augmented reality. Although several 3D hand reconstruction approaches leverage hand models as a strong prior to resolve ambiguities and achieve a more robust reconstruction, most existing models account only for the hand shape and poses and do not model the texture. To ﬁll this gap, in this work we present the ﬁrst parametric texture model of human hands. Our model spans several dimensions of hand appearance variability (e.g., related to gen- der, ethnicity, or age) and only requires a commodity camera for data acqui- sition. Experimentally, we demonstrate that our appearance model can be used to tackle a range of challenging problems such as 3D hand reconstruc- tion from a single monocular image. Furthermore, our appearance model can be used to deﬁne a neural rendering layer that enables training with a self-supervised photometric loss. We make our model publicly available. %K hand texture model, appearance modeling, hand tracking, 3D hand recon- struction %B Research Report %@ false

Rao, S., Stutz, D., and Schiele, B. 2020. Adversarial Training against Location-Optimized Adversarial Patches. https://arxiv.org/abs/2005.02313.

(arXiv: 2005.02313)

Abstract

Deep neural networks have been shown to be susceptible to adversarial examples -- small, imperceptible changes constructed to cause mis-classification in otherwise highly accurate image classifiers. As a practical alternative, recent work proposed so-called adversarial patches: clearly visible, but adversarially crafted rectangular patches in images. These patches can easily be printed and applied in the physical world. While defenses against imperceptible adversarial examples have been studied extensively, robustness against adversarial patches is poorly understood. In this work, we first devise a practical approach to obtain adversarial patches while actively optimizing their location within the image. Then, we apply adversarial training on these location-optimized adversarial patches and demonstrate significantly improved robustness on CIFAR10 and GTSRB. Additionally, in contrast to adversarial training on imperceptible adversarial examples, our adversarial patch training does not reduce accuracy.

Links

Export

BibTeX

@online{Rao_arXiv2005.02313, TITLE = {Adversarial Training against Location-Optimized Adversarial Patches}, AUTHOR = {Rao, Sukrut and Stutz, David and Schiele, Bernt}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2005.02313}, EPRINT = {2005.02313}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Deep neural networks have been shown to be susceptible to adversarial examples -- small, imperceptible changes constructed to cause mis-classification in otherwise highly accurate image classifiers. As a practical alternative, recent work proposed so-called adversarial patches: clearly visible, but adversarially crafted rectangular patches in images. These patches can easily be printed and applied in the physical world. While defenses against imperceptible adversarial examples have been studied extensively, robustness against adversarial patches is poorly understood. In this work, we first devise a practical approach to obtain adversarial patches while actively optimizing their location within the image. Then, we apply adversarial training on these location-optimized adversarial patches and demonstrate significantly improved robustness on CIFAR10 and GTSRB. Additionally, in contrast to adversarial training on imperceptible adversarial examples, our adversarial patch training does not reduce accuracy. }, }

Endnote

%0 Report %A Rao, Sukrut %A Stutz, David %A Schiele, Bernt %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Adversarial Training against Location-Optimized Adversarial Patches : %G eng %U http://hdl.handle.net/21.11116/0000-0007-80D0-C %U https://arxiv.org/abs/2005.02313 %D 2020 %X Deep neural networks have been shown to be susceptible to adversarial examples -- small, imperceptible changes constructed to cause mis-classification in otherwise highly accurate image classifiers. As a practical alternative, recent work proposed so-called adversarial patches: clearly visible, but adversarially crafted rectangular patches in images. These patches can easily be printed and applied in the physical world. While defenses against imperceptible adversarial examples have been studied extensively, robustness against adversarial patches is poorly understood. In this work, we first devise a practical approach to obtain adversarial patches while actively optimizing their location within the image. Then, we apply adversarial training on these location-optimized adversarial patches and demonstrate significantly improved robustness on CIFAR10 and GTSRB. Additionally, in contrast to adversarial training on imperceptible adversarial examples, our adversarial patch training does not reduce accuracy. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Cryptography and Security, cs.CR,Computer Science, Learning, cs.LG,Statistics, Machine Learning, stat.ML

Saberpour, A., Hersch, R.D., Fang, J., Zayer, R., Seidel, H.-P., and Babaei, V. 2020. Fabrication of Moiré on Curved Surfaces. Optics Express28, 13.

Links

Export

BibTeX

@article{Saberpour2020, TITLE = {Fabrication of Moir{\'e} on Curved Surfaces}, AUTHOR = {Saberpour, Artin and Hersch, Roger D. and Fang, Jiajing and Zayer, Rhaleb and Seidel, Hans-Peter and Babaei, Vahid}, LANGUAGE = {eng}, ISSN = {1094-4087}, DOI = {10.1364/OE.393843}, PUBLISHER = {Optical Society of America}, ADDRESS = {Washington, DC}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Optics Express}, VOLUME = {28}, NUMBER = {13}, PAGES = {19413--19427}, }

Endnote

%0 Journal Article %A Saberpour, Artin %A Hersch, Roger D. %A Fang, Jiajing %A Zayer, Rhaleb %A Seidel, Hans-Peter %A Babaei, Vahid %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Fabrication of Moiré on Curved Surfaces : %G eng %U http://hdl.handle.net/21.11116/0000-0006-D39D-B %R 10.1364/OE.393843 %7 2020 %D 2020 %J Optics Express %O Opt. Express %V 28 %N 13 %& 19413 %P 19413 - 19427 %I Optical Society of America %C Washington, DC %@ false

Sarkar, K., Mehta, D., Xu, W., Golyanik, V., and Theobalt, C. 2020. Neural Re-rendering of Humans from a Single Image. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Sarkar_ECCV20, TITLE = {Neural Re-rendering of Humans from a Single Image}, AUTHOR = {Sarkar, Kripasindhu and Mehta, Dushyant and Xu, Weipeng and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-58621-8}, DOI = {10.1007/978-3-030-58621-8_35}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {596--613}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12356}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Sarkar, Kripasindhu %A Mehta, Dushyant %A Xu, Weipeng %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Re-rendering of Humans from a Single Image : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D0A4-4 %R 10.1007/978-3-030-58621-8_35 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 596 - 613 %I Springer %@ 978-3-030-58621-8 %B Lecture Notes in Computer Science %N 12356

Seelbach Benkner, M., Golyanik, V., Theobalt, C., and Moeller, M. 2020. Adiabatic Quantum Graph Matching with Permutation Matrix Constraints. International Conference on 3D Vision, IEEE.

Links

Export

BibTeX

@inproceedings{SeelbachBenkner2020, TITLE = {Adiabatic Quantum Graph Matching with Permutation Matrix Constraints}, AUTHOR = {Seelbach Benkner, Marcel and Golyanik, Vladislav and Theobalt, Christian and Moeller, Michael}, LANGUAGE = {eng}, ISBN = {978-1-7281-8128-8}, DOI = {10.1109/3DV50981.2020.00068}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {International Conference on 3D Vision}, PAGES = {583--592}, ADDRESS = {Fukuoka, Japan (Virtual Event)}, }

Endnote

%0 Conference Proceedings %A Seelbach Benkner, Marcel %A Golyanik, Vladislav %A Theobalt, Christian %A Moeller, Michael %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Adiabatic Quantum Graph Matching with Permutation Matrix Constraints : %G eng %U http://hdl.handle.net/21.11116/0000-0008-0362-6 %R 10.1109/3DV50981.2020.00068 %D 2020 %B International Conference on 3D Vision %Z date of event: 2020-11-25 - 2020-11-28 %C Fukuoka, Japan (Virtual Event) %B International Conference on 3D Vision %P 583 - 592 %I IEEE %@ 978-1-7281-8128-8

Serrano, A., Martin, D., Gutierrez, D., Myszkowski, K., and Masia, B. 2020. Imperceptible Manipulation of Lateral Camera Motion for Improved Virtual Reality Applications. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Serrano2020, TITLE = {Imperceptible Manipulation of Lateral Camera Motion for Improved Virtual Reality Applications}, AUTHOR = {Serrano, Ana and Martin, Daniel and Gutierrez, Diego and Myszkowski, Karol and Masia, Belen}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417773}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {268}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Serrano, Ana %A Martin, Daniel %A Gutierrez, Diego %A Myszkowski, Karol %A Masia, Belen %+ External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Imperceptible Manipulation of Lateral Camera Motion for Improved Virtual Reality Applications : %G eng %U http://hdl.handle.net/21.11116/0000-0006-FBE8-A %R 10.1145/3414685.3417773 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 268 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Shahmirzadi, A.A., Babaei, V., and Seidel, H.-P. 2020. A Multispectral Dataset of Oil and Watercolor Paints. Electronic Imaging.

Links

Export

BibTeX

@article{shahmirzadi2020multispectral, TITLE = {A Multispectral Dataset of Oil and Watercolor Paints}, AUTHOR = {Shahmirzadi, Azadeh Asadi and Babaei, Vahid and Seidel, Hans-Peter}, LANGUAGE = {eng}, DOI = {10.2352/ISSN.2470-1173.2020.5.MAAP-107}, PUBLISHER = {IS\&T}, ADDRESS = {Springfield, VA}, YEAR = {2020}, JOURNAL = {Electronic Imaging}, PAGES = {1--4}, EID = {107}, }

Endnote

%0 Journal Article %A Shahmirzadi, Azadeh Asadi %A Babaei, Vahid %A Seidel, Hans-Peter %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T A Multispectral Dataset of Oil and Watercolor Paints : %G eng %U http://hdl.handle.net/21.11116/0000-0007-F064-9 %R 10.2352/ISSN.2470-1173.2020.5.MAAP-107 %7 2020 %D 2020 %J Electronic Imaging %& 1 %P 1 - 4 %Z sequence number: 107 %I IS&T %C Springfield, VA

Shimada, S., Golyanik, V., Xu, W., and Theobalt, C. 2020a. PhysCap: Physically Plausible Monocular 3D Motion Capture in Real Time. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Shimada_ToG2020, TITLE = {{PhysCap}: {P}hysically Plausible Monocular {3D} Motion Capture in Real Time}, AUTHOR = {Shimada, Soshi and Golyanik, Vladislav and Xu, Weipeng and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417877}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {235}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Shimada, Soshi %A Golyanik, Vladislav %A Xu, Weipeng %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T PhysCap: Physically Plausible Monocular 3D Motion Capture in Real Time : %G eng %U http://hdl.handle.net/21.11116/0000-0007-A709-3 %R 10.1145/3414685.3417877 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 235 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Shimada, S., Golyanik, V., Xu, W., and Theobalt, C. 2020b. PhysCap: Physically Plausible Monocular 3D Motion Capture in Real Time. https://arxiv.org/abs/2008.08880.

(arXiv: 2008.08880)

Abstract

Marker-less 3D human motion capture from a single colour camera has seen significant progress. However, it is a very challenging and severely ill-posed problem. In consequence, even the most accurate state-of-the-art approaches have significant limitations. Purely kinematic formulations on the basis of individual joints or skeletons, and the frequent frame-wise reconstruction in state-of-the-art methods greatly limit 3D accuracy and temporal stability compared to multi-view or marker-based motion capture. Further, captured 3D poses are often physically incorrect and biomechanically implausible, or exhibit implausible environment interactions (floor penetration, foot skating, unnatural body leaning and strong shifting in depth), which is problematic for any use case in computer graphics. We, therefore, present PhysCap, the first algorithm for physically plausible, real-time and marker-less human 3D motion capture with a single colour camera at 25 fps. Our algorithm first captures 3D human poses purely kinematically. To this end, a CNN infers 2D and 3D joint positions, and subsequently, an inverse kinematics step finds space-time coherent joint angles and global 3D pose. Next, these kinematic reconstructions are used as constraints in a real-time physics-based pose optimiser that accounts for environment constraints (e.g., collision handling and floor placement), gravity, and biophysical plausibility of human postures. Our approach employs a combination of ground reaction force and residual force for plausible root control, and uses a trained neural network to detect foot contact events in images. Our method captures physically plausible and temporally stable global 3D human motion, without physically implausible postures, floor penetrations or foot skating, from video in real time and in general scenes. The video is available at http://gvv.mpi-inf.mpg.de/projects/PhysCap

Links

Export

BibTeX

@online{Shimada_2008.08880, TITLE = {{PhysCap}: {P}hysically Plausible Monocular {3D} Motion Capture in Real Time}, AUTHOR = {Shimada, Soshi and Golyanik, Vladislav and Xu, Weipeng and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2008.08880}, EPRINT = {2008.08880}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Marker-less 3D human motion capture from a single colour camera has seen significant progress. However, it is a very challenging and severely ill-posed problem. In consequence, even the most accurate state-of-the-art approaches have significant limitations. Purely kinematic formulations on the basis of individual joints or skeletons, and the frequent frame-wise reconstruction in state-of-the-art methods greatly limit 3D accuracy and temporal stability compared to multi-view or marker-based motion capture. Further, captured 3D poses are often physically incorrect and biomechanically implausible, or exhibit implausible environment interactions (floor penetration, foot skating, unnatural body leaning and strong shifting in depth), which is problematic for any use case in computer graphics. We, therefore, present PhysCap, the first algorithm for physically plausible, real-time and marker-less human 3D motion capture with a single colour camera at 25 fps. Our algorithm first captures 3D human poses purely kinematically. To this end, a CNN infers 2D and 3D joint positions, and subsequently, an inverse kinematics step finds space-time coherent joint angles and global 3D pose. Next, these kinematic reconstructions are used as constraints in a real-time physics-based pose optimiser that accounts for environment constraints (e.g., collision handling and floor placement), gravity, and biophysical plausibility of human postures. Our approach employs a combination of ground reaction force and residual force for plausible root control, and uses a trained neural network to detect foot contact events in images. Our method captures physically plausible and temporally stable global 3D human motion, without physically implausible postures, floor penetrations or foot skating, from video in real time and in general scenes. The video is available at http://gvv.mpi-inf.mpg.de/projects/PhysCap }, }

Endnote

%0 Report %A Shimada, Soshi %A Golyanik, Vladislav %A Xu, Weipeng %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T PhysCap: Physically Plausible Monocular 3D Motion Capture in Real Time : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8F3-1 %U https://arxiv.org/abs/2008.08880 %D 2020 %X Marker-less 3D human motion capture from a single colour camera has seen significant progress. However, it is a very challenging and severely ill-posed problem. In consequence, even the most accurate state-of-the-art approaches have significant limitations. Purely kinematic formulations on the basis of individual joints or skeletons, and the frequent frame-wise reconstruction in state-of-the-art methods greatly limit 3D accuracy and temporal stability compared to multi-view or marker-based motion capture. Further, captured 3D poses are often physically incorrect and biomechanically implausible, or exhibit implausible environment interactions (floor penetration, foot skating, unnatural body leaning and strong shifting in depth), which is problematic for any use case in computer graphics. We, therefore, present PhysCap, the first algorithm for physically plausible, real-time and marker-less human 3D motion capture with a single colour camera at 25 fps. Our algorithm first captures 3D human poses purely kinematically. To this end, a CNN infers 2D and 3D joint positions, and subsequently, an inverse kinematics step finds space-time coherent joint angles and global 3D pose. Next, these kinematic reconstructions are used as constraints in a real-time physics-based pose optimiser that accounts for environment constraints (e.g., collision handling and floor placement), gravity, and biophysical plausibility of human postures. Our approach employs a combination of ground reaction force and residual force for plausible root control, and uses a trained neural network to detect foot contact events in images. Our method captures physically plausible and temporally stable global 3D human motion, without physically implausible postures, floor penetrations or foot skating, from video in real time and in general scenes. The video is available at http://gvv.mpi-inf.mpg.de/projects/PhysCap %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Sidhu, V., Tretschk, E., Golyanik, V., Agudo, A., and Theobalt, C. 2020. Neural Dense Non-Rigid Structure from Motion with Latent Space Constraints. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Sidhu_ECCV20, TITLE = {Neural Dense Non-Rigid Structure from Motion with Latent Space Constraints}, AUTHOR = {Sidhu, Vikramjit and Tretschk, Edgar and Golyanik, Vladislav and Agudo, Antonio and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-58516-7}, DOI = {10.1007/978-3-030-58517-4_13}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {204--222}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12361}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Sidhu, Vikramjit %A Tretschk, Edgar %A Golyanik, Vladislav %A Agudo, Antonio %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Dense Non-Rigid Structure from Motion with Latent Space Constraints : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D080-C %R 10.1007/978-3-030-58517-4_13 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 204 - 222 %I Springer %@ 978-3-030-58516-7 %B Lecture Notes in Computer Science %N 12361

Singh, G., Subr, K., Coeurjolly, D., Ostromoukhov, V., and Jarosz, W. 2020. Fourier Analysis of Correlated Monte Carlo Importance Sampling. Computer Graphics Forum39, 1.

Links

Export

BibTeX

@article{SinghCGF2020, TITLE = {Fourier Analysis of Correlated {Monte Carlo} Importance Sampling}, AUTHOR = {Singh, Gurprit and Subr, Kartic and Coeurjolly, David and Ostromoukhov, Victor and Jarosz, Wojciech}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.13613}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Computer Graphics Forum}, VOLUME = {39}, NUMBER = {1}, PAGES = {7--19}, }

Endnote

%0 Journal Article %A Singh, Gurprit %A Subr, Kartic %A Coeurjolly, David %A Ostromoukhov, Victor %A Jarosz, Wojciech %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T Fourier Analysis of Correlated Monte Carlo Importance Sampling : %G eng %U http://hdl.handle.net/21.11116/0000-0006-978D-1 %R 10.1111/cgf.13613 %7 2020 %D 2020 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 39 %N 1 %& 7 %P 7 - 19 %I Blackwell-Wiley %C Oxford %@ false

Stadlbauer, P., Mlakar, D., Seidel, H.-P., Steinberger, M., and Zayer, R. 2020. Interactive Modeling of Cellular Structures on Surfaces with Application to Additive Manufacturing. Computer Graphics Forum (Proc. EUROGRAPHICS 2020)39, 2.

Links

Export

BibTeX

@article{Stadlbauer_EG2020, TITLE = {Interactive Modeling of Cellular Structures on Surfaces with Application to Additive Manufacturing}, AUTHOR = {Stadlbauer, Pascal and Mlakar, Daniel and Seidel, Hans-Peter and Steinberger, Markus and Zayer, Rhaleb}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.13929}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {39}, NUMBER = {2}, PAGES = {277--289}, BOOKTITLE = {The European Association for Computer Graphics 41st Annual Conference (EUROGRAPHICS 2020)}, EDITOR = {Panozzo, Daniele and Assarsson, Ulf}, }

Endnote

%0 Journal Article %A Stadlbauer, Pascal %A Mlakar, Daniel %A Seidel, Hans-Peter %A Steinberger, Markus %A Zayer, Rhaleb %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Interactive Modeling of Cellular Structures on Surfaces with Application to Additive Manufacturing : %G eng %U http://hdl.handle.net/21.11116/0000-0006-DB8A-8 %R 10.1111/cgf.13929 %7 2020 %D 2020 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 39 %N 2 %& 277 %P 277 - 289 %I Blackwell-Wiley %C Oxford %@ false %B The European Association for Computer Graphics 41st Annual Conference %O EUROGRAPHICS 2020 EG 2020 The European Association for Computer Graphics 41st Annual Conference ; Norrköping, Sweden, May 25 – 29, 2020

Sultan, A.S., Elgharib, M., Tavares, T., Jessri, M., and Basile, J.R. 2020. The Use of Artificial Intelligence, Machine Learning and Deep Learning in Oncologic Histopathology. Journal of Oral Pathology & Medicine49, 9.

Links

Export

BibTeX

@article{Sultan2020, TITLE = {The Use of Artificial Intelligence, Machine Learning and Deep Learning in Oncologic Histopathology}, AUTHOR = {Sultan, Ahmed S. and Elgharib, Mohamed and Tavares, Tiffany and Jessri, Maryam and Basile, John R.}, LANGUAGE = {eng}, ISSN = {0904-2512}, DOI = {10.1111/jop.13042}, PUBLISHER = {Wiley-Blackwell}, ADDRESS = {Oxford}, YEAR = {2020}, JOURNAL = {Journal of Oral Pathology \& Medicine}, VOLUME = {49}, NUMBER = {9}, PAGES = {849--856}, }

Endnote

%0 Journal Article %A Sultan, Ahmed S. %A Elgharib, Mohamed %A Tavares, Tiffany %A Jessri, Maryam %A Basile, John R. %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations %T The Use of Artificial Intelligence, Machine Learning and Deep Learning in Oncologic Histopathology : %G eng %U http://hdl.handle.net/21.11116/0000-0006-A2C9-0 %R 10.1111/jop.13042 %7 2020 %D 2020 %J Journal of Oral Pathology & Medicine %V 49 %N 9 %& 849 %P 849 - 856 %I Wiley-Blackwell %C Oxford %@ false

Tewari, A., Elgharib, M., Bharaj, G., et al. 2020a. StyleRig: Rigging StyleGAN for 3D Control Over Portrait Images. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Tewari_CVPR2020, TITLE = {{StyleRig}: {R}igging {StyleGAN} for {3D} Control Over Portrait Images}, AUTHOR = {Tewari, Ayush and Elgharib, Mohamed and Bharaj, Gaurav and Bernard, Florian and Seidel, Hans-Peter and P{\'e}rez, Patrick and Zollh{\"o}fer, Michael and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00618}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {6141--6150}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Tewari, Ayush %A Elgharib, Mohamed %A Bharaj, Gaurav %A Bernard, Florian %A Seidel, Hans-Peter %A Pérez, Patrick %A Zollhöfer, Michael %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T StyleRig: Rigging StyleGAN for 3D Control Over Portrait Images : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B0E7-D %R 10.1109/CVPR42600.2020.00618 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 6141 - 6150 %I IEEE %@ 978-1-7281-7168-5

Tewari, A., Fried, O., Thies, J., et al. 2020b. State of the Art on Neural Rendering. https://arxiv.org/abs/2004.03805.

(arXiv: 2004.03805)

Abstract

Efficient rendering of photo-realistic virtual worlds is a long standing effort of computer graphics. Modern graphics techniques have succeeded in synthesizing photo-realistic images from hand-crafted scene representations. However, the automatic generation of shape, materials, lighting, and other aspects of scenes remains a challenging problem that, if solved, would make photo-realistic computer graphics more widely accessible. Concurrently, progress in computer vision and machine learning have given rise to a new approach to image synthesis and editing, namely deep generative models. Neural rendering is a new and rapidly emerging field that combines generative machine learning techniques with physical knowledge from computer graphics, e.g., by the integration of differentiable rendering into network training. With a plethora of applications in computer graphics and vision, neural rendering is poised to become a new area in the graphics community, yet no survey of this emerging field exists. This state-of-the-art report summarizes the recent trends and applications of neural rendering. We focus on approaches that combine classic computer graphics techniques with deep generative models to obtain controllable and photo-realistic outputs. Starting with an overview of the underlying computer graphics and machine learning concepts, we discuss critical aspects of neural rendering approaches. This state-of-the-art report is focused on the many important use cases for the described algorithms such as novel view synthesis, semantic photo manipulation, facial and body reenactment, relighting, free-viewpoint video, and the creation of photo-realistic avatars for virtual and augmented reality telepresence. Finally, we conclude with a discussion of the social implications of such technology and investigate open research problems.

Links

Export

BibTeX

@online{Tewari2004.03805, TITLE = {State of the Art on Neural Rendering}, AUTHOR = {Tewari, Ayush and Fried, Ohad and Thies, Justus and Sitzmann, Vincent and Lombardi, Stephen and Sunkavalli, Kalyan and Martin-Brualla, Ricardo and Simon, Tomas and Saragih, Jason and Nie{\ss}ner, Matthias and Pandey, Rohit and Fanello, Sean and Wetzstein, Gordon and Zhu, Jun-Yan and Theobalt, Christian and Agrawala, Maneesh and Shechtman, Eli and Goldman, Dan B and Zollh{\"o}fer, Michael}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2004.03805}, EPRINT = {2004.03805}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Efficient rendering of photo-realistic virtual worlds is a long standing effort of computer graphics. Modern graphics techniques have succeeded in synthesizing photo-realistic images from hand-crafted scene representations. However, the automatic generation of shape, materials, lighting, and other aspects of scenes remains a challenging problem that, if solved, would make photo-realistic computer graphics more widely accessible. Concurrently, progress in computer vision and machine learning have given rise to a new approach to image synthesis and editing, namely deep generative models. Neural rendering is a new and rapidly emerging field that combines generative machine learning techniques with physical knowledge from computer graphics, e.g., by the integration of differentiable rendering into network training. With a plethora of applications in computer graphics and vision, neural rendering is poised to become a new area in the graphics community, yet no survey of this emerging field exists. This state-of-the-art report summarizes the recent trends and applications of neural rendering. We focus on approaches that combine classic computer graphics techniques with deep generative models to obtain controllable and photo-realistic outputs. Starting with an overview of the underlying computer graphics and machine learning concepts, we discuss critical aspects of neural rendering approaches. This state-of-the-art report is focused on the many important use cases for the described algorithms such as novel view synthesis, semantic photo manipulation, facial and body reenactment, relighting, free-viewpoint video, and the creation of photo-realistic avatars for virtual and augmented reality telepresence. Finally, we conclude with a discussion of the social implications of such technology and investigate open research problems. }, }

Endnote

%0 Report %A Tewari, Ayush %A Fried, Ohad %A Thies, Justus %A Sitzmann, Vincent %A Lombardi, Stephen %A Sunkavalli, Kalyan %A Martin-Brualla, Ricardo %A Simon, Tomas %A Saragih, Jason %A Nießner, Matthias %A Pandey, Rohit %A Fanello, Sean %A Wetzstein, Gordon %A Zhu, Jun-Yan %A Theobalt, Christian %A Agrawala, Maneesh %A Shechtman, Eli %A Goldman, Dan B %A Zollhöfer, Michael %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T State of the Art on Neural Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E114-4 %U https://arxiv.org/abs/2004.03805 %D 2020 %X Efficient rendering of photo-realistic virtual worlds is a long standing effort of computer graphics. Modern graphics techniques have succeeded in synthesizing photo-realistic images from hand-crafted scene representations. However, the automatic generation of shape, materials, lighting, and other aspects of scenes remains a challenging problem that, if solved, would make photo-realistic computer graphics more widely accessible. Concurrently, progress in computer vision and machine learning have given rise to a new approach to image synthesis and editing, namely deep generative models. Neural rendering is a new and rapidly emerging field that combines generative machine learning techniques with physical knowledge from computer graphics, e.g., by the integration of differentiable rendering into network training. With a plethora of applications in computer graphics and vision, neural rendering is poised to become a new area in the graphics community, yet no survey of this emerging field exists. This state-of-the-art report summarizes the recent trends and applications of neural rendering. We focus on approaches that combine classic computer graphics techniques with deep generative models to obtain controllable and photo-realistic outputs. Starting with an overview of the underlying computer graphics and machine learning concepts, we discuss critical aspects of neural rendering approaches. This state-of-the-art report is focused on the many important use cases for the described algorithms such as novel view synthesis, semantic photo manipulation, facial and body reenactment, relighting, free-viewpoint video, and the creation of photo-realistic avatars for virtual and augmented reality telepresence. Finally, we conclude with a discussion of the social implications of such technology and investigate open research problems. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Tewari, A., Elgharib, M., Bharaj, G., et al. 2020c. StyleRig: Rigging StyleGAN for 3D Control over Portrait Images. https://arxiv.org/abs/2004.00121.

(arXiv: 2004.00121)

Abstract

StyleGAN generates photorealistic portrait images of faces with eyes, teeth, hair and context (neck, shoulders, background), but lacks a rig-like control over semantic face parameters that are interpretable in 3D, such as face pose, expressions, and scene illumination. Three-dimensional morphable face models (3DMMs) on the other hand offer control over the semantic parameters, but lack photorealism when rendered and only model the face interior, not other parts of a portrait image (hair, mouth interior, background). We present the first method to provide a face rig-like control over a pretrained and fixed StyleGAN via a 3DMM. A new rigging network, RigNet is trained between the 3DMM's semantic parameters and StyleGAN's input. The network is trained in a self-supervised manner, without the need for manual annotations. At test time, our method generates portrait images with the photorealism of StyleGAN and provides explicit control over the 3D semantic parameters of the face.

Links

Export

BibTeX

@online{Tewari_2004.00121, TITLE = {{StyleRig}: Rigging {StyleGAN} for {3D} Control over Portrait Images}, AUTHOR = {Tewari, Ayush and Elgharib, Mohamed and Bharaj, Gaurav and Bernard, Florian and Seidel, Hans-Peter and P{\'e}rez, Patrick and Zollh{\"o}fer, Michael and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2004.00121}, EPRINT = {2004.00121}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {StyleGAN generates photorealistic portrait images of faces with eyes, teeth, hair and context (neck, shoulders, background), but lacks a rig-like control over semantic face parameters that are interpretable in 3D, such as face pose, expressions, and scene illumination. Three-dimensional morphable face models (3DMMs) on the other hand offer control over the semantic parameters, but lack photorealism when rendered and only model the face interior, not other parts of a portrait image (hair, mouth interior, background). We present the first method to provide a face rig-like control over a pretrained and fixed StyleGAN via a 3DMM. A new rigging network, RigNet is trained between the 3DMM's semantic parameters and StyleGAN's input. The network is trained in a self-supervised manner, without the need for manual annotations. At test time, our method generates portrait images with the photorealism of StyleGAN and provides explicit control over the 3D semantic parameters of the face. }, }

Endnote

%0 Report %A Tewari, Ayush %A Elgharib, Mohamed %A Bharaj, Gaurav %A Bernard, Florian %A Seidel, Hans-Peter %A Pérez, Patrick %A Zollhöfer, Michael %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T StyleRig: Rigging StyleGAN for 3D Control over Portrait Images : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B0FC-6 %U https://arxiv.org/abs/2004.00121 %D 2020 %X StyleGAN generates photorealistic portrait images of faces with eyes, teeth, hair and context (neck, shoulders, background), but lacks a rig-like control over semantic face parameters that are interpretable in 3D, such as face pose, expressions, and scene illumination. Three-dimensional morphable face models (3DMMs) on the other hand offer control over the semantic parameters, but lack photorealism when rendered and only model the face interior, not other parts of a portrait image (hair, mouth interior, background). We present the first method to provide a face rig-like control over a pretrained and fixed StyleGAN via a 3DMM. A new rigging network, RigNet is trained between the 3DMM's semantic parameters and StyleGAN's input. The network is trained in a self-supervised manner, without the need for manual annotations. At test time, our method generates portrait images with the photorealism of StyleGAN and provides explicit control over the 3D semantic parameters of the face. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Tewari, A., Zollhöfer, M., Bernard, F., et al. 2020d. High-Fidelity Monocular Face Reconstruction based on an Unsupervised Model-based Face Autoencoder. IEEE Transactions on Pattern Analysis and Machine Intelligence42, 2.

Links

Export

BibTeX

@article{8496850, TITLE = {High-Fidelity Monocular Face Reconstruction based on an Unsupervised Model-based Face Autoencoder}, AUTHOR = {Tewari, Ayush and Zollh{\"o}fer, Michael and Bernard, Florian and Garrido, Pablo and Kim, Hyeongwoo and P{\'e}rez, Patrick and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0162-8828}, DOI = {10.1109/TPAMI.2018.2876842}, PUBLISHER = {IEEE}, ADDRESS = {Piscataway, NJ}, YEAR = {2020}, DATE = {2020}, JOURNAL = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, VOLUME = {42}, NUMBER = {2}, PAGES = {357--370}, }

Endnote

%0 Journal Article %A Tewari, Ayush %A Zollhöfer, Michael %A Bernard, Florian %A Garrido, Pablo %A Kim, Hyeongwoo %A Pérez, Patrick %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T High-Fidelity Monocular Face Reconstruction based on an Unsupervised Model-based Face Autoencoder : %G eng %U http://hdl.handle.net/21.11116/0000-0002-EF5B-C %R 10.1109/TPAMI.2018.2876842 %7 2018 %D 2020 %J IEEE Transactions on Pattern Analysis and Machine Intelligence %O IEEE Trans. Pattern Anal. Mach. Intell. %V 42 %N 2 %& 357 %P 357 - 370 %I IEEE %C Piscataway, NJ %@ false

Tewari, A., Fried, O., Thies, J., et al. 2020e. State of the Art on Neural Rendering. Computer Graphics Forum (Proc. EUROGRAPHICS 2020)39, 2.

Links

Export

BibTeX

@article{Tewari_EG2020, TITLE = {State of the Art on Neural Rendering}, AUTHOR = {Tewari, Ayush and Fried, Ohad and Thies, Justus and Sitzmann, Vincent and Lombardi, Stephen and Sunkavalli, Kalyan and Martin-Brualla, Ricardo and Simon, Tomas and Saragih, Jason and Nie{\ss}ner, Matthias and Pandey, Rohit and Fanello, Sean and Wetzstein, Gordon and Zhu, Jun-Yan and Theobalt, Christian and Agrawala, Maneesh and Shechtman, Eli and Goldman, Dan B. and Zollh{\"o}fer, Michael}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.14022}, PUBLISHER = {Blackwell-Wiley}, ADDRESS = {Oxford}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {39}, NUMBER = {2}, PAGES = {701--727}, BOOKTITLE = {The European Association for Computer Graphics 41st Annual Conference (EUROGRAPHICS 2020)}, EDITOR = {Panozzo, Daniele and Assarsson, Ulf}, }

Endnote

%0 Journal Article %A Tewari, Ayush %A Fried, Ohad %A Thies, Justus %A Sitzmann, Vincent %A Lombardi, Stephen %A Sunkavalli, Kalyan %A Martin-Brualla, Ricardo %A Simon, Tomas %A Saragih, Jason %A Nießner, Matthias %A Pandey, Rohit %A Fanello, Sean %A Wetzstein, Gordon %A Zhu, Jun-Yan %A Theobalt, Christian %A Agrawala, Maneesh %A Shechtman, Eli %A Goldman, Dan B. %A Zollhöfer, Michael %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations %T State of the Art on Neural Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0006-DB93-D %R 10.1111/cgf.14022 %7 2020 %D 2020 %J Computer Graphics Forum %O Computer Graphics Forum : journal of the European Association for Computer Graphics Comput. Graph. Forum %V 39 %N 2 %& 701 %P 701 - 727 %I Blackwell-Wiley %C Oxford %@ false %B The European Association for Computer Graphics 41st Annual Conference %O EUROGRAPHICS 2020 EG 2020 The European Association for Computer Graphics 41st Annual Conference ; Norrköping, Sweden, May 25 – 29, 2020

Tewari, A., Elgharib, M., Mallikarjun B R, et al. 2020f. PIE: Portrait Image Embedding for Semantic Control. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Tewari_ToG2020, TITLE = {{PIE}: {P}ortrait Image Embedding for Semantic Control}, AUTHOR = {Tewari, Ayush and Elgharib, Mohamed and Mallikarjun B R and Bernard, Florian and Seidel, Hans-Peter and P{\'e}rez, Patrick and Zollh{\"o}fer, Michael and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417803}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {223}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Tewari, Ayush %A Elgharib, Mohamed %A Mallikarjun B R, %A Bernard, Florian %A Seidel, Hans-Peter %A Pérez, Patrick %A Zollhöfer, Michael %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T PIE: Portrait Image Embedding for Semantic Control : %G eng %U http://hdl.handle.net/21.11116/0000-0007-9B0C-E %R 10.1145/3414685.3417803 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 223 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Tewari, A., Elgharib, M., Mallikarjun B R, et al. 2020g. PIE: Portrait Image Embedding for Semantic Control. https://arxiv.org/abs/2009.09485.

(arXiv: 2009.09485)

Abstract

Editing of portrait images is a very popular and important research topic with a large variety of applications. For ease of use, control should be provided via a semantically meaningful parameterization that is akin to computer animation controls. The vast majority of existing techniques do not provide such intuitive and fine-grained control, or only enable coarse editing of a single isolated control parameter. Very recently, high-quality semantically controlled editing has been demonstrated, however only on synthetically created StyleGAN images. We present the first approach for embedding real portrait images in the latent space of StyleGAN, which allows for intuitive editing of the head pose, facial expression, and scene illumination in the image. Semantic editing in parameter space is achieved based on StyleRig, a pretrained neural network that maps the control space of a 3D morphable face model to the latent space of the GAN. We design a novel hierarchical non-linear optimization problem to obtain the embedding. An identity preservation energy term allows spatially coherent edits while maintaining facial integrity. Our approach runs at interactive frame rates and thus allows the user to explore the space of possible edits. We evaluate our approach on a wide set of portrait photos, compare it to the current state of the art, and validate the effectiveness of its components in an ablation study.

Links

Export

BibTeX

@online{Tewari_2009.09485, TITLE = {{PIE}: {P}ortrait Image Embedding for Semantic Control}, AUTHOR = {Tewari, Ayush and Elgharib, Mohamed and Mallikarjun B R and Bernard, Florian and Seidel, Hans-Peter and P{\'e}rez, Patrick and Zollh{\"o}fer, Michael and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2009.09485}, EPRINT = {2009.09485}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Editing of portrait images is a very popular and important research topic with a large variety of applications. For ease of use, control should be provided via a semantically meaningful parameterization that is akin to computer animation controls. The vast majority of existing techniques do not provide such intuitive and fine-grained control, or only enable coarse editing of a single isolated control parameter. Very recently, high-quality semantically controlled editing has been demonstrated, however only on synthetically created StyleGAN images. We present the first approach for embedding real portrait images in the latent space of StyleGAN, which allows for intuitive editing of the head pose, facial expression, and scene illumination in the image. Semantic editing in parameter space is achieved based on StyleRig, a pretrained neural network that maps the control space of a 3D morphable face model to the latent space of the GAN. We design a novel hierarchical non-linear optimization problem to obtain the embedding. An identity preservation energy term allows spatially coherent edits while maintaining facial integrity. Our approach runs at interactive frame rates and thus allows the user to explore the space of possible edits. We evaluate our approach on a wide set of portrait photos, compare it to the current state of the art, and validate the effectiveness of its components in an ablation study. }, }

Endnote

%0 Report %A Tewari, Ayush %A Elgharib, Mohamed %A Mallikarjun B R, %A Bernard, Florian %A Seidel, Hans-Peter %A Pérez, Patrick %A Zollhöfer, Michael %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T PIE: Portrait Image Embedding for Semantic Control : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B117-7 %U https://arxiv.org/abs/2009.09485 %D 2020 %X Editing of portrait images is a very popular and important research topic with a large variety of applications. For ease of use, control should be provided via a semantically meaningful parameterization that is akin to computer animation controls. The vast majority of existing techniques do not provide such intuitive and fine-grained control, or only enable coarse editing of a single isolated control parameter. Very recently, high-quality semantically controlled editing has been demonstrated, however only on synthetically created StyleGAN images. We present the first approach for embedding real portrait images in the latent space of StyleGAN, which allows for intuitive editing of the head pose, facial expression, and scene illumination in the image. Semantic editing in parameter space is achieved based on StyleRig, a pretrained neural network that maps the control space of a 3D morphable face model to the latent space of the GAN. We design a novel hierarchical non-linear optimization problem to obtain the embedding. An identity preservation energy term allows spatially coherent edits while maintaining facial integrity. Our approach runs at interactive frame rates and thus allows the user to explore the space of possible edits. We evaluate our approach on a wide set of portrait photos, compare it to the current state of the art, and validate the effectiveness of its components in an ablation study. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Thies, J., Zollhöfer, M., Stamminger, M., Theobalt, C., and Nießner, M. 2020a. Face2Face: Real-time Face Capture and Reenactment of RGB Videos. https://arxiv.org/abs/2007.14808.

(arXiv: 2007.14808)

Abstract

We present Face2Face, a novel approach for real-time facial reenactment of a monocular target video sequence (e.g., Youtube video). The source sequence is also a monocular video stream, captured live with a commodity webcam. Our goal is to animate the facial expressions of the target video by a source actor and re-render the manipulated output video in a photo-realistic fashion. To this end, we first address the under-constrained problem of facial identity recovery from monocular video by non-rigid model-based bundling. At run time, we track facial expressions of both source and target video using a dense photometric consistency measure. Reenactment is then achieved by fast and efficient deformation transfer between source and target. The mouth interior that best matches the re-targeted expression is retrieved from the target sequence and warped to produce an accurate fit. Finally, we convincingly re-render the synthesized target face on top of the corresponding video stream such that it seamlessly blends with the real-world illumination. We demonstrate our method in a live setup, where Youtube videos are reenacted in real time.

Links

Export

BibTeX

@online{Thies_2007.14808, TITLE = {{Face2Face}: {R}eal-time Face Capture and Reenactment of {RGB} Videos}, AUTHOR = {Thies, Justus and Zollh{\"o}fer, Michael and Stamminger, Marc and Theobalt, Christian and Nie{\ss}ner, Matthias}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2007.14808}, EPRINT = {2007.14808}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present Face2Face, a novel approach for real-time facial reenactment of a monocular target video sequence (e.g., Youtube video). The source sequence is also a monocular video stream, captured live with a commodity webcam. Our goal is to animate the facial expressions of the target video by a source actor and re-render the manipulated output video in a photo-realistic fashion. To this end, we first address the under-constrained problem of facial identity recovery from monocular video by non-rigid model-based bundling. At run time, we track facial expressions of both source and target video using a dense photometric consistency measure. Reenactment is then achieved by fast and efficient deformation transfer between source and target. The mouth interior that best matches the re-targeted expression is retrieved from the target sequence and warped to produce an accurate fit. Finally, we convincingly re-render the synthesized target face on top of the corresponding video stream such that it seamlessly blends with the real-world illumination. We demonstrate our method in a live setup, where Youtube videos are reenacted in real time. }, }

Endnote

%0 Report %A Thies, Justus %A Zollhöfer, Michael %A Stamminger, Marc %A Theobalt, Christian %A Nießner, Matthias %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Face2Face: Real-time Face Capture and Reenactment of RGB Videos : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8E9-D %U https://arxiv.org/abs/2007.14808 %D 2020 %X We present Face2Face, a novel approach for real-time facial reenactment of a monocular target video sequence (e.g., Youtube video). The source sequence is also a monocular video stream, captured live with a commodity webcam. Our goal is to animate the facial expressions of the target video by a source actor and re-render the manipulated output video in a photo-realistic fashion. To this end, we first address the under-constrained problem of facial identity recovery from monocular video by non-rigid model-based bundling. At run time, we track facial expressions of both source and target video using a dense photometric consistency measure. Reenactment is then achieved by fast and efficient deformation transfer between source and target. The mouth interior that best matches the re-targeted expression is retrieved from the target sequence and warped to produce an accurate fit. Finally, we convincingly re-render the synthesized target face on top of the corresponding video stream such that it seamlessly blends with the real-world illumination. We demonstrate our method in a live setup, where Youtube videos are reenacted in real time. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Thies, J., Elgharib, M., Tewari, A., Theobalt, C., and Nießner, M. 2020b. Neural Voice Puppetry: Audio-Driven Facial Reenactment. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Thies_ECCV20, TITLE = {Neural Voice Puppetry: {A}udio-Driven Facial Reenactment}, AUTHOR = {Thies, Justus and Elgharib, Mohamed and Tewari, Ayush and Theobalt, Christian and Nie{\ss}ner, Matthias}, LANGUAGE = {eng}, ISBN = {978-3-030-58516-7}, DOI = {10.1007/978-3-030-58517-4_42}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {716--731}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12361}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Thies, Justus %A Elgharib, Mohamed %A Tewari, Ayush %A Theobalt, Christian %A Nießner, Matthias %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Neural Voice Puppetry: Audio-Driven Facial Reenactment : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D42F-6 %R 10.1007/978-3-030-58517-4_42 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 716 - 731 %I Springer %@ 978-3-030-58516-7 %B Lecture Notes in Computer Science %N 12361

Thies, J., Zollhöfer, M., Theobalt, C., Stamminger, M., and Nießner, M. 2020c. Image-guided Neural Object Rendering. International Conference on Learning Representations (ICLR 2020), OpenReview.net.

Links

Export

BibTeX

@inproceedings{Thies_ICLR2020, TITLE = {Image-guided Neural Object Rendering}, AUTHOR = {Thies, Justus and Zollh{\"o}fer, Michael and Theobalt, Christian and Stamminger, Marc and Nie{\ss}ner, Matthias}, LANGUAGE = {eng}, URL = {https://openreview.net/forum?id=Hyg9anEFPS; https://iclr.cc/Conferences/2020}, PUBLISHER = {OpenReview.net}, YEAR = {2020}, BOOKTITLE = {International Conference on Learning Representations (ICLR 2020)}, ADDRESS = {Addis Ababa, Ethopia}, }

Endnote

%0 Conference Proceedings %A Thies, Justus %A Zollhöfer, Michael %A Theobalt, Christian %A Stamminger, Marc %A Nießner, Matthias %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Image-guided Neural Object Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D431-2 %U https://openreview.net/forum?id=Hyg9anEFPS %D 2020 %B 8th International Conference on Learning Representations %Z date of event: 2020-04-26 - 2020-04-30 %C Addis Ababa, Ethopia %B International Conference on Learning Representations %I OpenReview.net %U https://openreview.net/forum?id=Hyg9anEFPS

Tong, X., Myszkowski, K., and Huang, J. 2020. Foreword to the Special Section on the International Conference on Computer-Aided Design and Computer Graphics (CAD/Graphics) 2019. Computers and Graphics86.

Links

Export

BibTeX

@article{Tong_CAD19, TITLE = {Foreword to the Special Section on the {International Conference on Computer-Aided Design and Computer Graphics (CAD/Graphics)} 2019}, AUTHOR = {Tong, Xin and Myszkowski, Karol and Huang, Jin}, LANGUAGE = {eng}, ISSN = {0097-8493}, DOI = {10.1016/j.cag.2019.12.002}, PUBLISHER = {Elsevier}, ADDRESS = {Amsterdam}, YEAR = {2020}, DATE = {2020}, JOURNAL = {Computers and Graphics}, VOLUME = {86}, PAGES = {A5--A6}, }

Endnote

%0 Journal Article %A Tong, Xin %A Myszkowski, Karol %A Huang, Jin %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Foreword to the Special Section on the International Conference on Computer-Aided Design and Computer Graphics (CAD/Graphics) 2019 : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CEAF-D %R 10.1016/j.cag.2019.12.002 %7 2019 %D 2020 %J Computers and Graphics %V 86 %& A5 %P A5 - A6 %I Elsevier %C Amsterdam %@ false

Tretschk, E., Tewari, A., Golyanik, V., Zollhöfer, M., Stoll, C., and Theobalt, C. 2020a. PatchNets: Patch-Based Generalizable Deep Implicit 3D Shape Representations. https://arxiv.org/abs/2008.01639.

(arXiv: 2008.01639)

Abstract

Implicit surface representations, such as signed-distance functions, combined with deep learning have led to impressive models which can represent detailed shapes of objects with arbitrary topology. Since a continuous function is learned, the reconstructions can also be extracted at any arbitrary resolution. However, large datasets such as ShapeNet are required to train such models. In this paper, we present a new mid-level patch-based surface representation. At the level of patches, objects across different categories share similarities, which leads to more generalizable models. We then introduce a novel method to learn this patch-based representation in a canonical space, such that it is as object-agnostic as possible. We show that our representation trained on one category of objects from ShapeNet can also well represent detailed shapes from any other category. In addition, it can be trained using much fewer shapes, compared to existing approaches. We show several applications of our new representation, including shape interpolation and partial point cloud completion. Due to explicit control over positions, orientations and scales of patches, our representation is also more controllable compared to object-level representations, which enables us to deform encoded shapes non-rigidly.

Links

Export

BibTeX

@online{Tretschk_2008.01639, TITLE = {{PatchNets}: {P}atch-Based Generalizable Deep Implicit {3D} Shape Representations}, AUTHOR = {Tretschk, Edgar and Tewari, Ayush and Golyanik, Vladislav and Zollh{\"o}fer, Michael and Stoll, Carsten and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2008.01639}, EPRINT = {2008.01639}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Implicit surface representations, such as signed-distance functions, combined with deep learning have led to impressive models which can represent detailed shapes of objects with arbitrary topology. Since a continuous function is learned, the reconstructions can also be extracted at any arbitrary resolution. However, large datasets such as ShapeNet are required to train such models. In this paper, we present a new mid-level patch-based surface representation. At the level of patches, objects across different categories share similarities, which leads to more generalizable models. We then introduce a novel method to learn this patch-based representation in a canonical space, such that it is as object-agnostic as possible. We show that our representation trained on one category of objects from ShapeNet can also well represent detailed shapes from any other category. In addition, it can be trained using much fewer shapes, compared to existing approaches. We show several applications of our new representation, including shape interpolation and partial point cloud completion. Due to explicit control over positions, orientations and scales of patches, our representation is also more controllable compared to object-level representations, which enables us to deform encoded shapes non-rigidly. }, }

Endnote

%0 Report %A Tretschk, Edgar %A Tewari, Ayush %A Golyanik, Vladislav %A Zollhöfer, Michael %A Stoll, Carsten %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T PatchNets: Patch-Based Generalizable Deep Implicit 3D Shape Representations : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E8ED-9 %U https://arxiv.org/abs/2008.01639 %D 2020 %X Implicit surface representations, such as signed-distance functions, combined with deep learning have led to impressive models which can represent detailed shapes of objects with arbitrary topology. Since a continuous function is learned, the reconstructions can also be extracted at any arbitrary resolution. However, large datasets such as ShapeNet are required to train such models. In this paper, we present a new mid-level patch-based surface representation. At the level of patches, objects across different categories share similarities, which leads to more generalizable models. We then introduce a novel method to learn this patch-based representation in a canonical space, such that it is as object-agnostic as possible. We show that our representation trained on one category of objects from ShapeNet can also well represent detailed shapes from any other category. In addition, it can be trained using much fewer shapes, compared to existing approaches. We show several applications of our new representation, including shape interpolation and partial point cloud completion. Due to explicit control over positions, orientations and scales of patches, our representation is also more controllable compared to object-level representations, which enables us to deform encoded shapes non-rigidly. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Tretschk, E., Tewari, A., Golyanik, V., Zollhöfer, M., Stoll, C., and Theobalt, C. 2020b. PatchNets: Patch-Based Generalizable Deep Implicit 3D Shape Representations. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Tretschk_ECCV20a, TITLE = {{PatchNets}: {P}atch-Based Generalizable Deep Implicit {3D} Shape Representations}, AUTHOR = {Tretschk, Edgar and Tewari, Ayush and Golyanik, Vladislav and Zollh{\"o}fer, Michael and Stoll, Carsten and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-58516-7}, DOI = {10.1007/978-3-030-58517-4_18}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {293--309}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12361}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Tretschk, Edgar %A Tewari, Ayush %A Golyanik, Vladislav %A Zollhöfer, Michael %A Stoll, Carsten %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T PatchNets: Patch-Based Generalizable Deep Implicit 3D Shape Representations : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D095-5 %R 10.1007/978-3-030-58517-4_18 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 293 - 309 %I Springer %@ 978-3-030-58516-7 %B Lecture Notes in Computer Science %N 12361

Tretschk, E., Tewari, A., Golyanik, V., Zollhöfer, M., Lassner, C., and Theobalt, C. 2020c. Non-Rigid Neural Radiance Fields: Reconstruction and Novel View Synthesis of a Deforming Scene from Monocular Video. https://arxiv.org/abs/2012.12247.

(arXiv: 2012.12247)

Abstract

In this tech report, we present the current state of our ongoing work on reconstructing Neural Radiance Fields (NERF) of general non-rigid scenes via ray bending. Non-rigid NeRF (NR-NeRF) takes RGB images of a deforming object (e.g., from a monocular video) as input and then learns a geometry and appearance representation that not only allows to reconstruct the input sequence but also to re-render any time step into novel camera views with high fidelity. In particular, we show that a consumer-grade camera is sufficient to synthesize convincing bullet-time videos of short and simple scenes. In addition, the resulting representation enables correspondence estimation across views and time, and provides rigidity scores for each point in the scene. We urge the reader to watch the supplemental videos for qualitative results. We will release our code.

Links

Export

BibTeX

@online{Tretschk_2012.12247, TITLE = {Non-Rigid Neural Radiance Fields: Reconstruction and Novel View Synthesis of a Deforming Scene from Monocular Video}, AUTHOR = {Tretschk, Edgar and Tewari, Ayush and Golyanik, Vladislav and Zollh{\"o}fer, Michael and Lassner, Christoph and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.12247}, EPRINT = {2012.12247}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {In this tech report, we present the current state of our ongoing work on reconstructing Neural Radiance Fields (NERF) of general non-rigid scenes via ray bending. Non-rigid NeRF (NR-NeRF) takes RGB images of a deforming object (e.g., from a monocular video) as input and then learns a geometry and appearance representation that not only allows to reconstruct the input sequence but also to re-render any time step into novel camera views with high fidelity. In particular, we show that a consumer-grade camera is sufficient to synthesize convincing bullet-time videos of short and simple scenes. In addition, the resulting representation enables correspondence estimation across views and time, and provides rigidity scores for each point in the scene. We urge the reader to watch the supplemental videos for qualitative results. We will release our code. }, }

Endnote

%0 Report %A Tretschk, Edgar %A Tewari, Ayush %A Golyanik, Vladislav %A Zollhöfer, Michael %A Lassner, Christoph %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Non-Rigid Neural Radiance Fields: Reconstruction and Novel View Synthesis of a Deforming Scene from Monocular Video : %G eng %U http://hdl.handle.net/21.11116/0000-0007-EA00-1 %U https://arxiv.org/abs/2012.12247 %D 2020 %X In this tech report, we present the current state of our ongoing work on reconstructing Neural Radiance Fields (NERF) of general non-rigid scenes via ray bending. Non-rigid NeRF (NR-NeRF) takes RGB images of a deforming object (e.g., from a monocular video) as input and then learns a geometry and appearance representation that not only allows to reconstruct the input sequence but also to re-render any time step into novel camera views with high fidelity. In particular, we show that a consumer-grade camera is sufficient to synthesize convincing bullet-time videos of short and simple scenes. In addition, the resulting representation enables correspondence estimation across views and time, and provides rigidity scores for each point in the scene. We urge the reader to watch the supplemental videos for qualitative results. We will release our code. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Tretschk, E., Tewari, A., Zollhöfer, M., Golyanik, V., and Theobalt, C. 2020d. DEMEA: Deep Mesh Autoencoders for Non-rigidly Deforming Objects. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Tretschk_ECCV20, TITLE = {{DEMEA}: {D}eep Mesh Autoencoders for Non-rigidly Deforming Objects}, AUTHOR = {Tretschk, Edgar and Tewari, Ayush and Zollh{\"o}fer, Michael and Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-58516-7}, DOI = {10.1007/978-3-030-58548-8_35}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {601--617}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12349}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Tretschk, Edgar %A Tewari, Ayush %A Zollhöfer, Michael %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T DEMEA: Deep Mesh Autoencoders for Non-rigidly Deforming Objects : %G eng %U http://hdl.handle.net/21.11116/0000-0007-D425-0 %R 10.1007/978-3-030-58548-8_35 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 601 - 617 %I Springer %@ 978-3-030-58516-7 %B Lecture Notes in Computer Science %N 12349

Wang, J., Mueller, F., Bernard, F., and Theobalt, C. 2020a. Generative Model-Based Loss to the Rescue: A Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation. 15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Wang_FG2020, TITLE = {Generative Model-Based Loss to the Rescue: {A} Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation}, AUTHOR = {Wang, Jiayi and Mueller, Franziska and Bernard, Florian and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-3079-8}, DOI = {10.1109/FG47880.2020.00013.}, PUBLISHER = {IEEE}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {15th IEEE International Conference on Automatic Face and Gesture Recognition (FG 2020)}, EDITOR = {{\v S}truc, Vitomir and G{\'o}mez-Fern{\'a}ndez, Francisco}, PAGES = {101--108}, ADDRESS = {Buenos Aires, Argentina}, }

Endnote

%0 Conference Proceedings %A Wang, Jiayi %A Mueller, Franziska %A Bernard, Florian %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Generative Model-Based Loss to the Rescue: A Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation : %G eng %U http://hdl.handle.net/21.11116/0000-0008-1687-7 %R 10.1109/FG47880.2020.00013. %D 2020 %B 15th IEEE International Conference on Automatic Face and Gesture Recognition %Z date of event: 2020-11-16 - 2020-11-20 %C Buenos Aires, Argentina %B 15th IEEE International Conference on Automatic Face and Gesture Recognition %E Štruc, Vitomir; Gómez-Fernández, Francisco %P 101 - 108 %I IEEE %@ 978-1-7281-3079-8

Wang, J., Mueller, F., Bernard, F., et al. 2020b. RGB2Hands: Real-Time Tracking of 3D Hand Interactions from Monocular RGB Video. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Wang_ToG2020, TITLE = {{RGB2Hands}: {R}eal-Time Tracking of {3D} Hand Interactions from Monocular {RGB} Video}, AUTHOR = {Wang, Jiayi and Mueller, Franziska and Bernard, Florian and Sorli, Suzanne and Sotnychenko, Oleksandr and Qian, Neng and Otaduy, Miguel A. and Casas, Dan and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417852}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {218}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Wang, Jiayi %A Mueller, Franziska %A Bernard, Florian %A Sorli, Suzanne %A Sotnychenko, Oleksandr %A Qian, Neng %A Otaduy, Miguel A. %A Casas, Dan %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T RGB2Hands: Real-Time Tracking of 3D Hand Interactions from Monocular RGB Video : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CF20-C %R 10.1145/3414685.3417852 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 218 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2020 %O ACM SIGGRAPH Asia 2020 SA'20 SA 2020

Wang, J., Mueller, F., Bernard, F., and Theobalt, C. 2020c. Generative Model-Based Loss to the Rescue: A Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation. https://arxiv.org/abs/2007.03073.

(arXiv: 2007.03073)

Abstract

We propose to use a model-based generative loss for training hand pose estimators on depth images based on a volumetric hand model. This additional loss allows training of a hand pose estimator that accurately infers the entire set of 21 hand keypoints while only using supervision for 6 easy-to-annotate keypoints (fingertips and wrist). We show that our partially-supervised method achieves results that are comparable to those of fully-supervised methods which enforce articulation consistency. Moreover, for the first time we demonstrate that such an approach can be used to train on datasets that have erroneous annotations, i.e. "ground truth" with notable measurement errors, while obtaining predictions that explain the depth images better than the given "ground truth".

Links

Export

BibTeX

@online{Wang_2007.03073, TITLE = {Generative Model-Based Loss to the Rescue: {A} Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation}, AUTHOR = {Wang, Jiayi and Mueller, Franziska and Bernard, Florian and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2007.03073}, EPRINT = {2007.03073}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We propose to use a model-based generative loss for training hand pose estimators on depth images based on a volumetric hand model. This additional loss allows training of a hand pose estimator that accurately infers the entire set of 21 hand keypoints while only using supervision for 6 easy-to-annotate keypoints (fingertips and wrist). We show that our partially-supervised method achieves results that are comparable to those of fully-supervised methods which enforce articulation consistency. Moreover, for the first time we demonstrate that such an approach can be used to train on datasets that have erroneous annotations, i.e. "ground truth" with notable measurement errors, while obtaining predictions that explain the depth images better than the given "ground truth". }, }

Endnote

%0 Report %A Wang, Jiayi %A Mueller, Franziska %A Bernard, Florian %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Generative Model-Based Loss to the Rescue: A Method to Overcome Annotation Errors for Depth-Based Hand Pose Estimation : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E89A-6 %U https://arxiv.org/abs/2007.03073 %D 2020 %X We propose to use a model-based generative loss for training hand pose estimators on depth images based on a volumetric hand model. This additional loss allows training of a hand pose estimator that accurately infers the entire set of 21 hand keypoints while only using supervision for 6 easy-to-annotate keypoints (fingertips and wrist). We show that our partially-supervised method achieves results that are comparable to those of fully-supervised methods which enforce articulation consistency. Moreover, for the first time we demonstrate that such an approach can be used to train on datasets that have erroneous annotations, i.e. "ground truth" with notable measurement errors, while obtaining predictions that explain the depth images better than the given "ground truth". %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Wang, P., Liu, L., Chen, N., Chu, H.-K., Theobalt, C., and Wang, W. 2020d. Vid2Curve: Simultaneous Camera Motion Estimation and Thin Structure Reconstruction from an RGB Video. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2020)39, 4.

Links

Export

BibTeX

@article{Wang_SIGGRAPH2020, TITLE = {{Vid2Curve}: {S}imultaneous Camera Motion Estimation and Thin Structure Reconstruction from an {RGB} Video}, AUTHOR = {Wang, Peng and Liu, Lingjie and Chen, Nenglun and Chu, Hung-Kuo and Theobalt, Christian and Wang, Wenping}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3386569.3392476}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {39}, NUMBER = {4}, EID = {132}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2020}, }

Endnote

%0 Journal Article %A Wang, Peng %A Liu, Lingjie %A Chen, Nenglun %A Chu, Hung-Kuo %A Theobalt, Christian %A Wang, Wenping %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Vid2Curve: Simultaneous Camera Motion Estimation and Thin Structure Reconstruction from an RGB Video : %G eng %U http://hdl.handle.net/21.11116/0000-0007-9A74-9 %R 10.1145/3386569.3392476 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 4 %Z sequence number: 132 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2020 %O ACM SIGGRAPH 2020 Virtual Conference ; 2020, 17-28 August

Wang, P., Liu, L., Chen, N., Chu, H.-K., Theobalt, C., and Wang, W. 2020e. Vid2Curve: Simultaneous Camera Motion Estimation and Thin Structure Reconstruction from an RGB Video. https://arxiv.org/abs/2005.03372.

(arXiv: 2005.03372)

Abstract

Thin structures, such as wire-frame sculptures, fences, cables, power lines, and tree branches, are common in the real world. It is extremely challenging to acquire their 3D digital models using traditional image-based or depth-based reconstruction methods because thin structures often lack distinct point features and have severe self-occlusion. We propose the first approach that simultaneously estimates camera motion and reconstructs the geometry of complex 3D thin structures in high quality from a color video captured by a handheld camera. Specifically, we present a new curve-based approach to estimate accurate camera poses by establishing correspondences between featureless thin objects in the foreground in consecutive video frames, without requiring visual texture in the background scene to lock on. Enabled by this effective curve-based camera pose estimation strategy, we develop an iterative optimization method with tailored measures on geometry, topology as well as self-occlusion handling for reconstructing 3D thin structures. Extensive validations on a variety of thin structures show that our method achieves accurate camera pose estimation and faithful reconstruction of 3D thin structures with complex shape and topology at a level that has not been attained by other existing reconstruction methods.

Links

Export

BibTeX

@online{Wang2005.03372, TITLE = {{Vid2Curve}: {S}imultaneous Camera Motion Estimation and Thin Structure Reconstruction from an {RGB} Video}, AUTHOR = {Wang, Peng and Liu, Lingjie and Chen, Nenglun and Chu, Hung-Kuo and Theobalt, Christian and Wang, Wenping}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2005.03372}, EPRINT = {2005.03372}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Thin structures, such as wire-frame sculptures, fences, cables, power lines, and tree branches, are common in the real world. It is extremely challenging to acquire their 3D digital models using traditional image-based or depth-based reconstruction methods because thin structures often lack distinct point features and have severe self-occlusion. We propose the first approach that simultaneously estimates camera motion and reconstructs the geometry of complex 3D thin structures in high quality from a color video captured by a handheld camera. Specifically, we present a new curve-based approach to estimate accurate camera poses by establishing correspondences between featureless thin objects in the foreground in consecutive video frames, without requiring visual texture in the background scene to lock on. Enabled by this effective curve-based camera pose estimation strategy, we develop an iterative optimization method with tailored measures on geometry, topology as well as self-occlusion handling for reconstructing 3D thin structures. Extensive validations on a variety of thin structures show that our method achieves accurate camera pose estimation and faithful reconstruction of 3D thin structures with complex shape and topology at a level that has not been attained by other existing reconstruction methods. }, }

Endnote

%0 Report %A Wang, Peng %A Liu, Lingjie %A Chen, Nenglun %A Chu, Hung-Kuo %A Theobalt, Christian %A Wang, Wenping %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Vid2Curve: Simultaneous Camera Motion Estimation and Thin Structure Reconstruction from an RGB Video : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E122-4 %U https://arxiv.org/abs/2005.03372 %D 2020 %X Thin structures, such as wire-frame sculptures, fences, cables, power lines, and tree branches, are common in the real world. It is extremely challenging to acquire their 3D digital models using traditional image-based or depth-based reconstruction methods because thin structures often lack distinct point features and have severe self-occlusion. We propose the first approach that simultaneously estimates camera motion and reconstructs the geometry of complex 3D thin structures in high quality from a color video captured by a handheld camera. Specifically, we present a new curve-based approach to estimate accurate camera poses by establishing correspondences between featureless thin objects in the foreground in consecutive video frames, without requiring visual texture in the background scene to lock on. Enabled by this effective curve-based camera pose estimation strategy, we develop an iterative optimization method with tailored measures on geometry, topology as well as self-occlusion handling for reconstructing 3D thin structures. Extensive validations on a variety of thin structures show that our method achieves accurate camera pose estimation and faithful reconstruction of 3D thin structures with complex shape and topology at a level that has not been attained by other existing reconstruction methods. %K Computer Science, Graphics, cs.GR,Computer Science, Computer Vision and Pattern Recognition, cs.CV,eess.IV

Xu, L., Xu, W., Golyanik, V., Habermann, M., Fang, L., and Theobalt, C. 2020a. EventCap: Monocular 3D Capture of High-Speed Human Motions Using an Event Camera. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{Xu_CVPR2020, TITLE = {{EventCap}: {M}onocular {3D} Capture of High-Speed Human Motions Using an Event Camera}, AUTHOR = {Xu, Lan and Xu, Weipeng and Golyanik, Vladislav and Habermann, Marc and Fang, Lu and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00502}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {4967--4977}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Xu, Lan %A Xu, Weipeng %A Golyanik, Vladislav %A Habermann, Marc %A Fang, Lu %A Theobalt, Christian %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T EventCap: Monocular 3D Capture of High-Speed Human Motions Using an Event Camera : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CF57-F %R 10.1109/CVPR42600.2020.00502 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 4967 - 4977 %I IEEE %@ 978-1-7281-7168-5

Xu, Y., Fan, T., Yuan, Y., and Singh, G. 2020b. Ladybird: Quasi-Monte Carlo Sampling for Deep Implicit Field Based 3D Reconstruction with Symmetry. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{Xu_ECCV20, TITLE = {Ladybird: {Quasi-Monte Carlo} Sampling for Deep Implicit Field Based {3D} Reconstruction with Symmetry}, AUTHOR = {Xu, Yifan and Fan, Tianqi and Yuan, Yi and Singh, Gurprit}, LANGUAGE = {eng}, ISBN = {978-3-030-58451-1}, DOI = {10.1007/978-3-030-58452-8_15}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {248--263}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12346}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Xu, Yifan %A Fan, Tianqi %A Yuan, Yi %A Singh, Gurprit %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Ladybird: Quasi-Monte Carlo Sampling for Deep Implicit Field Based 3D Reconstruction with Symmetry : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CEBE-C %R 10.1007/978-3-030-58452-8_15 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 248 - 263 %I Springer %@ 978-3-030-58451-1 %B Lecture Notes in Computer Science %N 12346

Xu, Y., Fan, T., Yuan, Y., and Singh, G. 2020c. Ladybird: Quasi-Monte Carlo Sampling for Deep Implicit Field Based 3D Reconstruction with Symmetry. https://arxiv.org/abs/2007.13393.

(arXiv: 2007.13393)

Abstract

Deep implicit field regression methods are effective for 3D reconstruction from single-view images. However, the impact of different sampling patterns on the reconstruction quality is not well-understood. In this work, we first study the effect of point set discrepancy on the network training. Based on Farthest Point Sampling algorithm, we propose a sampling scheme that theoretically encourages better generalization performance, and results in fast convergence for SGD-based optimization algorithms. Secondly, based on the reflective symmetry of an object, we propose a feature fusion method that alleviates issues due to self-occlusions which makes it difficult to utilize local image features. Our proposed system Ladybird is able to create high quality 3D object reconstructions from a single input image. We evaluate Ladybird on a large scale 3D dataset (ShapeNet) demonstrating highly competitive results in terms of Chamfer distance, Earth Mover's distance and Intersection Over Union (IoU).

Links

Export

BibTeX

@online{Xu_arXiv2007.13393, TITLE = {Ladybird: {Quasi-Monte Carlo} Sampling for Deep Implicit Field Based {3D} Reconstruction with Symmetry}, AUTHOR = {Xu, Yifan and Fan, Tianqi and Yuan, Yi and Singh, Gurprit}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2007.13393}, EPRINT = {2007.13393}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {Deep implicit field regression methods are effective for 3D reconstruction from single-view images. However, the impact of different sampling patterns on the reconstruction quality is not well-understood. In this work, we first study the effect of point set discrepancy on the network training. Based on Farthest Point Sampling algorithm, we propose a sampling scheme that theoretically encourages better generalization performance, and results in fast convergence for SGD-based optimization algorithms. Secondly, based on the reflective symmetry of an object, we propose a feature fusion method that alleviates issues due to self-occlusions which makes it difficult to utilize local image features. Our proposed system Ladybird is able to create high quality 3D object reconstructions from a single input image. We evaluate Ladybird on a large scale 3D dataset (ShapeNet) demonstrating highly competitive results in terms of Chamfer distance, Earth Mover's distance and Intersection Over Union (IoU). }, }

Endnote

%0 Report %A Xu, Yifan %A Fan, Tianqi %A Yuan, Yi %A Singh, Gurprit %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Ladybird: Quasi-Monte Carlo Sampling for Deep Implicit Field Based 3D Reconstruction with Symmetry : %G eng %U http://hdl.handle.net/21.11116/0000-0007-CECA-E %U https://arxiv.org/abs/2007.13393 %D 2020 %X Deep implicit field regression methods are effective for 3D reconstruction from single-view images. However, the impact of different sampling patterns on the reconstruction quality is not well-understood. In this work, we first study the effect of point set discrepancy on the network training. Based on Farthest Point Sampling algorithm, we propose a sampling scheme that theoretically encourages better generalization performance, and results in fast convergence for SGD-based optimization algorithms. Secondly, based on the reflective symmetry of an object, we propose a feature fusion method that alleviates issues due to self-occlusions which makes it difficult to utilize local image features. Our proposed system Ladybird is able to create high quality 3D object reconstructions from a single input image. We evaluate Ladybird on a large scale 3D dataset (ShapeNet) demonstrating highly competitive results in terms of Chamfer distance, Earth Mover's distance and Intersection Over Union (IoU). %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Yenamandra, T., Tewari, A., Bernard, F., et al. 2020. i3DMM: Deep Implicit 3D Morphable Model of Human Heads. https://arxiv.org/abs/2011.14143.

(arXiv: 2011.14143)

Abstract

We present the first deep implicit 3D morphable model (i3DMM) of full heads. Unlike earlier morphable face models it not only captures identity-specific geometry, texture, and expressions of the frontal face, but also models the entire head, including hair. We collect a new dataset consisting of 64 people with different expressions and hairstyles to train i3DMM. Our approach has the following favorable properties: (i) It is the first full head morphable model that includes hair. (ii) In contrast to mesh-based models it can be trained on merely rigidly aligned scans, without requiring difficult non-rigid registration. (iii) We design a novel architecture to decouple the shape model into an implicit reference shape and a deformation of this reference shape. With that, dense correspondences between shapes can be learned implicitly. (iv) This architecture allows us to semantically disentangle the geometry and color components, as color is learned in the reference space. Geometry is further disentangled as identity, expressions, and hairstyle, while color is disentangled as identity and hairstyle components. We show the merits of i3DMM using ablation studies, comparisons to state-of-the-art models, and applications such as semantic head editing and texture transfer. We will make our model publicly available.

Links

Export

BibTeX

@online{Yenamandra_arXiv2011.14143, TITLE = {i{3D}MM: Deep Implicit {3D} Morphable Model of Human Heads}, AUTHOR = {Yenamandra, Tarun and Tewari, Ayush and Bernard, Florian and Seidel, Hans-Peter and Elgharib, Mohamed and Cremers, Daniel and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2011.14143}, EPRINT = {2011.14143}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present the first deep implicit 3D morphable model (i3DMM) of full heads. Unlike earlier morphable face models it not only captures identity-specific geometry, texture, and expressions of the frontal face, but also models the entire head, including hair. We collect a new dataset consisting of 64 people with different expressions and hairstyles to train i3DMM. Our approach has the following favorable properties: (i) It is the first full head morphable model that includes hair. (ii) In contrast to mesh-based models it can be trained on merely rigidly aligned scans, without requiring difficult non-rigid registration. (iii) We design a novel architecture to decouple the shape model into an implicit reference shape and a deformation of this reference shape. With that, dense correspondences between shapes can be learned implicitly. (iv) This architecture allows us to semantically disentangle the geometry and color components, as color is learned in the reference space. Geometry is further disentangled as identity, expressions, and hairstyle, while color is disentangled as identity and hairstyle components. We show the merits of i3DMM using ablation studies, comparisons to state-of-the-art models, and applications such as semantic head editing and texture transfer. We will make our model publicly available. }, }

Endnote

%0 Report %A Yenamandra, Tarun %A Tewari, Ayush %A Bernard, Florian %A Seidel, Hans-Peter %A Elgharib, Mohamed %A Cremers, Daniel %A Theobalt, Christian %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T i3DMM: Deep Implicit 3D Morphable Model of Human Heads : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B702-8 %U https://arxiv.org/abs/2011.14143 %D 2020 %X We present the first deep implicit 3D morphable model (i3DMM) of full heads. Unlike earlier morphable face models it not only captures identity-specific geometry, texture, and expressions of the frontal face, but also models the entire head, including hair. We collect a new dataset consisting of 64 people with different expressions and hairstyles to train i3DMM. Our approach has the following favorable properties: (i) It is the first full head morphable model that includes hair. (ii) In contrast to mesh-based models it can be trained on merely rigidly aligned scans, without requiring difficult non-rigid registration. (iii) We design a novel architecture to decouple the shape model into an implicit reference shape and a deformation of this reference shape. With that, dense correspondences between shapes can be learned implicitly. (iv) This architecture allows us to semantically disentangle the geometry and color components, as color is learned in the reference space. Geometry is further disentangled as identity, expressions, and hairstyle, while color is disentangled as identity and hairstyle components. We show the merits of i3DMM using ablation studies, comparisons to state-of-the-art models, and applications such as semantic head editing and texture transfer. We will make our model publicly available. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Yoon, J.S., Liu, L., Golyanik, V., Sarkar, K., Park, H.S., and Theobalt, C. 2020. Pose-Guided Human Animation from a Single Image in the Wild. https://arxiv.org/abs/2012.03796.

(arXiv: 2012.03796)

Abstract

We present a new pose transfer method for synthesizing a human animation from a single image of a person controlled by a sequence of body poses. Existing pose transfer methods exhibit significant visual artifacts when applying to a novel scene, resulting in temporal inconsistency and failures in preserving the identity and textures of the person. To address these limitations, we design a compositional neural network that predicts the silhouette, garment labels, and textures. Each modular network is explicitly dedicated to a subtask that can be learned from the synthetic data. At the inference time, we utilize the trained network to produce a unified representation of appearance and its labels in UV coordinates, which remains constant across poses. The unified representation provides an incomplete yet strong guidance to generating the appearance in response to the pose change. We use the trained network to complete the appearance and render it with the background. With these strategies, we are able to synthesize human animations that can preserve the identity and appearance of the person in a temporally coherent way without any fine-tuning of the network on the testing scene. Experiments show that our method outperforms the state-of-the-arts in terms of synthesis quality, temporal coherence, and generalization ability.

Links

Export

BibTeX

@online{Yoon_2012.03796, TITLE = {Pose-Guided Human Animation from a Single Image in the Wild}, AUTHOR = {Yoon, Jae Shin and Liu, Lingjie and Golyanik, Vladislav and Sarkar, Kripasindhu and Park, Hyun Soo and Theobalt, Christian}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.03796}, EPRINT = {2012.03796}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present a new pose transfer method for synthesizing a human animation from a single image of a person controlled by a sequence of body poses. Existing pose transfer methods exhibit significant visual artifacts when applying to a novel scene, resulting in temporal inconsistency and failures in preserving the identity and textures of the person. To address these limitations, we design a compositional neural network that predicts the silhouette, garment labels, and textures. Each modular network is explicitly dedicated to a subtask that can be learned from the synthetic data. At the inference time, we utilize the trained network to produce a unified representation of appearance and its labels in UV coordinates, which remains constant across poses. The unified representation provides an incomplete yet strong guidance to generating the appearance in response to the pose change. We use the trained network to complete the appearance and render it with the background. With these strategies, we are able to synthesize human animations that can preserve the identity and appearance of the person in a temporally coherent way without any fine-tuning of the network on the testing scene. Experiments show that our method outperforms the state-of-the-arts in terms of synthesis quality, temporal coherence, and generalization ability. }, }

Endnote

%0 Report %A Yoon, Jae Shin %A Liu, Lingjie %A Golyanik, Vladislav %A Sarkar, Kripasindhu %A Park, Hyun Soo %A Theobalt, Christian %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Pose-Guided Human Animation from a Single Image in the Wild : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E9F3-0 %U https://arxiv.org/abs/2012.03796 %D 2020 %X We present a new pose transfer method for synthesizing a human animation from a single image of a person controlled by a sequence of body poses. Existing pose transfer methods exhibit significant visual artifacts when applying to a novel scene, resulting in temporal inconsistency and failures in preserving the identity and textures of the person. To address these limitations, we design a compositional neural network that predicts the silhouette, garment labels, and textures. Each modular network is explicitly dedicated to a subtask that can be learned from the synthetic data. At the inference time, we utilize the trained network to produce a unified representation of appearance and its labels in UV coordinates, which remains constant across poses. The unified representation provides an incomplete yet strong guidance to generating the appearance in response to the pose change. We use the trained network to complete the appearance and render it with the background. With these strategies, we are able to synthesize human animations that can preserve the identity and appearance of the person in a temporally coherent way without any fine-tuning of the network on the testing scene. Experiments show that our method outperforms the state-of-the-arts in terms of synthesis quality, temporal coherence, and generalization ability. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Yu, Y., Meka, A., Elgharib, M., Seidel, H.-P., Theobalt, C., and Smith, W.A.P. 2020. Self-supervised Outdoor Scene Relighting. Computer Vision -- ECCV 2020, Springer.

Links

Export

BibTeX

@inproceedings{yu_ECCV20, TITLE = {Self-supervised Outdoor Scene Relighting}, AUTHOR = {Yu, Ye and Meka, Abhimitra and Elgharib, Mohamed and Seidel, Hans-Peter and Theobalt, Christian and Smith, William A. P.}, LANGUAGE = {eng}, ISBN = {978-3-030-58541-9}, DOI = {10.1007/978-3-030-58542-6_6}, PUBLISHER = {Springer}, YEAR = {2020}, DATE = {2020}, BOOKTITLE = {Computer Vision -- ECCV 2020}, EDITOR = {Vedaldi, Andrea and Bischof, Horst and Brox, Thomas and Frahm, Jan-Michael}, PAGES = {84--101}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {12367}, ADDRESS = {Glasgow, UK}, }

Endnote

%0 Conference Proceedings %A Yu, Ye %A Meka, Abhimitra %A Elgharib, Mohamed %A Seidel, Hans-Peter %A Theobalt, Christian %A Smith, William A. P. %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Self-supervised Outdoor Scene Relighting : %G eng %U http://hdl.handle.net/21.11116/0000-0007-B0F6-C %R 10.1007/978-3-030-58542-6_6 %D 2020 %B 16th European Conference on Computer Vision %Z date of event: 2020-08-23 - 2020-08-28 %C Glasgow, UK %B Computer Vision -- ECCV 2020 %E Vedaldi, Andrea; Bischof, Horst; Brox, Thomas; Frahm, Jan-Michael %P 84 - 101 %I Springer %@ 978-3-030-58541-9 %B Lecture Notes in Computer Science %N 12367

Zheng, Q., Babaei, V., Wetzstein, G., Seidel, H.-P., Zwicker, M., and Singh, G. 2020. Neural Light Field 3D Printing. ACM Transactions on Graphics (Proc. SIGGRAPH Asia 2020)39, 6.

Links

Export

BibTeX

@article{Zheng_TOG2020, TITLE = {Neural Light Field {3D} Printing}, AUTHOR = {Zheng, Quan and Babaei, Vahid and Wetzstein, Gordon and Seidel, Hans-Peter and Zwicker, Matthias and Singh, Gurprit}, ISSN = {0730-0301}, DOI = {10.1145/3414685.3417879}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2020}, JOURNAL = {ACM Transactions on Graphics (Proc. SIGGRAPH Asia)}, VOLUME = {39}, NUMBER = {6}, EID = {207}, BOOKTITLE = {Proceedings of the SIGGRAPH Asia 2020}, EDITOR = {Myszkowski, Karol}, }

Endnote

%0 Journal Article %A Zheng, Quan %A Babaei, Vahid %A Wetzstein, Gordon %A Seidel, Hans-Peter %A Zwicker, Matthias %A Singh, Gurprit %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Light Field 3D Printing : %U http://hdl.handle.net/21.11116/0000-0007-9AA8-E %R 10.1145/3414685.3417879 %7 2020 %D 2020 %J ACM Transactions on Graphics %V 39 %N 6 %Z sequence number: 207 %I ACM %C New York, NY %@ false %B Proceedings of the SIGGRAPH Asia 2020 %O SIGGRAPH Asia 2020 SA'20 SA 2020

Zhou, Y., Habermann, M., Habibie, I., Tewari, A., Theobalt, C., and Xu, F. 2020a. Monocular Real-time Full Body Capture with Inter-part Correlations. https://arxiv.org/abs/2012.06087.

(arXiv: 2012.06087)

Abstract

We present the first method for real-time full body capture that estimates shape and motion of body and hands together with a dynamic 3D face model from a single color image. Our approach uses a new neural network architecture that exploits correlations between body and hands at high computational efficiency. Unlike previous works, our approach is jointly trained on multiple datasets focusing on hand, body or face separately, without requiring data where all the parts are annotated at the same time, which is much more difficult to create at sufficient variety. The possibility of such multi-dataset training enables superior generalization ability. In contrast to earlier monocular full body methods, our approach captures more expressive 3D face geometry and color by estimating the shape, expression, albedo and illumination parameters of a statistical face model. Our method achieves competitive accuracy on public benchmarks, while being significantly faster and providing more complete face reconstructions.

Links

Export

BibTeX

@online{Zhou_2012.06087, TITLE = {Monocular Real-time Full Body Capture with Inter-part Correlations}, AUTHOR = {Zhou, Yuxiao and Habermann, Marc and Habibie, Ikhsanul and Tewari, Ayush and Theobalt, Christian and Xu, Feng}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2012.06087}, EPRINT = {2012.06087}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present the first method for real-time full body capture that estimates shape and motion of body and hands together with a dynamic 3D face model from a single color image. Our approach uses a new neural network architecture that exploits correlations between body and hands at high computational efficiency. Unlike previous works, our approach is jointly trained on multiple datasets focusing on hand, body or face separately, without requiring data where all the parts are annotated at the same time, which is much more difficult to create at sufficient variety. The possibility of such multi-dataset training enables superior generalization ability. In contrast to earlier monocular full body methods, our approach captures more expressive 3D face geometry and color by estimating the shape, expression, albedo and illumination parameters of a statistical face model. Our method achieves competitive accuracy on public benchmarks, while being significantly faster and providing more complete face reconstructions. }, }

Endnote

%0 Report %A Zhou, Yuxiao %A Habermann, Marc %A Habibie, Ikhsanul %A Tewari, Ayush %A Theobalt, Christian %A Xu, Feng %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Monocular Real-time Full Body Capture with Inter-part Correlations : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E9FB-8 %U https://arxiv.org/abs/2012.06087 %D 2020 %X We present the first method for real-time full body capture that estimates shape and motion of body and hands together with a dynamic 3D face model from a single color image. Our approach uses a new neural network architecture that exploits correlations between body and hands at high computational efficiency. Unlike previous works, our approach is jointly trained on multiple datasets focusing on hand, body or face separately, without requiring data where all the parts are annotated at the same time, which is much more difficult to create at sufficient variety. The possibility of such multi-dataset training enables superior generalization ability. In contrast to earlier monocular full body methods, our approach captures more expressive 3D face geometry and color by estimating the shape, expression, albedo and illumination parameters of a statistical face model. Our method achieves competitive accuracy on public benchmarks, while being significantly faster and providing more complete face reconstructions. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Zhou, Y., Habermann, M., Xu, W., Habibie, I., Theobalt, C., and Xu, F. 2020b. Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020), IEEE.

Links

Export

BibTeX

@inproceedings{zhou2019monocular, TITLE = {Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data}, AUTHOR = {Zhou, Yuxiao and Habermann, Marc and Xu, Weipeng and Habibie, Ikhsanul and Theobalt, Christian and Xu, Feng}, LANGUAGE = {eng}, ISBN = {978-1-7281-7168-5}, DOI = {10.1109/CVPR42600.2020.00539}, PUBLISHER = {IEEE}, YEAR = {2020}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2020)}, PAGES = {5345--5354}, ADDRESS = {Seattle, WA, USA (Virtual)}, }

Endnote

%0 Conference Proceedings %A Zhou, Yuxiao %A Habermann, Marc %A Xu, Weipeng %A Habibie, Ikhsanul %A Theobalt, Christian %A Xu, Feng %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data : %G eng %U http://hdl.handle.net/21.11116/0000-0006-A89E-B %R 10.1109/CVPR42600.2020.00539 %D 2020 %B 33rd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2020-06-14 - 2020-06-19 %C Seattle, WA, USA (Virtual) %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 5345 - 5354 %I IEEE %@ 978-1-7281-7168-5

Zhou, Y., Habermann, M., Xu, W., Habibie, I., Theobalt, C., and Xu, F. 2020c. Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data. https://arxiv.org/abs/2003.09572.

(arXiv: 2003.09572)

Abstract

We present a novel method for monocular hand shape and pose estimation at unprecedented runtime performance of 100fps and at state-of-the-art accuracy. This is enabled by a new learning based architecture designed such that it can make use of all the sources of available hand training data: image data with either 2D or 3D annotations, as well as stand-alone 3D animations without corresponding image data. It features a 3D hand joint detection module and an inverse kinematics module which regresses not only 3D joint positions but also maps them to joint rotations in a single feed-forward pass. This output makes the method more directly usable for applications in computer vision and graphics compared to only regressing 3D joint positions. We demonstrate that our architectural design leads to a significant quantitative and qualitative improvement over the state of the art on several challenging benchmarks. Our model is publicly available for future research.

Links

Export

BibTeX

@online{Zhou2003.09572, TITLE = {Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data}, AUTHOR = {Zhou, Yuxiao and Habermann, Marc and Xu, Weipeng and Habibie, Ikhsanul and Theobalt, Christian and Xu, Feng}, LANGUAGE = {eng}, URL = {https://arxiv.org/abs/2003.09572}, EPRINT = {2003.09572}, EPRINTTYPE = {arXiv}, YEAR = {2020}, ABSTRACT = {We present a novel method for monocular hand shape and pose estimation at unprecedented runtime performance of 100fps and at state-of-the-art accuracy. This is enabled by a new learning based architecture designed such that it can make use of all the sources of available hand training data: image data with either 2D or 3D annotations, as well as stand-alone 3D animations without corresponding image data. It features a 3D hand joint detection module and an inverse kinematics module which regresses not only 3D joint positions but also maps them to joint rotations in a single feed-forward pass. This output makes the method more directly usable for applications in computer vision and graphics compared to only regressing 3D joint positions. We demonstrate that our architectural design leads to a significant quantitative and qualitative improvement over the state of the art on several challenging benchmarks. Our model is publicly available for future research. }, }

Endnote

%0 Report %A Zhou, Yuxiao %A Habermann, Marc %A Xu, Weipeng %A Habibie, Ikhsanul %A Theobalt, Christian %A Xu, Feng %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data : %G eng %U http://hdl.handle.net/21.11116/0000-0007-E0D3-D %U https://arxiv.org/abs/2003.09572 %D 2020 %X We present a novel method for monocular hand shape and pose estimation at unprecedented runtime performance of 100fps and at state-of-the-art accuracy. This is enabled by a new learning based architecture designed such that it can make use of all the sources of available hand training data: image data with either 2D or 3D annotations, as well as stand-alone 3D animations without corresponding image data. It features a 3D hand joint detection module and an inverse kinematics module which regresses not only 3D joint positions but also maps them to joint rotations in a single feed-forward pass. This output makes the method more directly usable for applications in computer vision and graphics compared to only regressing 3D joint positions. We demonstrate that our architectural design leads to a significant quantitative and qualitative improvement over the state of the art on several challenging benchmarks. Our model is publicly available for future research. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

2019

Aharon, I., Chen, R., Zorin, D., and Weber, O. 2019. Bounded Distortion Tetrahedral Metric Interpolation. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2019)38, 6.

Links

Export

BibTeX

@article{Aharon2019, TITLE = {Bounded Distortion Tetrahedral Metric Interpolation}, AUTHOR = {Aharon, Ido and Chen, Renjie and Zorin, Denis and Weber, Ofir}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3355089.3356569}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {38}, NUMBER = {6}, EID = {182}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2019}, }

Endnote

%0 Journal Article %A Aharon, Ido %A Chen, Renjie %A Zorin, Denis %A Weber, Ofir %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Bounded Distortion Tetrahedral Metric Interpolation : %G eng %U http://hdl.handle.net/21.11116/0000-0008-04BA-2 %R 10.1145/3355089.3356569 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 6 %Z sequence number: 182 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2019 %O ACM SIGGRAPH Asia 2019 Brisbane, Australia, 17 - 20 November 2019 SA'19 SA 2019

Alldieck, T., Pons-Moll, G., Theobalt, C., and Magnor, M.A. 2019a. Tex2Shape: Detailed Full Human Body Geometry from a Single Image. International Conference on Computer Vision (ICCV 2019), IEEE.

Abstract

We present a simple yet effective method to infer detailed full human body shape from only a single photograph. Our model can infer full-body shape including face, hair, and clothing including wrinkles at interactive frame-rates. Results feature details even on parts that are occluded in the input image. Our main idea is to turn shape regression into an aligned image-to-image translation problem. The input to our method is a partial texture map of the visible region obtained from off-the-shelf methods. From a partial texture, we estimate detailed normal and vector displacement maps, which can be applied to a low-resolution smooth body model to add detail and clothing. Despite being trained purely with synthetic data, our model generalizes well to real-world photographs. Numerous results demonstrate the versatility and robustness of our method.

Links

Export

BibTeX

@inproceedings{Alldieck_ICCV2019, TITLE = {{Tex2Shape}: Detailed Full Human Body Geometry from a Single Image}, AUTHOR = {Alldieck, Thiemo and Pons-Moll, Gerard and Theobalt, Christian and Magnor, Marcus A.}, LANGUAGE = {eng}, ISBN = {978-1-7281-4803-8}, DOI = {10.1109/ICCV.2019.00238}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, ABSTRACT = {We present a simple yet effective method to infer detailed full human body shape from only a single photograph. Our model can infer full-body shape including face, hair, and clothing including wrinkles at interactive frame-rates. Results feature details even on parts that are occluded in the input image. Our main idea is to turn shape regression into an aligned image-to-image translation problem. The input to our method is a partial texture map of the visible region obtained from off-the-shelf methods. From a partial texture, we estimate detailed normal and vector displacement maps, which can be applied to a low-resolution smooth body model to add detail and clothing. Despite being trained purely with synthetic data, our model generalizes well to real-world photographs. Numerous results demonstrate the versatility and robustness of our method. }, BOOKTITLE = {International Conference on Computer Vision (ICCV 2019)}, PAGES = {2293--2303}, ADDRESS = {Seoul, Korea}, }

Endnote

%0 Conference Proceedings %A Alldieck, Thiemo %A Pons-Moll, Gerard %A Theobalt, Christian %A Magnor, Marcus A. %+ External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Tex2Shape: Detailed Full Human Body Geometry from a Single Image : %G eng %U http://hdl.handle.net/21.11116/0000-0003-ECBE-E %R 10.1109/ICCV.2019.00238 %D 2019 %B International Conference on Computer Vision %Z date of event: 2019-10-27 - 2019-11-02 %C Seoul, Korea %X We present a simple yet effective method to infer detailed full human body shape from only a single photograph. Our model can infer full-body shape including face, hair, and clothing including wrinkles at interactive frame-rates. Results feature details even on parts that are occluded in the input image. Our main idea is to turn shape regression into an aligned image-to-image translation problem. The input to our method is a partial texture map of the visible region obtained from off-the-shelf methods. From a partial texture, we estimate detailed normal and vector displacement maps, which can be applied to a low-resolution smooth body model to add detail and clothing. Despite being trained purely with synthetic data, our model generalizes well to real-world photographs. Numerous results demonstrate the versatility and robustness of our method. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV %B International Conference on Computer Vision %P 2293 - 2303 %I IEEE %@ 978-1-7281-4803-8

Alldieck, T., Pons-Moll, G., Theobalt, C., and Magnor, M.A. 2019b. Tex2Shape: Detailed Full Human Body Geometry From a Single Image. http://arxiv.org/abs/1904.08645.

(arXiv: 1904.08645)

Abstract

Links

Export

BibTeX

@online{Alldieck_arXiv1904.08645, TITLE = {{Tex2Shape}: Detailed Full Human Body Geometry From a Single Image}, AUTHOR = {Alldieck, Thiemo and Pons-Moll, Gerard and Theobalt, Christian and Magnor, Marcus A.}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1904.08645}, EPRINT = {1904.08645}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {We present a simple yet effective method to infer detailed full human body shape from only a single photograph. Our model can infer full-body shape including face, hair, and clothing including wrinkles at interactive frame-rates. Results feature details even on parts that are occluded in the input image. Our main idea is to turn shape regression into an aligned image-to-image translation problem. The input to our method is a partial texture map of the visible region obtained from off-the-shelf methods. From a partial texture, we estimate detailed normal and vector displacement maps, which can be applied to a low-resolution smooth body model to add detail and clothing. Despite being trained purely with synthetic data, our model generalizes well to real-world photographs. Numerous results demonstrate the versatility and robustness of our method. }, }

Endnote

%0 Report %A Alldieck, Thiemo %A Pons-Moll, Gerard %A Theobalt, Christian %A Magnor, Marcus A. %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Tex2Shape: Detailed Full Human Body Geometry From a Single Image : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7CF6-B %U http://arxiv.org/abs/1904.08645 %D 2019 %X We present a simple yet effective method to infer detailed full human body shape from only a single photograph. Our model can infer full-body shape including face, hair, and clothing including wrinkles at interactive frame-rates. Results feature details even on parts that are occluded in the input image. Our main idea is to turn shape regression into an aligned image-to-image translation problem. The input to our method is a partial texture map of the visible region obtained from off-the-shelf methods. From a partial texture, we estimate detailed normal and vector displacement maps, which can be applied to a low-resolution smooth body model to add detail and clothing. Despite being trained purely with synthetic data, our model generalizes well to real-world photographs. Numerous results demonstrate the versatility and robustness of our method. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Alldieck, T., Magnor, M.A., Bhatnagar, B.L., Theobalt, C., and Pons-Moll, G. 2019c. Learning to Reconstruct People in Clothing from a Single RGB Camera. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2019), IEEE.

Links

Export

BibTeX

@inproceedings{alldieck19cvpr, TITLE = {Learning to Reconstruct People in Clothing from a Single {RGB} Camera}, AUTHOR = {Alldieck, Thiemo and Magnor, Marcus A. and Bhatnagar, Bharat Lal and Theobalt, Christian and Pons-Moll, Gerard}, ISBN = {978-1-7281-3293-8}, DOI = {10.1109/CVPR.2019.00127}, PUBLISHER = {IEEE}, YEAR = {2019}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2019)}, PAGES = {1175--1186}, ADDRESS = {Long Beach, CA, USA}, }

Endnote

%0 Conference Proceedings %A Alldieck, Thiemo %A Magnor, Marcus A. %A Bhatnagar, Bharat Lal %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Learning to Reconstruct People in Clothing from a Single RGB Camera : %U http://hdl.handle.net/21.11116/0000-0003-5F97-9 %R 10.1109/CVPR.2019.00127 %D 2019 %B 32nd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2019-06-15 - 2019-06-20 %C Long Beach, CA, USA %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 1175 - 1186 %I IEEE %@ 978-1-7281-3293-8

Alldieck, T., Magnor, M.A., Bhatnagar, B.L., Theobalt, C., and Pons-Moll, G. 2019d. Learning to Reconstruct People in Clothing from a Single RGB Camera. http://arxiv.org/abs/1903.05885.

(arXiv: 1903.05885)

Abstract

We present a learning-based model to infer the personalized 3D shape of people from a few frames (1-8) of a monocular video in which the person is moving, in less than 10 seconds with a reconstruction accuracy of 5mm. Our model learns to predict the parameters of a statistical body model and instance displacements that add clothing and hair to the shape. The model achieves fast and accurate predictions based on two key design choices. First, by predicting shape in a canonical T-pose space, the network learns to encode the images of the person into pose-invariant latent codes, where the information is fused. Second, based on the observation that feed-forward predictions are fast but do not always align with the input images, we predict using both, bottom-up and top-down streams (one per view) allowing information to flow in both directions. Learning relies only on synthetic 3D data. Once learned, the model can take a variable number of frames as input, and is able to reconstruct shapes even from a single image with an accuracy of 6mm. Results on 3 different datasets demonstrate the efficacy and accuracy of our approach.

Links

Export

BibTeX

@online{Alldieck_arXiv1903.05885, TITLE = {Learning to Reconstruct People in Clothing from a Single {RGB} Camera}, AUTHOR = {Alldieck, Thiemo and Magnor, Marcus A. and Bhatnagar, Bharat Lal and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1903.05885}, EPRINT = {1903.05885}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {We present a learning-based model to infer the personalized 3D shape of people from a few frames (1-8) of a monocular video in which the person is moving, in less than 10 seconds with a reconstruction accuracy of 5mm. Our model learns to predict the parameters of a statistical body model and instance displacements that add clothing and hair to the shape. The model achieves fast and accurate predictions based on two key design choices. First, by predicting shape in a canonical T-pose space, the network learns to encode the images of the person into pose-invariant latent codes, where the information is fused. Second, based on the observation that feed-forward predictions are fast but do not always align with the input images, we predict using both, bottom-up and top-down streams (one per view) allowing information to flow in both directions. Learning relies only on synthetic 3D data. Once learned, the model can take a variable number of frames as input, and is able to reconstruct shapes even from a single image with an accuracy of 6mm. Results on 3 different datasets demonstrate the efficacy and accuracy of our approach. }, }

Endnote

%0 Report %A Alldieck, Thiemo %A Magnor, Marcus A. %A Bhatnagar, Bharat Lal %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Learning to Reconstruct People in Clothing from a Single RGB Camera : %G eng %U http://hdl.handle.net/21.11116/0000-0003-FE01-E %U http://arxiv.org/abs/1903.05885 %D 2019 %X We present a learning-based model to infer the personalized 3D shape of people from a few frames (1-8) of a monocular video in which the person is moving, in less than 10 seconds with a reconstruction accuracy of 5mm. Our model learns to predict the parameters of a statistical body model and instance displacements that add clothing and hair to the shape. The model achieves fast and accurate predictions based on two key design choices. First, by predicting shape in a canonical T-pose space, the network learns to encode the images of the person into pose-invariant latent codes, where the information is fused. Second, based on the observation that feed-forward predictions are fast but do not always align with the input images, we predict using both, bottom-up and top-down streams (one per view) allowing information to flow in both directions. Learning relies only on synthetic 3D data. Once learned, the model can take a variable number of frames as input, and is able to reconstruct shapes even from a single image with an accuracy of 6mm. Results on 3 different datasets demonstrate the efficacy and accuracy of our approach. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Bemana, M., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2019a. Neural View-Interpolation for Sparse Light Field Video. http://arxiv.org/abs/1910.13921.

(arXiv: 1910.13921)

[arXiv]

[PuRe]

[BibTeX]

Abstract

We suggest representing light field (LF) videos as "one-off" neural networks (NN), i.e., a learned mapping from view-plus-time coordinates to high-resolution color values, trained on sparse views. Initially, this sounds like a bad idea for three main reasons: First, a NN LF will likely have less quality than a same-sized pixel basis representation. Second, only few training data, e.g., 9 exemplars per frame are available for sparse LF videos. Third, there is no generalization across LFs, but across view and time instead. Consequently, a network needs to be trained for each LF video. Surprisingly, these problems can turn into substantial advantages: Other than the linear pixel basis, a NN has to come up with a compact, non-linear i.e., more intelligent, explanation of color, conditioned on the sparse view and time coordinates. As observed for many NN however, this representation now is interpolatable: if the image output for sparse view coordinates is plausible, it is for all intermediate, continuous coordinates as well. Our specific network architecture involves a differentiable occlusion-aware warping step, which leads to a compact set of trainable parameters and consequently fast learning and fast execution.

Links

[PuRe]

[arXiv]

[BibTeX]

Export

BibTeX

@online{Bemana_arXiv1910.13921, TITLE = {Neural View-Interpolation for Sparse Light Field Video}, AUTHOR = {Bemana, Mojtaba and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1910.13921}, EPRINT = {1910.13921}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {We suggest representing light field (LF) videos as "one-off" neural networks (NN), i.e., a learned mapping from view-plus-time coordinates to high-resolution color values, trained on sparse views. Initially, this sounds like a bad idea for three main reasons: First, a NN LF will likely have less quality than a same-sized pixel basis representation. Second, only few training data, e.g., 9 exemplars per frame are available for sparse LF videos. Third, there is no generalization across LFs, but across view and time instead. Consequently, a network needs to be trained for each LF video. Surprisingly, these problems can turn into substantial advantages: Other than the linear pixel basis, a NN has to come up with a compact, non-linear i.e., more intelligent, explanation of color, conditioned on the sparse view and time coordinates. As observed for many NN however, this representation now is interpolatable: if the image output for sparse view coordinates is plausible, it is for all intermediate, continuous coordinates as well. Our specific network architecture involves a differentiable occlusion-aware warping step, which leads to a compact set of trainable parameters and consequently fast learning and fast execution. }, }

Endnote

%0 Report %A Bemana, Mojtaba %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Neural View-Interpolation for Sparse Light Field Video : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7B16-9 %U http://arxiv.org/abs/1910.13921 %D 2019 %X We suggest representing light field (LF) videos as "one-off" neural networks (NN), i.e., a learned mapping from view-plus-time coordinates to high-resolution color values, trained on sparse views. Initially, this sounds like a bad idea for three main reasons: First, a NN LF will likely have less quality than a same-sized pixel basis representation. Second, only few training data, e.g., 9 exemplars per frame are available for sparse LF videos. Third, there is no generalization across LFs, but across view and time instead. Consequently, a network needs to be trained for each LF video. Surprisingly, these problems can turn into substantial advantages: Other than the linear pixel basis, a NN has to come up with a compact, non-linear i.e., more intelligent, explanation of color, conditioned on the sparse view and time coordinates. As observed for many NN however, this representation now is interpolatable: if the image output for sparse view coordinates is plausible, it is for all intermediate, continuous coordinates as well. Our specific network architecture involves a differentiable occlusion-aware warping step, which leads to a compact set of trainable parameters and consequently fast learning and fast execution. %K Computer Science, Graphics, cs.GR,Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Learning, cs.LG,eess.IV

Bemana, M., Keinert, J., Myszkowski, K., et al. 2019b. Learning to Predict Image-based Rendering Artifacts with Respect to a Hidden Reference Image. Computer Graphics Forum (Proc. Pacific Graphics 2019)38, 7.

Links

Export

BibTeX

@article{Bemana_PG2019, TITLE = {Learning to Predict Image-based Rendering Artifacts with Respect to a Hidden Reference Image}, AUTHOR = {Bemana, Mojtaba and Keinert, Joachim and Myszkowski, Karol and B{\"a}tz, Michel and Ziegler, Matthias and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, ISSN = {1467-8659}, DOI = {10.1111/cgf.13862}, PUBLISHER = {Wiley-Blackwell}, ADDRESS = {Oxford, UK}, YEAR = {2019}, DATE = {2019}, JOURNAL = {Computer Graphics Forum (Proc. Pacific Graphics)}, VOLUME = {38}, NUMBER = {7}, PAGES = {579--589}, BOOKTITLE = {27th Annual International Conference on Computer Graphics and Applications (Pacific Graphics 2019)}, }

Endnote

%0 Journal Article %A Bemana, Mojtaba %A Keinert, Joachim %A Myszkowski, Karol %A Bätz, Michel %A Ziegler, Matthias %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Learning to Predict Image-based Rendering Artifacts with Respect to a Hidden Reference Image : %G eng %U http://hdl.handle.net/21.11116/0000-0004-9BC5-F %R 10.1111/cgf.13862 %7 2019 %D 2019 %J Computer Graphics Forum %V 38 %N 7 %& 579 %P 579 - 589 %I Wiley-Blackwell %C Oxford, UK %@ false %B 27th Annual International Conference on Computer Graphics and Applications %O Pacific Graphics 2019 PG 2019 Seoul, October 14-17, 2019

Bernard, F., Thunberg, J., Goncalves, J., and Theobalt, C. 2019a. Synchronisation of Partial Multi-Matchings via Non-negative Factorisations. Pattern Recognition92.

Links

Export

BibTeX

@article{Bernard2019, TITLE = {Synchronisation of Partial Multi-Matchings via Non-negative Factorisations}, AUTHOR = {Bernard, Florian and Thunberg, Johan and Goncalves, Jorge and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0031-3203}, DOI = {10.1016/j.patcog.2019.03.021}, PUBLISHER = {Pergamon}, ADDRESS = {Oxford}, YEAR = {2019}, DATE = {2019}, JOURNAL = {Pattern Recognition}, VOLUME = {92}, PAGES = {146--155}, }

Endnote

%0 Journal Article %A Bernard, Florian %A Thunberg, Johan %A Goncalves, Jorge %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Synchronisation of Partial Multi-Matchings via Non-negative Factorisations : %G eng %U http://hdl.handle.net/21.11116/0000-0003-B2EC-A %R 10.1016/j.patcog.2019.03.021 %7 2019-03-23 %D 2019 %J Pattern Recognition %O Pattern Recognit. %V 92 %& 146 %P 146 - 155 %I Pergamon %C Oxford %@ false

Bernard, F., Thunberg, J., Swoboda, P., and Theobalt, C. 2019b. HiPPI: Higher-Order Projected Power Iterations for Scalable Multi-Matching. International Conference on Computer Vision (ICCV 2019), IEEE.

Links

Export

BibTeX

@inproceedings{Bernard_ICCV2019, TITLE = {{HiPPI}: {H}igher-Order Projected Power Iterations for Scalable Multi-Matching}, AUTHOR = {Bernard, Florian and Thunberg, Johan and Swoboda, Paul and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-4803-8}, DOI = {10.1109/ICCV.2019.01038}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, BOOKTITLE = {International Conference on Computer Vision (ICCV 2019)}, PAGES = {10283--10292}, ADDRESS = {Seoul, Korea}, }

Endnote

%0 Conference Proceedings %A Bernard, Florian %A Thunberg, Johan %A Swoboda, Paul %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T HiPPI: Higher-Order Projected Power Iterations for Scalable Multi-Matching : %G eng %U http://hdl.handle.net/21.11116/0000-0006-DC81-0 %R 10.1109/ICCV.2019.01038 %D 2019 %B International Conference on Computer Vision %Z date of event: 2019-10-27 - 2019-11-02 %C Seoul, Korea %B International Conference on Computer Vision %P 10283 - 10292 %I IEEE %@ 978-1-7281-4803-8

Bhatnagar, B.L., Tiwari, G., Theobalt, C., and Pons-Moll, G. 2019a. Multi-Garment Net: Learning to Dress 3D People from Images. http://arxiv.org/abs/1908.06903.

(arXiv: 1908.06903)

Abstract

We present Multi-Garment Network (MGN), a method to predict body shape and clothing, layered on top of the SMPL model from a few frames (1-8) of a video. Several experiments demonstrate that this representation allows higher level of control when compared to single mesh or voxel representations of shape. Our model allows to predict garment geometry, relate it to the body shape, and transfer it to new body shapes and poses. To train MGN, we leverage a digital wardrobe containing 712 digital garments in correspondence, obtained with a novel method to register a set of clothing templates to a dataset of real 3D scans of people in different clothing and poses. Garments from the digital wardrobe, or predicted by MGN, can be used to dress any body shape in arbitrary poses. We will make publicly available the digital wardrobe, the MGN model, and code to dress SMPL with the garments.

Links

Export

BibTeX

@online{Bhatnagar_arXiv1908.06903, TITLE = {Multi-Garment Net: {L}earning to Dress {3D} People from Images}, AUTHOR = {Bhatnagar, Bharat Lal and Tiwari, Garvita and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1908.06903}, EPRINT = {1908.06903}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {We present Multi-Garment Network (MGN), a method to predict body shape and clothing, layered on top of the SMPL model from a few frames (1-8) of a video. Several experiments demonstrate that this representation allows higher level of control when compared to single mesh or voxel representations of shape. Our model allows to predict garment geometry, relate it to the body shape, and transfer it to new body shapes and poses. To train MGN, we leverage a digital wardrobe containing 712 digital garments in correspondence, obtained with a novel method to register a set of clothing templates to a dataset of real 3D scans of people in different clothing and poses. Garments from the digital wardrobe, or predicted by MGN, can be used to dress any body shape in arbitrary poses. We will make publicly available the digital wardrobe, the MGN model, and code to dress SMPL with the garments. }, }

Endnote

%0 Report %A Bhatnagar, Bharat Lal %A Tiwari, Garvita %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Multi-Garment Net: Learning to Dress 3D People from Images : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7D67-C %U http://arxiv.org/abs/1908.06903 %D 2019 %X We present Multi-Garment Network (MGN), a method to predict body shape and clothing, layered on top of the SMPL model from a few frames (1-8) of a video. Several experiments demonstrate that this representation allows higher level of control when compared to single mesh or voxel representations of shape. Our model allows to predict garment geometry, relate it to the body shape, and transfer it to new body shapes and poses. To train MGN, we leverage a digital wardrobe containing 712 digital garments in correspondence, obtained with a novel method to register a set of clothing templates to a dataset of real 3D scans of people in different clothing and poses. Garments from the digital wardrobe, or predicted by MGN, can be used to dress any body shape in arbitrary poses. We will make publicly available the digital wardrobe, the MGN model, and code to dress SMPL with the garments. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Bhatnagar, B.L., Tiwari, G., Theobalt, C., and Pons-Moll, G. 2019b. Multi-Garment Net: Learning to Dress 3D People from Images. International Conference on Computer Vision (ICCV 2019), IEEE.

Links

Export

BibTeX

@inproceedings{bhatnagar_ICCV2019, TITLE = {Multi-Garment {N}et: {L}earning to Dress {3D} People from Images}, AUTHOR = {Bhatnagar, Bharat Lal and Tiwari, Garvita and Theobalt, Christian and Pons-Moll, Gerard}, LANGUAGE = {eng}, ISBN = {978-1-7281-4803-8}, DOI = {10.1109/ICCV.2019.00552}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, BOOKTITLE = {International Conference on Computer Vision (ICCV 2019)}, PAGES = {5419--5429}, ADDRESS = {Seoul, Korea}, }

Endnote

%0 Conference Proceedings %A Bhatnagar, Bharat Lal %A Tiwari, Garvita %A Theobalt, Christian %A Pons-Moll, Gerard %+ Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society %T Multi-Garment Net: Learning to Dress 3D People from Images : %G eng %U http://hdl.handle.net/21.11116/0000-0004-89E8-C %R 10.1109/ICCV.2019.00552 %D 2019 %B International Conference on Computer Vision %Z date of event: 2019-10-27 - 2019-11-02 %C Seoul, Korea %B International Conference on Computer Vision %P 5419 - 5429 %I IEEE %@ 978-1-7281-4803-8

Bojja, A.K., Mueller, F., Malireddi, S.R., et al. 2019. HandSeg: An Automatically Labeled Dataset for Hand Segmentation from Depth Images. 16th Conference on Computer and Robot Vision (CRV 2019), IEEE.

Links

Export

BibTeX

@inproceedings{Malireddi_CRV2019, TITLE = {{HandSeg}: {An Automatically Labeled Dataset for Hand Segmentation from Depth Images}}, AUTHOR = {Bojja, Abhishake Kumar and Mueller, Franziska and Malireddi, Sri Raghu and Oberweger, Markus and Lepetit, Vincent and Theobalt, Christian and Yi, Kwang Moo and Tagliasacchi, Andrea}, LANGUAGE = {eng}, ISBN = {978-1-7281-1838-3}, DOI = {10.1109/CRV.2019.00028}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, BOOKTITLE = {16th Conference on Computer and Robot Vision (CRV 2019)}, PAGES = {151--158}, ADDRESS = {Kingston, Canada}, }

Endnote

%0 Conference Proceedings %A Bojja, Abhishake Kumar %A Mueller, Franziska %A Malireddi, Sri Raghu %A Oberweger, Markus %A Lepetit, Vincent %A Theobalt, Christian %A Yi, Kwang Moo %A Tagliasacchi, Andrea %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T HandSeg: An Automatically Labeled Dataset for Hand Segmentation from Depth Images : %G eng %U http://hdl.handle.net/21.11116/0000-0005-6BC4-6 %R 10.1109/CRV.2019.00028 %D 2019 %B 16th Conference on Computer and Robot Vision %Z date of event: 2019-05-29 - 2019-05-31 %C Kingston, Canada %B 16th Conference on Computer and Robot Vision %P 151 - 158 %I IEEE %@ 978-1-7281-1838-3

Božič, A., Zollhöfer, M., Theobalt, C., and Nießner, M. 2019. DeepDeform: Learning Non-rigid RGB-D Reconstruction with Semi-supervised Data. http://arxiv.org/abs/1912.04302.

(arXiv: 1912.04302)

Abstract

Applying data-driven approaches to non-rigid 3D reconstruction has been difficult, which we believe can be attributed to the lack of a large-scale training corpus. One recent approach proposes self-supervision based on non-rigid reconstruction. Unfortunately, this method fails for important cases such as highly non-rigid deformations. We first address this problem of lack of data by introducing a novel semi-supervised strategy to obtain dense inter-frame correspondences from a sparse set of annotations. This way, we obtain a large dataset of 400 scenes, over 390,000 RGB-D frames, and 2,537 densely aligned frame pairs; in addition, we provide a test set along with several metrics for evaluation. Based on this corpus, we introduce a data-driven non-rigid feature matching approach, which we integrate into an optimization-based reconstruction pipeline. Here, we propose a new neural network that operates on RGB-D frames, while maintaining robustness under large non-rigid deformations and producing accurate predictions. Our approach significantly outperforms both existing non-rigid reconstruction methods that do not use learned data terms, as well as learning-based approaches that only use self-supervision.

Links

Export

BibTeX

@online{Bozic_arXiv1912.04302, TITLE = {{DeepDeform}: Learning Non-rigid {RGB}-D Reconstruction with Semi-supervised Data}, AUTHOR = {Bo{\v z}i{\v c}, Alja{\v z} and Zollh{\"o}fer, Michael and Theobalt, Christian and Nie{\ss}ner, Matthias}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1912.04302}, EPRINT = {1912.04302}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Applying data-driven approaches to non-rigid 3D reconstruction has been difficult, which we believe can be attributed to the lack of a large-scale training corpus. One recent approach proposes self-supervision based on non-rigid reconstruction. Unfortunately, this method fails for important cases such as highly non-rigid deformations. We first address this problem of lack of data by introducing a novel semi-supervised strategy to obtain dense inter-frame correspondences from a sparse set of annotations. This way, we obtain a large dataset of 400 scenes, over 390,000 RGB-D frames, and 2,537 densely aligned frame pairs; in addition, we provide a test set along with several metrics for evaluation. Based on this corpus, we introduce a data-driven non-rigid feature matching approach, which we integrate into an optimization-based reconstruction pipeline. Here, we propose a new neural network that operates on RGB-D frames, while maintaining robustness under large non-rigid deformations and producing accurate predictions. Our approach significantly outperforms both existing non-rigid reconstruction methods that do not use learned data terms, as well as learning-based approaches that only use self-supervision. }, }

Endnote

%0 Report %A Božič, Aljaž %A Zollhöfer, Michael %A Theobalt, Christian %A Nießner, Matthias %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T DeepDeform: Learning Non-rigid RGB-D Reconstruction with Semi-supervised Data : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7DDE-6 %U http://arxiv.org/abs/1912.04302 %D 2019 %X Applying data-driven approaches to non-rigid 3D reconstruction has been difficult, which we believe can be attributed to the lack of a large-scale training corpus. One recent approach proposes self-supervision based on non-rigid reconstruction. Unfortunately, this method fails for important cases such as highly non-rigid deformations. We first address this problem of lack of data by introducing a novel semi-supervised strategy to obtain dense inter-frame correspondences from a sparse set of annotations. This way, we obtain a large dataset of 400 scenes, over 390,000 RGB-D frames, and 2,537 densely aligned frame pairs; in addition, we provide a test set along with several metrics for evaluation. Based on this corpus, we introduce a data-driven non-rigid feature matching approach, which we integrate into an optimization-based reconstruction pipeline. Here, we propose a new neural network that operates on RGB-D frames, while maintaining robustness under large non-rigid deformations and producing accurate predictions. Our approach significantly outperforms both existing non-rigid reconstruction methods that do not use learned data terms, as well as learning-based approaches that only use self-supervision. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR

Castelli Aleardi, L., Salihoglu, S., Singh, G., and Ovsjanikov, M. 2019. Spectral Measures of Distortion for Change Detection in Dynamic Graphs. Complex Networks and Their Applications VII, Springer.

Links

Export

BibTeX

@inproceedings{Castelli_COMPLEX2018, TITLE = {Spectral Measures of Distortion for Change Detection in Dynamic Graphs}, AUTHOR = {Castelli Aleardi, Luca and Salihoglu, Semih and Singh, Gurprit and Ovsjanikov, Maks}, LANGUAGE = {eng}, ISBN = {978-3-030-05413-7; 978-3-030-05414-4}, DOI = {10.1007/978-3-030-05414-4_5}, PUBLISHER = {Springer}, YEAR = {2018}, DATE = {2019}, BOOKTITLE = {Complex Networks and Their Applications VII}, EDITOR = {Aiello, Luca Maria and Cherifi, Chantal and Cherifi, Hocine and Lambiotte, Renaud and Li{\'o}, Pietro and Rocha, Luis M.}, PAGES = {54--66}, SERIES = {Studies in Computational Intelligence}, VOLUME = {813}, ADDRESS = {Cambridge, UK}, }

Endnote

%0 Conference Proceedings %A Castelli Aleardi, Luca %A Salihoglu, Semih %A Singh, Gurprit %A Ovsjanikov, Maks %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Spectral Measures of Distortion for Change Detection in Dynamic Graphs : %G eng %U http://hdl.handle.net/21.11116/0000-0003-F1F9-4 %R 10.1007/978-3-030-05414-4_5 %D 2019 %B 7th International Conference on Complex Networks and Their Applications %Z date of event: 2018-12-11 - 2018-12-13 %C Cambridge, UK %B Complex Networks and Their Applications VII %E Aiello, Luca Maria; Cherifi, Chantal; Cherifi, Hocine; Lambiotte, Renaud; Lió, Pietro; Rocha, Luis M. %P 54 - 66 %I Springer %@ 978-3-030-05413-7 978-3-030-05414-4 %B Studies in Computational Intelligence %N 813

Díaz Barros, J.M., Golyanik, V., Varanasi, K., and Stricker, D. 2019. Face It!: A Pipeline for Real-Time Performance-Driven Facial Animation. IEEE International Conference on Image Processing (ICIP 2019), IEEE.

Links

Export

BibTeX

@inproceedings{DiazBarros_ICIP2019, TITLE = {Face It!: {A} Pipeline for Real-Time Performance-Driven Facial Animation}, AUTHOR = {D{\'i}az Barros, Jilliam Mar{\'i}a and Golyanik, Vladislav and Varanasi, Kiran and Stricker, Didier}, LANGUAGE = {eng}, ISBN = {978-1-5386-6249-6}, DOI = {10.1109/ICIP.2019.8803330}, PUBLISHER = {IEEE}, YEAR = {2019}, BOOKTITLE = {IEEE International Conference on Image Processing (ICIP 2019)}, PAGES = {2209--2213}, ADDRESS = {Taipei, Taiwan}, }

Endnote

%0 Conference Proceedings %A Díaz Barros, Jilliam María %A Golyanik, Vladislav %A Varanasi, Kiran %A Stricker, Didier %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Face It!: A Pipeline for Real-Time Performance-Driven Facial Animation : %G eng %U http://hdl.handle.net/21.11116/0000-0005-982B-0 %R 10.1109/ICIP.2019.8803330 %D 2019 %B IEEE International Conference on Image Processing %Z date of event: 2019-09-22 - 2019-09-25 %C Taipei, Taiwan %B IEEE International Conference on Image Processing %P 2209 - 2213 %I IEEE %@ 978-1-5386-6249-6

Dokter, M., Hladký, J., Parger, M., Schmalstieg, D., Seidel, H.-P., and Steinberger, M. 2019. Hierarchical Rasterization of Curved Primitives for Vector Graphics Rendering on the GPU. Computer Graphics Forum (Proc. EUROGRAPHICS 2019)38, 2.

Links

Export

BibTeX

@article{Dokter_EG2019, TITLE = {Hierarchical Rasterization of Curved Primitives for Vector Graphics Rendering on the {GPU}}, AUTHOR = {Dokter, Mark and Hladk{\'y}, Jozef and Parger, Mathias and Schmalstieg, Dieter and Seidel, Hans-Peter and Steinberger, Markus}, LANGUAGE = {eng}, ISSN = {0167-7055}, DOI = {10.1111/cgf.13622}, PUBLISHER = {Wiley-Blackwell}, ADDRESS = {Oxford}, YEAR = {2019}, DATE = {2019}, JOURNAL = {Computer Graphics Forum (Proc. EUROGRAPHICS)}, VOLUME = {38}, NUMBER = {2}, PAGES = {93--103}, BOOKTITLE = {EUROGRAPHICS 2019 STAR -- State of The Art Reports}, }

Endnote

%0 Journal Article %A Dokter, Mark %A Hladký, Jozef %A Parger, Mathias %A Schmalstieg, Dieter %A Seidel, Hans-Peter %A Steinberger, Markus %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Hierarchical Rasterization of Curved Primitives for Vector Graphics Rendering on the GPU : %G eng %U http://hdl.handle.net/21.11116/0000-0002-FC80-1 %R 10.1111/cgf.13622 %7 2019 %D 2019 %J Computer Graphics Forum %V 38 %N 2 %& 93 %P 93 - 103 %I Wiley-Blackwell %C Oxford %@ false %B EUROGRAPHICS 2019 STAR – State of The Art Reports %O EUROGRAPHICS 2019 The 40th Annual Conference of the European Association for Computer Graphics ; Genova, Italy, May 6-10, 2019 EG 2019

Egger, B., Smith, W.A.P., Tewari, A., et al. 2019a. 3D Morphable Face Models -- Past, Present and Future. http://arxiv.org/abs/1909.01815.

(arXiv: 1909.01815)

Abstract

In this paper, we provide a detailed survey of 3D Morphable Face Models over the 20 years since they were first proposed. The challenges in building and applying these models, namely capture, modeling, image formation, and image analysis, are still active research topics, and we review the state-of-the-art in each of these areas. We also look ahead, identifying unsolved challenges, proposing directions for future research and highlighting the broad range of current and future applications.

Links

Export

BibTeX

@online{Egger_arXIv1909.01815, TITLE = {{3D} Morphable Face Models -- Past, Present and Future}, AUTHOR = {Egger, Bernhard and Smith, William A. P. and Tewari, Ayush and Wuhrer, Stefanie and Zollh{\"o}fer, Michael and Beeler, Thabo and Bernard, Florian and Bolkart, Timo and Kortylewski, Adam and Romdhani, Sami and Theobalt, Christian and Blanz, Volker and Vetter, Thomas}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1909.01815}, EPRINT = {1909.01815}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {In this paper, we provide a detailed survey of 3D Morphable Face Models over the 20 years since they were first proposed. The challenges in building and applying these models, namely capture, modeling, image formation, and image analysis, are still active research topics, and we review the state-of-the-art in each of these areas. We also look ahead, identifying unsolved challenges, proposing directions for future research and highlighting the broad range of current and future applications. }, }

Endnote

%0 Report %A Egger, Bernhard %A Smith, William A. P. %A Tewari, Ayush %A Wuhrer, Stefanie %A Zollhöfer, Michael %A Beeler, Thabo %A Bernard, Florian %A Bolkart, Timo %A Kortylewski, Adam %A Romdhani, Sami %A Theobalt, Christian %A Blanz, Volker %A Vetter, Thomas %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T 3D Morphable Face Models -- Past, Present and Future : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7D8E-0 %U http://arxiv.org/abs/1909.01815 %D 2019 %X In this paper, we provide a detailed survey of 3D Morphable Face Models over the 20 years since they were first proposed. The challenges in building and applying these models, namely capture, modeling, image formation, and image analysis, are still active research topics, and we review the state-of-the-art in each of these areas. We also look ahead, identifying unsolved challenges, proposing directions for future research and highlighting the broad range of current and future applications. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Egger, B., Smith, W., Theobalt, C., and Vetter, T., eds. 2019b. 3D Morphable Models. Schloss Dagstuhl.

Links

Export

BibTeX

@proceedings{Egger_2019, TITLE = {3D Morphable Models}, EDITOR = {Egger, Bernhard and Smith, William and Theobalt, Christian and Vetter, Thomas}, LANGUAGE = {eng}, URL = {urn:nbn:de:0030-drops-112894}, DOI = {10.4230/DagRep.9.3.16}, PUBLISHER = {Schloss Dagstuhl}, YEAR = {2019}, SERIES = {Dagstuhl Reports}, VOLUME = {9}, ISSUE = {3}, PAGES = {16--38}, ADDRESS = {Dagstuhl, Germany}, }

Endnote

%0 Conference Proceedings %E Egger, Bernhard %E Smith, William %E Theobalt, Christian %E Vetter, Thomas %+ External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T 3D Morphable Models : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7DF9-7 %U urn:nbn:de:0030-drops-112894 %R 10.4230/DagRep.9.3.16 %I Schloss Dagstuhl %D 2019 %B Dagstuhl Seminar 17201 "3D Morphable Models" %Z date of event: - %C Dagstuhl, Germany %S Dagstuhl Reports %V 9 %P 16 - 38 %U http://drops.dagstuhl.de/opus/volltexte/2019/11289/

Elgharib, M., Mallikarjun B R, Tewari, A., et al. 2019. EgoFace: Egocentric Face Performance Capture and Videorealistic Reenactment. http://arxiv.org/abs/1905.10822.

(arXiv: 1905.10822)

Abstract

Face performance capture and reenactment techniques use multiple cameras and sensors, positioned at a distance from the face or mounted on heavy wearable devices. This limits their applications in mobile and outdoor environments. We present EgoFace, a radically new lightweight setup for face performance capture and front-view videorealistic reenactment using a single egocentric RGB camera. Our lightweight setup allows operations in uncontrolled environments, and lends itself to telepresence applications such as video-conferencing from dynamic environments. The input image is projected into a low dimensional latent space of the facial expression parameters. Through careful adversarial training of the parameter-space synthetic rendering, a videorealistic animation is produced. Our problem is challenging as the human visual system is sensitive to the smallest face irregularities that could occur in the final results. This sensitivity is even stronger for video results. Our solution is trained in a pre-processing stage, through a supervised manner without manual annotations. EgoFace captures a wide variety of facial expressions, including mouth movements and asymmetrical expressions. It works under varying illuminations, background, movements, handles people from different ethnicities and can operate in real time.

Links

Export

BibTeX

@online{Elgharib_arXiv1905.10822, TITLE = {{EgoFace}: Egocentric Face Performance Capture and Videorealistic Reenactment}, AUTHOR = {Elgharib, Mohamed and Mallikarjun B R and Tewari, Ayush and Kim, Hyeongwoo and Liu, Wentao and Seidel, Hans-Peter and Theobalt, Christian}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1905.10822}, EPRINT = {1905.10822}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Face performance capture and reenactment techniques use multiple cameras and sensors, positioned at a distance from the face or mounted on heavy wearable devices. This limits their applications in mobile and outdoor environments. We present EgoFace, a radically new lightweight setup for face performance capture and front-view videorealistic reenactment using a single egocentric RGB camera. Our lightweight setup allows operations in uncontrolled environments, and lends itself to telepresence applications such as video-conferencing from dynamic environments. The input image is projected into a low dimensional latent space of the facial expression parameters. Through careful adversarial training of the parameter-space synthetic rendering, a videorealistic animation is produced. Our problem is challenging as the human visual system is sensitive to the smallest face irregularities that could occur in the final results. This sensitivity is even stronger for video results. Our solution is trained in a pre-processing stage, through a supervised manner without manual annotations. EgoFace captures a wide variety of facial expressions, including mouth movements and asymmetrical expressions. It works under varying illuminations, background, movements, handles people from different ethnicities and can operate in real time. }, }

Endnote

%0 Report %A Elgharib, Mohamed %A Mallikarjun B R, %A Tewari, Ayush %A Kim, Hyeongwoo %A Liu, Wentao %A Seidel, Hans-Peter %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T EgoFace: Egocentric Face Performance Capture and Videorealistic Reenactment : %G eng %U http://hdl.handle.net/21.11116/0000-0003-F1E6-9 %U http://arxiv.org/abs/1905.10822 %D 2019 %X Face performance capture and reenactment techniques use multiple cameras and sensors, positioned at a distance from the face or mounted on heavy wearable devices. This limits their applications in mobile and outdoor environments. We present EgoFace, a radically new lightweight setup for face performance capture and front-view videorealistic reenactment using a single egocentric RGB camera. Our lightweight setup allows operations in uncontrolled environments, and lends itself to telepresence applications such as video-conferencing from dynamic environments. The input image is projected into a low dimensional latent space of the facial expression parameters. Through careful adversarial training of the parameter-space synthetic rendering, a videorealistic animation is produced. Our problem is challenging as the human visual system is sensitive to the smallest face irregularities that could occur in the final results. This sensitivity is even stronger for video results. Our solution is trained in a pre-processing stage, through a supervised manner without manual annotations. EgoFace captures a wide variety of facial expressions, including mouth movements and asymmetrical expressions. It works under varying illuminations, background, movements, handles people from different ethnicities and can operate in real time. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR %U http://gvv.mpi-inf.mpg.de/projects/EgoFace/

Fried, O., Tewari, A., Zollhöfer, M., et al. 2019a. Text-based Editing of Talking-head Video. http://arxiv.org/abs/1906.01524.

(arXiv: 1906.01524)

Abstract

Editing talking-head video to change the speech content or to remove filler words is challenging. We propose a novel method to edit talking-head video based on its transcript to produce a realistic output video in which the dialogue of the speaker has been modified, while maintaining a seamless audio-visual flow (i.e. no jump cuts). Our method automatically annotates an input talking-head video with phonemes, visemes, 3D face pose and geometry, reflectance, expression and scene illumination per frame. To edit a video, the user has to only edit the transcript, and an optimization strategy then chooses segments of the input corpus as base material. The annotated parameters corresponding to the selected segments are seamlessly stitched together and used to produce an intermediate video representation in which the lower half of the face is rendered with a parametric face model. Finally, a recurrent video generation network transforms this representation to a photorealistic video that matches the edited transcript. We demonstrate a large variety of edits, such as the addition, removal, and alteration of words, as well as convincing language translation and full sentence synthesis.

Links

Export

BibTeX

@online{Fried_arXiv1906.01524, TITLE = {Text-based Editing of Talking-head Video}, AUTHOR = {Fried, Ohad and Tewari, Ayush and Zollh{\"o}fer, Michael and Finkelstein, Adam and Shechtman, Eli and Goldman, Dan B. and Genova, Kyle and Jin, Zeyu and Theobalt, Christian and Agrawala, Maneesh}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1906.01524}, EPRINT = {1906.01524}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Editing talking-head video to change the speech content or to remove filler words is challenging. We propose a novel method to edit talking-head video based on its transcript to produce a realistic output video in which the dialogue of the speaker has been modified, while maintaining a seamless audio-visual flow (i.e. no jump cuts). Our method automatically annotates an input talking-head video with phonemes, visemes, 3D face pose and geometry, reflectance, expression and scene illumination per frame. To edit a video, the user has to only edit the transcript, and an optimization strategy then chooses segments of the input corpus as base material. The annotated parameters corresponding to the selected segments are seamlessly stitched together and used to produce an intermediate video representation in which the lower half of the face is rendered with a parametric face model. Finally, a recurrent video generation network transforms this representation to a photorealistic video that matches the edited transcript. We demonstrate a large variety of edits, such as the addition, removal, and alteration of words, as well as convincing language translation and full sentence synthesis. }, }

Endnote

%0 Report %A Fried, Ohad %A Tewari, Ayush %A Zollhöfer, Michael %A Finkelstein, Adam %A Shechtman, Eli %A Goldman, Dan B. %A Genova, Kyle %A Jin, Zeyu %A Theobalt, Christian %A Agrawala, Maneesh %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Text-based Editing of Talking-head Video : %G eng %U http://hdl.handle.net/21.11116/0000-0003-FE15-8 %U http://arxiv.org/abs/1906.01524 %D 2019 %X Editing talking-head video to change the speech content or to remove filler words is challenging. We propose a novel method to edit talking-head video based on its transcript to produce a realistic output video in which the dialogue of the speaker has been modified, while maintaining a seamless audio-visual flow (i.e. no jump cuts). Our method automatically annotates an input talking-head video with phonemes, visemes, 3D face pose and geometry, reflectance, expression and scene illumination per frame. To edit a video, the user has to only edit the transcript, and an optimization strategy then chooses segments of the input corpus as base material. The annotated parameters corresponding to the selected segments are seamlessly stitched together and used to produce an intermediate video representation in which the lower half of the face is rendered with a parametric face model. Finally, a recurrent video generation network transforms this representation to a photorealistic video that matches the edited transcript. We demonstrate a large variety of edits, such as the addition, removal, and alteration of words, as well as convincing language translation and full sentence synthesis. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,Computer Science, Graphics, cs.GR,Computer Science, Learning, cs.LG

Fried, O., Tewari, A., Zollhöfer, M., et al. 2019b. Text-based Editing of Talking-head Video. ACM Transactions on Graphics (Proc. ACM SIGGRAPH 2019)38, 4.

Links

Export

BibTeX

@article{Fried_SIGGRAPH2019, TITLE = {Text-based Editing of Talking-head Video}, AUTHOR = {Fried, Ohad and Tewari, Ayush and Zollh{\"o}fer, Michael and Finkelstein, Adam and Shechtman, Eli and Goldman, Dan B. and Genova, Kyle and Jin, Zeyu and Theobalt, Christian and Agrawala, Maneesh}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3306346.3323028}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, DATE = {2019}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH)}, VOLUME = {38}, NUMBER = {4}, EID = {68}, BOOKTITLE = {Proceedings of ACM SIGGRAPH 2019}, }

Endnote

%0 Journal Article %A Fried, Ohad %A Tewari, Ayush %A Zollhöfer, Michael %A Finkelstein, Adam %A Shechtman, Eli %A Goldman, Dan B. %A Genova, Kyle %A Jin, Zeyu %A Theobalt, Christian %A Agrawala, Maneesh %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations External Organizations External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Text-based Editing of Talking-head Video : %G eng %U http://hdl.handle.net/21.11116/0000-0004-8458-4 %R 10.1145/3306346.3323028 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 4 %Z sequence number: 68 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH 2019 %O ACM SIGGRAPH 2019 Los Angeles, CA, USA, 28 July - 1 August

Golyanik, V. and Theobalt, C. 2019a. A Quantum Computational Approach to Correspondence Problems on Point Sets. http://arxiv.org/abs/1912.12296.

(arXiv: 1912.12296)

Abstract

Modern adiabatic quantum computers (AQC) are already used to solve difficult combinatorial optimisation problems in various domains of science. Currently, only a few applications of AQC in computer vision have been demonstrated. We review modern AQC and derive the first algorithm for transformation estimation and point set alignment suitable for AQC. Our algorithm has a subquadratic computational complexity of state preparation. We perform a systematic experimental analysis of the proposed approach and show several examples of successful point set alignment by simulated sampling. With this paper, we hope to boost the research on AQC for computer vision.

Links

Export

BibTeX

@online{Golyanik_arXiv1912.12296, TITLE = {A Quantum Computational Approach to Correspondence Problems on Point Sets}, AUTHOR = {Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1912.12296}, EPRINT = {1912.12296}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Modern adiabatic quantum computers (AQC) are already used to solve difficult combinatorial optimisation problems in various domains of science. Currently, only a few applications of AQC in computer vision have been demonstrated. We review modern AQC and derive the first algorithm for transformation estimation and point set alignment suitable for AQC. Our algorithm has a subquadratic computational complexity of state preparation. We perform a systematic experimental analysis of the proposed approach and show several examples of successful point set alignment by simulated sampling. With this paper, we hope to boost the research on AQC for computer vision. }, }

Endnote

%0 Report %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T A Quantum Computational Approach to Correspondence Problems on Point Sets : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7DF0-0 %U http://arxiv.org/abs/1912.12296 %D 2019 %X Modern adiabatic quantum computers (AQC) are already used to solve difficult combinatorial optimisation problems in various domains of science. Currently, only a few applications of AQC in computer vision have been demonstrated. We review modern AQC and derive the first algorithm for transformation estimation and point set alignment suitable for AQC. Our algorithm has a subquadratic computational complexity of state preparation. We perform a systematic experimental analysis of the proposed approach and show several examples of successful point set alignment by simulated sampling. With this paper, we hope to boost the research on AQC for computer vision. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV,cs.ET,Quantum Physics, quant-ph

Golyanik, V. and Theobalt, C. 2019b. Optimising for Scale in Globally Multiply-Linked Gravitational Point Set Registration Leads to Singularities. International Conference on 3D Vision, IEEE.

Links

Export

BibTeX

@inproceedings{Golyanik_3DV2019, TITLE = {Optimising for Scale in Globally Multiply-Linked Gravitational Point Set Registration Leads to Singularities}, AUTHOR = {Golyanik, Vladislav and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-3131-3}, DOI = {10.1109/3DV.2019.00027}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, BOOKTITLE = {International Conference on 3D Vision}, PAGES = {164--172}, ADDRESS = {Qu{\'e}bec City, Canada}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T Optimising for Scale in Globally Multiply-Linked Gravitational Point Set Registration Leads to Singularities : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7B4E-B %R 10.1109/3DV.2019.00027 %D 2019 %B International Conference on 3D Vision %Z date of event: 2019-09-16 - 2019-09-19 %C Québec City, Canada %B International Conference on 3D Vision %P 164 - 172 %I IEEE %@ 978-1-7281-3131-3

Golyanik, V., Jonas, A., Stricker, D., and Theobalt, C. 2019a. Intrinsic Dynamic Shape Prior for Fast, Sequential and Dense Non-Rigid Structure from Motion with Detection of Temporally-Disjoint Rigidity. http://arxiv.org/abs/1909.02468.

(arXiv: 1909.02468)

Abstract

While dense non-rigid structure from motion (NRSfM) has been extensively studied from the perspective of the reconstructability problem over the recent years, almost no attempts have been undertaken to bring it into the practical realm. The reasons for the slow dissemination are the severe ill-posedness, high sensitivity to motion and deformation cues and the difficulty to obtain reliable point tracks in the vast majority of practical scenarios. To fill this gap, we propose a hybrid approach that extracts prior shape knowledge from an input sequence with NRSfM and uses it as a dynamic shape prior for sequential surface recovery in scenarios with recurrence. Our Dynamic Shape Prior Reconstruction (DSPR) method can be combined with existing dense NRSfM techniques while its energy functional is optimised with stochastic gradient descent at real-time rates for new incoming point tracks. The proposed versatile framework with a new core NRSfM approach outperforms several other methods in the ability to handle inaccurate and noisy point tracks, provided we have access to a representative (in terms of the deformation variety) image sequence. Comprehensive experiments highlight convergence properties and the accuracy of DSPR under different disturbing effects. We also perform a joint study of tracking and reconstruction and show applications to shape compression and heart reconstruction under occlusions. We achieve state-of-the-art metrics (accuracy and compression ratios) in different scenarios.

Links

Export

BibTeX

@online{Golyanik_arXiv1909.02468, TITLE = {Intrinsic Dynamic Shape Prior for Fast, Sequential and Dense Non-Rigid Structure from Motion with Detection of Temporally-Disjoint Rigidity}, AUTHOR = {Golyanik, Vladislav and Jonas, Andr{\'e} and Stricker, Didier and Theobalt, Christian}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1909.02468}, EPRINT = {1909.02468}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {While dense non-rigid structure from motion (NRSfM) has been extensively studied from the perspective of the reconstructability problem over the recent years, almost no attempts have been undertaken to bring it into the practical realm. The reasons for the slow dissemination are the severe ill-posedness, high sensitivity to motion and deformation cues and the difficulty to obtain reliable point tracks in the vast majority of practical scenarios. To fill this gap, we propose a hybrid approach that extracts prior shape knowledge from an input sequence with NRSfM and uses it as a dynamic shape prior for sequential surface recovery in scenarios with recurrence. Our Dynamic Shape Prior Reconstruction (DSPR) method can be combined with existing dense NRSfM techniques while its energy functional is optimised with stochastic gradient descent at real-time rates for new incoming point tracks. The proposed versatile framework with a new core NRSfM approach outperforms several other methods in the ability to handle inaccurate and noisy point tracks, provided we have access to a representative (in terms of the deformation variety) image sequence. Comprehensive experiments highlight convergence properties and the accuracy of DSPR under different disturbing effects. We also perform a joint study of tracking and reconstruction and show applications to shape compression and heart reconstruction under occlusions. We achieve state-of-the-art metrics (accuracy and compression ratios) in different scenarios. }, }

Endnote

%0 Report %A Golyanik, Vladislav %A Jonas, André %A Stricker, Didier %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Intrinsic Dynamic Shape Prior for Fast, Sequential and Dense Non-Rigid Structure from Motion with Detection of Temporally-Disjoint Rigidity : %G eng %U http://hdl.handle.net/21.11116/0000-0005-7D9A-2 %U http://arxiv.org/abs/1909.02468 %D 2019 %X While dense non-rigid structure from motion (NRSfM) has been extensively studied from the perspective of the reconstructability problem over the recent years, almost no attempts have been undertaken to bring it into the practical realm. The reasons for the slow dissemination are the severe ill-posedness, high sensitivity to motion and deformation cues and the difficulty to obtain reliable point tracks in the vast majority of practical scenarios. To fill this gap, we propose a hybrid approach that extracts prior shape knowledge from an input sequence with NRSfM and uses it as a dynamic shape prior for sequential surface recovery in scenarios with recurrence. Our Dynamic Shape Prior Reconstruction (DSPR) method can be combined with existing dense NRSfM techniques while its energy functional is optimised with stochastic gradient descent at real-time rates for new incoming point tracks. The proposed versatile framework with a new core NRSfM approach outperforms several other methods in the ability to handle inaccurate and noisy point tracks, provided we have access to a representative (in terms of the deformation variety) image sequence. Comprehensive experiments highlight convergence properties and the accuracy of DSPR under different disturbing effects. We also perform a joint study of tracking and reconstruction and show applications to shape compression and heart reconstruction under occlusions. We achieve state-of-the-art metrics (accuracy and compression ratios) in different scenarios. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Golyanik, V., Theobalt, C., and Stricker, D. 2019b. Accelerated Gravitational Point Set Alignment with Altered Physical Laws. International Conference on Computer Vision (ICCV 2019), IEEE.

Links

Export

BibTeX

@inproceedings{BHRGA2019, TITLE = {Accelerated Gravitational Point Set Alignment with Altered Physical Laws}, AUTHOR = {Golyanik, Vladislav and Theobalt, Christian and Stricker, Didier}, LANGUAGE = {eng}, ISBN = {978-1-7281-4803-8}, DOI = {10.1109/ICCV.2019.00217}, PUBLISHER = {IEEE}, YEAR = {2019}, DATE = {2019}, BOOKTITLE = {International Conference on Computer Vision (ICCV 2019)}, PAGES = {2080--2089}, ADDRESS = {Seoul, Korea}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Theobalt, Christian %A Stricker, Didier %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Accelerated Gravitational Point Set Alignment with Altered Physical Laws : %G eng %U http://hdl.handle.net/21.11116/0000-0005-9473-2 %R 10.1109/ICCV.2019.00217 %D 2019 %B International Conference on Computer Vision %Z date of event: 2019-10-27 - 2019-11-02 %C Seoul, Korea %B International Conference on Computer Vision %P 2080 - 2089 %I IEEE %@ 978-1-7281-4803-8

Golyanik, V., Jonas, A., and Stricker, D. 2019c. Consolidating Segmentwise Non-Rigid Structure from Motion. Proceedings of the Sixteenth International Conference on Machine Vision Applications (MVA 2019), IEEE.

Links

Export

BibTeX

@inproceedings{Golyanik_MVA2019, TITLE = {Consolidating Segmentwise Non-Rigid Structure from Motion}, AUTHOR = {Golyanik, Vladislav and Jonas, Andr{\'e} and Stricker, Didier}, LANGUAGE = {eng}, ISBN = {978-4-901122-18-4}, DOI = {10.23919/MVA.2019.8757909}, PUBLISHER = {IEEE}, YEAR = {2019}, BOOKTITLE = {Proceedings of the Sixteenth International Conference on Machine Vision Applications (MVA 2019)}, PAGES = {1--6}, ADDRESS = {Tokyo, Japan}, }

Endnote

%0 Conference Proceedings %A Golyanik, Vladislav %A Jonas, André %A Stricker, Didier %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations %T Consolidating Segmentwise Non-Rigid Structure from Motion : %G eng %U http://hdl.handle.net/21.11116/0000-0005-9823-8 %R 10.23919/MVA.2019.8757909 %D 2019 %B Sixteenth International Conference on Machine Vision Applications %Z date of event: 2019-05-27 - 2019-05-31 %C Tokyo, Japan %B Proceedings of the Sixteenth International Conference on Machine Vision Applications %P 1 - 6 %I IEEE %@ 978-4-901122-18-4

Habermann, M., Xu, W., Zollhöfer, M., Pons-Moll, G., and Theobalt, C. 2019a. LiveCap: Real-time Human Performance Capture from Monocular Video. ACM Transactions on Graphics38, 2.

Links

Export

BibTeX

@article{Habermann_TOG19, TITLE = {{LiveCap}: {R}eal-time Human Performance Capture from Monocular Video}, AUTHOR = {Habermann, Marc and Xu, Weipeng and Zollh{\"o}fer, Michael and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3311970}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, DATE = {2019}, JOURNAL = {ACM Transactions on Graphics}, VOLUME = {38}, NUMBER = {2}, EID = {14}, }

Endnote

%0 Journal Article %A Habermann, Marc %A Xu, Weipeng %A Zollhöfer, Michael %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T LiveCap: Real-time Human Performance Capture from Monocular Video : %G eng %U http://hdl.handle.net/21.11116/0000-0002-B947-E %R 10.1145/3311970 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 2 %Z sequence number: 14 %I ACM %C New York, NY %@ false

Habermann, M., Xu, W., Rohdin, H., Zollhöfer, M., Pons-Moll, G., and Theobalt, C. 2019b. NRST: Non-rigid Surface Tracking from Monocular Video. Pattern Recognition (GCPR 2018), Springer.

Links

Export

BibTeX

@inproceedings{Habermann_GVPR18, TITLE = {{NRST}: {N}on-rigid Surface Tracking from Monocular Video}, AUTHOR = {Habermann, Marc and Xu, Weipeng and Rohdin, Helge and Zollh{\"o}fer, Michael and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-3-030-12938-5}, DOI = {10.1007/978-3-030-12939-2_23}, PUBLISHER = {Springer}, YEAR = {2018}, DATE = {2019}, BOOKTITLE = {Pattern Recognition (GCPR 2018)}, EDITOR = {Brox, Thomas and Bruhn, Andr{\'e}s and Fritz, Mario}, PAGES = {335--348}, SERIES = {Lecture Notes in Computer Science}, VOLUME = {11269}, ADDRESS = {Stuttgart, Germany}, }

Endnote

%0 Conference Proceedings %A Habermann, Marc %A Xu, Weipeng %A Rohdin, Helge %A Zollhöfer, Michael %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T NRST: Non-rigid Surface Tracking from Monocular Video : %G eng %U http://hdl.handle.net/21.11116/0000-0002-B94C-9 %R 10.1007/978-3-030-12939-2_23 %D 2019 %B 40th German Conference on Pattern Recognition %Z date of event: 2018-10-09 - 2018-10-12 %C Stuttgart, Germany %B Pattern Recognition %E Brox, Thomas; Bruhn, Andrés; Fritz, Mario %P 335 - 348 %I Springer %@ 978-3-030-12938-5 %B Lecture Notes in Computer Science %N 11269

Habibie, I., Xu, W., Mehta, D., Pons-Moll, G., and Theobalt, C. 2019a. In the Wild Human Pose Estimation using Explicit 2D Features and Intermediate 3D Representations. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2019), IEEE.

Links

Export

BibTeX

@inproceedings{habibieCVPR19, TITLE = {In the Wild Human Pose Estimation using Explicit {2D} Features and Intermediate {3D} Representations}, AUTHOR = {Habibie, Ikhsanul and Xu, Weipeng and Mehta, Dushyant and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, ISBN = {978-1-7281-3293-8}, DOI = {10.1109/CVPR.2019.01116}, PUBLISHER = {IEEE}, YEAR = {2019}, BOOKTITLE = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR 2019)}, PAGES = {10897--10906}, ADDRESS = {Long Beach, CA, USA}, }

Endnote

%0 Conference Proceedings %A Habibie, Ikhsanul %A Xu, Weipeng %A Mehta, Dushyant %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T In the Wild Human Pose Estimation using Explicit 2D Features and Intermediate 3D Representations : %G eng %U http://hdl.handle.net/21.11116/0000-0003-6520-7 %R 10.1109/CVPR.2019.01116 %D 2019 %B 32nd IEEE Conference on Computer Vision and Pattern Recognition %Z date of event: 2019-06-15 - 2019-06-20 %C Long Beach, CA, USA %B IEEE/CVF Conference on Computer Vision and Pattern Recognition %P 10897 - 10906 %I IEEE %@ 978-1-7281-3293-8

Habibie, I., Xu, W., Mehta, D., Pons-Moll, G., and Theobalt, C. 2019b. In the Wild Human Pose Estimation Using Explicit 2D Features and Intermediate 3D Representations. http://arxiv.org/abs/1904.03289.

(arXiv: 1904.03289)

Abstract

Convolutional Neural Network based approaches for monocular 3D human pose estimation usually require a large amount of training images with 3D pose annotations. While it is feasible to provide 2D joint annotations for large corpora of in-the-wild images with humans, providing accurate 3D annotations to such in-the-wild corpora is hardly feasible in practice. Most existing 3D labelled data sets are either synthetically created or feature in-studio images. 3D pose estimation algorithms trained on such data often have limited ability to generalize to real world scene diversity. We therefore propose a new deep learning based method for monocular 3D human pose estimation that shows high accuracy and generalizes better to in-the-wild scenes. It has a network architecture that comprises a new disentangled hidden space encoding of explicit 2D and 3D features, and uses supervision by a new learned projection model from predicted 3D pose. Our algorithm can be jointly trained on image data with 3D labels and image data with only 2D labels. It achieves state-of-the-art accuracy on challenging in-the-wild data.

Links

Export

BibTeX

@online{Habibie_arXiv1904.03289, TITLE = {In the Wild Human Pose Estimation Using Explicit {2D} Features and Intermediate {3D} Representations}, AUTHOR = {Habibie, Ikhsanul and Xu, Weipeng and Mehta, Dushyant and Pons-Moll, Gerard and Theobalt, Christian}, LANGUAGE = {eng}, URL = {http://arxiv.org/abs/1904.03289}, EPRINT = {1904.03289}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Convolutional Neural Network based approaches for monocular 3D human pose estimation usually require a large amount of training images with 3D pose annotations. While it is feasible to provide 2D joint annotations for large corpora of in-the-wild images with humans, providing accurate 3D annotations to such in-the-wild corpora is hardly feasible in practice. Most existing 3D labelled data sets are either synthetically created or feature in-studio images. 3D pose estimation algorithms trained on such data often have limited ability to generalize to real world scene diversity. We therefore propose a new deep learning based method for monocular 3D human pose estimation that shows high accuracy and generalizes better to in-the-wild scenes. It has a network architecture that comprises a new disentangled hidden space encoding of explicit 2D and 3D features, and uses supervision by a new learned projection model from predicted 3D pose. Our algorithm can be jointly trained on image data with 3D labels and image data with only 2D labels. It achieves state-of-the-art accuracy on challenging in-the-wild data. }, }

Endnote

%0 Report %A Habibie, Ikhsanul %A Xu, Weipeng %A Mehta, Dushyant %A Pons-Moll, Gerard %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Vision and Machine Learning, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society %T In the Wild Human Pose Estimation Using Explicit 2D Features and Intermediate 3D Representations : %G eng %U http://hdl.handle.net/21.11116/0000-0003-F76E-C %U http://arxiv.org/abs/1904.03289 %D 2019 %X Convolutional Neural Network based approaches for monocular 3D human pose estimation usually require a large amount of training images with 3D pose annotations. While it is feasible to provide 2D joint annotations for large corpora of in-the-wild images with humans, providing accurate 3D annotations to such in-the-wild corpora is hardly feasible in practice. Most existing 3D labelled data sets are either synthetically created or feature in-studio images. 3D pose estimation algorithms trained on such data often have limited ability to generalize to real world scene diversity. We therefore propose a new deep learning based method for monocular 3D human pose estimation that shows high accuracy and generalizes better to in-the-wild scenes. It has a network architecture that comprises a new disentangled hidden space encoding of explicit 2D and 3D features, and uses supervision by a new learned projection model from predicted 3D pose. Our algorithm can be jointly trained on image data with 3D labels and image data with only 2D labels. It achieves state-of-the-art accuracy on challenging in-the-wild data. %K Computer Science, Computer Vision and Pattern Recognition, cs.CV

Hladký, J., Seidel, H.-P., and Steinberger, M. 2019a. The Camera Offset Space: Real-time Potentially Visible Set Computations for Streaming Rendering. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2019)38, 6.

Links

Export

BibTeX

@article{Hladky_SA2019, TITLE = {The Camera Offset Space: Real-time Potentially Visible Set Computations for Streaming Rendering}, AUTHOR = {Hladk{\'y}, Jozef and Seidel, Hans-Peter and Steinberger, Markus}, LANGUAGE = {eng}, ISSN = {0730-0301}, ISBN = {978-1-4503-6008-1}, DOI = {10.1145/3355089.3356530}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {38}, NUMBER = {6}, EID = {231}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2019}, }

Endnote

%0 Journal Article %A Hladký, Jozef %A Seidel, Hans-Peter %A Steinberger, Markus %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T The Camera Offset Space: Real-time Potentially Visible Set Computations for Streaming Rendering : %G eng %U http://hdl.handle.net/21.11116/0000-0005-4E4F-D %R 10.1145/3355089.3356530 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 6 %Z sequence number: 231 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2019 %O ACM SIGGRAPH Asia 2019 Brisbane, Australia, 17 - 20 November 2019 SA'19 SA 2019 %@ 978-1-4503-6008-1

Hladký, J., Seidel, H.-P., and Steinberger, M. 2019b. Tessellated Shading Streaming. Computer Graphics Forum (Proc. Eurographics Symposium on Rendering 2019)38, 4.

Links

Export

BibTeX

@article{Hladky_EGSR2019, TITLE = {Tessellated Shading Streaming}, AUTHOR = {Hladk{\'y}, Jozef and Seidel, Hans-Peter and Steinberger, Markus}, LANGUAGE = {eng}, ISSN = {0167-7055}, URL = {https://diglib.eg.org/handle/10.1111/cgf13780}, DOI = {10.1111/cgf.13780}, PUBLISHER = {Wiley-Blackwell}, ADDRESS = {Oxford}, YEAR = {2019}, DATE = {2019}, JOURNAL = {Computer Graphics Forum (Proc. Eurographics Symposium on Rendering)}, VOLUME = {38}, NUMBER = {4}, PAGES = {171--182}, BOOKTITLE = {Eurographics Symposium on Rendering 2019}, EDITOR = {Boubekeur, Tamy and Sen, Pradeep}, }

Endnote

%0 Journal Article %A Hladký, Jozef %A Seidel, Hans-Peter %A Steinberger, Markus %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Tessellated Shading Streaming : %G eng %U http://hdl.handle.net/21.11116/0000-0004-4897-1 %R 10.1111/cgf.13780 %U https://diglib.eg.org/handle/10.1111/cgf13780 %7 2019 %D 2019 %J Computer Graphics Forum %V 38 %N 4 %& 171 %P 171 - 182 %I Wiley-Blackwell %C Oxford %@ false %B Eurographics Symposium on Rendering 2019 %O Eurographics Symposium on Rendering 2019 EGSR 2019 Strasbourg, France, July 10 - 12, 2109

Jiang, C., Tang, C., Seidel, H.-P., Chen, R., and Wonka, P. 2019. Computational Design of Lightweight Trusses. http://arxiv.org/abs/1901.05637.

(arXiv: 1901.05637)

Abstract

Trusses are load-carrying light-weight structures consisting of bars connected at joints ubiquitously applied in a variety of engineering scenarios. Designing optimal trusses that satisfy functional specifications with a minimal amount of material has interested both theoreticians and practitioners for more than a century. In this paper, we introduce two main ideas to improve upon the state of the art. First, we formulate an alternating linear programming problem for geometry optimization. Second, we introduce two sets of complementary topological operations, including a novel subdivision scheme for global topology refinement inspired by Michell's famed theoretical study. Based on these two ideas, we build an efficient computational framework for the design of lightweight trusses. \AD{We illustrate our framework with a variety of functional specifications and extensions. We show that our method achieves trusses with smaller volumes and is over two orders of magnitude faster compared with recent state-of-the-art approaches.

Links

Export

BibTeX

@online{Jiang_arXIv1901.05637, TITLE = {Computational Design of Lightweight Trusses}, AUTHOR = {Jiang, Caigui and Tang, Chengcheng and Seidel, Hans-Peter and Chen, Renjie and Wonka, Peter}, URL = {http://arxiv.org/abs/1901.05637}, EPRINT = {1901.05637}, EPRINTTYPE = {arXiv}, YEAR = {2019}, ABSTRACT = {Trusses are load-carrying light-weight structures consisting of bars connected at joints ubiquitously applied in a variety of engineering scenarios. Designing optimal trusses that satisfy functional specifications with a minimal amount of material has interested both theoreticians and practitioners for more than a century. In this paper, we introduce two main ideas to improve upon the state of the art. First, we formulate an alternating linear programming problem for geometry optimization. Second, we introduce two sets of complementary topological operations, including a novel subdivision scheme for global topology refinement inspired by Michell's famed theoretical study. Based on these two ideas, we build an efficient computational framework for the design of lightweight trusses. \AD{We illustrate our framework with a variety of functional specifications and extensions. We show that our method achieves trusses with smaller volumes and is over two orders of magnitude faster compared with recent state-of-the-art approaches. }, }

Endnote

%0 Report %A Jiang, Caigui %A Tang, Chengcheng %A Seidel, Hans-Peter %A Chen, Renjie %A Wonka, Peter %+ Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Computational Design of Lightweight Trusses : %U http://hdl.handle.net/21.11116/0000-0003-A7E9-A %U http://arxiv.org/abs/1901.05637 %D 2019 %X Trusses are load-carrying light-weight structures consisting of bars connected at joints ubiquitously applied in a variety of engineering scenarios. Designing optimal trusses that satisfy functional specifications with a minimal amount of material has interested both theoreticians and practitioners for more than a century. In this paper, we introduce two main ideas to improve upon the state of the art. First, we formulate an alternating linear programming problem for geometry optimization. Second, we introduce two sets of complementary topological operations, including a novel subdivision scheme for global topology refinement inspired by Michell's famed theoretical study. Based on these two ideas, we build an efficient computational framework for the design of lightweight trusses. \AD{We illustrate our framework with a variety of functional specifications and extensions. We show that our method achieves trusses with smaller volumes and is over two orders of magnitude faster compared with recent state-of-the-art approaches. %K Computer Science, Graphics, cs.GR

Kim, H. 2019. Learning-based Face Reconstruction and Editing. urn:nbn:de:bsz:291--ds-323942.

Links

Export

BibTeX

@phdthesis{Kim_2019, TITLE = {Learning-based Face Reconstruction and Editing}, AUTHOR = {Kim, Hyeongwoo}, LANGUAGE = {eng}, URL = {urn:nbn:de:bsz:291--ds-323942}, DOI = {10.22028/D291-32394}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2019}, DATE = {2019}, }

Endnote

%0 Thesis %A Kim, Hyeongwoo %Y Theobalt, Christian %A referee: Izadi, Shahram %A referee: Richardt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Learning-based Face Reconstruction and Editing : %G eng %U http://hdl.handle.net/21.11116/0000-0007-6FFA-4 %R 10.22028/D291-32394 %U urn:nbn:de:bsz:291--ds-323942 %F OTHER: hdl:20.500.11880/29769 %I Universität des Saarlandes %C Saarbrücken %D 2019 %P 124 p. %V phd %9 phd %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/29769

Kim, H., Elgharib, M., Zollhöfer, M., et al. 2019. Neural Style-preserving Visual Dubbing. ACM Transactions on Graphics38, 6.

Links

Export

BibTeX

@article{Kim2019, TITLE = {Neural Style-preserving Visual Dubbing}, AUTHOR = {Kim, Hyeongwoo and Elgharib, Mohamed and Zollh{\"o}fer, Michael and Seidel, Hans-Peter and Beeler, Thabo and Richardt, Christian and Theobalt, Christian}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3355089.3356500}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, DATE = {2019}, JOURNAL = {ACM Transactions on Graphics}, VOLUME = {38}, NUMBER = {6}, EID = {178}, }

Endnote

%0 Journal Article %A Kim, Hyeongwoo %A Elgharib, Mohamed %A Zollhöfer, Michael %A Seidel, Hans-Peter %A Beeler, Thabo %A Richardt, Christian %A Theobalt, Christian %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations Computer Graphics, MPI for Informatics, Max Planck Society %T Neural Style-preserving Visual Dubbing : %G eng %U http://hdl.handle.net/21.11116/0000-0005-6AC0-B %R 10.1145/3355089.3356500 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 6 %Z sequence number: 178 %I ACM %C New York, NY %@ false

Kovalenko, O., Golyanik, V., Malik, J., Elhayek, A., and Stricker, D. 2019. Structure from Articulated Motion: Accurate and Stable Monocular 3D Reconstruction without Training Data. Sensors19, 20.

Links

Export

BibTeX

@article{Kovalenko2019, TITLE = {Structure from Articulated Motion: {A}ccurate and Stable Monocular {3D} Reconstruction without Training Data}, AUTHOR = {Kovalenko, Onorina and Golyanik, Vladislav and Malik, Jameel and Elhayek, Ahmed and Stricker, Didier}, LANGUAGE = {eng}, ISSN = {1424-8220}, DOI = {10.3390/s19204603}, PUBLISHER = {MDPI}, YEAR = {2019}, JOURNAL = {Sensors}, VOLUME = {19}, NUMBER = {20}, EID = {4603}, }

Endnote

%0 Journal Article %A Kovalenko, Onorina %A Golyanik, Vladislav %A Malik, Jameel %A Elhayek, Ahmed %A Stricker, Didier %+ External Organizations Computer Graphics, MPI for Informatics, Max Planck Society External Organizations External Organizations External Organizations %T Structure from Articulated Motion: Accurate and Stable Monocular 3D Reconstruction without Training Data : %G eng %U http://hdl.handle.net/21.11116/0000-0005-5CB5-8 %R 10.3390/s19204603 %7 2019 %D 2019 %J Sensors %V 19 %N 20 %Z sequence number: 4603 %I MDPI %@ false

Leimkühler, T., Singh, G., Myszkowski, K., Seidel, H.-P., and Ritschel, T. 2019. Deep Point Correlation Design. ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia 2019)38, 6.

Links

Export

BibTeX

@article{Leimkuehler_SA2019, TITLE = {Deep Point Correlation Design}, AUTHOR = {Leimk{\"u}hler, Thomas and Singh, Gurprit and Myszkowski, Karol and Seidel, Hans-Peter and Ritschel, Tobias}, LANGUAGE = {eng}, ISSN = {0730-0301}, DOI = {10.1145/3355089.3356562}, PUBLISHER = {ACM}, ADDRESS = {New York, NY}, YEAR = {2019}, JOURNAL = {ACM Transactions on Graphics (Proc. ACM SIGGRAPH Asia)}, VOLUME = {38}, NUMBER = {6}, EID = {226}, BOOKTITLE = {Proceedings of ACM SIGGRAPH Asia 2019}, }

Endnote

%0 Journal Article %A Leimkühler, Thomas %A Singh, Gurprit %A Myszkowski, Karol %A Seidel, Hans-Peter %A Ritschel, Tobias %+ Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Deep Point Correlation Design : %G eng %U http://hdl.handle.net/21.11116/0000-0004-9BF3-B %R 10.1145/3355089.3356562 %7 2019 %D 2019 %J ACM Transactions on Graphics %V 38 %N 6 %Z sequence number: 226 %I ACM %C New York, NY %@ false %B Proceedings of ACM SIGGRAPH Asia 2019 %O ACM SIGGRAPH Asia 2019 Brisbane, Australia, 17 - 20 November 2019 SA'19 SA 2019

Leimkühler, T. 2019. Artificial Intelligence for Efficient Image-based View Synthesis. urn:nbn:de:bsz:291--ds-283793.

Abstract

Synthesizing novel views from image data is a widely investigated topic in both computer graphics and computer vision, and has many applications like stereo or multi-view rendering for virtual reality, light field reconstruction, and image post-processing. While image-based approaches have the advantage of reduced computational load compared to classical model-based rendering, efficiency is still a major concern. This thesis demonstrates how concepts and tools from artificial intelligence can be used to increase the efficiency of image-based view synthesis algorithms. In particular it is shown how machine learning can help to generate point patterns useful for a variety of computer graphics tasks, how path planning can guide image warping, how sparsity-enforcing optimization can lead to significant speedups in interactive distribution effect rendering, and how probabilistic inference can be used to perform real-time 2D-to-3D conversion.

Links

Export

BibTeX

@phdthesis{Leimphd2019, TITLE = {Artificial Intelligence for Efficient Image-based View Synthesis}, AUTHOR = {Leimk{\"u}hler, Thomas}, LANGUAGE = {eng}, URL = {urn:nbn:de:bsz:291--ds-283793}, DOI = {10.22028/D291-28379}, SCHOOL = {Universit{\"a}t des Saarlandes}, ADDRESS = {Saarbr{\"u}cken}, YEAR = {2019}, DATE = {2019}, ABSTRACT = {Synthesizing novel views from image data is a widely investigated topic in both computer graphics and computer vision, and has many applications like stereo or multi-view rendering for virtual reality, light field reconstruction, and image post-processing. While image-based approaches have the advantage of reduced computational load compared to classical model-based rendering, efficiency is still a major concern. This thesis demonstrates how concepts and tools from artificial intelligence can be used to increase the efficiency of image-based view synthesis algorithms. In particular it is shown how machine learning can help to generate point patterns useful for a variety of computer graphics tasks, how path planning can guide image warping, how sparsity-enforcing optimization can lead to significant speedups in interactive distribution effect rendering, and how probabilistic inference can be used to perform real-time 2D-to-3D conversion.}, }

Endnote

%0 Thesis %A Leimkühler, Thomas %Y Seidel, Hans-Peter %A referee: Ritschel, Tobias %A referee: Lensch, Hendrik %A referee: Drettakis, George %+ Computer Graphics, MPI for Informatics, Max Planck Society International Max Planck Research School, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society Computer Graphics, MPI for Informatics, Max Planck Society External Organizations %T Artificial Intelligence for Efficient Image-based View Synthesis : %G eng %U http://hdl.handle.net/21.11116/0000-0004-A589-7 %R 10.22028/D291-28379 %U urn:nbn:de:bsz:291--ds-283793 %F OTHER: hdl:20.500.11880/27664 %I Universität des Saarlandes %C Saarbrücken %D 2019 %P 136 p. %V phd %9 phd %X Synthesizing novel views from image data is a widely investigated topic in both computer graphics and computer vision, and has many applications like stereo or multi-view rendering for virtual reality, light field reconstruction, and image post-processing. While image-based approaches have the advantage of reduced computational load compared to classical model-based rendering, efficiency is still a major concern. This thesis demonstrates how concepts and tools from artificial intelligence can be used to increase the efficiency of image-based view synthesis algorithms. In particular it is shown how machine learning can help to generate point patterns useful for a variety of computer graphics tasks, how path planning can guide image warping, how sparsity-enforcing optimization can lead to significant speedups in interactive distribution effect rendering, and how probabilistic inference can be used to perform real-time 2D-to-3D conversion. %U https://publikationen.sulb.uni-saarland.de/handle/20.500.11880/27664

Liu, L., Xu, W., Zollhöfer, M., et al. 2019. Neural Rendering and Reenactment of Human A