File size: 12,568 Bytes
7e10b4a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<!-- Meta tags for social media banners, these should be filled in appropriately as they are your "business card" -->
<!-- Replace the content tag with appropriate information -->
<meta content="VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild"
property="og:title">
<meta content="VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild"
name="description" property="og:description">
<meta content="https://vinthony.github.io/video-retalking/" property="og:url">
<!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X630-->
<meta property="og:image" content="static/image/your_banner_image.png" />
<meta property="og:image:width" content="1200"/>
<meta property="og:image:height" content="630"/>
<meta name="twitter:title" content="TWITTER BANNER TITLE META TAG">
<meta name="twitter:description" content="TWITTER BANNER DESCRIPTION META TAG">
<!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X600-->
<meta name="twitter:image" content="static/images/your_twitter_banner_image.png">
<meta name="twitter:card" content="summary_large_image">
<!-- Keywords for your paper to be indexed by-->
<meta name="keywords" content="KEYWORDS SHOULD BE PLACED HERE">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>VideoRetalking</title>
<link rel="icon" type="image/x-icon" href="static/images/favicon.ico">
<link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
rel="stylesheet">
<link rel="stylesheet" href="static/css/bulma.min.css">
<link rel="stylesheet" href="static/css/bulma-carousel.min.css">
<link rel="stylesheet" href="static/css/bulma-slider.min.css">
<link rel="stylesheet" href="static/css/fontawesome.all.min.css">
<link rel="stylesheet"
href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
<link rel="stylesheet" href="static/css/index.css">
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src="https://documentcloud.adobe.com/view-sdk/main.js"></script>
<script defer src="static/js/fontawesome.all.min.js"></script>
<script src="static/js/bulma-carousel.min.js"></script>
<script src="static/js/bulma-slider.min.js"></script>
<script src="static/js/index.js"></script>
</head>
<body>
<section class="hero">
<div class="hero-body">
<div class="container is-max-desktop">
<div class="columns is-centered">
<div class="column has-text-centered">
<h1 class="xtitle is-1 publication-title">VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild</h1>
<br/>
<div class="is-size-5 publication-authors">
<!-- Paper authors -->
<span class="author-block">
<a href="#" target="_blank">Kun Cheng</a><sup>*,1,2</sup></span>
<span class="author-block">
<a href="https://vinthony.github.io" target="_blank">Xiaodong Cun</a><sup>*,2</sup></span>
<span class="author-block">
<a href="https://yzhang2016.github.io" target="_blank">Yong Zhang</a><sup>2</sup>
</span>
<span class="author-block">
<a href="https://menghanxia.github.io/" target="_blank">Menghan Xia</a><sup>2</sup>
</span>
<span class="author-block">
<a href="https://feiiyin.github.io/" target="_blank">Fei Yin</a><sup>2,3</sup>
</span>
</br>
<span class="author-block">
<a href="https://web.xidian.edu.cn/mrzhu/en/index.html" target="_blank">Mingrui Zhu</a><sup>1</sup>
</span>
<span class="author-block">
<a href="https://xuanwangvc.github.io/" target="_blank">Xuan Wang</a><sup>2</sup>
</span>
<span class="author-block">
<a href="https://juewang725.github.io/" target="_blank">Jue Wang</a><sup>2</sup>
</span>
<span class="author-block">
<a href="https://web.xidian.edu.cn/nnwang/en/index.html" target="_blank">Nannan Wang</a><sup>1</sup>
</span>
</div>
<br/>
<div class="is-size-5 publication-authors">
<span class="author-block">
<sup>1</sup> Xidian University
<sup>2</sup> Tencent AI Lab
<sup>3</sup> Tsinghua University
<br>SIGGRAPH Asia 2022 (Conference Track)</span>
<span class="eql-cntrb"><small><br><sup>*</sup>Indicates Equal Contribution</small></span>
</div>
<div class="column has-text-centered">
<div class="publication-links">
<!-- Arxiv PDF link -->
<span class="link-block">
<a href="https://arxiv.org/pdf/2211.14758.pdf" target="_blank"
class="external-link ">
<span class="icon">
<i class="fas fa-file-pdf"></i>
</span>
<span>Paper</span>
</a>
</span>
<!-- Github link -->
<span class="link-block">
<a href="https://github.com/vinthony/video-retalking/" target="_blank"
class="external-link ">
<span class="icon">
<i class="fab fa-github"></i>
</span>
<span>Code</span>
</a>
</span>
<!-- ArXiv abstract Link -->
<span class="link-block">
<a href="https://arxiv.org/abs/2211.14758" target="_blank"
class="external-link ">
<span class="icon">
<i class="ai ai-arxiv"></i>
</span>
<span>arXiv</span>
</a>
</span>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<!-- Teaser video-->
<section class="hero teaser">
<div class="container is-max-desktop">
<div class="hero-body-img">
<img src="./static/images/teaser.png" width="80%">
</div>
</div>
</section>
<!-- End teaser video -->
<!-- Paper abstract -->
<section class="section hero is-light">
<div class="container is-max-desktop">
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<h2 class="title is-3">Abstract</h2>
<div class="content has-text-justified">
<p>
We present VideoReTalking, a new system to edit the faces of a real-world talking head video according to input audio,
producing a high-quality and lip-syncing output video even with a different emotion. Our system disentangles this objective
into three sequential tasks: (1) face video generation with a canonical expression; (2) audio-driven lip-sync; and
(3) face enhancement for improving photo-realism. Given a talking-head video, we first modify the expression of each frame
according to the same expression template using the expression editing network, resulting in a video with the canonical
expression. This video, together with the given audio, is then fed into the lip-sync network to generate a lip-syncing video.
Finally, we improve the photo-realism of the synthesized faces through an identity-aware face enhancement network and
post-processing. We use learning-based approaches for all three steps and all our modules can be tackled in a sequential
pipeline without any user intervention.
</p>
</div>
</div>
</div>
</div>
</section>
<!-- End paper abstract -->
<!-- Youtube video -->
<section class="hero is-small is-light">
<div class="hero-body">
<div class="container">
<!-- Paper video. -->
<h2 class="title is-3">Pipeline</h2>
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<div class="hero-body-img">
<!-- Youtube embed code here -->
<img width='80%' src="static/images/pipeline.png">
</div>
</div>
</div>
</div>
</div>
</section>
<!-- End youtube video -->
<!-- Youtube video -->
<section class="hero is-small is-light">
<div class="hero-body">
<div class="container">
<!-- Paper video. -->
<h2 class="title is-3"><strong>Video1</strong>: Video Results in the Wild.</h2>
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<video controls="" width="100%">
<!-- t=0.001 is a hack to make iPhone show video thumbnail -->
<source src="./static/videos/Results_in_the_wild.mp4#t=0.001" type="video/mp4">
</video>
</div>
</div>
</div>
</div>
</section>
<!-- End youtube video
!-- Youtube video -->
<section class="hero is-small is-light">
<div class="hero-body">
<div class="container">
<!-- Paper video. -->
<h2 class="title is-3"><strong>Video2</strong>: Comparison with SOTA Methods.</h2>
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<video controls="" width="100%">
<!-- t=0.001 is a hack to make iPhone show video thumbnail -->
<source src="./static/videos/Comparison.mp4#t=0.001" type="video/mp4">
</video>
</div>
</div>
</div>
</div>
</section>
<section class="hero is-small is-light">
<div class="hero-body">
<div class="container">
<!-- Paper video. -->
<h2 class="title is-3"><strong>Video3</strong>: Ablation Study on Different Modules. </h2>
<div class="columns is-centered has-text-centered">
<div class="column is-four-fifths">
<video controls="" width="100%">
<!-- t=0.001 is a hack to make iPhone show video thumbnail -->
<source src="./static/videos/Ablation.mp4#0.001" type="video/mp4">
</video>
</div>
</div>
</div>
</div>
</section>
<!--BibTex citation -->
<section class="section" id="BibTeX">
<div class="container is-max-desktop content">
<h2 class="title">BibTeX</h2>
<pre><code>@misc{videoretalking,
title={VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild},
author={Kun Cheng and Xiaodong Cun and Yong Zhang and Menghan Xia and Fei Yin and Mingrui Zhu and Xuan Wang and Jue Wang and Nannan Wang},
year={2022},
eprint={2211.14758},
archivePrefix={arXiv},
primaryClass={cs.CV}
}</code></pre>
</div>
</section>
<!--End BibTex citation -->
<footer class="footer">
<div class="container">
<div class="columns is-centered">
<div class="column is-8">
<div class="content">
<p>
This page was built using the <a href="https://github.com/vinthony/project-page-template">modification version</a> of <a href="https://github.com/eliahuhorwitz/Academic-project-page-template" target="_blank">Academic Project Page Template</a> from <a href="https://github.com/vinthony">vinthony</a>.
You are free to borrow the of this website, we just ask that you link back to this page in the footer. <br> This website is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/" target="_blank">Creative
Commons Attribution-ShareAlike 4.0 International License</a>.
</p>
</div>
</div>
</div>
</div>
</footer>
<!-- Statcounter tracking code -->
<!-- You can add a tracker to track page visits by creating an account at statcounter.com -->
<!-- End of Statcounter Code -->
</body>
</html>
|