5dimension commited on
Commit
7744aeb
·
verified ·
1 Parent(s): 02ef535

Update index.html

Browse files
Files changed (1) hide show
  1. index.html +81 -270
index.html CHANGED
@@ -3,10 +3,10 @@
3
  <head>
4
  <meta charset="utf-8">
5
  <meta name="description"
6
- content="Deformable Neural Radiance Fields creates free-viewpoint portraits (nerfies) from casually captured videos.">
7
- <meta name="keywords" content="Nerfies, D-NeRF, NeRF">
8
  <meta name="viewport" content="width=device-width, initial-scale=1">
9
- <title>Nerfies: Deformable Neural Radiance Fields</title>
10
 
11
  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
12
  rel="stylesheet">
@@ -33,68 +33,22 @@
33
  <div class="container is-max-desktop">
34
  <div class="columns is-centered">
35
  <div class="column has-text-centered">
36
- <h1 class="title is-1 publication-title">Nerfies: Deformable Neural Radiance Fields</h1>
37
  <div class="is-size-5 publication-authors">
38
  <span class="author-block">
39
- <a href="https://keunhong.com" target="_blank">Keunhong Park</a><sup>1</sup>,</span>
40
- <span class="author-block">
41
- <a href="https://utkarshsinha.com" target="_blank">Utkarsh Sinha</a><sup>2</sup>,</span>
42
- <span class="author-block">
43
- <a href="https://jonbarron.info" target="_blank">Jonathan T. Barron</a><sup>2</sup>,
44
- </span>
45
- <span class="author-block">
46
- <a href="http://sofienbouaziz.com" target="_blank">Sofien Bouaziz</a><sup>2</sup>,
47
- </span>
48
- <span class="author-block">
49
- <a href="https://www.danbgoldman.com" target="_blank">Dan B Goldman</a><sup>2</sup>,
50
- </span>
51
- <span class="author-block">
52
- <a href="https://homes.cs.washington.edu/~seitz/" target="_blank">Steven M. Seitz</a><sup>1,2</sup>,
53
- </span>
54
- <span class="author-block">
55
- <a href="http://www.ricardomartinbrualla.com" target="_blank">Ricardo Martin-Brualla</a><sup>2</sup>
56
  </span>
57
  </div>
58
 
59
  <div class="is-size-5 publication-authors">
60
- <span class="author-block"><sup>1</sup>University of Washington,</span>
61
- <span class="author-block"><sup>2</sup>Google Research</span>
62
  </div>
63
 
64
  <div class="column has-text-centered">
65
  <div class="publication-links">
66
- <!-- PDF Link. -->
67
- <span class="link-block">
68
- <a href="https://arxiv.org/pdf/2011.12948" target="_blank"
69
- class="external-link button is-normal is-rounded is-dark">
70
- <span class="icon">
71
- <i class="fas fa-file-pdf"></i>
72
- </span>
73
- <span>Paper</span>
74
- </a>
75
- </span>
76
- <span class="link-block">
77
- <a href="https://arxiv.org/abs/2011.12948" target="_blank"
78
- class="external-link button is-normal is-rounded is-dark">
79
- <span class="icon">
80
- <i class="ai ai-arxiv"></i>
81
- </span>
82
- <span>arXiv</span>
83
- </a>
84
- </span>
85
- <!-- Video Link. -->
86
- <span class="link-block">
87
- <a href="https://www.youtube.com/watch?v=MrKrnHhk8IA" target="_blank"
88
- class="external-link button is-normal is-rounded is-dark">
89
- <span class="icon">
90
- <i class="fab fa-youtube"></i>
91
- </span>
92
- <span>Video</span>
93
- </a>
94
- </span>
95
  <!-- Code Link. -->
96
  <span class="link-block">
97
- <a href="https://github.com/google/nerfies" target="_blank"
98
  class="external-link button is-normal is-rounded is-dark">
99
  <span class="icon">
100
  <i class="fab fa-github"></i>
@@ -102,17 +56,7 @@
102
  <span>Code</span>
103
  </a>
104
  </span>
105
- <!-- Dataset Link. -->
106
- <span class="link-block">
107
- <a href="https://github.com/google/nerfies/releases/tag/0.1" target="_blank"
108
- class="external-link button is-normal is-rounded is-dark">
109
- <span class="icon">
110
- <i class="far fa-images"></i>
111
- </span>
112
- <span>Data</span>
113
- </a>
114
  </div>
115
-
116
  </div>
117
  </div>
118
  </div>
@@ -123,14 +67,10 @@
123
  <section class="hero teaser">
124
  <div class="container is-max-desktop">
125
  <div class="hero-body">
126
- <video id="teaser" autoplay muted loop playsinline height="100%">
127
- <source src="./static/videos/teaser.mp4"
128
- type="video/mp4">
129
- </video>
130
- <h2 class="subtitle has-text-centered">
131
- <span class="dnerf">Nerfies</span> turns selfie videos from your phone into
132
- free-viewpoint
133
- portraits.
134
  </h2>
135
  </div>
136
  </div>
@@ -141,53 +81,35 @@
141
  <div class="hero-body">
142
  <div class="container">
143
  <div id="results-carousel" class="carousel results-carousel">
144
- <div class="item item-steve">
145
- <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
146
- <source src="./static/videos/steve.mp4"
147
- type="video/mp4">
148
- </video>
149
- </div>
150
- <div class="item item-chair-tp">
151
- <video poster="" id="chair-tp" autoplay controls muted loop playsinline height="100%">
152
- <source src="./static/videos/chair-tp.mp4"
153
- type="video/mp4">
154
- </video>
155
- </div>
156
- <div class="item item-shiba">
157
- <video poster="" id="shiba" autoplay controls muted loop playsinline height="100%">
158
- <source src="./static/videos/shiba.mp4"
159
- type="video/mp4">
160
- </video>
161
- </div>
162
- <div class="item item-fullbody">
163
- <video poster="" id="fullbody" autoplay controls muted loop playsinline height="100%">
164
- <source src="./static/videos/fullbody.mp4"
165
- type="video/mp4">
166
- </video>
167
  </div>
168
- <div class="item item-blueshirt">
169
- <video poster="" id="blueshirt" autoplay controls muted loop playsinline height="100%">
170
- <source src="./static/videos/blueshirt.mp4"
171
- type="video/mp4">
172
- </video>
173
  </div>
174
- <div class="item item-mask">
175
- <video poster="" id="mask" autoplay controls muted loop playsinline height="100%">
176
- <source src="./static/videos/mask.mp4"
177
- type="video/mp4">
178
- </video>
179
  </div>
180
- <div class="item item-coffee">
181
- <video poster="" id="coffee" autoplay controls muted loop playsinline height="100%">
182
- <source src="./static/videos/coffee.mp4"
183
- type="video/mp4">
184
- </video>
185
  </div>
186
- <div class="item item-toby">
187
- <video poster="" id="toby" autoplay controls muted loop playsinline height="100%">
188
- <source src="./static/videos/toby2.mp4"
189
- type="video/mp4">
190
- </video>
191
  </div>
192
  </div>
193
  </div>
@@ -197,37 +119,20 @@
197
 
198
  <section class="section">
199
  <div class="container is-max-desktop">
200
- <!-- Abstract. -->
201
  <div class="columns is-centered has-text-centered">
202
  <div class="column is-four-fifths">
203
- <h2 class="title is-3">Abstract</h2>
204
  <div class="content has-text-justified">
205
  <p>
206
- We present the first method capable of photorealistically reconstructing a non-rigidly
207
- deforming scene using photos/videos captured casually from mobile phones.
208
  </p>
209
- <p>
210
- Our approach augments neural radiance fields
211
- (NeRF) by optimizing an
212
- additional continuous volumetric deformation field that warps each observed point into a
213
- canonical 5D NeRF.
214
- We observe that these NeRF-like deformation fields are prone to local minima, and
215
- propose a coarse-to-fine optimization method for coordinate-based models that allows for
216
- more robust optimization.
217
- By adapting principles from geometry processing and physical simulation to NeRF-like
218
- models, we propose an elastic regularization of the deformation field that further
219
- improves robustness.
220
  </p>
221
  <p>
222
- We show that <span class="dnerf">Nerfies</span> can turn casually captured selfie
223
- photos/videos into deformable NeRF
224
- models that allow for photorealistic renderings of the subject from arbitrary
225
- viewpoints, which we dub <i>"nerfies"</i>. We evaluate our method by collecting data
226
- using a
227
- rig with two mobile phones that take time-synchronized photos, yielding train/validation
228
- images of the same pose at different viewpoints. We show that our method faithfully
229
- reconstructs non-rigidly deforming scenes and reproduces unseen views with high
230
- fidelity.
231
  </p>
232
  </div>
233
  </div>
@@ -237,10 +142,17 @@
237
  <!-- Paper video. -->
238
  <div class="columns is-centered has-text-centered">
239
  <div class="column is-four-fifths">
240
- <h2 class="title is-3">Video</h2>
241
- <div class="publication-video">
242
- <iframe src="https://www.youtube.com/embed/MrKrnHhk8IA?rel=0&amp;showinfo=0"
243
- frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
 
 
 
 
 
 
 
244
  </div>
245
  </div>
246
  </div>
@@ -253,160 +165,63 @@
253
  <div class="container is-max-desktop">
254
 
255
  <div class="columns is-centered">
256
-
257
- <!-- Visual Effects. -->
258
  <div class="column">
259
  <div class="content">
260
- <h2 class="title is-3">Visual Effects</h2>
261
- <p>
262
- Using <i>nerfies</i> you can create fun visual effects. This Dolly zoom effect
263
- would be impossible without nerfies since it would require going through a wall.
264
- </p>
265
- <video id="dollyzoom" autoplay controls muted loop playsinline height="100%">
266
- <source src="./static/videos/dollyzoom-stacked.mp4"
267
- type="video/mp4">
268
- </video>
269
- </div>
270
- </div>
271
- <!--/ Visual Effects. -->
272
-
273
- <!-- Matting. -->
274
- <div class="column">
275
- <h2 class="title is-3">Matting</h2>
276
- <div class="columns is-centered">
277
- <div class="column content">
278
- <p>
279
- As a byproduct of our method, we can also solve the matting problem by ignoring
280
- samples that fall outside of a bounding box during rendering.
281
- </p>
282
- <video id="matting-video" controls playsinline height="100%">
283
- <source src="./static/videos/matting.mp4"
284
- type="video/mp4">
285
- </video>
286
- </div>
287
-
288
  </div>
289
  </div>
 
290
  </div>
291
- <!--/ Matting. -->
292
 
293
- <!-- Animation. -->
294
  <div class="columns is-centered">
295
  <div class="column is-full-width">
296
- <h2 class="title is-3">Animation</h2>
297
-
298
- <!-- Interpolating. -->
299
- <h3 class="title is-4">Interpolating states</h3>
300
  <div class="content has-text-justified">
301
  <p>
302
- We can also animate the scene by interpolating the deformation latent codes of two input
303
- frames. Use the slider here to linearly interpolate between the left frame and the right
304
- frame.
305
  </p>
306
  </div>
307
- <div class="columns is-vcentered interpolation-panel">
308
- <div class="column is-3 has-text-centered">
309
- <img src="./static/images/interpolate_start.jpg"
310
- class="interpolation-image"
311
- alt="Interpolate start reference image."/>
312
- <p>Start Frame</p>
313
- </div>
314
- <div class="column interpolation-video-column">
315
- <div id="interpolation-image-wrapper">
316
- Loading...
317
- </div>
318
- <input class="slider is-fullwidth is-large is-info"
319
- id="interpolation-slider"
320
- step="1" min="0" max="100" value="0" type="range">
321
- </div>
322
- <div class="column is-3 has-text-centered">
323
- <img src="./static/images/interpolate_end.jpg"
324
- class="interpolation-image"
325
- alt="Interpolation end reference image."/>
326
- <p class="is-bold">End Frame</p>
327
- </div>
328
  </div>
329
- <br/>
330
- <!--/ Interpolating. -->
331
 
332
- <!-- Re-rendering. -->
333
- <h3 class="title is-4">Re-rendering the input video</h3>
334
  <div class="content has-text-justified">
335
  <p>
336
- Using <span class="dnerf">Nerfies</span>, you can re-render a video from a novel
337
- viewpoint such as a stabilized camera by playing back the training deformations.
338
  </p>
339
  </div>
340
- <div class="content has-text-centered">
341
- <video id="replay-video"
342
- controls
343
- muted
344
- preload
345
- playsinline
346
- width="75%">
347
- <source src="./static/videos/replay.mp4"
348
- type="video/mp4">
349
- </video>
350
- </div>
351
- <!--/ Re-rendering. -->
352
-
353
- </div>
354
- </div>
355
- <!--/ Animation. -->
356
-
357
-
358
- <!-- Concurrent Work. -->
359
- <div class="columns is-centered">
360
- <div class="column is-full-width">
361
- <h2 class="title is-3">Related Links</h2>
362
-
363
  <div class="content has-text-justified">
364
  <p>
365
- There's a lot of excellent work that was introduced around the same time as ours.
366
- </p>
367
- <p>
368
- <a href="https://arxiv.org/abs/2104.09125" target="_blank">Progressive Encoding for Neural Optimization</a> introduces an idea similar to our windowed position encoding for coarse-to-fine optimization.
369
- </p>
370
- <p>
371
- <a href="https://www.albertpumarola.com/research/D-NeRF/index.html" target="_blank">D-NeRF</a> and <a href="https://gvv.mpi-inf.mpg.de/projects/nonrigid_nerf/" target="_blank">NR-NeRF</a>
372
- both use deformation fields to model non-rigid scenes.
373
- </p>
374
- <p>
375
- Some works model videos with a NeRF by directly modulating the density, such as <a href="https://video-nerf.github.io/" target="_blank">Video-NeRF</a>, <a href="https://www.cs.cornell.edu/~zl548/NSFF/" target="_blank">NSFF</a>, and <a href="https://neural-3d-video.github.io/" target="_blank">DyNeRF</a>
376
- </p>
377
- <p>
378
- There are probably many more by the time you are reading this. Check out <a href="https://dellaert.github.io/NeRF/" target="_blank">Frank Dellart's survey on recent NeRF papers</a>, and <a href="https://github.com/yenchenlin/awesome-NeRF" target="_blank">Yen-Chen Lin's curated list of NeRF papers</a>.
379
  </p>
380
  </div>
381
  </div>
382
  </div>
383
- <!--/ Concurrent Work. -->
384
 
385
  </div>
386
  </section>
387
 
388
 
389
- <section class="section" id="BibTeX">
390
- <div class="container is-max-desktop content">
391
- <h2 class="title">BibTeX</h2>
392
- <pre><code>@article{park2021nerfies,
393
- author = {Park, Keunhong and Sinha, Utkarsh and Barron, Jonathan T. and Bouaziz, Sofien and Goldman, Dan B and Seitz, Steven M. and Martin-Brualla, Ricardo},
394
- title = {Nerfies: Deformable Neural Radiance Fields},
395
- journal = {ICCV},
396
- year = {2021},
397
- }</code></pre>
398
- </div>
399
- </section>
400
-
401
-
402
  <footer class="footer">
403
  <div class="container">
404
  <div class="content has-text-centered">
405
- <a class="icon-link" target="_blank"
406
- href="./static/videos/nerfies_paper.pdf">
407
- <i class="fas fa-file-pdf"></i>
408
- </a>
409
- <a class="icon-link" href="https://github.com/keunhong" target="_blank" class="external-link" disabled>
410
  <i class="fab fa-github"></i>
411
  </a>
412
  </div>
@@ -419,11 +234,7 @@
419
  Commons Attribution-ShareAlike 4.0 International License</a>.
420
  </p>
421
  <p>
422
- This means you are free to borrow the <a target="_blank"
423
- href="https://github.com/nerfies/nerfies.github.io">source code</a> of this website,
424
- we just ask that you link back to this page in the footer.
425
- Please remember to remove the analytics code included in the header of the website which
426
- you do not want on your website.
427
  </p>
428
  </div>
429
  </div>
@@ -432,4 +243,4 @@
432
  </footer>
433
 
434
  </body>
435
- </html>
 
3
  <head>
4
  <meta charset="utf-8">
5
  <meta name="description"
6
+ content="MPCA is a novel, bio-inspired AI framework that moves beyond traditional machine learning models.">
7
+ <meta name="keywords" content="MPCA, MycoPhysarum, Cognitive Architecture, Bio-inspired AI, Slime Mold, Mycelium, Graph AI, Efficient AI">
8
  <meta name="viewport" content="width=device-width, initial-scale=1">
9
+ <title>MycoPhysarum Cognitive Architecture (MPCA)</title>
10
 
11
  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
12
  rel="stylesheet">
 
33
  <div class="container is-max-desktop">
34
  <div class="columns is-centered">
35
  <div class="column has-text-centered">
36
+ <h1 class="title is-1 publication-title">MycoPhysarum Cognitive Architecture (MPCA)</h1>
37
  <div class="is-size-5 publication-authors">
38
  <span class="author-block">
39
+ A Novel, Bio-Inspired AI Framework
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  </span>
41
  </div>
42
 
43
  <div class="is-size-5 publication-authors">
44
+ <span class="author-block">Inspired by the decentralized intelligence of slime molds and mycorrhizal networks.</span>
 
45
  </div>
46
 
47
  <div class="column has-text-centered">
48
  <div class="publication-links">
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  <!-- Code Link. -->
50
  <span class="link-block">
51
+ <a href="https://github.com/mlabonne/myco-physarum" target="_blank"
52
  class="external-link button is-normal is-rounded is-dark">
53
  <span class="icon">
54
  <i class="fab fa-github"></i>
 
56
  <span>Code</span>
57
  </a>
58
  </span>
 
 
 
 
 
 
 
 
 
59
  </div>
 
60
  </div>
61
  </div>
62
  </div>
 
67
  <section class="hero teaser">
68
  <div class="container is-max-desktop">
69
  <div class="hero-body">
70
+ <!-- Replaced video with a conceptual diagram -->
71
+ <img src="https://huggingface.co/mlabonne/myco-physarum/resolve/main/mpca.png" alt="MPCA Architecture Diagram" style="width: 100%; max-width: 800px; margin: auto; display: block; border-radius: 10px;">
72
+ <h2 class="subtitle has-text-centered" style="margin-top: 2rem;">
73
+ MPCA represents knowledge not as static data, but as a dynamic, living graph that evolves through interaction and self-reflection.
 
 
 
 
74
  </h2>
75
  </div>
76
  </div>
 
81
  <div class="hero-body">
82
  <div class="container">
83
  <div id="results-carousel" class="carousel results-carousel">
84
+ <div class="item">
85
+ <div class="content" style="padding: 2rem;">
86
+ <h3 class="title is-4">Mycelium</h3>
87
+ <p>The heart of the system. A <code>networkx</code> directed graph where nodes are concepts and edges represent the relationships between them. The strength of these connections is dynamic, changing based on usage and learning.</p>
88
+ </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  </div>
90
+ <div class="item">
91
+ <div class="content" style="padding: 2rem;">
92
+ <h3 class="title is-4">Builder</h3>
93
+ <p>The architect of the Mycelium. The Builder ingests raw text, uses <code>spaCy</code>'s dependency parser to understand grammatical structure, and translates it into a rich graph of nodes and relationships.</p>
94
+ </div>
95
  </div>
96
+ <div class="item">
97
+ <div class="content" style="padding: 2rem;">
98
+ <h3 class="title is-4">Solver</h3>
99
+ <p>The "consciousness" of MPCA. It traverses the Mycelium to find relevant pathways to answer queries. Implements Hebbian learning to reinforce successful paths and "dreaming" to form new speculative connections.</p>
100
+ </div>
101
  </div>
102
+ <div class="item">
103
+ <div class="content" style="padding: 2rem;">
104
+ <h3 class="title is-4">Node</h3>
105
+ <p>The fundamental unit of knowledge. Each node represents a word and has a <code>type</code> (concept, action, property) and a <code>strength</code>, indicating its importance in the network.</p>
106
+ </div>
107
  </div>
108
+ <div class="item">
109
+ <div class="content" style="padding: 2rem;">
110
+ <h3 class="title is-4">Spore</h3>
111
+ <p>A highly efficient persistence mechanism. A "spore" is a serialized (<code>pickle</code>) and compressed snapshot of the Mycelium, allowing the system's learned state to be saved and loaded from a tiny file (~9 MB).</p>
112
+ </div>
113
  </div>
114
  </div>
115
  </div>
 
119
 
120
  <section class="section">
121
  <div class="container is-max-desktop">
122
+ <!-- Abstract -->
123
  <div class="columns is-centered has-text-centered">
124
  <div class="column is-four-fifths">
125
+ <h2 class="title is-3">Core Philosophy</h2>
126
  <div class="content has-text-justified">
127
  <p>
128
+ The core of MPCA is the "Cognitive Mycelium," a graph-based knowledge structure. Unlike rigid, pre-trained models, the Mycelium is built from the ground up to understand the grammatical and conceptual relationships in language. It learns, forgets, and even "dreams" to form new connections, creating a resilient and emergent form of intelligence with a fraction of the computational overhead of conventional architectures.
 
129
  </p>
130
+ <h2 class="title is-3" style="margin-top: 2rem;">Radical Efficiency: The Spore Advantage</h2>
131
+ <p>
132
+ A key breakthrough of the MPCA is its incredible efficiency. A fully-formed Cognitive Mycelium, built from a large dataset (~100k entries, >2.9M sentences), can be compressed into a <code>spore</code> file of only <strong>~9 MB</strong>. This stands in stark contrast to conventional AI models like Transformers (GPT-2 is ~500 MB; modern models are many gigabytes).
 
 
 
 
 
 
 
 
133
  </p>
134
  <p>
135
+ This efficiency is a direct result of the architecture's design. Instead of storing billions of statistical weights to predict tokens, the Mycelium stores a compressed graph of concepts and their relationships. It captures knowledge, not just statistical patterns, leading to a powerful, lightweight, and truly novel form of intelligence.
 
 
 
 
 
 
 
 
136
  </p>
137
  </div>
138
  </div>
 
142
  <!-- Paper video. -->
143
  <div class="columns is-centered has-text-centered">
144
  <div class="column is-four-fifths">
145
+ <h2 class="title is-3">How to Run</h2>
146
+ <div class="content has-text-justified">
147
+ <p><strong>1. Install Dependencies:</strong></p>
148
+ <pre><code>pip install -r requirements.txt
149
+ python -m spacy download en_core_web_sm</code></pre>
150
+ <p><strong>2. Build a New Mycelium:</strong></p>
151
+ <p>To create a new knowledge graph from a dataset, run the main script with the <code>--build</code> flag. The default dataset is <code>mlabonne/FineTome-100k</code>.</p>
152
+ <pre><code>python main.py --build --spore-file mycelium_new.spore --limit 1000</code></pre>
153
+ <p><strong>3. Interact with an Existing Mycelium:</strong></p>
154
+ <p>To chat with a pre-built Mycelium, use the <code>--interactive</code> flag.</p>
155
+ <pre><code>python main.py --interactive --spore-file mycelium.spore</code></pre>
156
  </div>
157
  </div>
158
  </div>
 
165
  <div class="container is-max-desktop">
166
 
167
  <div class="columns is-centered">
168
+ <!-- Lifecycle -->
 
169
  <div class="column">
170
  <div class="content">
171
+ <h2 class="title is-3">The MPCA Lifecycle</h2>
172
+ <ol>
173
+ <li><strong>Genesis (Building):</strong> The <code>Builder</code> creates a Mycelium from a data source, performing grammatical parsing to construct a graph of nodes and relationships, which is then saved as a <code>.spore</code> file.</li>
174
+ <li><strong>Awakening (Loading):</strong> The system loads a <code>.spore</code> file into memory, awakening the Cognitive Mycelium.</li>
175
+ <li><strong>Interaction (Solving):</strong> A user asks a question. The <code>Solver</code> takes the core concepts and finds a thought-path through the Mycelium to construct an answer.</li>
176
+ <li><strong>Evolution (Learning):</strong> Successful thought-paths are reinforced via Hebbian learning, strengthening the system's knowledge.</li>
177
+ <li><strong>Introspection (Dreaming):</strong> During downtime, the system can dream to form new, speculative connections, expanding its creative potential.</li>
178
+ </ol>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
179
  </div>
180
  </div>
181
+ <!--/ Lifecycle. -->
182
  </div>
 
183
 
184
+ <!-- The Path Forward -->
185
  <div class="columns is-centered">
186
  <div class="column is-full-width">
187
+ <h2 class="title is-3">The Path Forward: Extending MPCA to a Multi-Modal World</h2>
 
 
 
188
  <div class="content has-text-justified">
189
  <p>
190
+ The current architecture is a powerful foundation for understanding language. Its true potential lies in extending this conceptual graph to understand and generate other forms of data. The core principle is that MPCA acts as a <strong>central orchestrator</strong>, connecting its abstract conceptual understanding to specialized external models for processing and generation.
 
 
191
  </p>
192
  </div>
193
+
194
+ <h3 class="title is-4">Image Understanding and Generation</h3>
195
+ <div class="content has-text-justified">
196
+ <p><strong>Understanding:</strong> A Vision-Language Model (VLM) like CLIP analyzes an image and outputs concept tags (e.g., "a red car on a street"). The <code>Builder</code> integrates these concepts into the Mycelium, linking an <code>ImageNode</code> to existing nodes like <code>Node('car')</code> and <code>Node('red')</code>. The graph learns <em>what's in the image</em>, not the pixels themselves.</p>
197
+ <p><strong>Generation:</strong> The <code>Solver</code> assembles a conceptual blueprint (e.g., <code>Node('boat') → Node('blue') → Node('ocean')</code>). This blueprint is passed as a highly-structured prompt to an external image generation model (like a VAE or Diffusion model) to render the final image.</p>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
  </div>
 
 
199
 
200
+ <h3 class="title is-4">Audio & Video</h3>
 
201
  <div class="content has-text-justified">
202
  <p>
203
+ A similar approach applies to audio and video. For audio, speech-to-text models provide text for integration, while event detection models can identify non-speech sounds ("dog barking"). For video, an analysis model tracks objects and actions over time, which the <code>Builder</code> represents as a complex, time-stamped sub-graph.
 
204
  </p>
205
  </div>
206
+
207
+ <h3 class="title is-4">Taking Action (Agency)</h3>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
208
  <div class="content has-text-justified">
209
  <p>
210
+ Actions are a native <code>Node</code> type in MPCA. To enable agency, these action nodes can be linked to real-world API calls or robotic functions. When the <code>Solver</code>'s thought-path traverses an <code>ActionNode</code> linked to an external function (e.g., <code>Node('turn_on_light')</code>), it triggers that function. This turns the MPCA from a passive knowledge base into an active agent that can perceive, reason about, and act upon its environment.
 
 
 
 
 
 
 
 
 
 
 
 
 
211
  </p>
212
  </div>
213
  </div>
214
  </div>
215
+ <!--/ The Path Forward -->
216
 
217
  </div>
218
  </section>
219
 
220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  <footer class="footer">
222
  <div class="container">
223
  <div class="content has-text-centered">
224
+ <a class="icon-link" href="https://github.com/mlabonne/myco-physarum" target="_blank" class="external-link" disabled>
 
 
 
 
225
  <i class="fab fa-github"></i>
226
  </a>
227
  </div>
 
234
  Commons Attribution-ShareAlike 4.0 International License</a>.
235
  </p>
236
  <p>
237
+ This page template was borrowed from the <a target="_blank" href="https://github.com/nerfies/nerfies.github.io">Nerfies</a> project website.
 
 
 
 
238
  </p>
239
  </div>
240
  </div>
 
243
  </footer>
244
 
245
  </body>
246
+ </html>