Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,235 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300005-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300006-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300007-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300008-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300009-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300010-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300011-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300012-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300013-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300014-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300015-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300016-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300017-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300018-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300019-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300020-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300021-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300022-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300023-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300024-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300025-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300026-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300027-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300028-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300029-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300030-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300031-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300033-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300034.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300035-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300036.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300037-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300039-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300040.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300041-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300043-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300045-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300053-simple.obj filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300056-simple.obj filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17278951300061-simple.obj filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390003-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390008.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390009-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390010.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390011-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390012.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390013-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390015-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390017-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390023-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390025-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390027-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390029-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390031-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390033-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390035-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390037-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390039-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390041-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390043-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390045-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390047-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390049-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390051-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390052.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390053-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390054.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390055-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390056.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390057-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390058.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390059-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390061-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390063-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390065-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390067-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390069-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390071-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390073-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390075-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390077-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390079-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390081-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390085-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390087-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17280589390089-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285060200001.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285060200002-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285060200003-top-bottom.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285371260002-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285859980001.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285859980002.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285859980003-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285861380002-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285861380003-left-right_video.avi filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17285861380004-depth_video.avi filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930002-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930003.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930004-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930005.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930006-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930010-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930012-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930016-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930018-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930020-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930026-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930028-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930036-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930046-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930050-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930052-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930053.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930054-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930055.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930056-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930057.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930058-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930059.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930060-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930061.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930062-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930063.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930064-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930066-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930070-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930072-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930080-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930082-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930084-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930120-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930126-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930132-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930142-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930147.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930152-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930154-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930156-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930158-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930160-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930162-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930164-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930166-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930168-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930170-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930172-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930174-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930176-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930178-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930180-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930182-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930184-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930186-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930188-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930190-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930194-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930196-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930198-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930199.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930200-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930202-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930204-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930206-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930208-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930210-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930212-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930213.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930214-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930216-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930218-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930220-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930222-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930224-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930226-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930228-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930230-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930232-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930234-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930236-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930238-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930240-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930242-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930244-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930246-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930248-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930250-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930252-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930253.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930254-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930256-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930258-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930260-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930261.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930262-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930263.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930264-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930265.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930266-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930268-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930270-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930272-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930274-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930276-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930278-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930280-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930282-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930284-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930286-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930288-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930290-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930292-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930294-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930296-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930298-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930300-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930302-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930304-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930306-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930308-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930310-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930312-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930316-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930318-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930322-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930324-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930326-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930328-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930330-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930332-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930334-left-right.png filter=lfs diff=lfs merge=lfs -text
+outputs/depthmap-17286927930336-left-right.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+venv/
+.idea/

CHANGELOG.md

@@ -0,0 +1,131 @@
+## Changelog
+### 0.4.8
+ * Depth Anything V2 support, thanks [@graemeniedermayer](https://github.com/graemeniedermayer)!
+### 0.4.7
+ * Tiling mode
+ * Reduced VRAM consumption for Depth Anything, as well as for ZoeDepth k and nk
+ * Some bugfixes
+### 0.4.6
+ * Support for [Depth Anything](https://github.com/LiheYoung/Depth-Anything).
+### 0.4.5
+ * Preliminary support for [Marigold](https://marigoldmonodepth.github.io). [PR #385](https://github.com/thygate/stable-diffusion-webui-depthmap-script/pull/385).
+### 0.4.4
+ * Compatibility with stable-diffusion-webui 1.6.0
+### 0.4.3 video processing tab
+ * Added an option to process videos directly from a video file. This leads to better results than batch-processing individual frames of a video. Allows generating depthmap videos, that can be used in further generations as custom depthmap videos.
+ * UI improvements.
+ * Extra stereoimage generation modes - enable in extension settings if you want to use them.
+ * New stereoimage generation parameter - offset exponent. Setting it to 1 may produce more realistic outputs.
+### 0.4.2
+ * Added UI options for 2 additional rembg models.
+ * Heatmap generation UI option is hidden - if you want to use it, please activate it in the extension settings.
+ * Bugfixes.
+### 0.4.1 standalone mode
+ * Added ability to run DepthMap without WebUI. (Use main.py. Make sure all the dependencies are installed. The support is not feature-complete.)
+ * NormalMap generation
+### 0.4.0 large code refactor
+ * UI improvements
+ * Improved Batch from Directory, Clip and renormalize DepthMap
+ * Slightly changed the behaviour of various options
+ * Extension may partially work even if some of the dependencies are unmet
+
+### 0.3.12
+ * Fixed stereo image generation
+ * Other bugfixes
+### 0.3.11
+ * 3D model viewer (Experimental!)
+ * simple and fast (occluded) 3D mesh generation, support for equirectangular projection
+      (accurate results with ZoeDepth models only, no boost, no custom maps)
+ * default output format is now obj for inpainted mesh and simple mesh
+### 0.3.10 
+ * ZoeDepth support (with boost), 3 new models, best results so far
+ * better heatmap
+### 0.3.9
+ * use existing/custom depthmaps in output dir for batch mode
+ * custom depthmap support for single file
+ * wavefront obj output support for inpainted mesh (enabled in settings)
+ * option to generate all stereo formats at once
+ * bugfix: convert single channel input image to rgb
+ * renamed midas imports to fix conflict with deforum
+ * ui cleanup
+### 0.3.8 bugfix
+ * bugfix in remove background path
+### 0.3.7 new features
+ * [rembg](https://github.com/danielgatis/rembg) Remove Background [PR](https://github.com/thygate/stable-diffusion-webui-depthmap-script/pull/78) by [@graemeniedermayer](https://github.com/graemeniedermayer) merged
+ * setting to flip Left/Right SBS images
+ * added missing parameter for 3d inpainting (repeat_inpaint_edge)
+ * option to generate demo videos with mesh
+### 0.3.6 new feature
+ * implemented binary ply file format for the inpainted 3D mesh, big reduction in filesize and save/load times. 
+ * added progress indicators to the inpainting process 
+### 0.3.5 bugfix
+ * create path to 3dphoto models before download (see [issue](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/76))
+### 0.3.4 new featues
+ * depth clipping option (original idea by [@Extraltodeus](https://github.com/Extraltodeus))
+ * by popular demand, 3D-Photo-Inpainting is now implemented
+ * generate inpainted 3D mesh (PLY) and videos of said mesh 
+### 0.3.3 bugfix and new midas models
+ * updated to midas 3.1, bringing 2 new depth models (the 512 one eats VRAM for breakfast!)
+ * fix Next-ViT dependency issue for new installs
+ * extension no longer clones repositories, all dependencies are now contained in the extension
+### 0.3.2 new feature and bugfixes
+ * several bug fixes for apple silicon and other machines without cuda
+ * NEW Stereo Image Generation techniques for gap filling by [@semjon00](https://github.com/semjon00) using polylines. (See [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/pull/56)) Significant improvement in quality.
+### 0.3.1 bugfix
+ * small speed increase for anaglyph creation
+ * clone midas repo before midas 3.1 to fix issue (see [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/55#issue-1510266008))
+### 0.3.0 improved stereo image generation
+ * New improved technique for generating stereo images and balancing distortion between eyes by [@semjon00](https://github.com/semjon00) (See [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/pull/51))
+ * Substantial speedup of stereo image generation code using numba JIT
+### 0.2.9 new feature 
+ * 3D Stereo (side-by-side) and red/cyan anaglyph image generation.   
+    (Thanks to [@sina-masoud-ansari](https://github.com/sina-masoud-ansari) for the tip! Discussion [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/discussions/45))
+### 0.2.8 bugfix
+ * boost (pix2pix) now also able to compute on cpu
+ * res101 able to compute on cpu
+### 0.2.7 separate tab
+ * Depth Tab now available for easier stand-alone (batch) processing
+### 0.2.6 ui layout and settings
+ * added link to repo so more people find their way to the instructions.
+ * boost rmax setting
+### 0.2.5 bugfix
+ * error checking on model download (now with progressbar)
+### 0.2.4 high resolution depthmaps
+ * multi-resolution merging is now implemented, significantly improving results!
+ * res101 can now also compute on CPU
+### 0.2.3 bugfix
+ * path error on linux fixed
+### 0.2.2 new features
+ * added (experimental) support for AdelaiDepth/LeReS (GPU Only!)
+ * new option to view depthmap as heatmap
+ * optimised ui layout
+### 0.2.1 bugfix
+ * Correct seed is now used in filename and pnginfo when running batches. (see [issue](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/35))
+### 0.2.0 upgrade
+ * the script is now an extension, enabling auto installation.
+### 0.1.9 bugfixes
+ * sd model moved to system memory while computing depthmap
+ * memory leak/fragmentation issue fixed
+ * recover from out of memory error
+### 0.1.8 new options
+ * net size can now be set as width and height, option to match input size, sliders now have the same range as generation parameters. (see usage below)
+ * better error handling
+### 0.1.7 bugfixes
+ * batch img2img now works (see [issue](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/21#issuecomment-1306445056))
+ * generation parameters now only saved when enabled in settings
+ * model memory freed explicitly at end of script
+### 0.1.6 new option
+ * option to invert depthmap (black=near, white=far), as required by some viewers.
+### 0.1.5 bugfix
+ * saving as any format other than PNG now always produces an 8 bit, 3 channel RGB image. A single channel 16 bit image is only supported when saving as PNG. (see [issue](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/15#issuecomment-1304909019))
+### 0.1.4 update
+ * added support for `--no-half`. Now also works with cards that don't support half precision like GTX 16xx. ([verified](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/12#issuecomment-1304656398))
+### 0.1.3 bugfix
+ * bugfix where some controls where not visible (see [issue](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/11#issuecomment-1304400537))
+### 0.1.2 new option
+ * network size slider. higher resolution depth maps (see usage below)
+### 0.1.1 bugfixes
+ * overflow issue (see [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/issues/10) for details and examples of artifacts)
+ * when not combining, depthmap is now saved as single channel 16 bit
+### 0.1.0
+ * initial version: script mode, supports generating depthmaps with 4 different midas models

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Bob Thiry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,244 @@
----
-title: Stable Diffusion Webui Depthmap Script
-emoji: 🔥
-colorFrom: pink
-colorTo: purple
-sdk: gradio
-sdk_version: 5.0.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: stable-diffusion-webui-depthmap-script
+app_file: main.py
+sdk: gradio
+sdk_version: 3.50.2
+---
+# High Resolution Depth Maps for Stable Diffusion WebUI
+This program is an addon for [AUTOMATIC1111's Stable Diffusion WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) that creates depth maps. Using either generated or custom depth maps, it can also create 3D stereo image pairs (side-by-side or anaglyph), normalmaps and 3D meshes. The outputs of the script can be viewed directly or used as an asset for a 3D engine. Please see [wiki](https://github.com/thygate/stable-diffusion-webui-depthmap-script/wiki/Viewing-Results) to learn more. The program has integration with [Rembg](https://github.com/danielgatis/rembg). It also supports batch processing, processing of videos, and can also be run in standalone mode, without Stable Diffusion WebUI.
+
+To generate realistic depth maps from individual images, this script uses code and models from the [Marigold](https://github.com/prs-eth/Marigold/) repository, from the [MiDaS](https://github.com/isl-org/MiDaS) and [ZoeDepth](https://github.com/isl-org/ZoeDepth) repositories by Intel ISL, or LeReS from the [AdelaiDepth](https://github.com/aim-uofa/AdelaiDepth) repository by Advanced Intelligent Machines. Multi-resolution merging as implemented by [BoostingMonocularDepth](https://github.com/compphoto/BoostingMonocularDepth) is used to generate high resolution depth maps.
+
+Stereoscopic images are created using a custom-written algorithm.
+
+3D Photography using Context-aware Layered Depth Inpainting by Virginia Tech Vision and Learning Lab, or [3D-Photo-Inpainting](https://github.com/vt-vl-lab/3d-photo-inpainting) is used to generate a `3D inpainted mesh` and render `videos` from said mesh.
+
+Rembg uses [U-2-Net](https://github.com/xuebinqin/U-2-Net) and [IS-Net](https://github.com/xuebinqin/DIS).
+
+## Depthmap Examples
+[![screenshot](examples.png)](https://raw.githubusercontent.com/thygate/stable-diffusion-webui-depthmap-script/main/examples.png)
+
+## 3D Photo Inpainting Examples
+[![video](https://img.youtube.com/vi/jRmVkIMS-SY/0.jpg)](https://www.youtube.com/watch?v=jRmVkIMS-SY)   
+video by [@graemeniedermayer](https://github.com/graemeniedermayer), more examples [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/discussions/50)
+
+## Stereo Image SBS and Anaglyph Examples
+![](https://user-images.githubusercontent.com/54073010/210012661-ef07986c-2320-4700-bc54-fad3899f0186.png)    
+images generated by [@semjon00](https://github.com/semjon00) from CC0 photos, more examples [here](https://github.com/thygate/stable-diffusion-webui-depthmap-script/pull/56#issuecomment-1367596463).
+
+## Install instructions
+### As extension
+The script can be installed directly from WebUI. Please navigate to `Extensions` tab, then click `Available`, `Load from` and then install the `Depth Maps` extension. Alternatively, the extension can be installed from the URL: `https://github.com/thygate/stable-diffusion-webui-depthmap-script`.
+
+### Updating
+In the WebUI, in the `Extensions` tab, in the `Installed` subtab, click `Check for Updates` and then `Apply and restart UI`.
+
+### Standalone
+Clone the repository, install the requirements from `requirements.txt`, launch using `main.py`.
+
+>Model weights will be downloaded automatically on their first use and saved to /models/midas, /models/leres and /models/pix2pix. Zoedepth models are stored in the torch cache folder.
+
+
+## Usage
+Select the "DepthMap" script from the script selection box in either txt2img or img2img, or go to the Depth tab when using existing images.
+![screenshot](options.png)
+
+The models can `Compute on` GPU and CPU, use CPU if low on VRAM.
+
+There are ten models available from the `Model` dropdown. For the first model, res101, see [AdelaiDepth/LeReS](https://github.com/aim-uofa/AdelaiDepth/tree/main/LeReS) for more info. The others are the midas models: dpt_beit_large_512, dpt_beit_large_384, dpt_large_384, dpt_hybrid_384, midas_v21, and midas_v21_small. See the [MiDaS](https://github.com/isl-org/MiDaS) repository for more info. The newest dpt_beit_large_512 model was trained on a 512x512 dataset but is VERY VRAM hungry. The last three models are [ZoeDepth](https://github.com/isl-org/ZoeDepth) models.
+
+Net size can be set with `net width` and `net height`, or will be the same as the input image when `Match input size` is enabled. There is a trade-off between structural consistency and high-frequency details with respect to net size (see [observations](https://github.com/compphoto/BoostingMonocularDepth#observations)).
+
+`Boost` will enable multi-resolution merging as implemented by [BoostingMonocularDepth](https://github.com/compphoto/BoostingMonocularDepth) and will significantly improve the results, mitigating the observations mentioned above, at the cost of much larger compute time. Best results with res101.
+
+`Clip and renormalize` allows for clipping the depthmap on the `near` and `far` side, the values in between will be renormalized to fit the available range. Set both values equal to get a b&w mask of a single depth plane at that value. This option works on the 16-bit depthmap and allows for 1000 steps to select the clip values.
+
+When enabled, `Invert DepthMap` will result in a depthmap with black near and white far.
+
+Regardless of global settings, `Save DepthMap` will always save the depthmap in the default txt2img or img2img directory with the filename suffix '_depth'. Generation parameters are saved with the image if enabled in settings. Files generated from the Depth tab are saved in the default extras-images directory.
+
+To see the generated output in the webui `Show DepthMap` should be enabled. When using Batch img2img this option should also be enabled.
+
+When `Combine into one image` is enabled, the depthmap will be combined with the original image, the orientation can be selected with `Combine axis`. When disabled, the depthmap will be saved as a 16 bit single channel PNG as opposed to a three channel (RGB), 8 bit per channel image when the option is enabled.
+
+When either `Generate Stereo` or `Generate anaglyph` is enabled, a stereo image pair will be generated. `Divergence` sets the amount of 3D effect that is desired. `Balance between eyes` determines where the (inevitable) distortion from filling up gaps will end up, -1 Left, +1 Right, and 0 balanced.  
+The different `Gap fill technique` options are : none (no gaps are filled), 
+naive (the original method), naive_interpolating (the original method with interpolation), polylines_soft and polylines_sharp are the latest technique, the last one being best quality and slowest. Note: All stereo image generation is done on CPU.
+
+To generate the mesh required to generate videos, enable `Generate 3D inpainted mesh`. This can be a lengthy process, from a few minutes for small images to an hour for very large images. This option is only available on the Depth tab. When enabled, the mesh in ply format and four demo video are generated. All files are saved to the extras directory.
+    
+Videos can be generated from the PLY mesh on the Depth Tab.
+It requires the mesh created by this extension, files created elsewhere might not work corectly, as some extra info is stored in the file (required value for dolly). Most options are self-explanatory, like `Number of frames` and `Framerate`. Two output `formats` are supported: mp4 and webm. Supersampling Anti-Aliasing (SSAA) can be used to get rid of jagged edges and flickering. The render size is scaled by this factor and then downsampled.    
+There are three `trajectories` to choose from : circle, straight-line, double-straight-line, to `translate` in three dimensions. The border can be `cropped` on four sides, and the `Dolly` option adjusts the FOV so the center subject will stay approximately the same size, like the dolly-zoom.
+
+Settings on WebUI Settings tab :  
+`Maximum wholesize for boost` sets the r_max value from the BoostingMonocularDepth paper, it relates to the max size that is chosen to render at internally, and directly influences the max amount of VRAM that could be used. The default value for this from the paper is 3000, I have lowered the value to 1600 so it will work more often with 8GB VRAM GPU's.
+If you often get out of memory errors when computing a depthmap on GPU while using Boost, you can try lowering this value. Note the 'wholeImage being processed in : xxxx' output when using boost, this number will never be greater than the r_max, but can be larger with a larger r_max. See the paper for more details.
+
+> 💡 Saving as any format other than PNG always produces an 8 bit, 3 channel RGB image. A single channel 16 bit image is only supported when saving as PNG. 
+
+## FAQ
+
+ * `Can I use this on existing images ?`
+    - Yes, you can use the Depth tab to easily process existing images.
+    - Another way of doing this would be to use img2img with denoising strength to 0. This will effectively skip stable diffusion and use the input image. You will still have to set the correct size, and need to select `Crop and resize` instead of `Just resize` when the input image resolution does not match the set size perfectly.
+ * `Can I run this on Google Colab?`
+    - You can run the MiDaS network on their colab linked here https://pytorch.org/hub/intelisl_midas_v2/
+    - You can run BoostingMonocularDepth on their colab linked here : https://colab.research.google.com/github/compphoto/BoostingMonocularDepth/blob/main/Boostmonoculardepth.ipynb
+    - Running this program on Colab is not officially supported, but it may work. Please look for more suitable ways of running this. If you still decide to try, standalone installation may be easier to manage.
+ * `What other depth-related projects could I check out?`
+    - [DepthFlow](https://github.com/BrokenSource/DepthFlow) by [@Tremeschin](https://github.com/Tremeschin) for a very fast generation of 2.5D videos from images (no need to create mesh beforehand!)
+    - Several [scripts](https://github.com/Extraltodeus?tab=repositories) by [@Extraltodeus](https://github.com/Extraltodeus) using depth maps.
+    - geo-11, [Depth3D](https://github.com/BlueSkyDefender/Depth3D) and [Geo3D](https://github.com/Flugan/Geo3D-Installer) for playing existing games in 3D.
+    - (Feel free to suggest more projects in the discussions!)
+ * `How can I know what changed in the new version of the script?`
+    - You can see the git history log or refer to the `CHANGELOG.md` file.
+
+## Help wanted!
+Developers wanted! Please help us fix the bugs and add new features by creating MRs.
+All help is heavily appreciated.
+Feel free to comment and share in the discussions and submit issues.
+
+## Acknowledgements
+
+This project relies on code and information from the following papers : 
+
+MiDaS :
+
+```
+@article {Ranftl2022,
+    author  = "Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun",
+    title   = "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-Shot Cross-Dataset Transfer",
+    journal = "IEEE Transactions on Pattern Analysis and Machine Intelligence",
+    year    = "2022",
+    volume  = "44",
+    number  = "3"
+}
+```
+
+Dense Prediction Transformers, DPT-based model :
+
+```
+@article{Ranftl2021,
+	author    = {Ren\'{e} Ranftl and Alexey Bochkovskiy and Vladlen Koltun},
+	title     = {Vision Transformers for Dense Prediction},
+	journal   = {ICCV},
+	year      = {2021},
+}
+```
+
+AdelaiDepth/LeReS :
+
+```
+@article{yin2022towards,
+	title={Towards Accurate Reconstruction of 3D Scene Shape from A Single Monocular Image},
+	author={Yin, Wei and Zhang, Jianming and Wang, Oliver and Niklaus, Simon and Chen, Simon and Liu, Yifan and Shen, Chunhua},
+	journal={TPAMI},
+	year={2022}
+}
+@inproceedings{Wei2021CVPR,
+	title     =  {Learning to Recover 3D Scene Shape from a Single Image},
+	author    =  {Wei Yin and Jianming Zhang and Oliver Wang and Simon Niklaus and Long Mai and Simon Chen and Chunhua Shen},
+	booktitle =  {Proc. IEEE Conf. Comp. Vis. Patt. Recogn. (CVPR)},
+	year      =  {2021}
+}
+```
+
+Boosting Monocular Depth Estimation Models to High-Resolution via Content-Adaptive Multi-Resolution Merging :
+
+```
+@inproceedings{Miangoleh2021Boosting,
+	title={Boosting Monocular Depth Estimation Models to High-Resolution via Content-Adaptive Multi-Resolution Merging},
+	author={S. Mahdi H. Miangoleh and Sebastian Dille and Long Mai and Sylvain Paris and Ya\u{g}{\i}z Aksoy},
+	journal={Proc. CVPR},
+	year={2021},
+}
+```
+
+3D Photography using Context-aware Layered Depth Inpainting :
+
+```
+@inproceedings{Shih3DP20,
+	author = {Shih, Meng-Li and Su, Shih-Yang and Kopf, Johannes and Huang, Jia-Bin},
+	title = {3D Photography using Context-aware Layered Depth Inpainting},
+	booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
+	year = {2020}
+}
+```
+
+U2-Net:
+
+```
+@InProceedings{Qin_2020_PR,
+    title = {U2-Net: Going Deeper with Nested U-Structure for Salient Object Detection},
+    author = {Qin, Xuebin and Zhang, Zichen and Huang, Chenyang and Dehghan, Masood and Zaiane, Osmar and Jagersand, Martin},
+    journal = {Pattern Recognition},
+    volume = {106},
+    pages = {107404},
+    year = {2020}
+}
+```
+
+IS-Net:
+
+```
+@InProceedings{qin2022,
+      author={Xuebin Qin and Hang Dai and Xiaobin Hu and Deng-Ping Fan and Ling Shao and Luc Van Gool},
+      title={Highly Accurate Dichotomous Image Segmentation},
+      booktitle={ECCV},
+      year={2022}
+}
+```
+
+
+ZoeDepth :
+
+```
+@misc{https://doi.org/10.48550/arxiv.2302.12288,
+  doi = {10.48550/ARXIV.2302.12288},
+  url = {https://arxiv.org/abs/2302.12288},
+  author = {Bhat, Shariq Farooq and Birkl, Reiner and Wofk, Diana and Wonka, Peter and Müller, Matthias},
+  keywords = {Computer Vision and Pattern Recognition (cs.CV), FOS: Computer and information sciences, FOS: Computer and information sciences},
+  title = {ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth},
+  publisher = {arXiv},
+  year = {2023},
+  copyright = {arXiv.org perpetual, non-exclusive license}
+}
+```
+
+Marigold - Repurposing Diffusion-Based Image Generators for Monocular Depth Estimation:
+
+```
+@misc{ke2023repurposing,
+      title={Repurposing Diffusion-Based Image Generators for Monocular Depth Estimation}, 
+      author={Bingxin Ke and Anton Obukhov and Shengyu Huang and Nando Metzger and Rodrigo Caye Daudt and Konrad Schindler},
+      year={2023},
+      eprint={2312.02145},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+```
+
+Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data
+
+```
+@misc{yang2024depth,
+      title={Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data}, 
+      author={Lihe Yang and Bingyi Kang and Zilong Huang and Xiaogang Xu and Jiashi Feng and Hengshuang Zhao},
+      year={2024},
+      eprint={2401.10891},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+```
+
+Depth Anything V2
+
+```bibtex
+@article{depth_anything_v2,
+  title={Depth Anything V2},
+  author={Yang, Lihe and Kang, Bingyi and Huang, Zilong and Zhao, Zhen and Xu, Xiaogang and Feng, Jiashi and Zhao, Hengshuang},
+  journal={arXiv:2406.09414},
+  year={2024}
+}
+```

bundled_sources.txt

@@ -0,0 +1,25 @@
+Since commit 110549b2 this extension bundles some code from other repositories.
+This was done to prevent possible upstream breakage and allow fixing breakage quicker.
+This file provides information about the original location of the code.
+*** Some of the bundled code was already modified. ***
+
+dmidas
+https://github.com/isl-org/MiDaS/tree/master/midas/
+
+dzoedepth
+https://github.com/isl-org/ZoeDepth/tree/main/zoedepth/
+
+inpaint
+https://github.com/vt-vl-lab/3d-photo-inpainting/
+
+lib
+https://github.com/aim-uofa/AdelaiDepth/tree/main/LeReS/Minist_Test/lib/
+
+pix2pix
+https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/
+
+Marigold
+https://github.com/prs-eth/Marigold/tree/22437a
+
+depth_anything_v2
+https://github.com/DepthAnything/Depth-Anything-V2/tree/bc0283

ddepth_anything_v2/DA-2K.md

@@ -0,0 +1,51 @@
+# DA-2K Evaluation Benchmark
+
+## Introduction
+
+![DA-2K](assets/DA-2K.png)
+
+DA-2K is proposed in [Depth Anything V2](https://depth-anything-v2.github.io) to evaluate the relative depth estimation capability. It encompasses eight representative scenarios of `indoor`, `outdoor`, `non_real`, `transparent_reflective`, `adverse_style`, `aerial`, `underwater`, and `object`. It consists of 1K diverse high-quality images and 2K precise pair-wise relative depth annotations.
+
+Please refer to our [paper](https://arxiv.org/abs/2406.09414) for details in constructing this benchmark.
+
+
+## Usage
+
+Please first [download the benchmark](https://huggingface.co/datasets/depth-anything/DA-2K/tree/main).
+
+All annotations are stored in `annotations.json`. The annotation file is a JSON object where each key is the path to an image file, and the value is a list of annotations associated with that image. Each annotation describes two points and identifies which point is closer to the camera. The structure is detailed below:
+
+```
+{
+  "image_path": [
+    {
+      "point1": [h1, w1], # (vertical position, horizontal position)
+      "point2": [h2, w2], # (vertical position, horizontal position)
+      "closer_point": "point1" # we always set "point1" as the closer one
+    },
+    ...
+  ],
+  ...
+}
+```
+
+To visualize the annotations:
+```bash
+python visualize.py [--scene-type <type>]
+```
+
+**Options**
+- `--scene-type <type>` (optional): Specify the scene type (`indoor`, `outdoor`, `non_real`, `transparent_reflective`, `adverse_style`, `aerial`, `underwater`, and `object`). Skip this argument or set <type> as `""` to include all scene types.
+
+## Citation
+
+If you find this benchmark useful, please consider citing:
+
+```bibtex
+@article{depth_anything_v2,
+  title={Depth Anything V2},
+  author={Yang, Lihe and Kang, Bingyi and Huang, Zilong and Zhao, Zhen and Xu, Xiaogang and Feng, Jiashi and Zhao, Hengshuang},
+  journal={arXiv:2406.09414},
+  year={2024}
+}
+```

ddepth_anything_v2/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

ddepth_anything_v2/README.md

@@ -0,0 +1,201 @@
+<div align="center">
+<h1>Depth Anything V2</h1>
+
+[**Lihe Yang**](https://liheyoung.github.io/)<sup>1</sup> · [**Bingyi Kang**](https://bingykang.github.io/)<sup>2&dagger;</sup> · [**Zilong Huang**](http://speedinghzl.github.io/)<sup>2</sup>
+<br>
+[**Zhen Zhao**](http://zhaozhen.me/) · [**Xiaogang Xu**](https://xiaogang00.github.io/) · [**Jiashi Feng**](https://sites.google.com/site/jshfeng/)<sup>2</sup> · [**Hengshuang Zhao**](https://hszhao.github.io/)<sup>1*</sup>
+
+<sup>1</sup>HKU&emsp;&emsp;&emsp;<sup>2</sup>TikTok
+<br>
+&dagger;project lead&emsp;*corresponding author  
+&dagger;[Bingyi Kang](https://bingykang.github.io/) proposed this project and advised in every aspect.  
+
+<a href="https://arxiv.org/abs/2406.09414"><img src='https://img.shields.io/badge/arXiv-Depth Anything V2-red' alt='Paper PDF'></a>
+<a href='https://depth-anything-v2.github.io'><img src='https://img.shields.io/badge/Project_Page-Depth Anything V2-green' alt='Project Page'></a>
+<a href='https://huggingface.co/spaces/depth-anything/Depth-Anything-V2'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Demo-blue'></a>
+<a href='https://huggingface.co/datasets/depth-anything/DA-2K'><img src='https://img.shields.io/badge/Benchmark-DA--2K-yellow' alt='Benchmark'></a>
+</div>
+
+This work presents Depth Anything V2. It significantly outperforms [V1](https://github.com/LiheYoung/Depth-Anything) in fine-grained details and robustness. Compared with SD-based models, it enjoys faster inference speed, fewer parameters, and higher depth accuracy.
+
+![teaser](assets/teaser.png)
+
+
+## News
+
+- **2024-07-06:** Depth Anything V2 is supported in [Transformers](https://github.com/huggingface/transformers/). See the [instructions](https://huggingface.co/docs/transformers/main/en/model_doc/depth_anything_v2) for convenient usage.
+- **2024-06-25:** Depth Anything is integrated into [Apple Core ML Models](https://developer.apple.com/machine-learning/models/). See the instructions ([V1](https://huggingface.co/apple/coreml-depth-anything-small), [V2](https://huggingface.co/apple/coreml-depth-anything-v2-small)) for usage.
+- **2024-06-22:** We release [smaller metric depth models](https://github.com/DepthAnything/Depth-Anything-V2/tree/main/metric_depth#pre-trained-models) based on Depth-Anything-V2-Small and Base.
+- **2024-06-20:** Our repository and project page are flagged by GitHub and removed from the public for 6 days. Sorry for the inconvenience.
+- **2024-06-14:** Paper, project page, code, models, demo, and benchmark are all released.
+
+
+## Pre-trained Models
+
+We provide **four models** of varying scales for robust relative depth estimation:
+
+| Model | Params | Checkpoint |
+|:-|-:|:-:|
+| Depth-Anything-V2-Small | 24.8M | [Download](https://huggingface.co/depth-anything/Depth-Anything-V2-Small/resolve/main/depth_anything_v2_vits.pth?download=true) |
+| Depth-Anything-V2-Base | 97.5M | [Download](https://huggingface.co/depth-anything/Depth-Anything-V2-Base/resolve/main/depth_anything_v2_vitb.pth?download=true) |
+| Depth-Anything-V2-Large | 335.3M | [Download](https://huggingface.co/depth-anything/Depth-Anything-V2-Large/resolve/main/depth_anything_v2_vitl.pth?download=true) |
+| Depth-Anything-V2-Giant | 1.3B | Coming soon |
+
+
+## Usage
+
+### Prepraration
+
+```bash
+git clone https://github.com/DepthAnything/Depth-Anything-V2
+cd Depth-Anything-V2
+pip install -r requirements.txt
+```
+
+Download the checkpoints listed [here](#pre-trained-models) and put them under the `checkpoints` directory.
+
+### Use our models
+```python
+import cv2
+import torch
+
+from depth_anything_v2.dpt import DepthAnythingV2
+
+DEVICE = 'cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'
+
+model_configs = {
+    'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
+    'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
+    'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
+    'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]}
+}
+
+encoder = 'vitl' # or 'vits', 'vitb', 'vitg'
+
+model = DepthAnythingV2(**model_configs[encoder])
+model.load_state_dict(torch.load(f'checkpoints/depth_anything_v2_{encoder}.pth', map_location='cpu'))
+model = model.to(DEVICE).eval()
+
+raw_img = cv2.imread('your/image/path')
+depth = model.infer_image(raw_img) # HxW raw depth map in numpy
+```
+
+If you do not want to clone this repository, you can also load our models through [Transformers](https://github.com/huggingface/transformers/). Below is a simple code snippet. Please refer to the [official page](https://huggingface.co/docs/transformers/main/en/model_doc/depth_anything_v2) for more details.
+
+- Note 1: Make sure you can connect to Hugging Face and have installed the latest Transformers.
+- Note 2: Due to the [upsampling difference](https://github.com/huggingface/transformers/pull/31522#issuecomment-2184123463) between OpenCV (we used) and Pillow (HF used), predictions may differ slightly. So you are more recommended to use our models through the way introduced above.
+```python
+from transformers import pipeline
+from PIL import Image
+
+pipe = pipeline(task="depth-estimation", model="depth-anything/Depth-Anything-V2-Small-hf")
+image = Image.open('your/image/path')
+depth = pipe(image)["depth"]
+```
+
+### Running script on *images*
+
+```bash
+python run.py \
+  --encoder <vits | vitb | vitl | vitg> \
+  --img-path <path> --outdir <outdir> \
+  [--input-size <size>] [--pred-only] [--grayscale]
+```
+Options:
+- `--img-path`: You can either 1) point it to an image directory storing all interested images, 2) point it to a single image, or 3) point it to a text file storing all image paths.
+- `--input-size` (optional): By default, we use input size `518` for model inference. ***You can increase the size for even more fine-grained results.***
+- `--pred-only` (optional): Only save the predicted depth map, without raw image.
+- `--grayscale` (optional): Save the grayscale depth map, without applying color palette.
+
+For example:
+```bash
+python run.py --encoder vitl --img-path assets/examples --outdir depth_vis
+```
+
+### Running script on *videos*
+
+```bash
+python run_video.py \
+  --encoder <vits | vitb | vitl | vitg> \
+  --video-path assets/examples_video --outdir video_depth_vis \
+  [--input-size <size>] [--pred-only] [--grayscale]
+```
+
+***Our larger model has better temporal consistency on videos.***
+
+### Gradio demo
+
+To use our gradio demo locally:
+
+```bash
+python app.py
+```
+
+You can also try our [online demo](https://huggingface.co/spaces/Depth-Anything/Depth-Anything-V2).
+
+***Note: Compared to V1, we have made a minor modification to the DINOv2-DPT architecture (originating from this [issue](https://github.com/LiheYoung/Depth-Anything/issues/81)).*** In V1, we *unintentionally* used features from the last four layers of DINOv2 for decoding. In V2, we use [intermediate features](https://github.com/DepthAnything/Depth-Anything-V2/blob/2cbc36a8ce2cec41d38ee51153f112e87c8e42d8/depth_anything_v2/dpt.py#L164-L169) instead. Although this modification did not improve details or accuracy, we decided to follow this common practice.
+
+
+## Fine-tuned to Metric Depth Estimation
+
+Please refer to [metric depth estimation](./metric_depth).
+
+
+## DA-2K Evaluation Benchmark
+
+Please refer to [DA-2K benchmark](./DA-2K.md).
+
+
+## Community Support
+
+**We sincerely appreciate all the community support for our Depth Anything series. Thank you a lot!**
+
+- Apple Core ML:
+    - https://developer.apple.com/machine-learning/models
+    - https://huggingface.co/apple/coreml-depth-anything-v2-small
+    - https://huggingface.co/apple/coreml-depth-anything-small
+- Transformers:
+    - https://huggingface.co/docs/transformers/main/en/model_doc/depth_anything_v2
+    - https://huggingface.co/docs/transformers/main/en/model_doc/depth_anything
+- TensorRT:
+    - https://github.com/spacewalk01/depth-anything-tensorrt
+    - https://github.com/zhujiajian98/Depth-Anythingv2-TensorRT-python
+- ONNX: https://github.com/fabio-sim/Depth-Anything-ONNX
+- ComfyUI: https://github.com/kijai/ComfyUI-DepthAnythingV2
+- Transformers.js (real-time depth in web): https://huggingface.co/spaces/Xenova/webgpu-realtime-depth-estimation
+- Android:
+  - https://github.com/shubham0204/Depth-Anything-Android
+  - https://github.com/FeiGeChuanShu/ncnn-android-depth_anything
+
+
+## Acknowledgement
+
+We are sincerely grateful to the awesome Hugging Face team ([@Pedro Cuenca](https://huggingface.co/pcuenq), [@Niels Rogge](https://huggingface.co/nielsr), [@Merve Noyan](https://huggingface.co/merve), [@Amy Roberts](https://huggingface.co/amyeroberts), et al.) for their huge efforts in supporting our models in Transformers and Apple Core ML.
+
+We also thank the [DINOv2](https://github.com/facebookresearch/dinov2) team for contributing such impressive models to our community.
+
+
+## LICENSE
+
+Depth-Anything-V2-Small model is under the Apache-2.0 license. Depth-Anything-V2-Base/Large/Giant models are under the CC-BY-NC-4.0 license.
+
+
+## Citation
+
+If you find this project useful, please consider citing:
+
+```bibtex
+@article{depth_anything_v2,
+  title={Depth Anything V2},
+  author={Yang, Lihe and Kang, Bingyi and Huang, Zilong and Zhao, Zhen and Xu, Xiaogang and Feng, Jiashi and Zhao, Hengshuang},
+  journal={arXiv:2406.09414},
+  year={2024}
+}
+
+@inproceedings{depth_anything_v1,
+  title={Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data}, 
+  author={Yang, Lihe and Kang, Bingyi and Huang, Zilong and Xu, Xiaogang and Feng, Jiashi and Zhao, Hengshuang},
+  booktitle={CVPR},
+  year={2024}
+}
+```

ddepth_anything_v2/__init__.py

@@ -0,0 +1 @@
+from .depth_anything_v2.dpt import DepthAnythingV2

ddepth_anything_v2/app.py

@@ -0,0 +1,88 @@
+import glob
+import gradio as gr
+import matplotlib
+import numpy as np
+from PIL import Image
+import torch
+import tempfile
+from gradio_imageslider import ImageSlider
+
+from depth_anything_v2.dpt import DepthAnythingV2
+
+css = """
+#img-display-container {
+    max-height: 100vh;
+}
+#img-display-input {
+    max-height: 80vh;
+}
+#img-display-output {
+    max-height: 80vh;
+}
+#download {
+    height: 62px;
+}
+"""
+DEVICE = 'cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'
+model_configs = {
+    'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
+    'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
+    'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
+    'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]}
+}
+encoder = 'vitl'
+model = DepthAnythingV2(**model_configs[encoder])
+state_dict = torch.load(f'checkpoints/depth_anything_v2_{encoder}.pth', map_location="cpu")
+model.load_state_dict(state_dict)
+model = model.to(DEVICE).eval()
+
+title = "# Depth Anything V2"
+description = """Official demo for **Depth Anything V2**.
+Please refer to our [paper](https://arxiv.org/abs/2406.09414), [project page](https://depth-anything-v2.github.io), or [github](https://github.com/DepthAnything/Depth-Anything-V2) for more details."""
+
+def predict_depth(image):
+    return model.infer_image(image)
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    gr.Markdown("### Depth Prediction demo")
+
+    with gr.Row():
+        input_image = gr.Image(label="Input Image", type='numpy', elem_id='img-display-input')
+        depth_image_slider = ImageSlider(label="Depth Map with Slider View", elem_id='img-display-output', position=0.5)
+    submit = gr.Button(value="Compute Depth")
+    gray_depth_file = gr.File(label="Grayscale depth map", elem_id="download",)
+    raw_file = gr.File(label="16-bit raw output (can be considered as disparity)", elem_id="download",)
+
+    cmap = matplotlib.colormaps.get_cmap('Spectral_r')
+
+    def on_submit(image):
+        original_image = image.copy()
+
+        h, w = image.shape[:2]
+
+        depth = predict_depth(image[:, :, ::-1])
+
+        raw_depth = Image.fromarray(depth.astype('uint16'))
+        tmp_raw_depth = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
+        raw_depth.save(tmp_raw_depth.name)
+
+        depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
+        depth = depth.astype(np.uint8)
+        colored_depth = (cmap(depth)[:, :, :3] * 255).astype(np.uint8)
+
+        gray_depth = Image.fromarray(depth)
+        tmp_gray_depth = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
+        gray_depth.save(tmp_gray_depth.name)
+
+        return [(original_image, colored_depth), tmp_gray_depth.name, tmp_raw_depth.name]
+
+    submit.click(on_submit, inputs=[input_image], outputs=[depth_image_slider, gray_depth_file, raw_file])
+
+    example_files = glob.glob('assets/examples/*')
+    examples = gr.Examples(examples=example_files, inputs=[input_image], outputs=[depth_image_slider, gray_depth_file, raw_file], fn=on_submit)
+
+
+if __name__ == '__main__':
+    demo.queue().launch()

ddepth_anything_v2/depth_anything_v2/dinov2.py

@@ -0,0 +1,415 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+from functools import partial
+import math
+import logging
+from typing import Sequence, Tuple, Union, Callable
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from torch.nn.init import trunc_normal_
+
+from .dinov2_layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
+
+
+logger = logging.getLogger("dinov2")
+
+
+def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
+    if not depth_first and include_root:
+        fn(module=module, name=name)
+    for child_name, child_module in module.named_children():
+        child_name = ".".join((name, child_name)) if name else child_name
+        named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
+    if depth_first and include_root:
+        fn(module=module, name=name)
+    return module
+
+
+class BlockChunk(nn.ModuleList):
+    def forward(self, x):
+        for b in self:
+            x = b(x)
+        return x
+
+
+class DinoVisionTransformer(nn.Module):
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        ffn_bias=True,
+        proj_bias=True,
+        drop_path_rate=0.0,
+        drop_path_uniform=False,
+        init_values=None,  # for layerscale: None or 0 => no layerscale
+        embed_layer=PatchEmbed,
+        act_layer=nn.GELU,
+        block_fn=Block,
+        ffn_layer="mlp",
+        block_chunks=1,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+    ):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            proj_bias (bool): enable bias for proj in attn if True
+            ffn_bias (bool): enable bias for ffn if True
+            drop_path_rate (float): stochastic depth rate
+            drop_path_uniform (bool): apply uniform drop rate across blocks
+            weight_init (str): weight init scheme
+            init_values (float): layer-scale init values
+            embed_layer (nn.Module): patch embedding layer
+            act_layer (nn.Module): MLP activation layer
+            block_fn (nn.Module): transformer block class
+            ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+            block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
+            num_register_tokens: (int) number of extra cls tokens (so-called "registers")
+            interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings
+            interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings
+        """
+        super().__init__()
+        norm_layer = partial(nn.LayerNorm, eps=1e-6)
+
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        self.num_tokens = 1
+        self.n_blocks = depth
+        self.num_heads = num_heads
+        self.patch_size = patch_size
+        self.num_register_tokens = num_register_tokens
+        self.interpolate_antialias = interpolate_antialias
+        self.interpolate_offset = interpolate_offset
+
+        self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
+        assert num_register_tokens >= 0
+        self.register_tokens = (
+            nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None
+        )
+
+        if drop_path_uniform is True:
+            dpr = [drop_path_rate] * depth
+        else:
+            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+
+        if ffn_layer == "mlp":
+            logger.info("using MLP layer as FFN")
+            ffn_layer = Mlp
+        elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
+            logger.info("using SwiGLU layer as FFN")
+            ffn_layer = SwiGLUFFNFused
+        elif ffn_layer == "identity":
+            logger.info("using Identity layer as FFN")
+
+            def f(*args, **kwargs):
+                return nn.Identity()
+
+            ffn_layer = f
+        else:
+            raise NotImplementedError
+
+        blocks_list = [
+            block_fn(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                proj_bias=proj_bias,
+                ffn_bias=ffn_bias,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                ffn_layer=ffn_layer,
+                init_values=init_values,
+            )
+            for i in range(depth)
+        ]
+        if block_chunks > 0:
+            self.chunked_blocks = True
+            chunked_blocks = []
+            chunksize = depth // block_chunks
+            for i in range(0, depth, chunksize):
+                # this is to keep the block index consistent if we chunk the block list
+                chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
+            self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
+        else:
+            self.chunked_blocks = False
+            self.blocks = nn.ModuleList(blocks_list)
+
+        self.norm = norm_layer(embed_dim)
+        self.head = nn.Identity()
+
+        self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+        self.init_weights()
+
+    def init_weights(self):
+        trunc_normal_(self.pos_embed, std=0.02)
+        nn.init.normal_(self.cls_token, std=1e-6)
+        if self.register_tokens is not None:
+            nn.init.normal_(self.register_tokens, std=1e-6)
+        named_apply(init_weights_vit_timm, self)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        previous_dtype = x.dtype
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        pos_embed = self.pos_embed.float()
+        class_pos_embed = pos_embed[:, 0]
+        patch_pos_embed = pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_size
+        h0 = h // self.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0
+        w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset
+        # w0, h0 = w0 + 0.1, h0 + 0.1
+        
+        sqrt_N = math.sqrt(N)
+        sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2),
+            scale_factor=(sx, sy),
+            # (int(w0), int(h0)), # to solve the upsampling shape issue
+            mode="bicubic",
+            antialias=self.interpolate_antialias
+        )
+        
+        assert int(w0) == patch_pos_embed.shape[-2]
+        assert int(h0) == patch_pos_embed.shape[-1]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype)
+
+    def prepare_tokens_with_masks(self, x, masks=None):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)
+        if masks is not None:
+            x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
+
+        x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        if self.register_tokens is not None:
+            x = torch.cat(
+                (
+                    x[:, :1],
+                    self.register_tokens.expand(x.shape[0], -1, -1),
+                    x[:, 1:],
+                ),
+                dim=1,
+            )
+
+        return x
+
+    def forward_features_list(self, x_list, masks_list):
+        x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)]
+        for blk in self.blocks:
+            x = blk(x)
+
+        all_x = x
+        output = []
+        for x, masks in zip(all_x, masks_list):
+            x_norm = self.norm(x)
+            output.append(
+                {
+                    "x_norm_clstoken": x_norm[:, 0],
+                    "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+                    "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+                    "x_prenorm": x,
+                    "masks": masks,
+                }
+            )
+        return output
+
+    def forward_features(self, x, masks=None):
+        if isinstance(x, list):
+            return self.forward_features_list(x, masks)
+
+        x = self.prepare_tokens_with_masks(x, masks)
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x_norm = self.norm(x)
+        return {
+            "x_norm_clstoken": x_norm[:, 0],
+            "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+            "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+            "x_prenorm": x,
+            "masks": masks,
+        }
+
+    def _get_intermediate_layers_not_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        # If n is an int, take the n last blocks. If it's a list, take them
+        output, total_block_len = [], len(self.blocks)
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if i in blocks_to_take:
+                output.append(x)
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def _get_intermediate_layers_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        output, i, total_block_len = [], 0, len(self.blocks[-1])
+        # If n is an int, take the n last blocks. If it's a list, take them
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for block_chunk in self.blocks:
+            for blk in block_chunk[i:]:  # Passing the nn.Identity()
+                x = blk(x)
+                if i in blocks_to_take:
+                    output.append(x)
+                i += 1
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def get_intermediate_layers(
+        self,
+        x: torch.Tensor,
+        n: Union[int, Sequence] = 1,  # Layers or n last layers to take
+        reshape: bool = False,
+        return_class_token: bool = False,
+        norm=True
+    ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
+        if self.chunked_blocks:
+            outputs = self._get_intermediate_layers_chunked(x, n)
+        else:
+            outputs = self._get_intermediate_layers_not_chunked(x, n)
+        if norm:
+            outputs = [self.norm(out) for out in outputs]
+        class_tokens = [out[:, 0] for out in outputs]
+        outputs = [out[:, 1 + self.num_register_tokens:] for out in outputs]
+        if reshape:
+            B, _, w, h = x.shape
+            outputs = [
+                out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
+                for out in outputs
+            ]
+        if return_class_token:
+            return tuple(zip(outputs, class_tokens))
+        return tuple(outputs)
+
+    def forward(self, *args, is_training=False, **kwargs):
+        ret = self.forward_features(*args, **kwargs)
+        if is_training:
+            return ret
+        else:
+            return self.head(ret["x_norm_clstoken"])
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):
+    """ViT weight initialization, original timm impl (for reproducibility)"""
+    if isinstance(module, nn.Linear):
+        trunc_normal_(module.weight, std=0.02)
+        if module.bias is not None:
+            nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=384,
+        depth=12,
+        num_heads=6,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_base(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_large(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs):
+    """
+    Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
+    """
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1536,
+        depth=40,
+        num_heads=24,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def DINOv2(model_name):
+    model_zoo = {
+        "vits": vit_small, 
+        "vitb": vit_base, 
+        "vitl": vit_large, 
+        "vitg": vit_giant2
+    }
+    
+    return model_zoo[model_name](
+        img_size=518,
+        patch_size=14,
+        init_values=1.0,
+        ffn_layer="mlp" if model_name != "vitg" else "swiglufused",
+        block_chunks=0,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1
+    )

ddepth_anything_v2/depth_anything_v2/dinov2_layers/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+from .block import NestedTensorBlock
+from .attention import MemEffAttention

ddepth_anything_v2/depth_anything_v2/dinov2_layers/attention.py

@@ -0,0 +1,83 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+
+from torch import Tensor
+from torch import nn
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+    from xformers.ops import memory_efficient_attention, unbind, fmha
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+
+        q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+        attn = q @ k.transpose(-2, -1)
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class MemEffAttention(Attention):
+    def forward(self, x: Tensor, attn_bias=None) -> Tensor:
+        if not XFORMERS_AVAILABLE:
+            assert attn_bias is None, "xFormers is required for nested tensors usage"
+            return super().forward(x)
+
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
+
+        q, k, v = unbind(qkv, 2)
+
+        x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+        x = x.reshape([B, N, C])
+
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+        

ddepth_anything_v2/depth_anything_v2/dinov2_layers/block.py

@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+from typing import Callable, List, Any, Tuple, Dict
+
+import torch
+from torch import nn, Tensor
+
+from .attention import Attention, MemEffAttention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+    from xformers.ops import fmha
+    from xformers.ops import scaled_index_add, index_select_cat
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+    ) -> None:
+        super().__init__()
+        # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+        self.norm1 = norm_layer(dim)
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+        )
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.sample_drop_ratio = drop_path
+
+    def forward(self, x: Tensor) -> Tensor:
+        def attn_residual_func(x: Tensor) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x)))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        if self.training and self.sample_drop_ratio > 0.1:
+            # the overhead is compensated only for a drop path rate larger than 0.1
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+        elif self.training and self.sample_drop_ratio > 0.0:
+            x = x + self.drop_path1(attn_residual_func(x))
+            x = x + self.drop_path1(ffn_residual_func(x))  # FIXME: drop_path2
+        else:
+            x = x + attn_residual_func(x)
+            x = x + ffn_residual_func(x)
+        return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: Tensor,
+    residual_func: Callable[[Tensor], Tensor],
+    sample_drop_ratio: float = 0.0,
+) -> Tensor:
+    # 1) extract subset using permutation
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    x_subset = x[brange]
+
+    # 2) apply residual_func to get residual
+    residual = residual_func(x_subset)
+
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+
+    residual_scale_factor = b / sample_subset_size
+
+    # 3) add the residual
+    x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    residual_scale_factor = b / sample_subset_size
+    return brange, residual_scale_factor
+
+
+def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
+    if scaling_vector is None:
+        x_flat = x.flatten(1)
+        residual = residual.flatten(1)
+        x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    else:
+        x_plus_residual = scaled_index_add(
+            x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
+        )
+    return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+    """
+    this will perform the index select, cat the tensors, and provide the attn_bias from cache
+    """
+    batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
+    all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+    if all_shapes not in attn_bias_cache.keys():
+        seqlens = []
+        for b, x in zip(batch_sizes, x_list):
+            for _ in range(b):
+                seqlens.append(x.shape[1])
+        attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+        attn_bias._batch_sizes = batch_sizes
+        attn_bias_cache[all_shapes] = attn_bias
+
+    if branges is not None:
+        cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
+    else:
+        tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+        cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+    return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+    x_list: List[Tensor],
+    residual_func: Callable[[Tensor, Any], Tensor],
+    sample_drop_ratio: float = 0.0,
+    scaling_vector=None,
+) -> Tensor:
+    # 1) generate random set of indices for dropping samples in the batch
+    branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
+    branges = [s[0] for s in branges_scales]
+    residual_scale_factors = [s[1] for s in branges_scales]
+
+    # 2) get attention bias and index+concat the tensors
+    attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+    # 3) apply residual_func to get residual, and split the result
+    residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias))  # type: ignore
+
+    outputs = []
+    for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
+        outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
+    return outputs
+
+
+class NestedTensorBlock(Block):
+    def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
+        """
+        x_list contains a list of tensors to nest together and run
+        """
+        assert isinstance(self.attn, MemEffAttention)
+
+        if self.training and self.sample_drop_ratio > 0.0:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.mlp(self.norm2(x))
+
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            return x_list
+        else:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls2(self.mlp(self.norm2(x)))
+
+            attn_bias, x = get_attn_bias_and_cat(x_list)
+            x = x + attn_residual_func(x, attn_bias=attn_bias)
+            x = x + ffn_residual_func(x)
+            return attn_bias.split(x)
+
+    def forward(self, x_or_x_list):
+        if isinstance(x_or_x_list, Tensor):
+            return super().forward(x_or_x_list)
+        elif isinstance(x_or_x_list, list):
+            assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage"
+            return self.forward_nested(x_or_x_list)
+        else:
+            raise AssertionError

ddepth_anything_v2/depth_anything_v2/dinov2_layers/drop_path.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0:
+        random_tensor.div_(keep_prob)
+    output = x * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob=None):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)