Spaces:
Running
Running
<html> | |
<head> | |
<title>Text to Terrain Generator</title> | |
<style> | |
body { margin: 0; } | |
canvas { display: block; } | |
#prompt-input { | |
position: absolute; | |
top: 10px; | |
left: 10px; | |
padding: 8px; | |
width: 200px; | |
border: 1px solid #ccc; | |
border-radius: 4px; | |
font-size: 14px; | |
} | |
#camera-toggle { | |
position: absolute; | |
top: 10px; | |
right: 10px; | |
padding: 8px 16px; | |
background-color: #3498db; | |
color: white; | |
border: none; | |
border-radius: 4px; | |
cursor: pointer; | |
font-size: 14px; | |
transition: background-color 0.3s; | |
} | |
#camera-toggle:hover { | |
background-color: #2980b9; | |
} | |
#heightmap { | |
display: none; | |
} | |
#loadingOverlay { | |
display: none; | |
position: fixed; | |
top: 0; | |
left: 0; | |
width: 100%; | |
height: 100%; | |
background-color: rgba(0, 0, 0, 0.7); | |
z-index: 9999; | |
justify-content: center; | |
align-items: center; | |
} | |
.loader { | |
border: 5px solid #f3f3f3; | |
border-top: 5px solid #3498db; | |
border-radius: 50%; | |
width: 50px; | |
height: 50px; | |
animation: spin 1s linear infinite; | |
} | |
@keyframes spin { | |
0% { transform: rotate(0deg); } | |
100% { transform: rotate(360deg); } | |
} | |
</style> | |
</head> | |
<body> | |
<input type="text" id="prompt-input" placeholder="Countryside house with trees.."> | |
<button id="camera-toggle">Toggle Camera</button> | |
<div id="loadingOverlay"> | |
<div class="loader"></div> | |
</div> | |
<canvas id="heightmap" width="256" height="256"></canvas> | |
<script type="importmap"> | |
{ | |
"imports": { | |
"three": "https://cdnjs.cloudflare.com/ajax/libs/three.js/0.172.0/three.module.min.js", | |
"three/addons/": "https://unpkg.com/[email protected]/examples/jsm/", | |
"three/examples/": "https://unpkg.com/[email protected]/examples/jsm/", | |
"@gradio/client": "https://cdn.jsdelivr.net/npm/@gradio/[email protected]/+esm" | |
} | |
} | |
</script> | |
<script type="module"> | |
import * as THREE from 'three'; | |
import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; | |
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; | |
import { Client } from "@gradio/client"; | |
let isLoading = false; | |
let lastPromptInputValue = ""; | |
let isDragging = false; | |
let mouseDownPosition = new THREE.Vector2(); | |
const objectCountByType = new Map(); | |
let terrainInstances; | |
let objectInstances = new Map(); | |
const TILE_SIZE = 4; | |
const GRID_SIZE = 10; | |
let currentTileType = null; | |
let hoveredTileIndex = -1; | |
const raycaster = new THREE.Raycaster(); | |
const mouse = new THREE.Vector2(); | |
let highlightMesh = null; | |
const tileTypes = new Map(); | |
const gltfLoader = new GLTFLoader(); | |
// Scene setup | |
const scene = new THREE.Scene(); | |
scene.background = new THREE.Color(0x87CEEB); | |
// Camera setup with both perspectives | |
const frustumSize = 40; | |
const aspect = window.innerWidth / window.innerHeight; | |
const orthoCamera = new THREE.OrthographicCamera( | |
-20, 20, 20, -20, | |
0.1, 2000 | |
); | |
orthoCamera.position.set(20, 20, -20); | |
orthoCamera.lookAt(0, 0, 0); | |
const perspCamera = new THREE.PerspectiveCamera( | |
75, window.innerWidth / window.innerHeight, 0.1, 2000 | |
); | |
perspCamera.position.set(20, 20, -20); | |
perspCamera.lookAt(0, 0, 0); | |
let currentCamera = orthoCamera; | |
let isOrthographic = true; | |
// Renderer setup | |
const renderer = new THREE.WebGLRenderer({ antialias: true }); | |
renderer.setSize(window.innerWidth, window.innerHeight); | |
renderer.shadowMap.enabled = true; | |
renderer.shadowMap.type = THREE.PCFSoftShadowMap; | |
document.body.appendChild(renderer.domElement); | |
// Controls | |
let controls = new OrbitControls(currentCamera, renderer.domElement); | |
controls.enableDamping = true; | |
// Lighting | |
const directionalLight = new THREE.DirectionalLight(0xffffff, 4.5); // Increased from 1.5 to 4.5 | |
directionalLight.position.set(20, 30, 20); | |
directionalLight.castShadow = true; | |
// Improve shadow quality | |
directionalLight.shadow.mapSize.width = 2048; | |
directionalLight.shadow.mapSize.height = 2048; | |
directionalLight.shadow.camera.near = 0.1; | |
directionalLight.shadow.camera.far = 100; | |
directionalLight.shadow.camera.left = -30; | |
directionalLight.shadow.camera.right = 30; | |
directionalLight.shadow.camera.top = 30; | |
directionalLight.shadow.camera.bottom = -30; | |
directionalLight.shadow.bias = -0.001; | |
// Add a brighter ambient light | |
scene.add(new THREE.AmbientLight(0x404040, 1.2)); | |
// Add lighter fog | |
scene.fog = new THREE.FogExp2(0x87CEEB, 0.015); | |
// Enable shadow mapping in the renderer | |
renderer.shadowMap.enabled = true; | |
renderer.shadowMap.type = THREE.PCFSoftShadowMap; | |
scene.add(directionalLight); | |
// Height map generation | |
const canvasH = document.getElementById('heightmap'); | |
const ctx = canvasH.getContext('2d'); | |
const heightMap = new THREE.CanvasTexture(canvasH); | |
function createSeamlessNoise(size, octaves) { | |
const noise = new Array(size * size).fill(0); | |
function smoothStep(t) { | |
return t * t * (3 - 2 * t); | |
} | |
function interpolate(a, b, t) { | |
return a + (b - a) * smoothStep(t); | |
} | |
for (let octave = 0; octave < octaves; octave++) { | |
const frequency = 1 << octave; | |
const amplitude = 1 / (1 << octave); | |
const grid = new Array((frequency + 1) * (frequency + 1)); | |
for (let i = 0; i <= frequency; i++) { | |
for (let j = 0; j <= frequency; j++) { | |
grid[i * (frequency + 1) + j] = Math.random(); | |
} | |
} | |
for (let i = 0; i <= frequency; i++) { | |
grid[i * (frequency + 1) + frequency] = grid[i * (frequency + 1)]; | |
} | |
for (let j = 0; j <= frequency; j++) { | |
grid[frequency * (frequency + 1) + j] = grid[j]; | |
} | |
grid[frequency * (frequency + 1) + frequency] = grid[0]; | |
const cellSize = size / frequency; | |
for (let y = 0; y < size; y++) { | |
for (let x = 0; x < size; x++) { | |
const gridX = Math.floor(x / cellSize); | |
const gridY = Math.floor(y / cellSize); | |
const fracX = (x % cellSize) / cellSize; | |
const fracY = (y % cellSize) / cellSize; | |
const v1 = grid[gridY * (frequency + 1) + gridX]; | |
const v2 = grid[gridY * (frequency + 1) + (gridX + 1)]; | |
const v3 = grid[(gridY + 1) * (frequency + 1) + gridX]; | |
const v4 = grid[(gridY + 1) * (frequency + 1) + (gridX + 1)]; | |
const i1 = interpolate(v1, v2, fracX); | |
const i2 = interpolate(v3, v4, fracX); | |
const value = interpolate(i1, i2, fracY); | |
noise[y * size + x] += value * amplitude; | |
} | |
} | |
} | |
return noise; | |
} | |
function createHeightMap() { | |
const size = 256; | |
const noise = createSeamlessNoise(size, 8); | |
const imageData = ctx.createImageData(size, size); | |
let min = Infinity, max = -Infinity; | |
for (let i = 0; i < noise.length; i++) { | |
min = Math.min(min, noise[i]); | |
max = Math.max(max, noise[i]); | |
} | |
for (let i = 0; i < noise.length; i++) { | |
const normalized = Math.floor(((noise[i] - min) / (max - min)) * 255); | |
imageData.data[i * 4] = normalized; | |
imageData.data[i * 4 + 1] = normalized; | |
imageData.data[i * 4 + 2] = normalized; | |
imageData.data[i * 4 + 3] = 255; | |
} | |
ctx.putImageData(imageData, 0, 0); | |
heightMap.needsUpdate = true; | |
if (terrainInstances?.material) { | |
terrainInstances.material.displacementMap = heightMap; | |
terrainInstances.material.needsUpdate = true; | |
} | |
} | |
// Create base terrain type | |
function createBaseTileType() { | |
const geometry = new THREE.PlaneGeometry(TILE_SIZE, TILE_SIZE, 64, 64); | |
const material = new THREE.MeshStandardMaterial({ | |
color: 0x46732a, | |
displacementMap: heightMap, | |
displacementScale: 0.8, | |
flatShading: false, | |
side: THREE.DoubleSide, | |
shadowSide: THREE.DoubleSide | |
}); | |
tileTypes.set('base', { geometry, material }); | |
} | |
function createHighlightMesh() { | |
const geometry = new THREE.PlaneGeometry(TILE_SIZE, TILE_SIZE); | |
const edges = new THREE.EdgesGeometry(geometry); | |
const material = new THREE.LineBasicMaterial({ | |
color: 0xffff00, | |
linewidth: 2, | |
transparent: true, | |
opacity: 0.9 | |
}); | |
highlightMesh = new THREE.LineSegments(edges, material); | |
highlightMesh.rotation.x = -Math.PI / 2; | |
highlightMesh.visible = false; | |
scene.add(highlightMesh); | |
} | |
function initializeTerrain() { | |
createBaseTileType(); | |
createHighlightMesh(); | |
const baseTile = tileTypes.get('base'); | |
const instanceCount = GRID_SIZE * GRID_SIZE; | |
terrainInstances = new THREE.InstancedMesh( | |
baseTile.geometry, | |
baseTile.material, | |
instanceCount | |
); | |
terrainInstances.castShadow = true; | |
terrainInstances.receiveShadow = true; | |
const matrix = new THREE.Matrix4(); | |
const position = new THREE.Vector3(); | |
const rotation = new THREE.Euler(); | |
const quaternion = new THREE.Quaternion(); | |
const scale = new THREE.Vector3(1, 1, 1); | |
let index = 0; | |
const offset = (GRID_SIZE * TILE_SIZE) / 2 - TILE_SIZE / 2; | |
for (let x = 0; x < GRID_SIZE; x++) { | |
for (let z = 0; z < GRID_SIZE; z++) { | |
position.set( | |
x * TILE_SIZE - offset, | |
0, | |
z * TILE_SIZE - offset | |
); | |
rotation.set(-Math.PI / 2, 0, 0); | |
quaternion.setFromEuler(rotation); | |
matrix.compose(position, quaternion, scale); | |
terrainInstances.setMatrixAt(index, matrix); | |
index++; | |
} | |
} | |
scene.add(terrainInstances); | |
} | |
async function createGLBTileType(modelUrl, typeName) { | |
return new Promise((resolve, reject) => { | |
gltfLoader.load(modelUrl, (gltf) => { | |
const model = gltf.scene; | |
const box = new THREE.Box3().setFromObject(model); | |
const size = box.getSize(new THREE.Vector3()); | |
const geometry = new THREE.BoxGeometry(1, 1, 1); | |
const material = new THREE.MeshStandardMaterial({ visible: false }); | |
tileTypes.set(typeName, { | |
geometry, | |
material, | |
model: model.clone() | |
}); | |
resolve(); | |
}, undefined, reject); | |
}); | |
} | |
// First, create a function to find the Y position of the widest cross-section | |
function findWidestCrossSection(mesh) { | |
// Track the maximum width/depth we find and its Y position | |
let maxArea = 0; | |
let maxAreaY = 0; | |
// Get all vertices from the mesh and its children | |
const vertices = []; | |
mesh.traverse((child) => { | |
if (child.isMesh && child.geometry) { | |
const positions = child.geometry.attributes.position; | |
const vertexCount = positions.count; | |
// Transform vertices to world space | |
const matrix = child.matrixWorld; | |
for (let i = 0; i < vertexCount; i++) { | |
const vertex = new THREE.Vector3(); | |
vertex.fromBufferAttribute(positions, i); | |
vertex.applyMatrix4(matrix); | |
vertices.push(vertex); | |
} | |
} | |
}); | |
if (vertices.length === 0) return 0; | |
// Find Y range to analyze | |
const yValues = vertices.map(v => v.y); | |
const minY = Math.min(...yValues); | |
const maxY = Math.max(...yValues); | |
// Sample Y positions at regular intervals | |
const steps = 128; // Number of cross-sections to check | |
const yStep = (maxY - minY) / steps; | |
for (let i = 0; i <= steps; i++) { | |
const currentY = minY + (i * yStep); | |
// Find vertices near this Y level (within small threshold) | |
const threshold = yStep / 2; | |
const sectionVertices = vertices.filter(v => | |
Math.abs(v.y - currentY) < threshold | |
); | |
if (sectionVertices.length > 0) { | |
// Calculate bounding area of this cross-section | |
const xValues = sectionVertices.map(v => v.x); | |
const zValues = sectionVertices.map(v => v.z); | |
const width = Math.max(...xValues) - Math.min(...xValues); | |
const depth = Math.max(...zValues) - Math.min(...zValues); | |
const area = width * depth; | |
if (area > maxArea) { | |
maxArea = area; | |
maxAreaY = currentY; | |
} | |
} | |
} | |
return maxAreaY; | |
} | |
function replaceTileInstance(instanceIndex, newTypeName) { | |
const newType = tileTypes.get(newTypeName); | |
if (!newType) return; | |
// Get original transformation matrix from terrain instance | |
const matrix = new THREE.Matrix4(); | |
terrainInstances.getMatrixAt(instanceIndex, matrix); | |
// Extract position from matrix | |
const position = new THREE.Vector3(); | |
const rotation = new THREE.Quaternion(); | |
const scale = new THREE.Vector3(); | |
matrix.decompose(position, rotation, scale); | |
// Check if there's an existing object of the same type | |
if (objectInstances.has(instanceIndex)) { | |
const existing = objectInstances.get(instanceIndex); | |
if (existing.typeName === newTypeName) { | |
// Rotate existing object's rotation group by 90 degrees | |
const currentRotationY = (existing.rotationY || 0) + Math.PI / 2; | |
const normalizedRotation = currentRotationY % (Math.PI * 2); | |
existing.rotationGroup.rotation.y = (Math.PI / 4) + normalizedRotation; | |
existing.rotationY = normalizedRotation; | |
return; // Exit early as we just rotated the existing object | |
} else { | |
// Remove existing object if it's a different type | |
scene.remove(existing.rotationGroup); | |
} | |
} | |
// Create new mesh from model | |
const newMesh = newType.model.clone(); | |
// Create hierarchy of groups for different transformations | |
const rotationGroup = new THREE.Group(); // Handles user-controlled Y rotation | |
const orientationGroup = new THREE.Group(); // Handles initial orientation | |
// Build hierarchy | |
scene.add(rotationGroup); | |
rotationGroup.add(orientationGroup); | |
orientationGroup.add(newMesh); | |
// Position the top-level group at the tile location | |
rotationGroup.position.copy(position); | |
// Enable shadows | |
newMesh.traverse((child) => { | |
if (child.isMesh) { | |
child.castShadow = true; | |
child.receiveShadow = true; | |
} | |
if (child.material) { | |
child.material.shadowSide = THREE.DoubleSide; | |
child.material.needsUpdate = true; | |
} | |
}); | |
// Create a bounding box to calculate the mesh's dimensions | |
const bbox = new THREE.Box3().setFromObject(newMesh); | |
const meshCenter = bbox.getCenter(new THREE.Vector3()); | |
// Center the mesh on its local origin | |
newMesh.position.sub(meshCenter); | |
// Apply initial orientation using clean, separate rotations | |
orientationGroup.rotation.x = 0.6; // fix mesh generation tiling | |
orientationGroup.rotation.y = 0; | |
orientationGroup.rotation.z = 0; | |
// Find the Y position of widest cross-section and adjust position | |
const baseY = findWidestCrossSection(newMesh); | |
newMesh.position.y -= baseY; | |
newMesh.position.y += 1.2; | |
rotationGroup.rotation.y = Math.PI / 4; | |
// Apply scale to the mesh | |
newMesh.scale.set(5.6, 5.6, 5.6); | |
// Adjust XZ position to ensure centering after rotation | |
bbox.setFromObject(newMesh); | |
const rotatedCenter = bbox.getCenter(new THREE.Vector3()); | |
//newMesh.position.x -= 1.0;//rotatedCenter.x; | |
//newMesh.position.z -= 0;//rotatedCenter.z; | |
// Add new object to scene and track it with metadata | |
objectInstances.set(instanceIndex, { | |
rotationGroup: rotationGroup, | |
orientationGroup: orientationGroup, | |
mesh: newMesh, | |
typeName: newTypeName, | |
rotationY: 0 // Start with 0 rotation for the group | |
}); | |
const currentCount = objectCountByType.get(newTypeName) || 0; | |
objectCountByType.set(newTypeName, currentCount + 1); | |
} | |
let gradioClient = null; | |
async function initializeGradioClient() { | |
try { | |
gradioClient = await Client.connect("jbilcke-hf/text-to-3d"); | |
} catch (error) { | |
console.error("Failed to connect to Gradio:", error); | |
} | |
} | |
async function generateTile(prompt) { | |
if (!gradioClient) { | |
await initializeGradioClient(); | |
} | |
if (!gradioClient) { | |
console.error("Gradio client not initialized"); | |
return; | |
} | |
if (prompt === lastPromptInputValue) { | |
return; | |
} | |
lastPromptInputValue = prompt; | |
try { | |
isLoading = true; | |
loadingOverlay.style.display = 'flex'; | |
const result = await gradioClient.predict("/generate", { | |
prompt: `isometric videogame asset, ${prompt.trim()}`, | |
}); | |
const modelUrl = result.data[0].url; | |
//const modelUrl = "/models/eiffel_tower.glb"; | |
const tileTypeName = `custom_${Date.now()}`; | |
await createGLBTileType(modelUrl, tileTypeName); | |
// Instead of random placement, store the current tile type | |
currentTileType = tileTypeName; | |
// Optional: place one instance to show the new type | |
replaceTileInstance(Math.floor(GRID_SIZE * GRID_SIZE / 2), tileTypeName); | |
} catch (error) { | |
console.error("Failed to generate/load terrain:", error); | |
} finally { | |
isLoading = false; | |
loadingOverlay.style.display = 'none'; | |
} | |
} | |
// Handle window resize | |
window.addEventListener('resize', () => { | |
const aspect = window.innerWidth / window.innerHeight; | |
const frustumSize = 40; | |
camera.left = -frustumSize * aspect / 2; | |
camera.right = frustumSize * aspect / 2; | |
camera.top = frustumSize / 2; | |
camera.bottom = -frustumSize / 2; | |
camera.updateProjectionMatrix(); | |
renderer.setSize(window.innerWidth, window.innerHeight); | |
}); | |
// Text input handling | |
const promptInput = document.getElementById('prompt-input'); | |
// Event listeners | |
promptInput.addEventListener('keypress', async (e) => { | |
if (e.key === 'Enter') { | |
await generateTile(promptInput.value); | |
} | |
}); | |
promptInput.addEventListener('blur', async () => { | |
await generateTile(promptInput.value); | |
}); | |
canvasH.addEventListener('click', createHeightMap); | |
// Initialize and start | |
createHeightMap(); | |
initializeTerrain(); | |
function onMouseDown(event) { | |
if (event.button === 0) { // Left click | |
mouseDownPosition.x = event.clientX; | |
mouseDownPosition.y = event.clientY; | |
isDragging = false; | |
} | |
} | |
// Update raycasting to use current camera | |
function onMouseMove(event) { | |
if (event.buttons === 1) { | |
const deltaX = Math.abs(event.clientX - mouseDownPosition.x); | |
const deltaY = Math.abs(event.clientY - mouseDownPosition.y); | |
if (deltaX > 5 || deltaY > 5) { | |
isDragging = true; | |
} | |
} | |
mouse.x = (event.clientX / window.innerWidth) * 2 - 1; | |
mouse.y = -(event.clientY / window.innerHeight) * 2 + 1; | |
raycaster.setFromCamera(mouse, currentCamera); | |
const intersects = raycaster.intersectObject(terrainInstances); | |
if (intersects.length > 0) { | |
const instanceId = intersects[0].instanceId; | |
if (instanceId !== hoveredTileIndex) { | |
hoveredTileIndex = instanceId; | |
const matrix = new THREE.Matrix4(); | |
terrainInstances.getMatrixAt(instanceId, matrix); | |
const position = new THREE.Vector3(); | |
matrix.decompose(position, new THREE.Quaternion(), new THREE.Vector3()); | |
highlightMesh.position.set(position.x, 0.4, position.z); | |
highlightMesh.visible = true; | |
} | |
} else { | |
hoveredTileIndex = -1; | |
highlightMesh.visible = false; | |
} | |
} | |
function onMouseUp(event) { | |
if (event.button === 0) { // Left click | |
if (!isDragging && hoveredTileIndex !== -1 && currentTileType) { | |
replaceTileInstance(hoveredTileIndex, currentTileType); | |
} | |
} | |
} | |
function onClick(event) { | |
if (event.button === 2) { // Right click | |
if (hoveredTileIndex !== -1 && objectInstances.has(hoveredTileIndex)) { | |
const existing = objectInstances.get(hoveredTileIndex); | |
const typeName = existing.typeName; | |
if (typeName) { | |
const currentCount = objectCountByType.get(typeName) || 0; | |
if (currentCount > 1) { | |
objectCountByType.set(typeName, currentCount - 1); | |
} else { | |
objectCountByType.delete(typeName); | |
} | |
} | |
scene.remove(existing.rotationGroup); | |
objectInstances.delete(hoveredTileIndex); | |
} | |
} | |
} | |
// Add camera toggle button handler | |
const cameraToggle = document.getElementById('camera-toggle'); | |
cameraToggle.addEventListener('click', () => { | |
isOrthographic = !isOrthographic; | |
currentCamera = isOrthographic ? orthoCamera : perspCamera; | |
// Update controls | |
controls.dispose(); | |
controls = new OrbitControls(currentCamera, renderer.domElement); | |
controls.enableDamping = true; | |
// Sync camera positions | |
const oldPos = isOrthographic ? perspCamera.position : orthoCamera.position; | |
currentCamera.position.copy(oldPos); | |
currentCamera.lookAt(controls.target); | |
// Update projection if needed | |
if (!isOrthographic) { | |
perspCamera.aspect = window.innerWidth / window.innerHeight; | |
perspCamera.updateProjectionMatrix(); | |
} | |
}); | |
// Update window resize handler | |
window.addEventListener('resize', () => { | |
const aspect = window.innerWidth / window.innerHeight; | |
if (isOrthographic) { | |
const frustumSize = 40; | |
orthoCamera.left = -frustumSize * aspect / 2; | |
orthoCamera.right = frustumSize * aspect / 2; | |
orthoCamera.top = frustumSize / 2; | |
orthoCamera.bottom = -frustumSize / 2; | |
orthoCamera.updateProjectionMatrix(); | |
} else { | |
perspCamera.aspect = aspect; | |
perspCamera.updateProjectionMatrix(); | |
} | |
renderer.setSize(window.innerWidth, window.innerHeight); | |
}); | |
renderer.domElement.removeEventListener('click', onClick); | |
renderer.domElement.addEventListener('mousedown', onMouseDown); | |
renderer.domElement.addEventListener('mousemove', onMouseMove); | |
renderer.domElement.addEventListener('mouseup', onMouseUp); | |
renderer.domElement.addEventListener('contextmenu', (event) => { | |
event.preventDefault(); | |
onClick(event); | |
}); | |
function animate() { | |
requestAnimationFrame(animate); | |
controls.update(); | |
renderer.render(scene, currentCamera); | |
} | |
animate(); | |
</script> | |
</body> | |
</html> |