Spaces:
Running
on
Zero
Running
on
Zero
File size: 65,461 Bytes
3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 c0fe80d 3172319 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 |
import os
import re
import json
import random
import numpy as np
from typing import Dict, List, Tuple, Any, Optional
from scene_type import SCENE_TYPES
from scene_detail_templates import SCENE_DETAIL_TEMPLATES
from object_template_fillers import OBJECT_TEMPLATE_FILLERS
from lighting_conditions import LIGHTING_CONDITIONS
from viewpoint_templates import VIEWPOINT_TEMPLATES
from cultural_templates import CULTURAL_TEMPLATES
from confifence_templates import CONFIDENCE_TEMPLATES
class EnhancedSceneDescriber:
"""
Enhanced scene description generator with improved template handling,
viewpoint awareness, and cultural context recognition.
Provides detailed natural language descriptions of scenes based on
detection results and scene classification.
"""
def __init__(self, templates_db: Optional[Dict] = None, scene_types: Optional[Dict] = None):
"""
Initialize the enhanced scene describer.
Args:
templates_db: Optional custom templates database
scene_types: Dictionary of scene type definitions
"""
# Load or use provided scene types
self.scene_types = scene_types or self._load_default_scene_types()
# Load templates database
self.templates = templates_db or self._load_templates()
# Initialize viewpoint detection parameters
self._initialize_viewpoint_parameters()
def _load_default_scene_types(self) -> Dict:
"""
Load default scene types.
Returns:
Dict: Scene type definitions
"""
return SCENE_TYPES
def _load_templates(self) -> Dict:
"""
Load description templates from imported Python modules.
Returns:
Dict: Template collections for different description components
"""
templates = {}
# 直接從導入的 Python 模組中獲取模板
templates["scene_detail_templates"] = SCENE_DETAIL_TEMPLATES
templates["object_template_fillers"] = OBJECT_TEMPLATE_FILLERS
templates["viewpoint_templates"] = VIEWPOINT_TEMPLATES
templates["cultural_templates"] = CULTURAL_TEMPLATES
# 從 LIGHTING_CONDITIONS 獲取照明模板
templates["lighting_templates"] = {
key: data["general"] for key, data in LIGHTING_CONDITIONS.get("time_descriptions", {}).items()
}
# 設置默認的置信度模板
templates["confidence_templates"] = {
"high": "{description} {details}",
"medium": "This appears to be {description} {details}",
"low": "This might be {description}, but the confidence is low. {details}"
}
# 初始化其他必要的模板(現在這個函數簡化了很多)
self._initialize_default_templates(templates)
return templates
def _initialize_default_templates(self, templates: Dict):
"""
檢查模板字典並填充任何缺失的默認模板。
在將模板移至專門的模組後,此方法主要作為安全機制,
確保即使導入失敗或某些模板未在外部定義,系統仍能正常運行。
Args:
templates: 要檢查和更新的模板字典
"""
# 檢查關鍵模板類型是否存在,如果不存在則添加默認值
# 置信度模板 - 用於控制描述的語氣
if "confidence_templates" not in templates:
templates["confidence_templates"] = {
"high": "{description} {details}",
"medium": "This appears to be {description} {details}",
"low": "This might be {description}, but the confidence is low. {details}"
}
# 場景細節模板 - 如果未從外部導入
if "scene_detail_templates" not in templates:
templates["scene_detail_templates"] = {
"default": ["A space with various objects."]
}
# 物體填充模板 - 用於生成物體描述
if "object_template_fillers" not in templates:
templates["object_template_fillers"] = {
"default": ["various items"]
}
# 視角模板 - 雖然我們現在從專門模組導入,但作為備份
if "viewpoint_templates" not in templates:
# 使用簡化版的默認視角模板
templates["viewpoint_templates"] = {
"eye_level": {
"prefix": "From eye level, ",
"observation": "the scene is viewed straight on."
},
"aerial": {
"prefix": "From above, ",
"observation": "the scene is viewed from a bird's-eye perspective."
}
}
# 文化模板
if "cultural_templates" not in templates:
templates["cultural_templates"] = {
"asian": {
"elements": ["cultural elements"],
"description": "The scene has Asian characteristics."
},
"european": {
"elements": ["architectural features"],
"description": "The scene has European characteristics."
}
}
# 照明模板 - 用於描述光照條件
if "lighting_templates" not in templates:
templates["lighting_templates"] = {
"day_clear": "The scene is captured during daylight.",
"night": "The scene is captured at night.",
"unknown": "The lighting conditions are not easily determined."
}
def _initialize_viewpoint_parameters(self):
"""
Initialize parameters used for viewpoint detection.
"""
self.viewpoint_params = {
# Parameters for detecting aerial views
"aerial_threshold": 0.7, # High object density viewed from top
"aerial_size_variance_threshold": 0.15, # Low size variance in aerial views
# Parameters for detecting low angle views
"low_angle_threshold": 0.3, # Bottom-heavy object distribution
"vertical_size_ratio_threshold": 1.8, # Vertical objects appear taller
# Parameters for detecting elevated views
"elevated_threshold": 0.6, # Objects mostly in middle/bottom
"elevated_top_threshold": 0.3 # Few objects at top of frame
}
def generate_description(self,
scene_type: str,
detected_objects: List[Dict],
confidence: float,
lighting_info: Optional[Dict] = None,
functional_zones: Optional[Dict] = None) -> str:
"""
Generate enhanced scene description based on detection results, scene type,
and additional contextual information.
This is the main entry point that replaces the original _generate_scene_description.
Args:
scene_type: Identified scene type
detected_objects: List of detected objects
confidence: Scene classification confidence
lighting_info: Optional lighting condition information
functional_zones: Optional identified functional zones
Returns:
str: Natural language description of the scene
"""
# Handle unknown scene type or very low confidence
if scene_type == "unknown" or confidence < 0.4:
return self._format_final_description(self._generate_generic_description(detected_objects, lighting_info))
# Detect viewpoint
viewpoint = self._detect_viewpoint(detected_objects)
# Process aerial viewpoint scene types
if viewpoint == "aerial":
if "intersection" in scene_type or self._is_intersection(detected_objects):
scene_type = "aerial_view_intersection"
elif any(keyword in scene_type for keyword in ["commercial", "shopping", "retail"]):
scene_type = "aerial_view_commercial_area"
elif any(keyword in scene_type for keyword in ["plaza", "square"]):
scene_type = "aerial_view_plaza"
else:
scene_type = "aerial_view_intersection"
# Detect cultural context - only for non-aerial viewpoints
cultural_context = None
if viewpoint != "aerial":
cultural_context = self._detect_cultural_context(scene_type, detected_objects)
# Select appropriate template based on confidence
if confidence > 0.75:
confidence_level = "high"
elif confidence > 0.5:
confidence_level = "medium"
else:
confidence_level = "low"
# Get base description for the scene type
if viewpoint == "aerial":
if 'base_description' not in locals():
base_description = "An aerial view showing the layout and movement patterns from above"
elif scene_type in self.scene_types:
base_description = self.scene_types[scene_type].get("description", "A scene")
else:
base_description = "A scene"
# Generate detailed scene information
scene_details = self._generate_scene_details(
scene_type,
detected_objects,
lighting_info,
viewpoint
)
# Start with the base description
description = base_description
# If there's a secondary description from the scene type template, append it properly
if scene_type in self.scene_types and "secondary_description" in self.scene_types[scene_type]:
secondary_desc = self.scene_types[scene_type]["secondary_description"]
if secondary_desc:
description = self._smart_append(description, secondary_desc)
# Improve description based on people count
people_objs = [obj for obj in detected_objects if obj["class_id"] == 0] # Person class
if people_objs:
people_count = len(people_objs)
if people_count > 5:
people_phrase = f"numerous people ({people_count})"
else:
people_phrase = f"{people_count} {'people' if people_count > 1 else 'person'}"
# Add people information to the scene details if not already mentioned
if "people" not in description.lower() and "pedestrian" not in description.lower():
description = self._smart_append(description, f"The scene includes {people_phrase}")
# Apply cultural context if detected (only for non-aerial viewpoints)
if cultural_context and viewpoint != "aerial":
cultural_elements = self._generate_cultural_elements(cultural_context)
if cultural_elements:
description = self._smart_append(description, cultural_elements)
# Now append the detailed scene information if available
if scene_details:
# Use smart_append to ensure proper formatting between base description and details
description = self._smart_append(description, scene_details)
# Include lighting information if available
lighting_description = ""
if lighting_info and "time_of_day" in lighting_info:
lighting_type = lighting_info["time_of_day"]
if lighting_type in self.templates.get("lighting_templates", {}):
lighting_description = self.templates["lighting_templates"][lighting_type]
# Add lighting description if available
if lighting_description and lighting_description not in description:
description = self._smart_append(description, lighting_description)
# Process viewpoint information
if viewpoint != "eye_level" and viewpoint in self.templates.get("viewpoint_templates", {}):
viewpoint_template = self.templates["viewpoint_templates"][viewpoint]
# Special handling for viewpoint prefix
prefix = viewpoint_template.get('prefix', '')
if prefix and not description.startswith(prefix):
# Prefix is a phrase like "From above, " that should precede the description
if description and description[0].isupper():
# Maintain the flow by lowercasing the first letter after the prefix
description = prefix + description[0].lower() + description[1:]
else:
description = prefix + description
# Get appropriate scene elements description based on viewpoint
if viewpoint == "aerial":
scene_elements = "the crossing patterns and pedestrian movement"
else:
scene_elements = "objects and layout"
viewpoint_desc = viewpoint_template.get("observation", "").format(
scene_elements=scene_elements
)
# Add viewpoint observation if not already included
if viewpoint_desc and viewpoint_desc not in description:
description = self._smart_append(description, viewpoint_desc)
# Add information about functional zones if available
if functional_zones and len(functional_zones) > 0:
zones_desc = self._describe_functional_zones(functional_zones)
if zones_desc:
description = self._smart_append(description, zones_desc)
# Calculate actual people count
people_count = len([obj for obj in detected_objects if obj["class_id"] == 0])
# Check for inconsistencies in people count descriptions
if people_count > 5:
# Identify fragments that might contain smaller people counts
small_people_patterns = [
r"Area with \d+ people\.",
r"Area with \d+ person\.",
r"with \d+ people",
r"with \d+ person"
]
# Check and remove each pattern
filtered_description = description
for pattern in small_people_patterns:
matches = re.findall(pattern, filtered_description)
for match in matches:
# Extract the number from the match
number_match = re.search(r'\d+', match)
if number_match:
try:
people_mentioned = int(number_match.group())
# If the mentioned count is less than total, remove the entire sentence
if people_mentioned < people_count:
# Split description into sentences
sentences = re.split(r'(?<=[.!?])\s+', filtered_description)
# Remove sentences containing the match
filtered_sentences = []
for sentence in sentences:
if match not in sentence:
filtered_sentences.append(sentence)
# Recombine the description
filtered_description = " ".join(filtered_sentences)
except ValueError:
# Failed number conversion, continue processing
continue
# Use the filtered description
description = filtered_description
# Final formatting to ensure correct punctuation and capitalization
description = self._format_final_description(description)
return description
def _smart_append(self, current_text: str, new_fragment: str) -> str:
"""
Intelligently append a new text fragment to the current text,
handling punctuation and capitalization correctly.
Args:
current_text: The existing text to append to
new_fragment: The new text fragment to append
Returns:
str: The combined text with proper formatting
"""
# Handle empty cases
if not new_fragment:
return current_text
if not current_text:
# Ensure first character is uppercase for the first fragment
return new_fragment[0].upper() + new_fragment[1:] if new_fragment else ""
# Clean up existing text
current_text = current_text.rstrip()
# Check for ending punctuation
ends_with_sentence = current_text.endswith(('.', '!', '?'))
ends_with_comma = current_text.endswith(',')
# Specifically handle the "A xxx A yyy" pattern that's causing issues
if (current_text.startswith("A ") or current_text.startswith("An ")) and \
(new_fragment.startswith("A ") or new_fragment.startswith("An ")):
return current_text + ". " + new_fragment
# Decide how to join the texts
if ends_with_sentence:
# After a sentence, start with uppercase and add proper spacing
joined_text = current_text + " " + (new_fragment[0].upper() + new_fragment[1:])
elif ends_with_comma:
# After a comma, maintain flow with lowercase unless it's a proper noun or special case
if new_fragment.startswith(('I ', 'I\'', 'A ', 'An ', 'The ')) or new_fragment[0].isupper():
joined_text = current_text + " " + new_fragment
else:
joined_text = current_text + " " + new_fragment[0].lower() + new_fragment[1:]
elif "scene is" in new_fragment.lower() or "scene includes" in new_fragment.lower():
# When adding a new sentence about the scene, use a period
joined_text = current_text + ". " + new_fragment
else:
# For other cases, decide based on the content
if self._is_related_phrases(current_text, new_fragment):
if new_fragment.startswith(('I ', 'I\'', 'A ', 'An ', 'The ')) or new_fragment[0].isupper():
joined_text = current_text + ", " + new_fragment
else:
joined_text = current_text + ", " + new_fragment[0].lower() + new_fragment[1:]
else:
# Use period for unrelated phrases
joined_text = current_text + ". " + (new_fragment[0].upper() + new_fragment[1:])
return joined_text
def _is_related_phrases(self, text1: str, text2: str) -> bool:
"""
Determine if two phrases are related and should be connected with a comma
rather than separated with a period.
Args:
text1: The first text fragment
text2: The second text fragment to be appended
Returns:
bool: Whether the phrases appear to be related
"""
# Check if either phrase starts with "A" or "An" - these are likely separate descriptions
if (text1.startswith("A ") or text1.startswith("An ")) and \
(text2.startswith("A ") or text2.startswith("An ")):
return False # These are separate descriptions, not related phrases
# Check if the second phrase starts with a connecting word
connecting_words = ["which", "where", "who", "whom", "whose", "with", "without",
"this", "these", "that", "those", "and", "or", "but"]
first_word = text2.split()[0].lower() if text2 else ""
if first_word in connecting_words:
return True
# Check if the first phrase ends with something that suggests continuity
ending_patterns = ["such as", "including", "like", "especially", "particularly",
"for example", "for instance", "namely", "specifically"]
for pattern in ending_patterns:
if text1.lower().endswith(pattern):
return True
# Check if both phrases are about the scene
if "scene" in text1.lower() and "scene" in text2.lower():
return False # Separate statements about the scene should be separate sentences
return False
def _format_final_description(self, text: str) -> str:
"""
Format the final description text to ensure correct punctuation,
capitalization, and spacing.
Args:
text: The text to format
Returns:
str: The properly formatted text
"""
import re
if not text:
return ""
# 1. 特別處理連續以"A"開頭的片段 (這是一個常見問題)
text = re.sub(r'(A\s[^.!?]+?)\s+(A\s)', r'\1. \2', text, flags=re.IGNORECASE)
text = re.sub(r'(An\s[^.!?]+?)\s+(An?\s)', r'\1. \2', text, flags=re.IGNORECASE)
# 2. 確保第一個字母大寫
text = text[0].upper() + text[1:] if text else ""
# 3. 修正詞之間的空格問題
text = re.sub(r'\s{2,}', ' ', text) # 多個空格改為一個
text = re.sub(r'([a-z])([A-Z])', r'\1 \2', text) # 小寫後大寫間加空格
# 4. 修正詞連接問題
text = re.sub(r'([a-zA-Z])and', r'\1 and', text) # "xxx"和"and"間加空格
text = re.sub(r'([a-zA-Z])with', r'\1 with', text) # "xxx"和"with"間加空格
text = re.sub(r'plants(and|with|or)', r'plants \1', text) # 修正"plantsand"這類問題
# 5. 修正標點符號後的大小寫問題
text = re.sub(r'\.(\s+)([a-z])', lambda m: f'.{m.group(1)}{m.group(2).upper()}', text) # 句號後大寫
# 6. 修正逗號後接大寫單詞的問題
def fix_capitalization_after_comma(match):
word = match.group(2)
# 例外情況:保留專有名詞、人稱代詞等的大寫
if word in ["I", "I'm", "I've", "I'd", "I'll"]:
return match.group(0) # 保持原樣
# 保留月份、星期、地名等專有名詞的大寫
proper_nouns = ["January", "February", "March", "April", "May", "June", "July",
"August", "September", "October", "November", "December",
"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
if word in proper_nouns:
return match.group(0) # 保持原樣
# 其他情況:將首字母改為小寫
return match.group(1) + word[0].lower() + word[1:]
# 匹配逗號後接空格再接大寫單詞的模式
text = re.sub(r'(,\s+)([A-Z][a-zA-Z]*)', fix_capitalization_after_comma, text)
common_phrases = [
(r'Social or seating area', r'social or seating area'),
(r'Sleeping area', r'sleeping area'),
(r'Dining area', r'dining area'),
(r'Living space', r'living space')
]
for phrase, replacement in common_phrases:
# 只修改句中的術語,保留句首的大寫
text = re.sub(r'(?<=[.!?]\s)' + phrase, replacement, text)
# 修改句中的術語,但保留句首的大寫
text = re.sub(r'(?<=,\s)' + phrase, replacement, text)
# 7. 確保標點符號後有空格
text = re.sub(r'\s+([.,;:!?])', r'\1', text) # 標點符號前不要空格
text = re.sub(r'([.,;:!?])([a-zA-Z0-9])', r'\1 \2', text) # 標點符號後要有空格
# 8. 修正重複標點符號
text = re.sub(r'\.{2,}', '.', text) # 多個句號變一個
text = re.sub(r',{2,}', ',', text) # 多個逗號變一個
# 9. 確保文本以標點結束
if text and not text[-1] in '.!?':
text += '.'
return text
def _is_intersection(self, detected_objects: List[Dict]) -> bool:
"""
通過分析物體分佈來判斷場景是否為十字路口
"""
# 檢查行人分佈模式
pedestrians = [obj for obj in detected_objects if obj["class_id"] == 0]
if len(pedestrians) >= 8: # 需要足夠的行人來形成十字路口
# 抓取行人位置
positions = [obj.get("normalized_center", (0, 0)) for obj in pedestrians]
# 分析 x 和 y 坐標分佈
x_coords = [pos[0] for pos in positions]
y_coords = [pos[1] for pos in positions]
# 計算 x 和 y 坐標的變異數
x_variance = np.var(x_coords) if len(x_coords) > 1 else 0
y_variance = np.var(y_coords) if len(y_coords) > 1 else 0
# 計算範圍
x_range = max(x_coords) - min(x_coords)
y_range = max(y_coords) - min(y_coords)
# 如果 x 和 y 方向都有較大範圍且範圍相似,那就有可能是十字路口
if x_range > 0.5 and y_range > 0.5 and 0.7 < (x_range / y_range) < 1.3:
return True
return False
def _generate_generic_description(self, detected_objects: List[Dict], lighting_info: Optional[Dict] = None) -> str:
"""
Generate a generic description when scene type is unknown or confidence is very low.
Args:
detected_objects: List of detected objects
lighting_info: Optional lighting condition information
Returns:
str: Generic description based on detected objects
"""
# Count object occurrences
obj_counts = {}
for obj in detected_objects:
class_name = obj["class_name"]
if class_name not in obj_counts:
obj_counts[class_name] = 0
obj_counts[class_name] += 1
# Get top objects by count
top_objects = sorted(obj_counts.items(), key=lambda x: x[1], reverse=True)[:5]
if not top_objects:
base_desc = "No clearly identifiable objects are visible in this scene."
else:
# Format object list
objects_text = []
for name, count in top_objects:
if count > 1:
objects_text.append(f"{count} {name}s")
else:
objects_text.append(name)
if len(objects_text) == 1:
objects_list = objects_text[0]
elif len(objects_text) == 2:
objects_list = f"{objects_text[0]} and {objects_text[1]}"
else:
objects_list = ", ".join(objects_text[:-1]) + f", and {objects_text[-1]}"
base_desc = f"This scene contains {objects_list}."
# Add lighting information if available
if lighting_info and "time_of_day" in lighting_info:
lighting_type = lighting_info["time_of_day"]
if lighting_type in self.templates.get("lighting_templates", {}):
lighting_desc = self.templates["lighting_templates"][lighting_type]
base_desc += f" {lighting_desc}"
return base_desc
def _generate_scene_details(self,
scene_type: str,
detected_objects: List[Dict],
lighting_info: Optional[Dict] = None,
viewpoint: str = "eye_level") -> str:
"""
Generate detailed description based on scene type and detected objects.
Args:
scene_type: Identified scene type
detected_objects: List of detected objects
lighting_info: Optional lighting condition information
viewpoint: Detected viewpoint (aerial, eye_level, etc.)
Returns:
str: Detailed scene description
"""
# Get scene-specific templates
scene_details = ""
scene_templates = self.templates.get("scene_detail_templates", {})
# Handle specific scene types
if scene_type in scene_templates:
# Select a template appropriate for the viewpoint if available
viewpoint_key = f"{scene_type}_{viewpoint}"
if viewpoint_key in scene_templates:
# We have a viewpoint-specific template
templates_list = scene_templates[viewpoint_key]
else:
# Fall back to general templates for this scene type
templates_list = scene_templates[scene_type]
# Select a random template from the list
if templates_list:
detail_template = random.choice(templates_list)
# Fill the template with object information
scene_details = self._fill_detail_template(
detail_template,
detected_objects,
scene_type
)
else:
# Use default templates if specific ones aren't available
if "default" in scene_templates:
detail_template = random.choice(scene_templates["default"])
scene_details = self._fill_detail_template(
detail_template,
detected_objects,
"default"
)
else:
# Fall back to basic description if no templates are available
scene_details = self._generate_basic_details(scene_type, detected_objects)
return scene_details
def _fill_detail_template(self, template: str, detected_objects: List[Dict], scene_type: str) -> str:
"""
Fill a template with specific details based on detected objects.
Args:
template: Template string with placeholders
detected_objects: List of detected objects
scene_type: Identified scene type
Returns:
str: Filled template
"""
# Find placeholders in the template using simple {placeholder} syntax
import re
placeholders = re.findall(r'\{([^}]+)\}', template)
filled_template = template
# Get object template fillers
fillers = self.templates.get("object_template_fillers", {})
# 為所有可能的變數設置默認值
default_replacements = {
# 室內相關
"furniture": "various furniture pieces",
"seating": "comfortable seating",
"electronics": "entertainment devices",
"bed_type": "a bed",
"bed_location": "room",
"bed_description": "sleeping arrangements",
"extras": "personal items",
"table_setup": "a dining table and chairs",
"table_description": "a dining surface",
"dining_items": "dining furniture and tableware",
"appliances": "kitchen appliances",
"kitchen_items": "cooking utensils and dishware",
"cooking_equipment": "cooking equipment",
"office_equipment": "work-related furniture and devices",
"desk_setup": "a desk and chair",
"computer_equipment": "electronic devices",
# 室外/城市相關
"traffic_description": "vehicles and pedestrians",
"people_and_vehicles": "people and various vehicles",
"street_elements": "urban infrastructure",
"park_features": "benches and greenery",
"outdoor_elements": "natural features",
"park_description": "outdoor amenities",
"store_elements": "merchandise displays",
"shopping_activity": "customers browse and shop",
"store_items": "products for sale",
# 高級餐廳相關
"design_elements": "elegant decor",
"lighting": "stylish lighting fixtures",
# 亞洲商業街相關
"storefront_features": "compact shops",
"pedestrian_flow": "people walking",
"asian_elements": "distinctive cultural elements",
"cultural_elements": "traditional design features",
"signage": "colorful signs",
"street_activities": "busy urban activity",
# 金融區相關
"buildings": "tall buildings",
"traffic_elements": "vehicles",
"skyscrapers": "high-rise buildings",
"road_features": "wide streets",
"architectural_elements": "modern architecture",
"city_landmarks": "prominent structures",
# 十字路口相關
"crossing_pattern": "marked pedestrian crossings",
"pedestrian_behavior": "careful walking",
"pedestrian_density": "groups of pedestrians",
"traffic_pattern": "regulated traffic flow",
# 交通樞紐相關
"transit_vehicles": "public transportation vehicles",
"passenger_activity": "commuter movement",
"transportation_modes": "various transit options",
"passenger_needs": "waiting areas",
"transit_infrastructure": "transit facilities",
"passenger_movement": "commuter flow",
# 購物區相關
"retail_elements": "shops and displays",
"store_types": "various retail establishments",
"walkway_features": "pedestrian pathways",
"commercial_signage": "store signs",
"consumer_behavior": "shopping activities",
# 空中視角相關
"commercial_layout": "organized retail areas",
"pedestrian_pattern": "people movement patterns",
"gathering_features": "public gathering spaces",
"movement_pattern": "crowd flow patterns",
"urban_elements": "city infrastructure",
"public_activity": "social interaction",
# 文化特定元素
"stall_elements": "vendor booths",
"lighting_features": "decorative lights",
"food_elements": "food offerings",
"vendor_stalls": "market stalls",
"nighttime_activity": "evening commerce",
"cultural_lighting": "traditional lighting",
"night_market_sounds": "lively market sounds",
"evening_crowd_behavior": "nighttime social activity",
"architectural_elements": "cultural buildings",
"religious_structures": "sacred buildings",
"decorative_features": "ornamental designs",
"cultural_practices": "traditional activities",
"temple_architecture": "religious structures",
"sensory_elements": "atmospheric elements",
"visitor_activities": "cultural experiences",
"ritual_activities": "ceremonial practices",
"cultural_symbols": "meaningful symbols",
"architectural_style": "historical buildings",
"historic_elements": "traditional architecture",
"urban_design": "city planning elements",
"social_behaviors": "public interactions",
"european_features": "European architectural details",
"tourist_activities": "visitor activities",
"local_customs": "regional practices",
# 時間特定元素
"lighting_effects": "artificial lighting",
"shadow_patterns": "light and shadow",
"urban_features": "city elements",
"illuminated_elements": "lit structures",
"evening_activities": "nighttime activities",
"light_sources": "lighting points",
"lit_areas": "illuminated spaces",
"shadowed_zones": "darker areas",
"illuminated_signage": "bright signs",
"colorful_lighting": "multicolored lights",
"neon_elements": "neon signs",
"night_crowd_behavior": "evening social patterns",
"light_displays": "lighting installations",
"building_features": "architectural elements",
"nightlife_activities": "evening entertainment",
"lighting_modifier": "bright",
# 混合環境元素
"transitional_elements": "connecting features",
"indoor_features": "interior elements",
"outdoor_setting": "exterior spaces",
"interior_amenities": "inside comforts",
"exterior_features": "outside elements",
"inside_elements": "interior design",
"outside_spaces": "outdoor areas",
"dual_environment_benefits": "combined settings",
"passenger_activities": "waiting behaviors",
"transportation_types": "transit vehicles",
"sheltered_elements": "covered areas",
"exposed_areas": "open sections",
"waiting_behaviors": "passenger activities",
"indoor_facilities": "inside services",
"platform_features": "transit platform elements",
"transit_routines": "transportation procedures",
# 專門場所元素
"seating_arrangement": "spectator seating",
"playing_surface": "athletic field",
"sporting_activities": "sports events",
"spectator_facilities": "viewer accommodations",
"competition_space": "sports arena",
"sports_events": "athletic competitions",
"viewing_areas": "audience sections",
"field_elements": "field markings and equipment",
"game_activities": "competitive play",
"construction_equipment": "building machinery",
"building_materials": "construction supplies",
"construction_activities": "building work",
"work_elements": "construction tools",
"structural_components": "building structures",
"site_equipment": "construction gear",
"raw_materials": "building supplies",
"construction_process": "building phases",
"medical_elements": "healthcare equipment",
"clinical_activities": "medical procedures",
"facility_design": "healthcare layout",
"healthcare_features": "medical facilities",
"patient_interactions": "care activities",
"equipment_types": "medical devices",
"care_procedures": "health services",
"treatment_spaces": "clinical areas",
"educational_furniture": "learning furniture",
"learning_activities": "educational practices",
"instructional_design": "teaching layout",
"classroom_elements": "school equipment",
"teaching_methods": "educational approaches",
"student_engagement": "learning participation",
"learning_spaces": "educational areas",
"educational_tools": "teaching resources",
"knowledge_transfer": "learning exchanges"
}
# For each placeholder, try to fill with appropriate content
for placeholder in placeholders:
if placeholder in fillers:
# Get random filler for this placeholder
options = fillers[placeholder]
if options:
# Select 1-3 items from the options list
num_items = min(len(options), random.randint(1, 3))
selected_items = random.sample(options, num_items)
# Create a formatted list
if len(selected_items) == 1:
replacement = selected_items[0]
elif len(selected_items) == 2:
replacement = f"{selected_items[0]} and {selected_items[1]}"
else:
replacement = ", ".join(selected_items[:-1]) + f", and {selected_items[-1]}"
# Replace the placeholder
filled_template = filled_template.replace(f"{{{placeholder}}}", replacement)
else:
# Try to fill with scene-specific logic
replacement = self._generate_placeholder_content(placeholder, detected_objects, scene_type)
if replacement:
filled_template = filled_template.replace(f"{{{placeholder}}}", replacement)
elif placeholder in default_replacements:
# Use default replacement if available
filled_template = filled_template.replace(f"{{{placeholder}}}", default_replacements[placeholder])
else:
# Last resort default
filled_template = filled_template.replace(f"{{{placeholder}}}", "various items")
return filled_template
def _generate_placeholder_content(self, placeholder: str, detected_objects: List[Dict], scene_type: str) -> str:
"""
Generate content for a template placeholder based on scene-specific logic.
Args:
placeholder: Template placeholder
detected_objects: List of detected objects
scene_type: Identified scene type
Returns:
str: Content for the placeholder
"""
# Handle different types of placeholders with custom logic
if placeholder == "furniture":
# Extract furniture items
furniture_ids = [56, 57, 58, 59, 60, 61] # Example furniture IDs
furniture_objects = [obj for obj in detected_objects if obj["class_id"] in furniture_ids]
if furniture_objects:
furniture_names = [obj["class_name"] for obj in furniture_objects[:3]]
return ", ".join(set(furniture_names))
return "various furniture items"
elif placeholder == "electronics":
# Extract electronic items
electronics_ids = [62, 63, 64, 65, 66, 67, 68, 69, 70] # Example electronics IDs
electronics_objects = [obj for obj in detected_objects if obj["class_id"] in electronics_ids]
if electronics_objects:
electronics_names = [obj["class_name"] for obj in electronics_objects[:3]]
return ", ".join(set(electronics_names))
return "electronic devices"
elif placeholder == "people_count":
# Count people
people_count = len([obj for obj in detected_objects if obj["class_id"] == 0])
if people_count == 0:
return "no people"
elif people_count == 1:
return "one person"
elif people_count < 5:
return f"{people_count} people"
else:
return "several people"
elif placeholder == "seating":
# Extract seating items
seating_ids = [56, 57] # chair, sofa
seating_objects = [obj for obj in detected_objects if obj["class_id"] in seating_ids]
if seating_objects:
seating_names = [obj["class_name"] for obj in seating_objects[:2]]
return ", ".join(set(seating_names))
return "seating arrangements"
# Default case - empty string
return ""
def _generate_basic_details(self, scene_type: str, detected_objects: List[Dict]) -> str:
"""
Generate basic details when templates aren't available.
Args:
scene_type: Identified scene type
detected_objects: List of detected objects
Returns:
str: Basic scene details
"""
# Handle specific scene types with custom logic
if scene_type == "living_room":
tv_objs = [obj for obj in detected_objects if obj["class_id"] == 62] # TV
sofa_objs = [obj for obj in detected_objects if obj["class_id"] == 57] # Sofa
if tv_objs and sofa_objs:
tv_region = tv_objs[0]["region"]
sofa_region = sofa_objs[0]["region"]
arrangement = f"The TV is in the {tv_region.replace('_', ' ')} of the image, "
arrangement += f"while the sofa is in the {sofa_region.replace('_', ' ')}. "
return f"{arrangement}This appears to be a space designed for relaxation and entertainment."
elif scene_type == "bedroom":
bed_objs = [obj for obj in detected_objects if obj["class_id"] == 59] # Bed
if bed_objs:
bed_region = bed_objs[0]["region"]
extra_items = []
for obj in detected_objects:
if obj["class_id"] == 74: # Clock
extra_items.append("clock")
elif obj["class_id"] == 73: # Book
extra_items.append("book")
extras = ""
if extra_items:
extras = f" There is also a {' and a '.join(extra_items)} visible."
return f"The bed is located in the {bed_region.replace('_', ' ')} of the image.{extras}"
elif scene_type in ["dining_area", "kitchen"]:
# Count food and dining-related items
food_items = []
for obj in detected_objects:
if obj["class_id"] in [39, 41, 42, 43, 44, 45]: # Kitchen items
food_items.append(obj["class_name"])
food_str = ""
if food_items:
unique_items = list(set(food_items))
if len(unique_items) <= 3:
food_str = f" with {', '.join(unique_items)}"
else:
food_str = f" with {', '.join(unique_items[:3])} and other items"
return f"{food_str}."
elif scene_type == "city_street":
# Count people and vehicles
people_count = len([obj for obj in detected_objects if obj["class_id"] == 0])
vehicle_count = len([obj for obj in detected_objects
if obj["class_id"] in [1, 2, 3, 5, 7]]) # Bicycle, car, motorbike, bus, truck
traffic_desc = ""
if people_count > 0 and vehicle_count > 0:
traffic_desc = f" with {people_count} {'people' if people_count > 1 else 'person'} and "
traffic_desc += f"{vehicle_count} {'vehicles' if vehicle_count > 1 else 'vehicle'}"
elif people_count > 0:
traffic_desc = f" with {people_count} {'people' if people_count > 1 else 'person'}"
elif vehicle_count > 0:
traffic_desc = f" with {vehicle_count} {'vehicles' if vehicle_count > 1 else 'vehicle'}"
return f"{traffic_desc}."
# Handle more specialized scenes
elif scene_type == "asian_commercial_street":
# Look for key urban elements
people_count = len([obj for obj in detected_objects if obj["class_id"] == 0])
vehicle_count = len([obj for obj in detected_objects if obj["class_id"] in [1, 2, 3]])
# Analyze pedestrian distribution
people_positions = []
for obj in detected_objects:
if obj["class_id"] == 0: # Person
people_positions.append(obj["normalized_center"])
# Check if people are distributed along a line (indicating a walking path)
structured_path = False
if len(people_positions) >= 3:
# Simplified check - see if y-coordinates are similar for multiple people
y_coords = [pos[1] for pos in people_positions]
y_mean = sum(y_coords) / len(y_coords)
y_variance = sum((y - y_mean)**2 for y in y_coords) / len(y_coords)
if y_variance < 0.05: # Low variance indicates linear arrangement
structured_path = True
street_desc = "A commercial street with "
if people_count > 0:
street_desc += f"{people_count} {'pedestrians' if people_count > 1 else 'pedestrian'}"
if vehicle_count > 0:
street_desc += f" and {vehicle_count} {'vehicles' if vehicle_count > 1 else 'vehicle'}"
elif vehicle_count > 0:
street_desc += f"{vehicle_count} {'vehicles' if vehicle_count > 1 else 'vehicle'}"
else:
street_desc += "various commercial elements"
if structured_path:
street_desc += ". The pedestrians appear to be following a defined walking path"
# Add cultural elements
street_desc += ". The signage and architectural elements suggest an Asian urban setting."
return street_desc
# Default general description
return "The scene contains various elements characteristic of this environment."
def _detect_viewpoint(self, detected_objects: List[Dict]) -> str:
"""
改進視角檢測,特別加強對空中俯視視角的識別。
Args:
detected_objects: 檢測到的物體列表
Returns:
str: 檢測到的視角類型
"""
if not detected_objects:
return "eye_level" # default
# 提取物體位置和大小
top_region_count = 0
bottom_region_count = 0
total_objects = len(detected_objects)
# 追蹤大小分布以檢測空中視角
sizes = []
# 垂直大小比例用於低角度檢測
height_width_ratios = []
# 用於檢測規則圖案的變數
people_positions = []
crosswalk_pattern_detected = False
for obj in detected_objects:
# 計算頂部/底部區域中的物體
region = obj["region"]
if "top" in region:
top_region_count += 1
elif "bottom" in region:
bottom_region_count += 1
# 計算標準化大小(面積)
if "normalized_area" in obj:
sizes.append(obj["normalized_area"])
# 計算高度/寬度比例
if "normalized_size" in obj:
width, height = obj["normalized_size"]
if width > 0:
height_width_ratios.append(height / width)
# 收集人的位置用於圖案檢測
if obj["class_id"] == 0: # 人
if "normalized_center" in obj:
people_positions.append(obj["normalized_center"])
# 專門為斑馬線十字路口添加檢測邏輯
# 檢查是否有明顯的垂直和水平行人分布
people_objs = [obj for obj in detected_objects if obj["class_id"] == 0] # 人
if len(people_objs) >= 8: # 需要足夠多的人才能形成十字路口模式
# 檢查是否有斑馬線模式 - 新增功能
if len(people_positions) >= 4:
# 對位置進行聚類分析,尋找線性分布
x_coords = [pos[0] for pos in people_positions]
y_coords = [pos[1] for pos in people_positions]
# 計算 x 和 y 坐標的變異數和範圍
x_variance = np.var(x_coords) if len(x_coords) > 1 else 0
y_variance = np.var(y_coords) if len(y_coords) > 1 else 0
x_range = max(x_coords) - min(x_coords)
y_range = max(y_coords) - min(y_coords)
# 嘗試檢測十字形分布
# 如果 x 和 y 方向都有較大範圍,且範圍相似,可能是十字路口
if x_range > 0.5 and y_range > 0.5 and 0.7 < (x_range / y_range) < 1.3:
# 計算到中心點的距離
center_x = np.mean(x_coords)
center_y = np.mean(y_coords)
# 將點映射到十字架的軸上(水平和垂直)
x_axis_distance = [abs(x - center_x) for x in x_coords]
y_axis_distance = [abs(y - center_y) for y in y_coords]
# 點應該接近軸線(水平或垂直)
# 對於每個點,檢查它是否接近水平或垂直軸線
close_to_axis_count = 0
for i in range(len(x_coords)):
if x_axis_distance[i] < 0.1 or y_axis_distance[i] < 0.1:
close_to_axis_count += 1
# 如果足夠多的點接近軸線,認為是十字路口
if close_to_axis_count >= len(x_coords) * 0.6:
crosswalk_pattern_detected = True
# 如果沒有檢測到十字形,嘗試檢測線性聚類分布
if not crosswalk_pattern_detected:
# 檢查 x 和 y 方向的聚類
x_clusters = self._detect_linear_clusters(x_coords)
y_clusters = self._detect_linear_clusters(y_coords)
# 如果在 x 和 y 方向上都有多個聚類,可能是交叉的斑馬線
if len(x_clusters) >= 2 and len(y_clusters) >= 2:
crosswalk_pattern_detected = True
# 檢測斑馬線模式 - 優先判斷
if crosswalk_pattern_detected:
return "aerial"
# 檢測行人分布情況
if len(people_objs) >= 10:
people_region_counts = {}
for obj in people_objs:
region = obj["region"]
if region not in people_region_counts:
people_region_counts[region] = 0
people_region_counts[region] += 1
# 計算不同區域中的行人數量
region_count = len([r for r, c in people_region_counts.items() if c >= 2])
# 如果行人分布在多個區域中,可能是空中視角
if region_count >= 4:
# 檢查行人分布的模式
# 特別是檢查不同區域中行人數量的差異
region_counts = list(people_region_counts.values())
region_counts_variance = np.var(region_counts) if len(region_counts) > 1 else 0
region_counts_mean = np.mean(region_counts) if region_counts else 0
# 如果行人分布較為均勻(變異係數小),可能是空中視角
if region_counts_mean > 0:
variation_coefficient = region_counts_variance / region_counts_mean
if variation_coefficient < 0.5:
return "aerial"
# 計算指標
top_ratio = top_region_count / total_objects if total_objects > 0 else 0
bottom_ratio = bottom_region_count / total_objects if total_objects > 0 else 0
# 大小變異數(標準化)
size_variance = 0
if sizes:
mean_size = sum(sizes) / len(sizes)
size_variance = sum((s - mean_size) ** 2 for s in sizes) / len(sizes)
size_variance = size_variance / (mean_size ** 2) # 標準化
# 平均高度/寬度比例
avg_height_width_ratio = sum(height_width_ratios) / len(height_width_ratios) if height_width_ratios else 1.0
# 空中視角:低大小差異,物體均勻分布,底部很少或沒有物體
if (size_variance < self.viewpoint_params["aerial_size_variance_threshold"] and
bottom_ratio < 0.3 and top_ratio > self.viewpoint_params["aerial_threshold"]):
return "aerial"
# 低角度視角:物體傾向於比寬高,頂部較多物體
elif (avg_height_width_ratio > self.viewpoint_params["vertical_size_ratio_threshold"] and
top_ratio > self.viewpoint_params["low_angle_threshold"]):
return "low_angle"
# 高視角:底部較多物體,頂部較少
elif (bottom_ratio > self.viewpoint_params["elevated_threshold"] and
top_ratio < self.viewpoint_params["elevated_top_threshold"]):
return "elevated"
# 默認:平視角
return "eye_level"
def _detect_linear_clusters(self, coords, threshold=0.05):
"""
檢測坐標中的線性聚類
Args:
coords: 一維坐標列表
threshold: 聚類閾值
Returns:
list: 聚類列表
"""
if not coords:
return []
# 排序坐標
sorted_coords = sorted(coords)
clusters = []
current_cluster = [sorted_coords[0]]
for i in range(1, len(sorted_coords)):
# 如果當前坐標與前一個接近,添加到當前聚類
if sorted_coords[i] - sorted_coords[i-1] < threshold:
current_cluster.append(sorted_coords[i])
else:
# 否則開始新的聚類
if len(current_cluster) >= 2: # 至少需要2個點形成聚類
clusters.append(current_cluster)
current_cluster = [sorted_coords[i]]
# 添加最後一個cluster
if len(current_cluster) >= 2:
clusters.append(current_cluster)
return clusters
def _detect_cultural_context(self, scene_type: str, detected_objects: List[Dict]) -> Optional[str]:
"""
Detect the likely cultural context of the scene.
Args:
scene_type: Identified scene type
detected_objects: List of detected objects
Returns:
Optional[str]: Detected cultural context (asian, european, etc.) or None
"""
# Scene types with explicit cultural contexts
cultural_scene_mapping = {
"asian_commercial_street": "asian",
"asian_night_market": "asian",
"asian_temple_area": "asian",
"european_plaza": "european"
}
# Check if scene type directly indicates cultural context
if scene_type in cultural_scene_mapping:
return cultural_scene_mapping[scene_type]
# No specific cultural context detected
return None
def _generate_cultural_elements(self, cultural_context: str) -> str:
"""
Generate description of cultural elements for the detected context.
Args:
cultural_context: Detected cultural context
Returns:
str: Description of cultural elements
"""
# Get template for this cultural context
cultural_templates = self.templates.get("cultural_templates", {})
if cultural_context in cultural_templates:
template = cultural_templates[cultural_context]
elements = template.get("elements", [])
if elements:
# Select 1-2 random elements
num_elements = min(len(elements), random.randint(1, 2))
selected_elements = random.sample(elements, num_elements)
# Format elements list
elements_text = " and ".join(selected_elements) if num_elements == 2 else selected_elements[0]
# Fill template
return template.get("description", "").format(elements=elements_text)
return ""
def _optimize_object_description(self, description: str) -> str:
"""
優化物品描述,避免重複列舉相同物品
"""
import re
# 處理床鋪重複描述
if "bed in the room" in description:
description = description.replace("a bed in the room", "a bed")
# 處理重複的物品列表
# 尋找格式如 "item, item, item" 的模式
object_lists = re.findall(r'with ([^\.]+?)(?:\.|\band\b)', description)
for obj_list in object_lists:
# 計算每個物品出現次數
items = re.findall(r'([a-zA-Z\s]+)(?:,|\band\b|$)', obj_list)
item_counts = {}
for item in items:
item = item.strip()
if item and item not in ["and", "with"]:
if item not in item_counts:
item_counts[item] = 0
item_counts[item] += 1
# 生成優化後的物品列表
if item_counts:
new_items = []
for item, count in item_counts.items():
if count > 1:
new_items.append(f"{count} {item}s")
else:
new_items.append(item)
# 格式化新列表
if len(new_items) == 1:
new_list = new_items[0]
elif len(new_items) == 2:
new_list = f"{new_items[0]} and {new_items[1]}"
else:
new_list = ", ".join(new_items[:-1]) + f", and {new_items[-1]}"
# 替換原始列表
description = description.replace(obj_list, new_list)
return description
def _describe_functional_zones(self, functional_zones: Dict) -> str:
"""
生成場景功能區域的描述,優化處理行人區域、人數統計和物品重複問題。
Args:
functional_zones: 識別出的功能區域字典
Returns:
str: 功能區域描述
"""
if not functional_zones:
return ""
# 計算場景中的總人數
total_people_count = 0
people_by_zone = {}
# 計算每個區域的人數並累計總人數
for zone_name, zone_info in functional_zones.items():
if "objects" in zone_info:
zone_people_count = zone_info["objects"].count("person")
people_by_zone[zone_name] = zone_people_count
total_people_count += zone_people_count
# 分類區域為行人區域和其他區域
pedestrian_zones = []
other_zones = []
for zone_name, zone_info in functional_zones.items():
# 檢查是否是行人相關區域
if any(keyword in zone_name.lower() for keyword in ["pedestrian", "crossing", "people"]):
pedestrian_zones.append((zone_name, zone_info))
else:
other_zones.append((zone_name, zone_info))
# 獲取最重要的行人區域和其他區域
main_pedestrian_zones = sorted(pedestrian_zones,
key=lambda z: people_by_zone.get(z[0], 0),
reverse=True)[:1] # 最多1個主要行人區域
top_other_zones = sorted(other_zones,
key=lambda z: len(z[1].get("objects", [])),
reverse=True)[:2] # 最多2個其他區域
# 合併區域
top_zones = main_pedestrian_zones + top_other_zones
if not top_zones:
return ""
# 生成匯總描述
summary = ""
max_mentioned_people = 0 # 跟踪已經提到的最大人數
# 如果總人數顯著且還沒在主描述中提到,添加總人數描述
if total_people_count > 5:
summary = f"The scene contains a significant number of pedestrians ({total_people_count} people). "
max_mentioned_people = total_people_count # 更新已提到的最大人數
# 處理每個區域的描述,確保人數信息的一致性
processed_zones = []
for zone_name, zone_info in top_zones:
zone_desc = zone_info.get("description", "a functional zone")
zone_people_count = people_by_zone.get(zone_name, 0)
# 檢查描述中是否包含人數信息
contains_people_info = "with" in zone_desc and ("person" in zone_desc.lower() or "people" in zone_desc.lower())
# 如果描述包含人數信息,且人數較小(小於已提到的最大人數),則修改描述
if contains_people_info and zone_people_count < max_mentioned_people:
parts = zone_desc.split("with")
if len(parts) > 1:
# 移除人數部分
zone_desc = parts[0].strip() + " area"
processed_zones.append((zone_name, {"description": zone_desc}))
# 根據處理後的區域數量生成最終描述
final_desc = ""
if len(processed_zones) == 1:
_, zone_info = processed_zones[0]
zone_desc = zone_info["description"]
final_desc = summary + f"The scene includes {zone_desc}."
elif len(processed_zones) == 2:
_, zone1_info = processed_zones[0]
_, zone2_info = processed_zones[1]
zone1_desc = zone1_info["description"]
zone2_desc = zone2_info["description"]
final_desc = summary + f"The scene is divided into two main areas: {zone1_desc} and {zone2_desc}."
else:
zones_desc = ["The scene contains multiple functional areas including"]
zone_descriptions = [z[1]["description"] for z in processed_zones]
# 格式化最終的多區域描述
if len(zone_descriptions) == 3:
formatted_desc = f"{zone_descriptions[0]}, {zone_descriptions[1]}, and {zone_descriptions[2]}"
else:
formatted_desc = ", ".join(zone_descriptions[:-1]) + f", and {zone_descriptions[-1]}"
final_desc = summary + f"{zones_desc[0]} {formatted_desc}."
return self._optimize_object_description(final_desc)
|