@DaKenpachi (DaKenpachi) posted a new topic, subject "Problems with rendering a video stream onto a texture/material". It read:

   Hi together,
    
    I´m pretty new to Unreal. I want to render a RGB video from a Realsense D435 to a texture/material (so I can see it within the game).
    
    So I wrote a plugin (because Intel sadly does not provide one any more) for accessing the realsense api. It works so far as I can now grab images from the camera. But I have problems showing it in the game.
    
    I used some tutorials and questions to get this far. This is what I did:  
    I created a dynamic material within a blueprint, and I´m trying to update its texture (which is a Param2d defining the base color of the material, called “VideoTexture”) every tick with a fresh grabbed frame.
    
    250724-ue-realsense-dynamic-mat-update-bp.png1120×1040 211 KB
    
    Here is my cpp file (I tried the build-in `UpdateTextureRegions()` as well as the older one using an older tutorial):
    [code] 
        // Sets default values for this component's properties
        ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069RealSenseComponent()
        {
        
        }
        
        
        
        
        void ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069reateUpdateableTexture(int width, int height)
        {
        	TextureFromVideo = UTextur8b7c:f320:99b9:690f:4595:cd17:293a:c069CreateTransient(width, height);
        	TextureFromVideo->AddToRoot();
        	TextureFromVideo->UpdateResource();
        	textureVideoRegion = new FUpdateTextureRegion2D(0, 0, 0, 0, width, height);
        
        }
        
        UTexture2D * ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069ReceiveRGBFrame()
        {
        	if (cameraWorks) {
        		TextureFromVideo->UpdateResource();
        		if (!receiveFrame()) {
        			UE_LOG(RealSenseLog, Error, TEXT("Could not receive a Frame from camera!"));
        		}
        		else {
        			UE_LOG(RealSenseLog, Log, TEXT("Received Frame!"))
        		}
        	}
        	else {
        		UE_LOG(RealSenseLog, Log, TEXT("CameraWorks is false"));
        	}
        	return TextureFromVideo;
        }
        
        // Called when the game starts
        void ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069ginPlay()
        {
        	Super8b7c:f320:99b9:690f:4595:cd17:293a:c069ginPlay();
        
        	if (IRealSenseModul8b7c:f320:99b9:690f:4595:cd17:293a:c069IsAvailable()) {
        		int x = IRealSenseModul8b7c:f320:99b9:690f:4595:cd17:293a:c069Get().CheckRealsenseCamera();
        		UE_LOG(RealSenseLog, Log, TEXT("CheckRealsenseCamera returned: %d"), x);
        	}
        	else {
        		UE_LOG(RealSenseLog, Warning, TEXT("RealSenseModule not available"));
        	}
        
        
        	try {
        
        		pipeline = new rs8b7c:f320:99b9:690f:4595:cd17:293a:c069pipeline();
        		pipeline->start();
        		cameraWorks = true;
        	}
        	catch (st8b7c:f320:99b9:690f:4595:cd17:293a:c069exception e) {
        		UE_LOG(RealSenseLog, Error, TEXT("Realsense initialization error: %s"), e.what());
        	}
        
        	try {
        		rs8b7c:f320:99b9:690f:4595:cd17:293a:c069frameset frames = pipeline->wait_for_frames();
        		rs8b7c:f320:99b9:690f:4595:cd17:293a:c069video_frame colorFrame = frames.get_color_frame();
        		int width = colorFrame.get_width();
        		int height = colorFrame.get_height();
        		CreateUpdateableTexture(width, height);
        
        	}
        	catch (st8b7c:f320:99b9:690f:4595:cd17:293a:c069exception e) {
        		UE_LOG(RealSenseLog, Error, TEXT("Realsense video stream error: %s"), e.what());
        		cameraWorks = false;
        	}
        }
        
        bool ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069receiveFrame()
        {
        	try {
        		rs8b7c:f320:99b9:690f:4595:cd17:293a:c069frameset frames = pipeline->wait_for_frames();
        		rs8b7c:f320:99b9:690f:4595:cd17:293a:c069video_frame colorFrame = frames.get_color_frame();
        		uint8* data = (uint8*)(colorFrame.get_data());
        		int height = colorFrame.get_height();
        		int width = colorFrame.get_width();
        		int channels = colorFrame.get_bytes_per_pixel();
        		int bits = colorFrame.get_bits_per_pixel();
        		rs2_format format = colorFrame.get_profile().format();
        		int end = width * height * channels;
        
        		UE_LOG(RealSenseLog, Log, TEXT("Image Received. Resolution: %d/%d, Channels: %d, Format: %s"), width, height, channels, *FString(rs2_format_to_string(format)));
        		UE_LOG(RealSenseLog, Log, TEXT("First/Last pixel: (%d/%d/%d), (%d/%d/%d)"), data[0], data[1], data[2], data[end - 3], data[end - 2], data[end - 1]);
        
        		// tried differend updating methods
        		TextureFromVideo->UpdateTextureRegions(DBL_MAX_10_EXP, 1, textureVideoRegion, static_cast<uint32>(width * bits),
        			bits, data, texCleanUpFP);
        
        		//UpdateTextureRegions(TextureFromVideo, DBL_MAX_10_EXP, 1, textureVideoRegion, static_cast<uint32>(width * bits), bits, data, false);
        	}
        	catch (const rs8b7c:f320:99b9:690f:4595:cd17:293a:c069error & e) {
        
        		UE_LOG(RealSenseLog, Error, TEXT("%s"), *FString(e.what()));
        		cameraWorks = false;
        	}
        	catch (const st8b7c:f320:99b9:690f:4595:cd17:293a:c069exception& e) {
        
        		UE_LOG(RealSenseLog, Error, TEXT("%s"), *FString(e.what()));
        		cameraWorks = false;
        	}
        
        	return cameraWorks;
        }
        
        
        // Called every frame
        void ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069Tick(float DeltaTime)
        {
        	//TextureFromVideo->UpdateResource();
        	Super8b7c:f320:99b9:690f:4595:cd17:293a:c069Tick(DeltaTime);
        
        }
        
        void ARealSenseComponent8b7c:f320:99b9:690f:4595:cd17:293a:c069UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData)
        {
        	if (Texture && Texture->Resource)
        	{
        		struct FUpdateTextureRegionsData
        		{
        			FTexture2DResource* Texture2DResource;
        			int32 MipIndex;
        			uint32 NumRegions;
        			FUpdateTextureRegion2D* Regions;
        			uint32 SrcPitch;
        			uint32 SrcBpp;
        			uint8* SrcData;
        		};
        
        		FUpdateTextureRegionsData* RegionData = new FUpdateTextureRegionsData;
        
        		RegionData->Texture2DResource = (FTexture2DResource*)Texture->Resource;
        		RegionData->MipIndex = MipIndex;
        		RegionData->NumRegions = NumRegions;
        		RegionData->Regions = Regions;
        		RegionData->SrcPitch = SrcPitch;
        		RegionData->SrcBpp = SrcBpp;
        		RegionData->SrcData = SrcData;
        
        		ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER(
        			UpdateTextureRegionsData,
        			FUpdateTextureRegionsData*, RegionData, RegionData,
        			bool, bFreeData, bFreeData,
        			{
        				for (uint32 RegionIndex = 0; RegionIndex < RegionData->NumRegions; ++RegionIndex)
        				{
        					int32 CurrentFirstMip = RegionData->Texture2DResource->GetCurrentFirstMip();
        					if (RegionData->MipIndex >= CurrentFirstMip)
        					{
        						RHIUpdateTexture2D(
        							RegionData->Texture2DResource->GetTexture2DRHI(),
        							RegionData->MipIndex - CurrentFirstMip,
        							RegionData->Regions[RegionIndex],
        							RegionData->SrcPitch,
        							RegionData->SrcData
        							+ RegionData->Regions[RegionIndex].SrcY * RegionData->SrcPitch
        							+ RegionData->Regions[RegionIndex].SrcX * RegionData->SrcBpp
        						);
        					}
        				}
        		if (bFreeData)
        		{
        			FMemory8b7c:f320:99b9:690f:4595:cd17:293a:c069ree(RegionData->Regions);
        			FMemory8b7c:f320:99b9:690f:4595:cd17:293a:c069ree(RegionData->SrcData);
        		}
        		delete RegionData;
        			});
        	}
        }
        
    [/code]
    
    So as result, I can see in the log that new frames are comming. But the texture does not get updated, I can´t see the video ingame:
    
    250725-ue-realsense-receiving-not-updating.jpg1399×1136 230 KB
    
    Any suggestions?

After which, @DaKenpachi (DaKenpachi) added

    Ah sorry here is the header file:
    [code] 
        #pragma once
        
        #include "CoreMinimal.h"
        #include "GameFramework/Actor.h"
        #include <Engine/Texture2D.h>
        #include <rs.hpp>
        #include "RealSenseComponent.generated.h"
        
        
        
        UCLASS( ClassGroup=(Custom))
        class REALSENSEPLUGIN_API ARealSenseComponent : public AActor
        {
        	GENERATED_BODY()
        
        public:	
        	// Sets default values for this component's properties
        	ARealSenseComponent();
        
        	UFUNCTION(BlueprintCallable, Category = "RealSense")
        		void CreateUpdateableTexture(int img_width, int img_height);
        
        	UFUNCTION(BlueprintCallable, Category = "RealSense")
        		UTexture2D* ReceiveRGBFrame();
        
        protected:
        	// Called when the game starts
        	virtual void BeginPlay() override;
        
        	/** Reveice a frame from realsense device and update texture*/
        	bool receiveFrame();
        
        	// texture to update
        	UTexture2D* TextureFromVideo = nullptr;
        	FUpdateTextureRegion2D* textureVideoRegion = nullptr;
        
        	// Realsense device pipeline
        	rs8b7c:f320:99b9:690f:4595:cd17:293a:c069pipeline* pipeline = nullptr;
        
        	/** Update texture region from https://wiki.unrealengine.com/Dynamic_Textures */
        	void UpdateTextureRegions(UTexture2D* Texture, int32 MipIndex, uint32 NumRegions, FUpdateTextureRegion2D* Regions, uint32 SrcPitch, uint32 SrcBpp, uint8* SrcData, bool bFreeData);
        
        	// from SRWORKS plugin
        	void TextureRegionCleanUp(uint8* rawData, const FUpdateTextureRegion2D* region) {};
        	TFunction<void(uint8*, const FUpdateTextureRegion2D*)> texCleanUpFP = [this](uint8* rawData, const FUpdateTextureRegion2D* region) { TextureRegionCleanUp(rawData, region); };
        
        
        public:	
        	// Called every frame
        	virtual void Tick(float DeltaTime) override;
        
        	bool cameraWorks = false;
        	
        };
    [/code]

So @E1andri1 (E1andri1) replied

    Hello,
    
    how did you set up your material? And if you debug the event tick, what is passed to the value?

So @DaKenpachi (DaKenpachi) replied

    Hi, this is my Material right now (Because I found a tutorial for streaming a SceneCapture2D onto a Material, they did set it up like this):
    
    251005-ue-video-material-notworking.jpg1161×878 101 KB
    
    But I also tried it with a “Default Lit” Shading Model, and with the Texture being either attached to the base color or to the emissive color (like above). None of it changed anything

After which, @DaKenpachi (DaKenpachi) added

    So, I finally found the Problem.  
    It was a dump mixture of copy-paste-error and misunderstanding.  
    The problem is the line
    [code] 
        TextureFromVideo->UpdateTextureRegions(DBL_MAX_10_EXP, 1, textureVideoRegion, static_cast<uint32>(width * bits), bits, data, texCleanUpFP);
        
    [/code]
    
    Which I copied from the SRWorks SDK:
    [code] 
         textureDistortedLeft->UpdateTextureRegions(0, 1, textureDistortedRegion, static_cast<uint32>(DistortedImageWidth * sizeof(uint8) * DistortedImageChannel), sizeof(uint8) * DistortedImageChannel, RawDistortedLeftBGRA, texCleanUpFP);
        
    [/code]
    
    So somewhere in the copying process, the first parameter somehow became `DBL_MAX_10_EXP` (Still don´t know how the ■■■■ that could happen), and I got confused by the complete unnessecary `sizeof(uint8)` (which is 1, obviously, but I forgot that and thought it was the count of bits in one byte).
    
    So in the end this is the solution:
    [code] 
        int bytes = colorFrame.get_bytes_per_pixel();
        
        TextureFromVideo->UpdateTextureRegions(0, 1, textureVideoRegion, static_cast<uint32>(width * bytes), bytes, data, texCleanUpFP);
        
    [/code]
    
    So if anyone is interested in one or another thing I was doing here, you can take a look at the github repository. =)


{'source': 'discourse', 'url': 'https://forums.unrealengine.com/t/problems-with-rendering-a-video-stream-onto-a-texture-material/431812', 'title': 'Problems with rendering a video stream onto a texture/material', 'views': 584, 'has_accepted_answer': True, 'word_count': 1106, 'like_count': 0, 'reply_count': 4, 'participant_count': 2, 'posts': 5, 'tags': 'question UE4 CPP Plugins Textures dynamic-material-ins', 'author': '@DaKenpachi (DaKenpachi)', 'participants': '@DaKenpachi (DaKenpachi), @E1andri1 (E1andri1)', 'image_url': ['https://d3kjluh73b9h9o.cloudfront.net/optimized/4X/4/f/1/4f1e1cbae9b6bb10a2b2c9ac5c855a29193eee98_2_1024x950.png'], 'created_at': '2018-08-17T08:19:03.000Z', 'last_posted_at': '2018-08-22T13:07:34.000Z'}