How to set a value for FloatVectorProperty from a function?
Elaborate From brockmann comment and Blender API:

bpy.props.FloatVectorProperty(
min=sys.float_info.min, 
max=sys.float_info.max, 
soft_min=sys.float_info.min, 
soft_max=sys.float_info.max)



  
  min (float) – Hard minimum, trying to assign a value below will silently assign this minimum instead.
  max (float) – Hard maximum, trying to assign a value above will silently assign this maximum instead.
  soft_min (float) – Soft minimum (>= min), user won’t be able to drag the widget below this value in the UI.
  soft_max (float) – Soft maximum (&lt;= max), user won’t be able to drag the widget above this value in the UI.
  


You need to set min = 0.0 and max=1.0 (or soft min/max) for the color subtype property.
So it will look like this:

a_color : bpy.props.FloatVectorProperty(name='Color',subtype='COLOR_GAMMA',size=4,default=(0.5,0.5,0.9,1.0),min = 0.0, max = 1.0)

b_color : bpy.props.FloatVectorProperty(name='Color',subtype='COLOR',min = 0.0, max = 1.0)


It seems like color also support by light system. So the min max wasn't set by default, causing the silder getting a ridiculous range and break the panel



Operator based on this answer:

import bpy


class SimplePropConfirmOperator(bpy.types.Operator):
    """Really?"""
    bl_idname = "my_category.custom_confirm_dialog"
    bl_label = "Do you really want to do that?"
    bl_options = {'REGISTER', 'INTERNAL'}

    def update_func(self, context):
        #print("my test function", self)
        self.my_color = (0.5,0.5,0.9, 1.0) # Alpha

    my_enum : bpy.props.EnumProperty(
        items = (("RND", "Randomize", ""),("SET", "Set", "")),
        update = update_func)

    my_color : bpy.props.FloatVectorProperty(
        subtype='COLOR', 
        min=0.0, 
        max=1.0,
        size=4) # Alpha

    @classmethod
    def poll(cls, context):
        return True

    def execute(self, context):
        self.report({'INFO'}, "YES!")
        return {'FINISHED'}

    '''
    def check(self, context): 
        return True
    '''

    def invoke(self, context, event):
        return context.window_manager.invoke_props_dialog(self)

    def draw(self, context):
        row = self.layout
        row.prop(self, "my_enum", text="Property A")
        if self.my_enum == "SET":
            row.prop(self, "my_color", text="Property B")


class OBJECT_PT_CustomPanel(bpy.types.Panel):
    bl_label = "My Panel"
    bl_idname = "OBJECT_PT_custom_panel"
    bl_space_type = "VIEW_3D"   
    bl_region_type = "UI"
    bl_category = "Tools"
    bl_context = "objectmode"

    def draw(self, context):
        layout = self.layout
        layout.operator(SimplePropConfirmOperator.bl_idname)

def register():
    bpy.utils.register_class(OBJECT_PT_CustomPanel)
    bpy.utils.register_class(SimplePropConfirmOperator)

def unregister():
    bpy.utils.unregister_class(SimplePropConfirmOperator)
    bpy.utils.unregister_class(OBJECT_PT_CustomPanel)

if __name__ == "__main__":
    register()