How can I change the size of a modal operator popup dialog box?
Just pass a custom width (integer) when calling invoke_props_dialog(operator, width=300) or invoke_popup(operator, width=300). The height is determined dynamically by the size of elements placed onto the props_dialog and there is no option to set it. From the docs:

Parameters

operator (Operator) – Operator to call
width (int in [0, inf], (optional)) – Width of the popup



Demo based on the code of How to call a confirmation dialog box?

import bpy

class SimplePropConfirmOperator(bpy.types.Operator):
    bl_idname = &quot;my_category.custom_confirm_dialog&quot;
    bl_label = &quot;Do you really want to do that?&quot;

    prop1: bpy.props.BoolProperty()
    prop2: bpy.props.BoolProperty()

    def execute(self, context):
        self.report({'INFO'}, &quot;YES!&quot;)
        return {'FINISHED'}

    def invoke(self, context, event):
        return context.window_manager.invoke_props_dialog(self, width=750)

    def draw(self, context):
        row = self.layout
        row.prop(self, &quot;prop1&quot;, text=&quot;Property A&quot;)
        row.prop(self, &quot;prop2&quot;, text=&quot;Property B&quot;)

def register():
    bpy.utils.register_class(SimplePropConfirmOperator)

def unregister():
    bpy.utils.unregister_class(SimplePropConfirmOperator)

if __name__ == &quot;__main__&quot;:
    register()
    
    # test call
    bpy.ops.my_category.custom_confirm_dialog('INVOKE_DEFAULT')