How can I add a panel to the properties area of the file browser?
You would have to add a draw() method along with a pass statement to the operator in order to skip drawing and declare a Panel class, get the actual operator from that class in order to draw it's properties. Demo based on the Operator Export template that comes with Blender:

import bpy

# ExportHelper is a helper class, defines filename and
# invoke() function which calls the file selector.
from bpy_extras.io_utils import ExportHelper
from bpy.props import StringProperty, BoolProperty, EnumProperty
from bpy.types import Operator, Panel


class ExportSomeData(Operator, ExportHelper):
    &quot;&quot;&quot;This appears in the tooltip of the operator and in the generated docs&quot;&quot;&quot;
    bl_idname = &quot;export_test.some_data&quot;  # important since its how bpy.ops.import_test.some_data is constructed
    bl_label = &quot;Export Some Data&quot;

    # ExportHelper mixin class uses this
    filename_ext = &quot;.txt&quot;

    filter_glob: StringProperty(
        default=&quot;*.txt&quot;,
        options={'HIDDEN'},
        maxlen=255,  # Max internal buffer length, longer would be clamped.
    )

    # List of operator properties, the attributes will be assigned
    # to the class instance from the operator settings before calling.
    use_setting: BoolProperty(
        name=&quot;Example Boolean&quot;,
        description=&quot;Example Tooltip&quot;,
        default=True,
    )

    type: EnumProperty(
        name=&quot;Example Enum&quot;,
        description=&quot;Choose between two items&quot;,
        items=(
            ('OPT_A', &quot;First Option&quot;, &quot;Description one&quot;),
            ('OPT_B', &quot;Second Option&quot;, &quot;Description two&quot;),
        ),
        default='OPT_A',
    )

    def execute(self, context):
        return {'FINISHED'}
    
    def draw(self, context):
        pass


class CUSTOM_PT_export_settings(Panel):
    bl_space_type = 'FILE_BROWSER'
    bl_region_type = 'TOOL_PROPS'
    bl_label = &quot;Settings Panel&quot;
    bl_options = {'DEFAULT_CLOSED'}

    @classmethod
    def poll(cls, context):
        sfile = context.space_data
        operator = sfile.active_operator
        return operator.bl_idname == &quot;EXPORT_TEST_OT_some_data&quot;

    def draw(self, context):
        layout = self.layout
        layout.use_property_split = True
        layout.use_property_decorate = False  # No animation.

        sfile = context.space_data
        operator = sfile.active_operator

        layout.prop(operator, 'type')
        layout.prop(operator, 'use_setting')
        

# Only needed if you want to add into a dynamic menu
def menu_func_export(self, context):
    self.layout.operator(ExportSomeData.bl_idname, text=&quot;Text Export Operator&quot;)


def register():
    bpy.utils.register_class(ExportSomeData)
    bpy.utils.register_class(CUSTOM_PT_export_settings)
    bpy.types.TOPBAR_MT_file_export.append(menu_func_export)


def unregister():
    bpy.utils.unregister_class(CUSTOM_PT_export_settings)
    bpy.utils.unregister_class(ExportSomeData)
    bpy.types.TOPBAR_MT_file_export.remove(menu_func_export)


if __name__ == &quot;__main__&quot;:
    register()

    # test call
    bpy.ops.export_test.some_data('INVOKE_DEFAULT')

Make sure to add a poll() method to the panel, testing for the actual bl_idname of the operator otherwise the panel will appear for all other export operators as well.
@classmethod
def poll(cls, context):
    sfile = context.space_data
    operator = sfile.active_operator
    return operator.bl_idname == &quot;EXPORT_TEST_OT_some_data&quot;

Note: Once the export operator is registered you can use the python console and call its idname() method getting the proper bl_idname:
&gt;&gt;&gt; bpy.ops.export_test.some_data.idname()
'EXPORT_TEST_OT_some_data'


Just for the sake, adding sub panels is possible as well:

import bpy

# ExportHelper is a helper class, defines filename and
# invoke() function which calls the file selector.
from bpy_extras.io_utils import ExportHelper
from bpy.props import StringProperty, BoolProperty, EnumProperty
from bpy.types import Operator, Panel


class ExportSomeData(Operator, ExportHelper):
    &quot;&quot;&quot;This appears in the tooltip of the operator and in the generated docs&quot;&quot;&quot;
    bl_idname = &quot;export_test.some_data&quot;  # important since its how bpy.ops.import_test.some_data is constructed
    bl_label = &quot;Export Some Data&quot;
    bl_options = {'PRESET', 'UNDO'}

    # ExportHelper mixin class uses this
    filename_ext = &quot;.txt&quot;

    filter_glob: StringProperty(
        default=&quot;*.txt&quot;,
        options={'HIDDEN'},
        maxlen=255,  # Max internal buffer length, longer would be clamped.
    )

    # List of operator properties, the attributes will be assigned
    # to the class instance from the operator settings before calling.
    use_setting: BoolProperty(
        name=&quot;Example Boolean&quot;,
        description=&quot;Example Tooltip&quot;,
        default=True,
    )

    type: EnumProperty(
        name=&quot;Example Enum&quot;,
        description=&quot;Choose between two items&quot;,
        items=(
            ('OPT_A', &quot;First Option&quot;, &quot;Description one&quot;),
            ('OPT_B', &quot;Second Option&quot;, &quot;Description two&quot;),
        ),
        default='OPT_A',
    )

    def execute(self, context):
        return {'FINISHED'}
    
    def draw(self, context):
        self.layout.label(text=&quot;Hello World&quot;)
        #self.layout.prop(self, 'type')
        #pass


class EXAMPLE_PT_panel_1(Panel):
    bl_space_type = 'FILE_BROWSER'
    bl_region_type = 'TOOL_PROPS'
    bl_label = &quot;Settings Panel 1&quot;
    #bl_parent_id = &quot;FILE_PT_operator&quot; # Optional

    @classmethod
    def poll(cls, context):
        sfile = context.space_data
        operator = sfile.active_operator
        return operator.bl_idname == &quot;EXPORT_TEST_OT_some_data&quot;

    def draw(self, context):
        layout = self.layout

        sfile = context.space_data
        operator = sfile.active_operator
        
        layout.label(text=&quot;This is the first Panel&quot;)
        layout.prop(operator, 'type')
        layout.prop(operator, 'use_setting')


class EXAMPLE_PT_panel_2(Panel):
    bl_space_type = 'FILE_BROWSER'
    bl_region_type = 'TOOL_PROPS'
    bl_label = &quot;Settings Panel 2&quot;
    bl_parent_id = &quot;EXAMPLE_PT_panel_1&quot;
    bl_options = {'DEFAULT_CLOSED'}

    @classmethod
    def poll(cls, context):
        sfile = context.space_data
        operator = sfile.active_operator
        return operator.bl_idname == &quot;EXPORT_TEST_OT_some_data&quot;

    def draw(self, context):
        layout = self.layout
        
        sfile = context.space_data
        operator = sfile.active_operator
        
        layout.label(text=&quot;This is the second Panel&quot;)
        layout.prop(operator, 'type')


# Only needed if you want to add into a dynamic menu
def menu_func_export(self, context):
    self.layout.operator(ExportSomeData.bl_idname, text=&quot;Text Export Operator&quot;)


def register():
    bpy.utils.register_class(ExportSomeData)
    bpy.utils.register_class(EXAMPLE_PT_panel_1)
    bpy.utils.register_class(EXAMPLE_PT_panel_2)
    bpy.types.TOPBAR_MT_file_export.append(menu_func_export)


def unregister():
    bpy.utils.unregister_class(EXAMPLE_PT_panel_2)
    bpy.utils.unregister_class(EXAMPLE_PT_panel_1)
    bpy.utils.unregister_class(ExportSomeData)
    bpy.types.TOPBAR_MT_file_export.remove(menu_func_export)


if __name__ == &quot;__main__&quot;:
    register()

    # test call
    bpy.ops.export_test.some_data('INVOKE_DEFAULT')

Related: How do a create a foldout UI panel?

If you'd like to add a checkbox to one of the panel headers, declare a BoolProperty in your operator (alternatively use the scope of the window manager):
class ExportSomeData(Operator, ExportHelper):
    ...
    bake = BoolProperty(default=False)
    ...

And call Layout.prop() in the dedicated draw_header() method of the panel:
def draw_header(self, context):
    sfile = context.space_data
    operator = sfile.active_operator

    self.layout.prop(operator, &quot;bake&quot;, text=&quot;&quot;)

Related: How to add a checkbox to a panel header?