Confused with how to stack bpy calls
You can avoid the ambiguity of the situation by explicitly setting the object's mode to what you want in each stage:

class QuickOrigin(bpy.types.Operator):
    bl_idname = "quick.origin"
    bl_label = "Set Origin"
    bl_options = {'REGISTER'}

    def execute(self, context):
        if context.object.mode == 'OBJECT':
            bpy.ops.object.mode_set( mode = 'EDIT' )
            return{'FINISHED'}

        bpy.ops.view3d.snap_cursor_to_selected()
        bpy.ops.object.mode_set( mode = 'OBJECT' )
        bpy.ops.object.origin_set(type='ORIGIN_CURSOR')
        bpy.ops.object.mode_set( mode = 'EDIT' )
        return{'FINISHED'}