How to stop the timer when the ESC key is pressed
You will have to unregister your timer function in the cancel method.

To do so I had to bring printFunction out of the operator class, because otherwise it needs self argument, and I haven't found how to bring it with the function to the timer register operator. So now the idea is to set returnValue not as class's instance's variable (using self.returnValue), but as class's variable (using bpy.types.YOUR_OPERATOR_CLASS instead of self) - in execute, cancel, printFunction and any other method or function. 

import bpy

bl_info = {
    "name": "TestPanel",
    "author": "taichi",
    "version": (1, 0),
    "blender": (2, 80, 0),
    "location": "3DView",
    "description": "test",
    "warning": "",
    "support": "COMMUNITY",
    "wiki_url": "",
    "tracker_url": "",
    "category": "Print"
}

class TEST_PT_Panel(bpy.types.Panel):
    blidname = "TEST_PT_Panel"
    bl_label = "Test Panel"
    bl_category = "test"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"

    def draw(self,context):
        layout = self.layout

        row = layout.row()
        row.operator('wm.modal_test_operator',text ='TestButton')

def printFunction():
    print("test")
    return bpy.types.WM_OT_modal_test_operator.returnValue

class Test_OT_Operator(bpy.types.Operator):
    bl_idname = "wm.modal_test_operator"
    bl_label = "Modal test Operator"
    _timer = None
    returnValue = 0.1

    def modal(self, context, event):
        if event.type in {'ESC'}:
            self.cancel(context)
            return{'CANCELLED'}
        return{'PASS_THROUGH'}

    def execute(self, context):
        bpy.types.WM_OT_modal_test_operator.returnValue = 0.1
        wm = context.window_manager
        bpy.app.timers.register(printFunction)
        wm.modal_handler_add(self)

        return {'RUNNING_MODAL'}

    def cancel(self, context):
        bpy.app.timers.unregister(printFunction)
        bpy.types.WM_OT_modal_test_operator.returnValue = 0


classes = (TEST_PT_Panel,Test_OT_Operator)
register,unregister = bpy.utils.register_classes_factory(classes)

if __name__ == "__main__":
    register()


Note that once the variable is changed as class variable it keeps its last meaning even after operator is over. You can for example reset it as class variable in cancel method. You can also add for example set_time: FloatProperty(name="Set Time", default=0.1) to your Operator class and in the execute method set bpy.types.WM_OT_modal_test_operator.returnValue = self.set_time - so you could change it at the begining of every launch of your operator, like this: bpy.ops.wm.modal_test_operator(set_time=0.5) and also to use this property in ui panel.

UPDATE:
Found much easier solution with less changes to your code. Set your printFunction to self._timer variable and register and unregister self._timer

import bpy

class TEST_PT_Panel(bpy.types.Panel):
    blidname = "TEST_PT_Panel"
    bl_label = "Test Panel"
    bl_category = "test"
    bl_space_type = "VIEW_3D"
    bl_region_type = "UI"

    def draw(self,context):
        layout = self.layout

        row = layout.row()
        row.operator('wm.modal_test_operator',text ='TestButton')

class Test_OT_Operator(bpy.types.Operator):
    bl_idname = "wm.modal_test_operator"
    bl_label = "Modal test Operator"
    _timer = None
    returnValue = 0.1

    def modal(self, context, event):
        if event.type in {'ESC'}:
            self.cancel(context)
            return{'CANCELLED'}
        return{'PASS_THROUGH'}

    def execute(self, context):
        def printFunction():
            print("test")
            return self.returnValue
        self._timer = printFunction
        wm = context.window_manager
        bpy.app.timers.register(self._timer)
        wm.modal_handler_add(self)

        return {'RUNNING_MODAL'}

    def cancel(self, context):
        bpy.app.timers.unregister(self._timer)
        self.returnValue = 0


classes = (TEST_PT_Panel,Test_OT_Operator)
register,unregister = bpy.utils.register_classes_factory(classes)

if __name__ == "__main__":
    register()