       Logger.debug(fn -> "A probe with name '#{name}' already exists. Overriding that one." end)
     end
 
-    {:noreply, Map.put(probes, name, probe)}
+    {:ok, pid} = DynamicSupervisor.start_child(Appsignal.Probes.DynamicSupervisor, probe)
+
+    {:noreply, Map.put(probes, name, pid)}
   end
 
   def handle_cast({:unregister, name}, probes) do
-    {:noreply, Map.delete(probes, name)}
-  end
-
-  def handle_info(:run_probes, probes) do
-    if Appsignal.Config.minutely_probes_enabled?() do
-      Enum.each(probes, fn {name, probe} ->
-        Task.start(fn ->
-          try do
-            probe.()
-          rescue
-            e ->
-              Logger.error("Error while calling probe #{name}: #{inspect(e)}")
-          end
-        end)
-      end)
+    with {:ok, pid} <- Map.fetch(probes, name) do
+      DynamicSupervisor.terminate_child(Appsignal.Probes.DynamicSupervisor, pid)
     end
 
-    schedule_probes()
-    {:noreply, probes}
+    {:noreply, Map.delete(probes, name)}
   end
 
   def child_spec(_) do

  end

  def init([]) do
    schedule_probes()
    {:ok, %{}}
  end


  def register(name, probe) do
    if genserver_running?() do
      if is_function(probe) do
        GenServer.cast(__MODULE__, {:register, {name, {Appsignal.Probes.FunctionProbeWrapper, [probe]}}})
        :ok
      else
        Logger.debug(fn ->
