How to add a view mode caching dependency in my custom block?
Found it out :)

Using entity_get_display to actually load the display view settings of the current node and then just merge the tags:

&lt;?php

public function getCacheTags() {
  if ($node = $this-&gt;getNode()) {
    $cacheTags = Cache::mergeTags(parent::getCacheTags(), array('node:' . $node-&gt;id()));
    $display = $this-&gt;getEntityViewDisplay(); // function which uses entity_get_display
    $cacheTags = Cache::mergeTags($display-&gt;getCacheTags(), $cacheTags);
  }
  else {
    $cacheTags = parent::getCacheTags();
  }

  return $cacheTags;
}