relatedTo a category in a matrix field
Figured it out:

    {# Fetch entries with a Matrix block to get non empty categories #}
    {% for block in entry.teamMember.all() %}
    {% set category = block.department.one() %}

    &lt;section id="{{ category.title }}"&gt;
        {% if category.title != 'Ownership' %}&lt;h1&gt;{{ category.title }}&lt;/h1&gt;{% endif %}
        {% for block in entry.teamMember.relatedTo(category).all() %}
        &lt;section class="member"&gt;
            {% set image = block.photograph.one() %}
            {% if image %}
            &lt;div style="background-image: url('{{ image.getUrl('image500') }}');"&gt;&amp;nbsp;&lt;/div&gt;
            {% endif %}

            &lt;h1&gt;{{ block.fullName }} &lt;em&gt;{{ block.jobTitle|markdown }}&lt;/em&gt;&lt;/h1&gt;
            {{ block.biography }}
        &lt;/section&gt;
        {% endfor %}

    &lt;/section&gt;
    {% endfor %}