How do form #states interact with dialog action buttons, if at all?
This doesn't answer the question itself, but since I struggled for a while to get this all syncd up I figured I should drop the work into something that people might hit from google. To get the attributes all syncd up you can use the following js file:

(function ($, Drupal, drupalSettings) {
  var handler = Drupal.behaviors.commentsRatingFlexible = {
    attach: function (context, settings) {
      // If our form is loaded in a modal, then the context is
      // the form itself. Therefore we cannot look for the form
      // inside of context. As such, we need to check for forms
      // that exist in both situations an handle accordingly.
      var formInstances = [];

      // This would be the class for your particular form
      $('.comment-rating-flexible-form', context).each(function () {
        formInstances.push($(this));
      })

      if ($(context).hasClass('comment-rating-flexible-form')) {
        formInstances.push($(context));
      }

      // Lookup any rating-flexible-forms and disable the submit
      // button by default, since it shouldn't allow an empty val
      // for the fivestar field (if it's present)
      for (var k in formInstances) {
        var $form = formInstances[k];
        var $submit = $('.form-submit', $form);
        var $fivestarSelect = $('[name="field_fivestar_rating[0][rating]"]', $form);

        if (!$fivestarSelect.length) {
          return
        }

        handler.dialogSubmitButtonHandler($form); //must be called first
        handler.observeAttributes($submit);

        if ($fivestarSelect.val() == '-' || $fivestarSelect.val() == '0') {
          // #states don't apply on form load, only on change, 
          // so we need to disable the form button here and 
          // sync that onto the dialog's button
          $submit.attr('disabled', "");
        }
      }
    },
    /**
     * Syncs attributes from real form-submit onto dialog submit btn.
     *
     * @param $form
     *   The form contained in the dialog
     */
    dialogSubmitButtonHandler: function ($form) {
      var $drupalModal = $form.parent();

      // If the form is loaded outside the context of a modal, don't execute
      if ($drupalModal.attr('id') !== 'drupal-modal') {
        return;
      }

      // The actual modal and the rest of its contents won't exists
      // on the initial load. As such, we need to add a watcher to 
      // the DOM to find out when our dialog action buttons exist
      // so that we can act on them then.
      var observer = new MutationObserver(function (mutations) {
        mutations.forEach(function (mutation) {
          if (!mutation.addedNodes.length) {
            return
          }

          mutation.addedNodes.forEach(function (node) {
            var $node = $(node);
            if (!$node.hasClass('ui-dialog-buttonpane')) {
              return;
            }

            var $formSubmitButton = $('.form-submit', $form);
            var $dialogSubmitButton = $('.form-submit', $node);

            // Watch for our custom $.event from observeAttributes
            $formSubmitButton.on('attributeUpdated', function (e, attrData) {
              $dialogSubmitButton.attr(
                attrData.attributeName,
                attrData.attributeNewValue
              );
            })

            observer.disconnect();
          })
        })
      })

      observer.observe(document.body, {
        childList: true,
        subtree: true,
        attributes: false,
        characterData: false
      })
    },
    /**
     * Fires an event when an element's attributes change.
     *
     * @param $el
     *   The jQuery element being observed
     */
    observeAttributes: function ($el) {
      var observer = new MutationObserver(function (mutationList, observer) {
        var eventData = [];

        mutationList.forEach(function (mutation) {
          if (mutation.attributeName === 'style') {
            return;
          }

          var attributeName = (mutation.attributeName === 'class') ? 'classList' : mutation.attributeName;

          $el.trigger('attributeUpdated', {
            'attributeName': mutation.attributeName,
            'attributeNewValue': mutation.target[attributeName],
            'attributeOldValue': mutation.oldValue
          })
        })
      });

      observer.observe($el[0], {
        attributes: true,
        attributeOldValue: true,
      });
    },
  }
})(jQuery, Drupal, drupalSettings);