How do I validate form elements in a modal without closing?
This is a excellent article which you can read for "fixing" your issue:
http://www.mediacurrent.com/blog/loading-and-rendering-modal-forms-drupal-8

Keep in mind this lines:

First, we wrap our form with a div

public function buildForm(array $form, FormStateInterface $form_state, $options = NULL) {
  // It is important this id
  $form['#prefix'] = '&lt;div id="modal_example_form"&gt;';
  $form['#suffix'] = '&lt;/div&gt;';

  // The status messages that will contain any form errors.
  $form['status_messages'] = [
    '#type' =&gt; 'status_messages',
    '#weight' =&gt; -10,
  ];

  // more code

  $form['actions']['send'] = [
      '#type' =&gt; 'submit',
      '#value' =&gt; $this-&gt;t('Submit modal form'),
      '#attributes' =&gt; [
        'class' =&gt; [
          'use-ajax',
        ],
      ],
      // Function to call when the users click on submit button.
      '#ajax' =&gt; [
        'callback' =&gt; [$this, 'submitModalFormAjax'],
        'event' =&gt; 'click',
      ],
    ];

    // more code
}


Second, we show the errors:

/**
 * AJAX callback handler that displays any errors or a success message.
 */
public function submitModalFormAjax(array $form, FormStateInterface $form_state) {
  $response = new AjaxResponse();

  // If there are any form errors, re-display the form.
  if ($form_state-&gt;hasAnyErrors()) {
    // Remember the previous id ? Here it is
    $response-&gt;addCommand(new ReplaceCommand('#modal_example_form', $form));
  }
  else {
    $response-&gt;addCommand(new OpenModalDialogCommand("Success!", 'The modal form has been submitted.', ['width' =&gt; 800]));
  }

  return $response;
}