title: NSTokenField crashes when editing tokens
body: <p>I'm using <code>NSTokenField</code> and a corresponding <code>NSTokenFieldDelegate</code> to treat all text that is surrounded by dash characters <code>-</code> as rounded tokens and all other text as plain text.</p>
  
  <p>The rounded tokens are displayed without the surrounding dashes and upon double-clicking on one of the tokens, the token switches in plain-text mode and shows the dashes.</p>
  
  <p>In my <code>NSTokenFieldDelegate</code> implementation of </p>
  
  <p><a href="https://developer.apple.com/documentation/appkit/nstokenfielddelegate/1524376-tokenfield" rel="nofollow noreferrer"><code>tokenField(NSTokenField, shouldAdd: [Any], at: Int) -&gt; [Any]</code></a></p>
  
  <p>I'm inspecting the given tokens and create rounded tokens if necessary.</p>
  
  <p>In certain situations, e.g., when committing the text change after editing by pressing the return key, the application crashes (tested on macOS Mojave 10.14.6). The demo project reports an index out of range exception with a stack trace I'm posting below for reference.</p>
  
  <p>To reproduce the crash, follow these simple steps</p>
  
  <ol>
  <li>Run the demo application</li>
  <li>Enter "-Dash1-. -Dash2-" into the token field</li>
  <li>Press return, the text changes to "<code>Dash1</code>. <code>Dash2</code>"</li>
  <li>Double-click <code>Dash1</code> which then turns into "-Dash1-" and is selected</li>
  <li>Press return</li>
  </ol>
  
  <p>You can find the demo project at <a href="https://github.com/fheidenreich/token-test" rel="nofollow noreferrer">https://github.com/fheidenreich/token-test</a> and I'm referencing the relevant code here:</p>
  
  <pre class="lang-swift prettyprint-override"><code>import Cocoa
  
  class ViewController: NSViewController {
      @IBOutlet weak var tokenField: NSTokenField!
  }
  
  class Token: Codable {
      let text: String
      let isRounded: Bool
  
      init(text: String, isRounded: Bool) {
          self.text = text
          self.isRounded = isRounded
      }
  }
  
  extension ViewController: NSTokenFieldDelegate {
  
      func tokenField(_ tokenField: NSTokenField, displayStringForRepresentedObject representedObject: Any) -&gt; String? {
          if let token = representedObject as? Token {
              if token.isRounded {
                  return token.text.trimmingCharacters(in: CharacterSet(charactersIn: "-")).capitalized(with: .current)
              } else {
                  return token.text
              }
          }
          return representedObject as? String
      }
  
      func tokenField(_ tokenField: NSTokenField, styleForRepresentedObject representedObject: Any) -&gt; NSTokenField.TokenStyle {
          if let token = representedObject as? Token {
              return token.isRounded ? .rounded : .none
          } else {
              return .none
          }
      }
  
      func tokenField(_ tokenField: NSTokenField, representedObjectForEditing editingString: String) -&gt; Any? {
          var isRounded = editingString.hasPrefix("-") &amp;&amp; editingString.hasSuffix("-") &amp;&amp; editingString.count &gt; 1
          if isRounded {
              // We treat it only as rounded token if we find no dashes inside the editing string
              let startIndex = editingString.index(after: editingString.startIndex)
              let endIndex = editingString.index(before: editingString.endIndex)
              let searchRange = startIndex..&lt;endIndex
              let range = editingString.rangeOfCharacter(from: CharacterSet(charactersIn: "-"), options: [], range: searchRange)
              isRounded = range == nil
          }
          return Token(text: editingString, isRounded: isRounded)
      }
  
      func tokenField(_ tokenField: NSTokenField, editingStringForRepresentedObject representedObject: Any) -&gt; String? {
          if let token = representedObject as? Token {
              return token.text
          }
          return representedObject as? String
      }
  
      func tokenField(_ tokenField: NSTokenField, shouldAdd tokens: [Any], at index: Int) -&gt; [Any] {
          if let tokens = tokens as? [Token] {
              var added = [Any]()
  
              for token in tokens {
                  if token.isRounded {
                      added.append(token)
                      continue
                  }
  
                  let newTokens = createTokens(token.text)
                  added.append(contentsOf: newTokens)
              }
  
              return added
          }
          return tokens
      }
  
      func createTokens(_ text: String) -&gt; [Token] {
          guard let exp = try? NSRegularExpression(pattern: "-(.+?)-", options: []) else {
              return []
          }
  
          var tokens = [Token]()
          var previousEndLocation = 0
          let matches = exp.matches(in: text, options: [], range: NSRange(location: 0, length: text.count))
          for match in matches {
              let range0 = match.range(at: 0) // Range that denotes the part before the match
              let range1 = match.range(at: 1) // Range that denotes the match
  
              if previousEndLocation &lt; range0.location {
                  let rangePrefix = NSRange(location: previousEndLocation, length: range0.location - previousEndLocation)
                  // We found some text that prefixes the matches
                  if rangePrefix.length &gt; 0 {
                      if let swiftRange = Range(rangePrefix, in: text) {
                          let name = String(text[swiftRange])
                          tokens.append(Token(text: name, isRounded: false))
                      }
                  }
              }
  
              if let swiftRange = Range(range1, in: text) {
                  let name = "-" + text[swiftRange] + "-"
                  tokens.append(Token(text: name, isRounded: true))
              }
  
              previousEndLocation = range0.location + range0.length
          }
  
          // We found some text that postfixes the matches
          if previousEndLocation &lt; text.count {
              let range = NSRange(location: previousEndLocation, length: text.count - previousEndLocation)
              if let swiftRange = Range(range, in: text) {
                  let name = String(text[swiftRange])
                  tokens.append(Token(text: name, isRounded: false))
              }
          }
  
          return tokens
      }
  }
  </code></pre>
  
  <p>Can anyone explain why the application is crashing and possibly give hints in fixing or circumventing the issue?</p>
  
  <pre><code>2019-11-23 22:05:54.155614+0100 TokenTest[52583:3146671] !!! _NSGlyphTreeGlyphAtIndex missing glyphs
  2019-11-23 22:05:54.160776+0100 TokenTest[52583:3146671] [General] An uncaught exception was raised
  2019-11-23 22:05:54.160940+0100 TokenTest[52583:3146671] [General] *** -[NSBigMutableString _getBlockStart:end:contentsEnd:forRange:stopAtLineSeparators:]: Range {0, 5} out of bounds; string length 0
  2019-11-23 22:05:54.200975+0100 TokenTest[52583:3146671] [General] (
      0   CoreFoundation                      0x00007fff398e5bc9 __exceptionPreprocess + 256
      1   libobjc.A.dylib                     0x00007fff640893c6 objc_exception_throw + 48
      2   CoreFoundation                      0x00007fff398e59fb +[NSException raise:format:] + 193
      3   Foundation                          0x00007fff3bb23da2 -[NSString _getBlockStart:end:contentsEnd:forRange:stopAtLineSeparators:] + 214
      4   Foundation                          0x00007fff3bb23cc5 -[NSString getParagraphStart:end:contentsEnd:forRange:] + 31
      5   UIFoundation                        0x00007fff60551488 _NSFastFillAllLayoutHolesForGlyphRange + 1425
      6   UIFoundation                        0x00007fff6053e260 -[NSLayoutManager(NSPrivate) _firstPassGlyphRangeForBoundingRect:inTextContainer:okToFillHoles:] + 285
      7   UIFoundation                        0x00007fff6053d26c -[NSLayoutManager(NSPrivate) _glyphRangeForBoundingRect:inTextContainer:fast:okToFillHoles:] + 851
      8   UIFoundation                        0x00007fff6053cf12 -[NSLayoutManager glyphRangeForBoundingRect:inTextContainer:] + 67
      9   AppKit                              0x00007fff36e7fdd8 -[NSTextView setNeedsDisplayInRect:avoidAdditionalLayout:] + 1205
      10  AppKit                              0x00007fff36e7f91b -[NSTextView setNeedsDisplayInRect:] + 41
      11  AppKit                              0x00007fff36e5d109 -[NSView setNeedsDisplay:] + 79
      12  AppKit                              0x00007fff370f26df -[NSTextView(NSSharing) setSelectedTextAttributes:] + 323
      13  AppKit                              0x00007fff370f1daa _NSEditTextCellWithOptions + 1313
      14  AppKit                              0x00007fff370f179f -[NSTextFieldCell _selectOrEdit:inView:target:editor:event:start:end:] + 357
      15  AppKit                              0x00007fff3721123e -[NSTokenFieldCell _selectOrEdit:inView:target:editor:event:start:end:] + 147
      16  AppKit                              0x00007fff370f1634 -[NSCell selectWithFrame:inView:editor:delegate:start:length:] + 46
      17  AppKit                              0x00007fff370f15fe __26-[NSTextField selectText:]_block_invoke + 94
      18  AppKit                              0x00007fff36e47849 +[NSAppearance _performWithCurrentAppearance:usingBlock:] + 84
      19  AppKit                              0x00007fff370f129a -[NSTextField selectText:] + 231
      20  AppKit                              0x00007fff37105719 -[NSTextField textDidEndEditing:] + 1062
      21  CoreFoundation                      0x00007fff398920ea __CFNOTIFICATIONCENTER_IS_CALLING_OUT_TO_AN_OBSERVER__ + 12
      22  CoreFoundation                      0x00007fff39892064 ___CFXRegistrationPost_block_invoke + 63
      23  CoreFoundation                      0x00007fff39891fce _CFXRegistrationPost + 404
      24  CoreFoundation                      0x00007fff3989a3fd ___CFXNotificationPost_block_invoke + 87
      25  CoreFoundation                      0x00007fff3980340c -[_CFXNotificationRegistrar find:object:observer:enumerator:] + 1834
      26  CoreFoundation                      0x00007fff398026b7 _CFXNotificationPost + 840
      27  Foundation                          0x00007fff3baa6a7b -[NSNotificationCenter postNotificationName:object:userInfo:] + 66
      28  AppKit                              0x00007fff37215efb -[NSTextView(NSPrivate) _giveUpFirstResponder:] + 415
      29  AppKit                              0x00007fff376b2aee -[NSTokenTextView insertNewline:] + 354
      30  AppKit                              0x00007fff37142ca6 -[NSTextView doCommandBySelector:] + 194
      31  AppKit                              0x00007fff37142bba -[NSTextInputContext(NSInputContext_WithCompletion) doCommandBySelector:completionHandler:] + 228
      32  AppKit                              0x00007fff37137e1a -[NSKeyBindingManager(NSKeyBindingManager_MultiClients) interpretEventAsCommand:forClient:] + 2972
      33  AppKit                              0x00007fff37803fdf __84-[NSTextInputContext _handleEvent:options:allowingSyntheticEvent:completionHandler:]_block_invoke_5 + 341
      34  AppKit                              0x00007fff37803e75 __84-[NSTextInputContext _handleEvent:options:allowingSyntheticEvent:completionHandler:]_block_invoke_3.784 + 74
      35  AppKit                              0x00007fff3713ef23 -[NSTextInputContext tryHandleEvent_HasMarkedText_withDispatchCondition:dispatchWork:continuation:] + 87
      36  AppKit                              0x00007fff37803dfb __84-[NSTextInputContext _handleEvent:options:allowingSyntheticEvent:completionHandler:]_block_invoke.781 + 237
      37  HIToolbox                           0x00007fff38ae7efb __TSMProcessRawKeyEventWithOptionsAndCompletionHandler_block_invoke_5 + 70
      38  HIToolbox                           0x00007fff38ae6db9 ___ZL23DispatchEventToHandlersP14EventTargetRecP14OpaqueEventRefP14HandlerCallRec_block_invoke + 110
      39  AppKit                              0x00007fff377fe70e __55-[NSTextInputContext handleTSMEvent:completionHandler:]_block_invoke.265 + 575
      40  AppKit                              0x00007fff37139512 __55-[NSTextInputContext handleTSMEvent:completionHandler:]_block_invoke_2 + 74
      41  AppKit                              0x00007fff3713949a -[NSTextInputContext tryHandleTSMEvent_HasMarkedText_withDispatchCondition:dispatchWork:continuation:] + 87
      42  AppKit                              0x00007fff37138c92 -[NSTextInputContext handleTSMEvent:completionHandler:] + 1749
      43  AppKit                              0x00007fff37138545 _NSTSMEventHandler + 306
      44  HIToolbox                           0x00007fff38a7d22e _ZL23DispatchEventToHandlersP14EventTargetRecP14OpaqueEventRefP14HandlerCallRec + 1422
      45  HIToolbox                           0x00007fff38a7c5df _ZL30SendEventToEventTargetInternalP14OpaqueEventRefP20OpaqueEventTargetRefP14HandlerCallRec + 371
      46  HIToolbox                           0x00007fff38a7c465 SendEventToEventTargetWithOptions + 45
      47  HIToolbox                           0x00007fff38ae3f8f SendTSMEvent_WithCompletionHandler + 383
      48  HIToolbox                           0x00007fff38ae43fa __SendUnicodeTextAEToUnicodeDoc_WithCompletionHandler_block_invoke + 387
      49  HIToolbox                           0x00007fff38ae4268 __SendFilterTextEvent_WithCompletionHandler_block_invoke + 221
      50  HIToolbox                           0x00007fff38ae3fde SendTSMEvent_WithCompletionHandler + 462
      51  HIToolbox                           0x00007fff38ae3de3 SendFilterTextEvent_WithCompletionHandler + 225
      52  HIToolbox                           0x00007fff38ae3aa4 SendUnicodeTextAEToUnicodeDoc_WithCompletionHandler + 280
      53  HIToolbox                           0x00007fff38ae384e __utDeliverTSMEvent_WithCompletionHandler_block_invoke_2 + 283
      54  HIToolbox                           0x00007fff38ae36ad __utDeliverTSMEvent_WithCompletionHandler_block_invoke + 355
      55  HIToolbox                           0x00007fff38ae34cb TSMKeyEvent_WithCompletionHandler + 598
      56  HIToolbox                           0x00007fff38ae325a __TSMProcessRawKeyEventWithOptionsAndCompletionHandler_block_invoke_4 + 250
      57  HIToolbox                           0x00007fff38ae3089 __TSMProcessRawKeyEventWithOptionsAndCompletionHandler_block_invoke_3 + 257
      58  HIToolbox                           0x00007fff38ae2dce __TSMProcessRawKeyEventWithOptionsAndCompletionHandler_block_invoke_2 + 282
      59  HIToolbox                           0x00007fff38ae2b32 __TSMProcessRawKeyEventWithOptionsAndCompletionHandler_block_invoke + 274
      60  HIToolbox                           0x00007fff38ae2127 TSMProcessRawKeyEventWithOptionsAndCompletionHandler + 3398
      61  AppKit                              0x00007fff37803cd9 __84-[NSTextInputContext _handleEvent:options:allowingSyntheticEvent:completionHandler:]_block_invoke_3.779 + 110
      62  AppKit                              0x00007fff378032d6 __204-[NSTextInputContext tryTSMProcessRawKeyEvent_orSubstitution:dispatchCondition:setupForDispatch:furtherCondition:doubleSpaceSubstitutionCondition:doubleSpaceSubstitutionWork:dispatchTSMWork:continuation:]_block_invoke.734 + 115
      63  AppKit                              0x00007fff378031c7 -[NSTextInputContext tryTSMProcessRawKeyEvent_orSubstitution:dispatchCondition:setupForDispatch:furtherCondition:doubleSpaceSubstitutionCondition:doubleSpaceSubstitutionWork:dispatchTSMWork:continuation:] + 245
      64  AppKit                              0x00007fff37803892 -[NSTextInputContext _handleEvent:options:allowingSyntheticEvent:completionHandler:] + 1286
      65  AppKit                              0x00007fff37803086 -[NSTextInputContext _handleEvent:allowingSyntheticEvent:] + 107
      66  AppKit                              0x00007fff37137004 -[NSView interpretKeyEvents:] + 209
      67  AppKit                              0x00007fff37136e2d -[NSTextView keyDown:] + 726
      68  AppKit                              0x00007fff36f84367 -[NSWindow(NSEventRouting) _reallySendEvent:isDelayedEvent:] + 6840
      69  AppKit                              0x00007fff36f82667 -[NSWindow(NSEventRouting) sendEvent:] + 478
      70  AppKit                              0x00007fff36e22889 -[NSApplication(NSEvent) sendEvent:] + 2953
      71  AppKit                              0x00007fff36e105c0 -[NSApplication run] + 755
      72  AppKit                              0x00007fff36dffac8 NSApplicationMain + 777
      73  TokenTest                           0x0000000100008cad main + 13
      74  libdyld.dylib                       0x00007fff6584f3d5 start + 1
  )
  </code></pre>