func tokenize(line string) []*Token { //nolint:gocyclo
	line = strings.TrimSpace(line)

	var (
		tokens []*Token
		beg    = 0
		size   = len(line)
	)

out:
	for beg < size-1 {
		var (
			end     int
			tagType = getTagType(line, beg)
		)

		switch tagType {
		case typeTagTextEffect:
			end = beg + 1
			for end < len(line) && line[end] != line[beg] {
				end++
			}

			var word string
			if end < size-1 {
				word = line[beg : end+1]
			} else {
				word = line[beg:end]
			}

			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagTextEffect,
				startIdx: beg,
				endIdx:   end,
			})
			end++
		case typeTagHeading:
			fallthrough
		case typeTagInlineQuote:
			end = beg + 1
			for end < len(line) && line[end] != '.' {
				end++
			}
			word := line[beg : end+1]

			tokens = append(tokens, &Token{
				tag:      word,
				family:   tagType,
				startIdx: beg,
				endIdx:   end,
			})
			break out
		case typeTagList:
			end = beg + 1
			for end < len(line) && line[end] == line[beg] {
				end++
			}
			word := line[beg:end]

			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagList,
				startIdx: beg,
				endIdx:   end,
			})
			end++
		case typeTagReferenceLink:
			end = beg + 1
			for end < len(line) && line[end] != ']' {
				end++
			}
			word := line[beg : end+1]

			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagReferenceLink,
				startIdx: beg,
				endIdx:   end,
			})
		case typeTagTable:
			end = len(line) - 1

			tokens = append(tokens, &Token{
				tag:      line,
				family:   typeTagTable,
				startIdx: beg,
				endIdx:   end,
			})
		default:
			end = beg + 1
			for end < size && line[end] != '*' && line[end] != '{' && line[end] != '}' && line[end] != '[' && line[end] != ']' {
				end++
			}

			if end != size && line[end] != '*' && line[end] != '{' && line[end] != '[' {
				end++
			}

			word := line[beg:end]
			word, attrs := extractAttributes(word)

			if isToken(word) {
				fam := typeTagOther
				if word == TagCodeBlock || word == TagNoFormat {
					fam = typeTagFencedCode
				}

				tokens = append(tokens, &Token{
					tag:      word,
					family:   fam,
					attrs:    attrs,
					startIdx: beg,
					endIdx:   end - 1,
				})
			}
		}

		beg = end
	}

	return tokens
}
// Mark tokens in a given string.
Mark tokens in a given string.
// Mark tokens in a given string .
func tokenize(line string) []*Token { 
	line = strings.TrimSpace(line)
	var (
		tokens []*Token
		beg    = 0
		size   = len(line)
	)
out:
	for beg < size-1 {
		var (
			end     int
			tagType = getTagType(line, beg)
		)
		switch tagType {
		case typeTagTextEffect:
			end = beg + 1
			for end < len(line) && line[end] != line[beg] {
				end++
			}
			var word string
			if end < size-1 {
				word = line[beg : end+1]
			} else {
				word = line[beg:end]
			}
			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagTextEffect,
				startIdx: beg,
				endIdx:   end,
			})
			end++
		case typeTagHeading:
			fallthrough
		case typeTagInlineQuote:
			end = beg + 1
			for end < len(line) && line[end] != '.' {
				end++
			}
			word := line[beg : end+1]
			tokens = append(tokens, &Token{
				tag:      word,
				family:   tagType,
				startIdx: beg,
				endIdx:   end,
			})
			break out
		case typeTagList:
			end = beg + 1
			for end < len(line) && line[end] == line[beg] {
				end++
			}
			word := line[beg:end]
			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagList,
				startIdx: beg,
				endIdx:   end,
			})
			end++
		case typeTagReferenceLink:
			end = beg + 1
			for end < len(line) && line[end] != ']' {
				end++
			}
			word := line[beg : end+1]
			tokens = append(tokens, &Token{
				tag:      word,
				family:   typeTagReferenceLink,
				startIdx: beg,
				endIdx:   end,
			})
		case typeTagTable:
			end = len(line) - 1
			tokens = append(tokens, &Token{
				tag:      line,
				family:   typeTagTable,
				startIdx: beg,
				endIdx:   end,
			})
		default:
			end = beg + 1
			for end < size && line[end] != '*' && line[end] != '{' && line[end] != '}' && line[end] != '[' && line[end] != ']' {
				end++
			}
			if end != size && line[end] != '*' && line[end] != '{' && line[end] != '[' {
				end++
			}
			word := line[beg:end]
			word, attrs := extractAttributes(word)
			if isToken(word) {
				fam := typeTagOther
				if word == TagCodeBlock || word == TagNoFormat {
					fam = typeTagFencedCode
				}
				tokens = append(tokens, &Token{
					tag:      word,
					family:   fam,
					attrs:    attrs,
					startIdx: beg,
					endIdx:   end - 1,
				})
			}
		}
		beg = end
	}
	return tokens
}
func tokenize ( line string ) [ ] * Token { line = strings . TrimSpace ( line ) var ( tokens [ ] * Token beg = 0 size = len ( line ) ) out : for beg < size - 1 { var ( end int tagType = getTagType ( line , beg ) ) switch tagType { case typeTagTextEffect : end = beg + 1 for end < len ( line ) && line [ end ] != line [ beg ] { end ++ } var word string if end < size - 1 { word = line [ beg : end + 1 ] } else { word = line [ beg : end ] } tokens = append ( tokens , & Token { tag : word , family : typeTagTextEffect , startIdx : beg , endIdx : end , } ) end ++ case typeTagHeading : fallthrough case typeTagInlineQuote : end = beg + 1 for end < len ( line ) && line [ end ] != '.' { end ++ } word := line [ beg : end + 1 ] tokens = append ( tokens , & Token { tag : word , family : tagType , startIdx : beg , endIdx : end , } ) break out case typeTagList : end = beg + 1 for end < len ( line ) && line [ end ] == line [ beg ] { end ++ } word := line [ beg : end ] tokens = append ( tokens , & Token { tag : word , family : typeTagList , startIdx : beg , endIdx : end , } ) end ++ case typeTagReferenceLink : end = beg + 1 for end < len ( line ) && line [ end ] != ']' { end ++ } word := line [ beg : end + 1 ] tokens = append ( tokens , & Token { tag : word , family : typeTagReferenceLink , startIdx : beg , endIdx : end , } ) case typeTagTable : end = len ( line ) - 1 tokens = append ( tokens , & Token { tag : line , family : typeTagTable , startIdx : beg , endIdx : end , } ) default : end = beg + 1 for end < size && line [ end ] != '*' && line [ end ] != '{' && line [ end ] != '}' && line [ end ] != '[' && line [ end ] != ']' { end ++ } if end != size && line [ end ] != '*' && line [ end ] != '{' && line [ end ] != '[' { end ++ } word := line [ beg : end ] word , attrs := extractAttributes ( word ) if isToken ( word ) { fam := typeTagOther if word == TagCodeBlock || word == TagNoFormat { fam = typeTagFencedCode } tokens = append ( tokens , & Token { tag : word , family : fam , attrs : attrs , startIdx : beg , endIdx : end - 1 , } ) } } beg = end } return tokens }
Mark tokens in a given string.
Mark tokens in a given string .
//nolint:gocyclo