I'm writing a compiler for a couple of months now, this is the tokenization part of the lexer.I would like a code review to improve my coding style and learn new techniques to pretty up my code and make it easier to maintain.Also because I did not actually study compiler design, I do not know if the structure makes sense at all, so feel free to give any kind of criticism.This class is a compiler instance, today I will only discuss the tokenization part of it.using System;using System.Collections.Generic;using System.Diagnostics;using System.IO;namespace ShirLanguageCompiler{    public class ShirEnvironment    {        public ShirEnvironment(string inputpath)        {            this.code = File.ReadAllText(inputpath);            Console.WriteLine(Starting tokenizer);            this.Tokenizer = new Tokenizer(this);            Console.WriteLine(Starting parser);            this.Parser = new Parser(this);            Console.WriteLine(Starting Compiling Environment);            this.Factory = new ILFactory(this);            Console.WriteLine(Starting Virtual Machine);            this.VirtualMachine = new VM(this);        }        public void Compile()        {            Tokenizer.Tokenize();            Console.WriteLine(String.Join(Environment.NewLine, Tokens));            Parser.Parse();            Debug.Assert(Tokenizer.AsCode() == code);            Factory.Generate();            VirtualMachine.Execute();        }        public Tokenizer Tokenizer;        public Parser Parser;        public ILFactory Factory;        public VM VirtualMachine;        public ILEnv Intermidiate = new ILEnv();        public List<Token> Tokens = new List<Token>();        public List<SyntaxNode> Nodes = new List<SyntaxNode>();        public ProgramNode Program = new ProgramNode();        public string code;    }}The Tokenizer:using System.Collections.Generic;using System.Linq;using System.Text.RegularExpressions;namespace ShirLanguageCompiler{    public class Tokenizer    {        public Tokenizer(ShirEnvironment _env)        {            this.env = _env;        }        ShirEnvironment env;        int cursor;        int line = 0, col = 0;        private readonly Dictionary<string, Regex> Patterns = new Dictionary<string, Regex>()        {            {CharPattern,     new Regex([\\$a-zA-Z])},            {StringPattern,   new Regex([^\])      },            {NumCharPattern,  new Regex([[0-9a-zA-Z])},            {NumberPattern,   new Regex([0-9\\.])   }        };        private static readonly Dictionary<string, SyntaxKind> Keywords = new Dictionary<string, SyntaxKind>()        {            {True,    SyntaxKind.LiteralTrueKeyword   },            {False,   SyntaxKind.LiteralFalseKeyword  },            {ref,     SyntaxKind.RefKeyword           },            {val,     SyntaxKind.ValKeyword           },            {return,  SyntaxKind.ReturnKeyword        },            {bind,    SyntaxKind.BindKeyword          },            {boolean, SyntaxKind.BooleanKeyword       },            {number,  SyntaxKind.NumberKeyword        },            {letter,  SyntaxKind.LetterKeyword        },            {string,  SyntaxKind.StringKeyword        }        };        private static readonly Dictionary<SyntaxKind, Regex> Definitions = new Dictionary<SyntaxKind, Regex>()        {            { SyntaxKind.ColonToken,                 new Regex(:)             },            { SyntaxKind.SemiColonToken,            new Regex(;)             },            { SyntaxKind.AssignmentToken,            new Regex(=>)         },            { SyntaxKind.AccessorToken,             new Regex(->)         },            { SyntaxKind.LiteralCharToken,             new Regex(\')         },            { SyntaxKind.LiteralStringToken,         new Regex(\)         },            { SyntaxKind.LiteralNumberToken,         new Regex([0-9])        },            { SyntaxKind.OpenCurlyBracketToken,     new Regex(\\{)         },            { SyntaxKind.CloseCurlyBracketToken,     new Regex(\\})        },            { SyntaxKind.OpenParenthesisToken,         new Regex(\\()         },            { SyntaxKind.CloseParenthesisToken,     new Regex(\\))         },            { SyntaxKind.OpenSquareBracketToken,     new Regex(\\[)         },            { SyntaxKind.CloseSquareBracketToken,    new Regex(\\])         },            { SyntaxKind.CommaToken,                 new Regex(,)             },            { SyntaxKind.EOLToken,                     new Regex([\\r\\n])     },            { SyntaxKind.WhitespaceToken,             new Regex(\\s)         },            { SyntaxKind.QuestionMarkToken,         new Regex(\\?)        },            { SyntaxKind.PlusOperationToken,         new Regex(\\+)        },            { SyntaxKind.MinusOperationToken,         new Regex(\\-)        },            { SyntaxKind.MultiplyOperationToken,     new Regex(\\*)        },            { SyntaxKind.PowerOperationToken,       new Regex(\\*\\*)     },            { SyntaxKind.RootOperationToken,        new Regex(\\/\\/)     },            { SyntaxKind.DivideOperationToken,         new Regex(\\/)        },            { SyntaxKind.EqualToken,                 new Regex(==)            },            { SyntaxKind.InEqualToken,              new Regex(!=)         }        };        static readonly SyntaxKind[] LiteralTokens =        {            SyntaxKind.EOLToken                 ,            SyntaxKind.WhitespaceToken          ,            SyntaxKind.QuestionMarkToken        ,            SyntaxKind.ColonToken               ,            SyntaxKind.SemiColonToken           ,            SyntaxKind.CommaToken               ,            SyntaxKind.OpenParenthesisToken     ,            SyntaxKind.CloseParenthesisToken    ,            SyntaxKind.OpenSquareBracketToken   ,            SyntaxKind.CloseSquareBracketToken  ,            SyntaxKind.OpenCurlyBracketToken    ,            SyntaxKind.CloseCurlyBracketToken   ,            SyntaxKind.PlusOperationToken       ,            SyntaxKind.MinusOperationToken      ,            SyntaxKind.MultiplyOperationToken   ,            SyntaxKind.DivideOperationToken        };        /*         * Implementing a generic tokenizer here         * this might seem not standart, but im not following any standarts here. that would be boring wouldn't it         */        private bool MatchesPattern(Regex expression, int size = 1) => env.code.Length >= cursor + size && expression.IsMatch(env.code.Substring(cursor, size));        private bool MatchesDefition(SyntaxKind kind, int size = 1) => MatchesPattern(Definitions[kind], size);        public void Tokenize()        {            for (cursor = 0; cursor < env.code.Length;)            {                int savecursor = cursor;                if (MatchesDefition(SyntaxKind.PowerOperationToken, 2))                {                    MakeToken(SyntaxKind.PowerOperationToken, cursor, 2);                    cursor += 2;                    continue;                }                if (MatchesDefition(SyntaxKind.RootOperationToken, 2))                {                    MakeToken(SyntaxKind.RootOperationToken, cursor, 2);                    cursor += 2;                    continue;                }                if (MatchesDefition(SyntaxKind.EqualToken, 2))                {                    MakeToken(SyntaxKind.EqualToken, cursor, 2);                    cursor += 2;                    continue;                }                if (MatchesDefition(SyntaxKind.InEqualToken, 2))                {                    MakeToken(SyntaxKind.InEqualToken, cursor, 2);                    cursor += 2;                    continue;                }                if (MatchesDefition(SyntaxKind.AccessorToken, 2))                {                    MakeToken(SyntaxKind.AccessorToken, cursor, 2);                    cursor += 2;                    continue;                }                if (LiteralTokens.Any(n=>MatchesDefition(n)))                {                    MakeToken(LiteralTokens.First(n=>MatchesDefition(n)));                    cursor++;                    continue;                }                if (MatchesDefition(SyntaxKind.AssignmentToken,2))                {                    MakeToken(SyntaxKind.AssignmentToken,cursor,2);                    cursor+=2;                    continue;                }                if (MatchesDefition(SyntaxKind.LiteralCharToken))                {                    int oldcursor = cursor;                    do                    {                        cursor++;                    }                    while (MatchesPattern(Patterns[NumberPattern]));                    if (MatchesDefition(SyntaxKind.LiteralCharToken))                        MakeToken(SyntaxKind.LiteralCharToken,oldcursor,cursor - oldcursor + 1);                    else                        throw new ShirException.TokenizerException.CountNotTokenizeCharException($char: {env.code.Substring(oldcursor, cursor - oldcursor + 1)} could not be tokenized);                    cursor++;                    continue;                }                if (MatchesDefition(SyntaxKind.LiteralStringToken))                {                    int oldcursor = cursor;                    do                    {                        cursor++;                    }                    while (MatchesPattern(Patterns[StringPattern]));                    if (MatchesDefition(SyntaxKind.LiteralStringToken))                        MakeToken(SyntaxKind.LiteralStringToken, oldcursor, cursor - oldcursor + 1);                    else                        throw new ShirException.TokenizerException.CountNotTokenizeCharException($char: {env.code.Substring(oldcursor, cursor - oldcursor + 1)} could not be tokenized);                    cursor++;                    continue;                }                if (MatchesDefition(SyntaxKind.LiteralNumberToken))                {                    int oldcursor = cursor;                    do                    {                        cursor++;                    }                    while (MatchesPattern(Patterns[NumberPattern]));                    MakeToken(SyntaxKind.LiteralNumberToken, oldcursor, cursor - oldcursor);                    continue;                }                if (MatchesPattern(Patterns[CharPattern]))                {                    int oldcursor = cursor;                    do                    {                        cursor++;                    }                    while (MatchesPattern(Patterns[NumCharPattern]));                    int len = cursor - oldcursor;                    string TokenString = env.code.Substring(oldcursor, len);                    if(Keywords.ContainsKey(TokenString))                    MakeToken(Keywords[TokenString], oldcursor, len);                    else                    {                        char nextchar = env.code[cursor];                        if (nextchar == '(')                            MakeToken(SyntaxKind.FunctionNameToken, oldcursor, len);                        else                            MakeToken(SyntaxKind.VariableNameToken, oldcursor, len);                    }                    continue;                }                if (savecursor == cursor)                    throw new ShirException.TokenizerException.CountNotTokenizeCharException($char: {env.code[cursor]} could not be tokenized);                cursor++;            }        }        public string AsCode()        {            return string.Join(,env.Tokens.Select(n=>n.GetValue()));        }        private void MakeToken(SyntaxKind type, int oldcursor, int length)        {            col += length;            if (type == SyntaxKind.EOLToken)            {                line++;                col = 0;            }            env.Tokens.Add(new Token(oldcursor, length, type, env,line,col));        }        private void MakeToken(SyntaxKind type, int length = 1)        {            col += length;            if (type == SyntaxKind.EOLToken)            {                line++;                col = 0;            }            env.Tokens.Add(new Token(cursor,length,type, env,line,col));        }    }}A Token:using System.Text.RegularExpressions;namespace ShirLanguageCompiler{    public enum SyntaxKind    {        // math operators        PlusOperationToken,        MinusOperationToken,        DivideOperationToken,        MultiplyOperationToken,        PowerOperationToken,        RootOperationToken,        //boolean tokens        EqualToken,        InEqualToken,        VariableNameToken,        FunctionNameToken,        // Variable type rokens        NumberKeyword,        BooleanKeyword,        LetterKeyword,        StringKeyword,        // Function related rokens        BindKeyword,        ReturnKeyword,        RefKeyword,        ValKeyword,        // Literal Values        LiteralTrueKeyword,        LiteralFalseKeyword,        LiteralNumberToken,        LiteralCharToken,        LiteralStringToken,        EOLToken,        QuoteToken,        ColonToken,        SemiColonToken,        CommaToken,        QuestionMarkToken,        WhitespaceToken,        AssignmentToken,        AccessorToken,        OpenParenthesisToken,        CloseParenthesisToken,        OpenCurlyBracketToken,        CloseCurlyBracketToken,        OpenSquareBracketToken,        CloseSquareBracketToken,    }    public class Token    {        ShirEnvironment env;        public int start { get; private set; }        public int length { get; private set; }        public int line, col;        public SyntaxKind type { get; private set; }        public Token(int _start, int _length, SyntaxKind _type,ShirEnvironment _env,int line,int col) {            this.start = _start;            this.length = _length;            this.type = _type;            this.env = _env;            this.line = line;            this.col = col;        }        public string GetLocation() => $<line:{line},col:{col}>;        public override string ToString() => $<{type}> start: {start} length: {length} value: {evaluate()};        public string evaluate() => $'{Regex.Escape(env.code.Substring(start, length))}';        public string GetValue() => env.code.Substring(start, length); //type == SyntaxKind.EOLToken ? Environment.NewLine :    }}
Compiler Tokenizer implementation in C#
c#;compiler;lexical analysis