/*

 This is a modified version of TypeScript's scanner that doesn't perform as much bookkeeping, simply
 skips over trivia and provides a more general mark/reset mechanism for the parser to utilize on
 ambiguous tokens.

 next()                 advances the token
 peek()                 peeks for the next token
 skip(token)            skips over a token if possible
 mark()                 marks at current token
 reset()                resets to marked state
 range()                gets the range of the current token

 readFloat()            on FLOATLITERAL
 readIdentifier()       on IDENTIFIER
 readInteger()          on INTEGERLITERAL
 readString()           on STRINGLITERAL

*/

import {
  DiagnosticCode,
  DiagnosticMessage,
  DiagnosticEmitter
} from "./diagnostics";

import {
  Source,
  CommentKind
} from "./ast";

import {
  CharCode,
  isLineBreak,
  isWhiteSpace,
  isIdentifierStart,
  isIdentifierPart,
  isDecimalDigit,
  isOctalDigit,
  isKeywordCharacter
} from "./util/charcode";

/** Named token types. */
export enum Token {

  // keywords
  // discarded: ANY, BOOLEAN, NEVER, NUMBER, STRING, SYMBOL, UNDEFINED, LESSTHAN_SLASH

  ABSTRACT,
  AS,
  ASYNC,
  AWAIT,        // ES2017
  BREAK,        // ES2017
  CASE,         // ES2017
  CATCH,        // ES2017
  CLASS,        // ES2017
  CONST,        // ES2017
  CONTINUE,     // ES2017
  CONSTRUCTOR,
  DEBUGGER,     // ES2017
  DECLARE,
  DEFAULT,      // ES2017
  DELETE,       // ES2017
  DO,           // ES2017
  ELSE,         // ES2017
  ENUM,         // ES2017 future
  EXPORT,       // ES2017
  EXTENDS,      // ES2017
  FALSE,        // ES
  FINALLY,      // ES2017
  FOR,          // ES2017
  FROM,         // AS possible identifier
  FUNCTION,     // ES2017
  GET,
  IF,           // ES2017
  IMPLEMENTS,   // ES2017 non-lexical
  IMPORT,       // ES2017
  IN,           // ES2017
  INSTANCEOF,   // ES2017
  INTERFACE,    // ES2017 non-lexical
  IS,
  KEYOF,
  LET,          // ES2017 non-lexical
  MODULE,       // AS possible identifier
  NAMESPACE,    // AS possible identifier
  NEW,          // ES2017
  NULL,         // ES
  OF,
  PACKAGE,      // ES2017 non-lexical
  PRIVATE,      // ES2017 non-lexical
  PROTECTED,    // ES2017 non-lexical
  PUBLIC,       // ES2017 non-lexical
  READONLY,
  RETURN,       // ES2017
  SET,
  STATIC,       // ES2017 non-lexical
  SUPER,        // ES2017
  SWITCH,       // ES2017
  THIS,         // ES2017
  THROW,        // ES2017
  TRUE,         // ES
  TRY,          // ES2017
  TYPE,         // AS possible identifier
  TYPEOF,       // ES2017
  VAR,          // ES2017
  VOID,         // ES2017
  WHILE,        // ES2017
  WITH,         // ES2017
  YIELD,        // ES2017

  // punctuation

  OPENBRACE,
  CLOSEBRACE,
  OPENPAREN,
  CLOSEPAREN,
  OPENBRACKET,
  CLOSEBRACKET,
  DOT,
  DOT_DOT_DOT,
  SEMICOLON,
  COMMA,
  LESSTHAN,
  GREATERTHAN,
  LESSTHAN_EQUALS,
  GREATERTHAN_EQUALS,
  EQUALS_EQUALS,
  EXCLAMATION_EQUALS,
  EQUALS_EQUALS_EQUALS,
  EXCLAMATION_EQUALS_EQUALS,
  EQUALS_GREATERTHAN,
  PLUS,
  MINUS,
  ASTERISK_ASTERISK,
  ASTERISK,
  SLASH,
  PERCENT,
  PLUS_PLUS,
  MINUS_MINUS,
  LESSTHAN_LESSTHAN,
  GREATERTHAN_GREATERTHAN,
  GREATERTHAN_GREATERTHAN_GREATERTHAN,
  AMPERSAND,
  BAR,
  CARET,
  EXCLAMATION,
  TILDE,
  AMPERSAND_AMPERSAND,
  BAR_BAR,
  QUESTION,
  COLON,
  EQUALS,
  PLUS_EQUALS,
  MINUS_EQUALS,
  ASTERISK_EQUALS,
  ASTERISK_ASTERISK_EQUALS,
  SLASH_EQUALS,
  PERCENT_EQUALS,
  LESSTHAN_LESSTHAN_EQUALS,
  GREATERTHAN_GREATERTHAN_EQUALS,
  GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS,
  AMPERSAND_EQUALS,
  BAR_EQUALS,
  CARET_EQUALS,
  AT,

  // literals

  IDENTIFIER,
  STRINGLITERAL,
  INTEGERLITERAL,
  FLOATLITERAL,

  // meta

  INVALID,
  ENDOFFILE
}

export function tokenFomKeyword(text: string): Token {
  switch (text) {
    case "abstract": return Token.ABSTRACT;
    case "as": return Token.AS;
    case "async": return Token.ASYNC;
    case "await": return Token.AWAIT;
    case "break": return Token.BREAK;
    case "case": return Token.CASE;
    case "catch": return Token.CATCH;
    case "class": return Token.CLASS;
    case "continue": return Token.CONTINUE;
    case "const": return Token.CONST;
    case "constructor": return Token.CONSTRUCTOR;
    case "debugger": return Token.DEBUGGER;
    case "declare": return Token.DECLARE;
    case "default": return Token.DEFAULT;
    case "delete": return Token.DELETE;
    case "do": return Token.DO;
    case "else": return Token.ELSE;
    case "enum": return Token.ENUM;
    case "export": return Token.EXPORT;
    case "extends": return Token.EXTENDS;
    case "false": return Token.FALSE;
    case "finally": return Token.FINALLY;
    case "for": return Token.FOR;
    case "from": return Token.FROM;
    case "function": return Token.FUNCTION;
    case "get": return Token.GET;
    case "if": return Token.IF;
    case "implements": return Token.IMPLEMENTS;
    case "import": return Token.IMPORT;
    case "in": return Token.IN;
    case "instanceof": return Token.INSTANCEOF;
    case "interface": return Token.INTERFACE;
    case "is": return Token.IS;
    case "keyof": return Token.KEYOF;
    case "let": return Token.LET;
    case "module": return Token.MODULE;
    case "namespace": return Token.NAMESPACE;
    case "new": return Token.NEW;
    case "null": return Token.NULL;
    case "of": return Token.OF;
    case "package": return Token.PACKAGE;
    case "private": return Token.PRIVATE;
    case "protected": return Token.PROTECTED;
    case "public": return Token.PUBLIC;
    case "readonly": return Token.READONLY;
    case "return": return Token.RETURN;
    case "set": return Token.SET;
    case "static": return Token.STATIC;
    case "super": return Token.SUPER;
    case "switch": return Token.SWITCH;
    case "this": return Token.THIS;
    case "throw": return Token.THROW;
    case "true": return Token.TRUE;
    case "try": return Token.TRY;
    case "type": return Token.TYPE;
    case "typeof": return Token.TYPEOF;
    case "var": return Token.VAR;
    case "void": return Token.VOID;
    case "while": return Token.WHILE;
    case "with": return Token.WITH;
    case "yield": return Token.YIELD;
    default: return Token.INVALID;
  }
}

export function tokenIsAlsoIdentifier(token: Token): bool {
  switch (token) {
    case Token.ABSTRACT:
    case Token.AS:
    case Token.CONSTRUCTOR:
    case Token.DECLARE:
    case Token.DELETE:
    case Token.FROM:
    case Token.GET:
    case Token.IS:
    case Token.KEYOF:
    case Token.MODULE:
    case Token.NAMESPACE:
    case Token.READONLY:
    case Token.SET:
    case Token.TYPE: return true;
    default: return false;
  }
}

export function operatorTokenToString(token: Token): string {
  switch (token) {
    case Token.DELETE: return "delete";
    case Token.IN: return "in";
    case Token.INSTANCEOF: return "instanceof";
    case Token.NEW: return "new";
    case Token.TYPEOF: return "typeof";
    case Token.VOID: return "void";
    case Token.YIELD: return "yield";
    case Token.DOT_DOT_DOT: return "...";
    case Token.COMMA: return ",";
    case Token.LESSTHAN: return "<";
    case Token.GREATERTHAN: return ">";
    case Token.LESSTHAN_EQUALS: return "<=";
    case Token.GREATERTHAN_EQUALS: return ">=";
    case Token.EQUALS_EQUALS: return "==";
    case Token.EXCLAMATION_EQUALS: return "!=";
    case Token.EQUALS_EQUALS_EQUALS: return "===";
    case Token.EXCLAMATION_EQUALS_EQUALS: return "!==";
    case Token.PLUS: return "+";
    case Token.MINUS: return "-";
    case Token.ASTERISK_ASTERISK: return "**";
    case Token.ASTERISK: return "*";
    case Token.SLASH: return "/";
    case Token.PERCENT: return "%";
    case Token.PLUS_PLUS: return "++";
    case Token.MINUS_MINUS: return "--";
    case Token.LESSTHAN_LESSTHAN: return "<<";
    case Token.GREATERTHAN_GREATERTHAN: return ">>";
    case Token.GREATERTHAN_GREATERTHAN_GREATERTHAN: return ">>>";
    case Token.AMPERSAND: return "&";
    case Token.BAR: return "|";
    case Token.CARET: return "^";
    case Token.EXCLAMATION: return "!";
    case Token.TILDE: return "~";
    case Token.AMPERSAND_AMPERSAND: return "&&";
    case Token.BAR_BAR: return "||";
    case Token.EQUALS: return "=";
    case Token.PLUS_EQUALS: return "+=";
    case Token.MINUS_EQUALS: return "-=";
    case Token.ASTERISK_EQUALS: return "*=";
    case Token.ASTERISK_ASTERISK_EQUALS: return "**=";
    case Token.SLASH_EQUALS: return "/=";
    case Token.PERCENT_EQUALS: return "%=";
    case Token.LESSTHAN_LESSTHAN_EQUALS: return "<<=";
    case Token.GREATERTHAN_GREATERTHAN_EQUALS: return ">>=";
    case Token.GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS: return ">>>=";
    case Token.AMPERSAND_EQUALS: return "&=";
    case Token.BAR_EQUALS: return "|=";
    case Token.CARET_EQUALS: return "^=";
    default: {
      assert(false);
      return "";
    }
  }
}

export class Range {

  source: Source;
  start: i32;
  end: i32;

  // TODO: set these while tokenizing
  // line: i32;
  // column: i32;

  constructor(source: Source, start: i32, end: i32) {
    this.source = source;
    this.start = start;
    this.end = end;
  }

  static join(a: Range, b: Range): Range {
    if (a.source != b.source) throw new Error("source mismatch");
    return new Range(a.source,
      a.start < b.start ? a.start : b.start,
      a.end > b.end ? a.end : b.end
    );
  }

  get atStart(): Range {
    return new Range(this.source, this.start, this.start);
  }
  get atEnd(): Range {
    return new Range(this.source, this.end, this.end);
  }

  get line(): i32 {
    var text = this.source.text;
    var line = 1;
    for (let pos = this.start; pos >= 0; --pos) {
      if (text.charCodeAt(pos) == CharCode.LINEFEED) line++;
    }
    return line;
  }

  get column(): i32 {
    var text = this.source.text;
    var column = 0;
    for (let pos = this.start - 1; pos >= 0; --pos) {
      if (text.charCodeAt(pos) == CharCode.LINEFEED) break;
      ++column;
    }
    return column;
  }

  toString(): string {
    return this.source.text.substring(this.start, this.end);
  }

  debugInfoRef: usize = 0;
}

declare function parseFloat(str: string): f64;

export class Tokenizer extends DiagnosticEmitter {

  source: Source;
  end: i32 = 0;

  pos: i32 = 0;
  token: Token = -1;
  tokenPos: i32 = 0;

  nextToken: Token = -1;
  nextTokenPos: i32 = 0;
  nextTokenOnNewLine: bool = false;

  onComment: ((kind: CommentKind, text: string, range: Range) => void) | null = null;

  constructor(source: Source, diagnostics: DiagnosticMessage[] | null = null) {
    super(diagnostics);
    this.source = source;
    this.pos = 0;
    this.end = source.text.length;
    this.diagnostics = diagnostics ? diagnostics : new Array();

    var text = source.text;

    // skip bom
    if (
      this.pos < this.end &&
      text.charCodeAt(this.pos) == CharCode.BYTEORDERMARK
    ) {
      ++this.pos;
    }

    // skip shebang
    if (
      this.pos + 1 < this.end &&
      text.charCodeAt(this.pos) == CharCode.HASH &&
      text.charCodeAt(this.pos + 1) == CharCode.EXCLAMATION
    ) {
      this.pos += 2;
      while (
        this.pos < this.end &&
        text.charCodeAt(this.pos) != CharCode.LINEFEED
      ) {
        ++this.pos;
      }
      // 'next' now starts at lf or eof
    }
  }

  next(preferIdentifier: bool = false): Token {
    this.nextToken = -1;
    return this.token = this.unsafeNext(preferIdentifier);
  }

  private unsafeNext(preferIdentifier: bool = false, maxTokenLength: i32 = i32.MAX_VALUE): Token {
    var text = this.source.text;
    while (this.pos < this.end) {
      this.tokenPos = this.pos;
      let c = text.charCodeAt(this.pos);
      switch (c) {
        case CharCode.CARRIAGERETURN: {
          if (
            ++this.pos < this.end &&
            text.charCodeAt(this.pos) == CharCode.LINEFEED
          ) {
            ++this.pos;
          }
          break;
        }
        case CharCode.LINEFEED:
        case CharCode.TAB:
        case CharCode.VERTICALTAB:
        case CharCode.FORMFEED:
        case CharCode.SPACE: {
          ++this.pos;
          break;
        }
        case CharCode.EXCLAMATION: {
          ++this.pos;
          if (
            maxTokenLength > 1 && this.pos < this.end &&
            text.charCodeAt(this.pos) == CharCode.EQUALS
          ) {
            ++this.pos;
            if (
              maxTokenLength > 2 && this.pos < this.end &&
              text.charCodeAt(this.pos) == CharCode.EQUALS
            ) {
              ++this.pos;
              return Token.EXCLAMATION_EQUALS_EQUALS;
            }
            return Token.EXCLAMATION_EQUALS;
          }
          return Token.EXCLAMATION;
        }
        case CharCode.DOUBLEQUOTE:
        case CharCode.SINGLEQUOTE:
        case CharCode.BACKTICK: { // TODO
          return Token.STRINGLITERAL; // expects a call to readString
        }
        case CharCode.PERCENT: {
          ++this.pos;
          if (
            maxTokenLength > 1 && this.pos < this.end &&
            text.charCodeAt(this.pos) == CharCode.EQUALS
          ) {
            ++this.pos;
            return Token.PERCENT_EQUALS;
          }
          return Token.PERCENT;
        }
        case CharCode.AMPERSAND: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.AMPERSAND) {
              ++this.pos;
              return Token.AMPERSAND_AMPERSAND;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.AMPERSAND_EQUALS;
            }
          }
          return Token.AMPERSAND;
        }
        case CharCode.OPENPAREN: {
          ++this.pos;
          return Token.OPENPAREN;
        }
        case CharCode.CLOSEPAREN: {
          ++this.pos;
          return Token.CLOSEPAREN;
        }
        case CharCode.ASTERISK: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.ASTERISK_EQUALS;
            }
            if (text.charCodeAt(this.pos) == CharCode.ASTERISK) {
              ++this.pos;
              if (
                maxTokenLength > 2 && this.pos < this.end &&
                text.charCodeAt(this.pos) == CharCode.EQUALS
              ) {
                ++this.pos;
                return Token.ASTERISK_ASTERISK_EQUALS;
              }
              return Token.ASTERISK_ASTERISK;
            }
          }
          return Token.ASTERISK;
        }
        case CharCode.PLUS: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.PLUS) {
              ++this.pos;
              return Token.PLUS_PLUS;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.PLUS_EQUALS;
            }
          }
          return Token.PLUS;
        }
        case CharCode.COMMA: {
          ++this.pos;
          return Token.COMMA;
        }
        case CharCode.MINUS: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.MINUS) {
              ++this.pos;
              return Token.MINUS_MINUS;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.MINUS_EQUALS;
            }
          }
          return Token.MINUS;
        }
        case CharCode.DOT: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (isDecimalDigit(text.charCodeAt(this.pos))) {
              --this.pos;
              return Token.FLOATLITERAL; // expects a call to readFloat
            }
            if (
              maxTokenLength > 2 && this.pos + 1 < this.end &&
              text.charCodeAt(this.pos) == CharCode.DOT &&
              text.charCodeAt(this.pos + 1) == CharCode.DOT
            ) {
              this.pos += 2;
              return Token.DOT_DOT_DOT;
            }
          }
          return Token.DOT;
        }
        case CharCode.SLASH: {
          let commentStartPos = this.pos;
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.SLASH) { // single-line
              let commentKind = CommentKind.LINE;
              if (
                this.pos + 1 < this.end &&
                text.charCodeAt(this.pos + 1) == CharCode.SLASH
              ) {
                ++this.pos;
                commentKind = CommentKind.TRIPLE;
              }
              while (++this.pos < this.end) {
                if (text.charCodeAt(this.pos) == CharCode.LINEFEED) {
                  ++this.pos;
                  break;
                }
              }
              if (this.onComment) {
                this.onComment(
                  commentKind,
                  text.substring(commentStartPos, this.pos),
                  this.range(commentStartPos, this.pos)
                );
              }
              break;
            }
            if (text.charCodeAt(this.pos) == CharCode.ASTERISK) { // multi-line
              let closed = false;
              while (++this.pos < this.end) {
                c = text.charCodeAt(this.pos);
                if (
                  c == CharCode.ASTERISK &&
                  this.pos + 1 < this.end &&
                  text.charCodeAt(this.pos + 1) == CharCode.SLASH
                ) {
                  this.pos += 2;
                  closed = true;
                  break;
                }
              }
              if (!closed) {
                this.error(
                  DiagnosticCode._0_expected,
                  this.range(this.pos), "*/"
                );
              } else if (this.onComment) {
                this.onComment(
                  CommentKind.BLOCK,
                  text.substring(commentStartPos, this.pos),
                  this.range(commentStartPos, this.pos)
                );
              }
              break;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.SLASH_EQUALS;
            }
          }
          return Token.SLASH;
        }
        case CharCode._0:
        case CharCode._1:
        case CharCode._2:
        case CharCode._3:
        case CharCode._4:
        case CharCode._5:
        case CharCode._6:
        case CharCode._7:
        case CharCode._8:
        case CharCode._9: {
          return this.testInteger()
            ? Token.INTEGERLITERAL // expects a call to readInteger
            : Token.FLOATLITERAL;  // expects a call to readFloat
        }
        case CharCode.COLON: {
          ++this.pos;
          return Token.COLON;
        }
        case CharCode.SEMICOLON: {
          ++this.pos;
          return Token.SEMICOLON;
        }
        case CharCode.LESSTHAN: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.LESSTHAN) {
              ++this.pos;
              if (
                maxTokenLength > 2 &&
                this.pos < this.end &&
                text.charCodeAt(this.pos) == CharCode.EQUALS
              ) {
                ++this.pos;
                return Token.LESSTHAN_LESSTHAN_EQUALS;
              }
              return Token.LESSTHAN_LESSTHAN;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.LESSTHAN_EQUALS;
            }
          }
          return Token.LESSTHAN;
        }
        case CharCode.EQUALS: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              if (
                maxTokenLength > 2 &&
                this.pos < this.end &&
                text.charCodeAt(this.pos) == CharCode.EQUALS
              ) {
                ++this.pos;
                return Token.EQUALS_EQUALS_EQUALS;
              }
              return Token.EQUALS_EQUALS;
            }
            if (text.charCodeAt(this.pos) == CharCode.GREATERTHAN) {
              ++this.pos;
              return Token.EQUALS_GREATERTHAN;
            }
          }
          return Token.EQUALS;
        }
        case CharCode.GREATERTHAN: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.GREATERTHAN) {
              ++this.pos;
              if (maxTokenLength > 2 && this.pos < this.end) {
                if (text.charCodeAt(this.pos) == CharCode.GREATERTHAN) {
                  ++this.pos;
                  if (
                    maxTokenLength > 3 && this.pos < this.end &&
                    text.charCodeAt(this.pos) == CharCode.EQUALS
                  ) {
                    ++this.pos;
                    return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS;
                  }
                  return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN;
                }
                if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
                  ++this.pos;
                  return Token.GREATERTHAN_GREATERTHAN_EQUALS;
                }
              }
              return Token.GREATERTHAN_GREATERTHAN;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.GREATERTHAN_EQUALS;
            }
          }
          return Token.GREATERTHAN;
        }
        case CharCode.QUESTION: {
          ++this.pos;
          return Token.QUESTION;
        }
        case CharCode.OPENBRACKET: {
          ++this.pos;
          return Token.OPENBRACKET;
        }
        case CharCode.CLOSEBRACKET: {
          ++this.pos;
          return Token.CLOSEBRACKET;
        }
        case CharCode.CARET: {
          ++this.pos;
          if (
            maxTokenLength > 1 && this.pos < this.end &&
            text.charCodeAt(this.pos) == CharCode.EQUALS
          ) {
            ++this.pos;
            return Token.CARET_EQUALS;
          }
          return Token.CARET;
        }
        case CharCode.OPENBRACE: {
          ++this.pos;
          return Token.OPENBRACE;
        }
        case CharCode.BAR: {
          ++this.pos;
          if (maxTokenLength > 1 && this.pos < this.end) {
            if (text.charCodeAt(this.pos) == CharCode.BAR) {
              ++this.pos;
              return Token.BAR_BAR;
            }
            if (text.charCodeAt(this.pos) == CharCode.EQUALS) {
              ++this.pos;
              return Token.BAR_EQUALS;
            }
          }
          return Token.BAR;
        }
        case CharCode.CLOSEBRACE: {
          ++this.pos;
          return Token.CLOSEBRACE;
        }
        case CharCode.TILDE: {
          ++this.pos;
          return Token.TILDE;
        }
        case CharCode.AT: {
          ++this.pos;
          return Token.AT;
        }
        default: {
          if (isIdentifierStart(c)) {
            if (isKeywordCharacter(c)) {
              let posBefore = this.pos;
              while (
                ++this.pos < this.end &&
                isIdentifierPart(c = text.charCodeAt(this.pos))
              ) {
                if (!isKeywordCharacter(c)) {
                  this.pos = posBefore;
                  return Token.IDENTIFIER;
                }
              }
              let keywordText = text.substring(posBefore, this.pos);
              let keywordToken = tokenFomKeyword(keywordText);
              if (
                keywordToken != Token.INVALID &&
                !(preferIdentifier && tokenIsAlsoIdentifier(keywordToken))
              ) {
                return keywordToken;
              }
              this.pos = posBefore;
            }
            return Token.IDENTIFIER; // expects a call to readIdentifier
          } else if (isWhiteSpace(c)) {
            ++this.pos;
            break;
          }
          this.error(
            DiagnosticCode.Invalid_character,
            this.range(this.pos, this.pos + 1)
          );
          ++this.pos;
          return Token.INVALID;
        }
      }
    }
    return Token.ENDOFFILE;
  }

  peek(
    checkOnNewLine: bool = false,
    preferIdentifier: bool = false,
    maxCompoundLength: i32 = i32.MAX_VALUE
  ): Token {
    var text = this.source.text;
    if (this.nextToken < 0) {
      let posBefore = this.pos;
      let tokenBefore = this.token;
      let tokenPosBefore = this.tokenPos;
      this.nextToken = this.unsafeNext(preferIdentifier, maxCompoundLength);
      this.nextTokenPos = this.tokenPos;
      if (checkOnNewLine) {
        this.nextTokenOnNewLine = false;
        while (--this.tokenPos > posBefore) {
          if (isLineBreak(text.charCodeAt(this.tokenPos))) {
            this.nextTokenOnNewLine = true;
            break;
          }
        }
      }
      this.pos = posBefore;
      this.token = tokenBefore;
      this.tokenPos = tokenPosBefore;
    }
    return this.nextToken;
  }

  skip(token: Token): bool {
    var posBefore = this.pos;
    var tokenBefore = this.token;
    var tokenPosBefore = this.tokenPos;
    var maxCompoundLength = i32.MAX_VALUE;
    switch (token) {
      case Token.GREATERTHAN: { // where parsing type arguments
        maxCompoundLength = 1;
        break;
      }
    }
    this.token = this.unsafeNext(token == Token.IDENTIFIER, maxCompoundLength);
    if (this.token == token) {
      this.nextToken = -1;
      return true;
    } else {
      this.pos = posBefore;
      this.token = tokenBefore;
      this.tokenPos = tokenPosBefore;
      return false;
    }
  }

  mark(): State {
    var state: State;
    if (reusableState) {
      state = reusableState;
      reusableState = null;
    } else {
      state = new State();
    }
    state.pos = this.pos;
    state.token = this.token;
    state.tokenPos = this.tokenPos;
    return state;
  }

  discard(state: State): void {
    reusableState = state;
  }

  reset(state: State): void {
    this.pos = state.pos;
    this.token = state.token;
    this.tokenPos = state.tokenPos;
    this.nextToken = -1;
    reusableState = state;
  }

  range(start: i32 = -1, end: i32 = -1): Range {
    if (start < 0) {
      start = this.tokenPos;
      end = this.pos;
    } else if (end < 0) {
      end = start;
    }
    return new Range(this.source, start, end);
  }

  readIdentifier(): string {
    var text = this.source.text;
    var start = this.pos;
    while (
      ++this.pos < this.end &&
      isIdentifierPart(text.charCodeAt(this.pos))
    );
    return text.substring(start, this.pos);
  }

  readString(): string {
    var text = this.source.text;
    var quote = text.charCodeAt(this.pos++);
    var start = this.pos;
    var result = "";
    while (true) {
      if (this.pos >= this.end) {
        result += text.substring(start, this.pos);
        this.error(
          DiagnosticCode.Unterminated_string_literal,
          this.range(start - 1, this.end)
        );
        break;
      }
      let c = text.charCodeAt(this.pos);
      if (c == quote) {
        result += text.substring(start, this.pos++);
        break;
      }
      if (c == CharCode.BACKSLASH) {
        result += text.substring(start, this.pos);
        result += this.readEscapeSequence();
        start = this.pos;
        continue;
      }
      if (isLineBreak(c)) {
        result += text.substring(start, this.pos);
        this.error(
          DiagnosticCode.Unterminated_string_literal,
          this.range(start - 1, this.pos)
        );
        break;
      }
      ++this.pos;
    }
    return result;
  }

  readEscapeSequence(): string {
    if (++this.pos >= this.end) {
      this.error(
        DiagnosticCode.Unexpected_end_of_text,
        this.range(this.end)
      );
      return "";
    }

    var text = this.source.text;
    var c = text.charCodeAt(this.pos++);
    switch (c) {
      case CharCode._0: return "\0";
      case CharCode.b: return "\b";
      case CharCode.t: return "\t";
      case CharCode.n: return "\n";
      case CharCode.v: return "\v";
      case CharCode.f: return "\f";
      case CharCode.r: return "\r";
      case CharCode.SINGLEQUOTE: return "'";
      case CharCode.DOUBLEQUOTE: return "\"";
      case CharCode.u: {
        if (
          this.pos < this.end &&
          text.charCodeAt(this.pos) == CharCode.OPENBRACE
        ) {
          ++this.pos;
          return this.readExtendedUnicodeEscape(); // \u{DDDDDDDD}
        }
        return this.readUnicodeEscape(); // \uDDDD
      }
      case CharCode.CARRIAGERETURN: {
        if (
          this.pos < this.end &&
          text.charCodeAt(this.pos) == CharCode.LINEFEED
        ) {
          ++this.pos;
        }
        // fall through
      }
      case CharCode.LINEFEED:
      case CharCode.LINESEPARATOR:
      case CharCode.PARAGRAPHSEPARATOR: return "";
      default: return String.fromCharCode(c);
    }
  }

  readRegexpPattern(): string {
    var text = this.source.text;
    var start = this.pos;
    var escaped = false;
    while (true) {
      if (this.pos >= this.end) {
        this.error(
          DiagnosticCode.Unterminated_regular_expression_literal,
          this.range(start, this.end)
        );
        break;
      }
      if (text.charCodeAt(this.pos) == CharCode.BACKSLASH) {
        ++this.pos;
        escaped = true;
        continue;
      }
      let c = text.charCodeAt(this.pos);
      if (c == CharCode.SLASH && !escaped) break;
      if (isLineBreak(c)) {
        this.error(
          DiagnosticCode.Unterminated_regular_expression_literal,
          this.range(start, this.pos)
        );
        break;
      }
      ++this.pos;
      escaped = false;
    }
    return text.substring(start, this.pos);
  }

  readRegexpFlags(): string {
    var text = this.source.text;
    var start = this.pos;
    var flags = 0;
    while (this.pos < this.end) {
      let c: i32 = text.charCodeAt(this.pos);
      if (!isIdentifierPart(c)) break;
      ++this.pos;

      // make sure each supported flag is unique
      switch (c) {
        case CharCode.g: {
          flags |= flags & 1 ? -1 : 1;
          break;
        }
        case CharCode.i: {
          flags |= flags & 2 ? -1 : 2;
          break;
        }
        case CharCode.m: {
          flags |= flags & 4 ? -1 : 4;
          break;
        }
        default: {
          flags = -1;
          break;
        }
      }
    }
    if (flags == -1) {
      this.error(
        DiagnosticCode.Invalid_regular_expression_flags,
        this.range(start, this.pos)
      );
    }
    return text.substring(start, this.pos);
  }

  testInteger(): bool {
    var text = this.source.text;
    if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 1 < this.end) {
      switch (text.charCodeAt(this.pos + 2)) {
        case CharCode.X:
        case CharCode.x:
        case CharCode.B:
        case CharCode.b:
        case CharCode.O:
        case CharCode.o: return true;
      }
    }
    var pos = this.pos;
    while (pos < this.end) {
      let c = text.charCodeAt(pos);
      if (c == CharCode.DOT || c == CharCode.E || c == CharCode.e) {
        return false;
      }
      if (c < CharCode._0 || c > CharCode._9) break;
      pos++;
    }
    return true;
  }

  readInteger(): I64 {
    var text = this.source.text;
    if (text.charCodeAt(this.pos) == CharCode._0 && this.pos + 2 < this.end) {
      switch (text.charCodeAt(this.pos + 1)) {
        case CharCode.X:
        case CharCode.x: {
          this.pos += 2;
          return this.readHexInteger();
        }
        case CharCode.B:
        case CharCode.b: {
          this.pos += 2;
          return this.readBinaryInteger();
        }
        case CharCode.O:
        case CharCode.o: {
          this.pos += 2;
          return this.readOctalInteger();
        }
      }
      if (isOctalDigit(text.charCodeAt(this.pos + 1))) {
        let start = this.pos;
        ++this.pos;
        let value = this.readOctalInteger();
        this.error(
          DiagnosticCode.Octal_literals_are_not_allowed_in_strict_mode,
          this.range(start, this.pos)
        );
        return value;
      }
    }
    return this.readDecimalInteger();
  }

  readHexInteger(): I64 {
    var text = this.source.text;
    var start = this.pos;
    var value = i64_new(0, 0);
    var i64_16 = i64_new(16, 0);
    while (this.pos < this.end) {
      let c = text.charCodeAt(this.pos);
      if (c >= CharCode._0 && c <= CharCode._9) {
        // value = value * 16 + c - CharCode._0;
        value = i64_add(
          i64_mul(value, i64_16),
          i64_new(c - CharCode._0, 0)
        );
       } else if (c >= CharCode.A && c <= CharCode.F) {
        // value = value * 16 + 10 + c - CharCode.A;
        value = i64_add(
          i64_mul(value, i64_16),
          i64_new(10 + c - CharCode.A, 0)
        );
      } else if (c >= CharCode.a && c <= CharCode.f) {
        // value = value * 16 + 10 + c - CharCode.a;
        value = i64_add(
          i64_mul(value, i64_16),
          i64_new(10 + c - CharCode.a, 0)
        );
      } else {
        break;
      }
      ++this.pos;
    }
    if (this.pos == start) {
      this.error(
        DiagnosticCode.Hexadecimal_digit_expected,
        this.range(start)
      );
    }
    return value;
  }

  readDecimalInteger(): I64 {
    var text = this.source.text;
    var start = this.pos;
    var value = i64_new(0, 0);
    var i64_10 = i64_new(10, 0);
    while (this.pos < this.end) {
      let c = text.charCodeAt(this.pos);
      if (c >= CharCode._0 && c <= CharCode._9) {
        // value = value * 10 + c - CharCode._0;
        value = i64_add(
          i64_mul(value, i64_10),
          i64_new(c - CharCode._0, 0)
        );
      } else {
        break;
      }
      ++this.pos;
    }
    if (this.pos == start) {
      this.error(
        DiagnosticCode.Digit_expected,
        this.range(start)
      );
    }
    return value;
  }

  readOctalInteger(): I64 {
    var text = this.source.text;
    var start = this.pos;
    var value = i64_new(0, 0);
    var i64_8 = i64_new(8, 0);
    while (this.pos < this.end) {
      let c = text.charCodeAt(this.pos);
      if (c >= CharCode._0 && c <= CharCode._7) {
        // value = value * 8 + c - CharCode._0;
        value = i64_add(
          i64_mul(value, i64_8),
          i64_new(c - CharCode._0, 0)
        );
      } else {
        break;
      }
      ++this.pos;
    }
    if (this.pos == start) {
      this.error(
        DiagnosticCode.Octal_digit_expected,
        this.range(start)
      );
    }
    return value;
  }

  readBinaryInteger(): I64 {
    var text = this.source.text;
    var start = this.pos;
    var value = i64_new(0, 0);
    var i64_2 = i64_new(2, 0);
    var i64_1 = i64_new(1, 0);
    while (this.pos < this.end) {
      let c = text.charCodeAt(this.pos);
      if (c == CharCode._0) {
        // value = value * 2;
        value = i64_mul(
          value,
          i64_2
        );
      } else if (c == CharCode._1) {
        // value = value * 2 + 1;
        value = i64_add(
          i64_mul(value, i64_2),
          i64_1
        );
      } else {
        break;
      }
      ++this.pos;
    }
    if (this.pos == start) {
      this.error(
        DiagnosticCode.Binary_digit_expected,
        this.range(start)
      );
    }
    return value;
  }

  readFloat(): f64 {
    var start = this.pos;
    var text = this.source.text;
    while (this.pos < this.end && isDecimalDigit(text.charCodeAt(this.pos))) {
      ++this.pos;
    }
    if (this.pos < this.end && text.charCodeAt(this.pos) == CharCode.DOT) {
      ++this.pos;
      while (this.pos < this.end && isDecimalDigit(text.charCodeAt(this.pos))) {
        ++this.pos;
      }
    }
    if (this.pos < this.end) {
      let c = text.charCodeAt(this.pos);
      if (c == CharCode.E || c == CharCode.e) {
        if (
          ++this.pos < this.end && (
            text.charCodeAt(this.pos) == CharCode.MINUS ||
            text.charCodeAt(this.pos) == CharCode.PLUS
          ) &&
          isDecimalDigit(text.charCodeAt(this.pos + 1))
        ) {
          ++this.pos;
        }
        while (this.pos < this.end && isDecimalDigit(text.charCodeAt(this.pos))) {
          ++this.pos;
        }
      }
    }
    return parseFloat(text.substring(start, this.pos));
  }

  readUnicodeEscape(): string {
    var remain = 4;
    var value = 0;
    var text = this.source.text;
    while (this.pos < this.end) {
      let c = text.charCodeAt(this.pos++);
      if (c >= CharCode._0 && c <= CharCode._9) {
        value = value * 16 + c - CharCode._0;
      } else if (c >= CharCode.A && c <= CharCode.F) {
        value = value * 16 + 10 + c - CharCode.A;
      } else if (c >= CharCode.a && c <= CharCode.f) {
        value = value * 16 + 10 + c - CharCode.a;
      } else {
        this.error(
          DiagnosticCode.Hexadecimal_digit_expected,
          this.range(this.pos - 1, this.pos)
        );
        return "";
      }
      if (--remain == 0) break;
    }
    if (remain) {
      this.error(
        DiagnosticCode.Unexpected_end_of_text,
        this.range(this.pos)
      );
      return "";
    }
    return String.fromCharCode(value);
  }

  private readExtendedUnicodeEscape(): string {
    var start = this.pos;
    var value = this.readHexInteger();
    var value32 = i64_low(value);
    var invalid = false;

    assert(!i64_high(value));
    if (value32 > 0x10FFFF) {
      this.error(
        DiagnosticCode.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive,
        this.range(start, this.pos)
      );
      invalid = true;
    }

    var text = this.source.text;
    if (this.pos >= this.end) {
      this.error(
        DiagnosticCode.Unexpected_end_of_text,
        this.range(start, this.end)
      );
      invalid = true;
    } else if (text.charCodeAt(this.pos) == CharCode.CLOSEBRACE) {
      ++this.pos;
    } else {
      this.error(
        DiagnosticCode.Unterminated_Unicode_escape_sequence,
        this.range(start, this.pos)
      );
      invalid = true;
    }

    if (invalid) return "";
    return value32 < 65536
      ? String.fromCharCode(value32)
      : String.fromCharCode(
        (((value32 - 65536) / 1024 | 0) + 0xD800) as i32,
        ( (value32 - 65536) % 1024      + 0xDC00) as i32
      );
  }

  finish(): void {
  }
}

/** Tokenizer state as returned by {@link Tokenizer#mark} and consumed by {@link Tokenizer#reset}. */
export class State {
  /** Current position. */
  pos: i32;
  /** Current token. */
  token: Token;
  /** Current token's position. */
  tokenPos: i32;
}

// Reusable state object to reduce allocations
var reusableState: State | null = null;