Allow intercepting comments when tokenizing

It appears that this isn't necessary for the compiler at this point, but might be good to have for future tooling.

.gitignore

@@ -1,5 +1,4 @@
 node_modules/
 npm-debug.*
-dist/
 out/
 raw/

src/ast.ts

@@ -87,7 +87,8 @@ export enum NodeKind {
   // special
   DECORATOR,
   EXPORTMEMBER,
-  SWITCHCASE
+  SWITCHCASE,
+  COMMENT
 }
 
 /** Base class of all nodes. */
@@ -220,6 +221,18 @@ export abstract class Node {
     return stmt;
   }
 
+  static createComment(
+    text: string,
+    kind: CommentKind,
+    range: Range
+  ): CommentNode {
+    var node = new CommentNode();
+    node.range = range;
+    node.commentKind = kind;
+    node.text = text;
+    return node;
+  }
+
   // expressions
 
   static createIdentifierExpression(
@@ -1042,7 +1055,7 @@ export class SignatureNode extends CommonTypeNode {
 // special
 
 /** Built-in decorator kinds. */
-export const enum DecoratorKind {
+export enum DecoratorKind {
   CUSTOM,
   GLOBAL,
   OPERATOR,
@@ -1050,7 +1063,7 @@ export const enum DecoratorKind {
   OFFSET
 }
 
-/** Depresents a decorator. */
+/** Represents a decorator. */
 export class DecoratorNode extends Node {
   kind = NodeKind.DECORATOR;
 
@@ -1062,6 +1075,26 @@ export class DecoratorNode extends Node {
   arguments: Expression[] | null;
 }
 
+/** Comment kinds. */
+export enum CommentKind {
+  /** Line comment. */
+  LINE,
+  /** Triple-slash comment. */
+  TRIPLE,
+  /** Block comment. */
+  BLOCK
+}
+
+/** Represents a comment. */
+export class CommentNode extends Node {
+  kind = NodeKind.COMMENT;
+
+  /** Comment kind. */
+  commentKind: CommentKind;
+  /** Comment text. */
+  text: string;
+}
+
 // expressions
 
 /** Base class of all expression nodes. */
@@ -1076,7 +1109,7 @@ export class IdentifierExpression extends Expression {
 }
 
 /** Indicates the kind of a literal. */
-export const enum LiteralKind {
+export enum LiteralKind {
   FLOAT,
   INTEGER,
   STRING,
@@ -1102,7 +1135,7 @@ export class ArrayLiteralExpression extends LiteralExpression {
 }
 
 /** Indicates the kind of an assertion. */
-export const enum AssertionKind {
+export enum AssertionKind {
   PREFIX,
   AS
 }

src/tokenizer.ts

@@ -25,7 +25,8 @@ import {
 } from "./diagnostics";
 
 import {
-  Source
+  Source,
+  CommentKind
 } from "./ast";
 
 import {
@@ -392,6 +393,8 @@ export class Tokenizer extends DiagnosticEmitter {
   nextTokenPos: i32 = 0;
   nextTokenOnNewLine: bool = false;
 
+  onComment: ((kind: CommentKind, text: string, range: Range) => void) | null = null;
+
   constructor(source: Source, diagnostics: DiagnosticMessage[] | null = null) {
     super(diagnostics);
     this.source = source;
@@ -582,21 +585,31 @@ export class Tokenizer extends DiagnosticEmitter {
           return Token.DOT;
         }
         case CharCode.SLASH: {
+          let commentStartPos = this.pos;
           ++this.pos;
           if (maxTokenLength > 1 && this.pos < this.end) {
             if (text.charCodeAt(this.pos) == CharCode.SLASH) { // single-line
-              // TODO: triple-slash?
-              // if (
-              //   this.pos + 1 < this.end &&
-              //   text.charCodeAt(this.pos + 1) == CharCode.SLASH
-              // ) {
-              // }
+              let commentKind = CommentKind.LINE;
+              if (
+                this.pos + 1 < this.end &&
+                text.charCodeAt(this.pos + 1) == CharCode.SLASH
+              ) {
+                ++this.pos;
+                commentKind = CommentKind.TRIPLE;
+              }
               while (++this.pos < this.end) {
-                if (isLineBreak(text.charCodeAt(this.pos))) {
+                if (text.charCodeAt(this.pos) == CharCode.LINEFEED) {
                   ++this.pos;
                   break;
                 }
               }
+              if (this.onComment) {
+                this.onComment(
+                  commentKind,
+                  text.substring(commentStartPos, this.pos),
+                  this.range(commentStartPos, this.pos)
+                );
+              }
               break;
             }
             if (text.charCodeAt(this.pos) == CharCode.ASTERISK) { // multi-line
@@ -618,6 +631,12 @@ export class Tokenizer extends DiagnosticEmitter {
                   DiagnosticCode._0_expected,
                   this.range(this.pos), "*/"
                 );
+              } else if (this.onComment) {
+                this.onComment(
+                  CommentKind.BLOCK,
+                  text.substring(commentStartPos, this.pos),
+                  this.range(commentStartPos, this.pos)
+                );
               }
               break;
             }