fluencelabs/assemblyscript
1
0
Fork 0
mirror of https://github.com/fluencelabs/assemblyscript synced 2025-06-23 19:51:47 +00:00
Code Issues Projects Releases Wiki Activity

Handle numeric separators in integer literals while tokenizing

Browse Source
This commit is contained in:
dcodeIO
2018-06-02 13:39:24 +02:00
parent f69bccfe09
commit 0054eae491
7 changed files with 126 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/diagnosticMessages.generated.ts
View File

@ -111,7 +111,9 @@ export enum DiagnosticCode {
Expected_0_type_arguments_but_got_1 = 2558,
A_member_initializer_in_a_enum_declaration_cannot_reference_members_declared_after_it_including_members_defined_in_other_enums = 2651,
Namespace_0_has_no_exported_member_1 = 2694,
File_0_not_found = 6054
File_0_not_found = 6054,
Numeric_separators_are_not_allowed_here = 6188,
Multiple_consecutive_numeric_separators_are_not_permitted = 6189
}
/** Translates a diagnostic code to its respective string. */
@ -222,6 +224,8 @@ export function diagnosticCodeToString(code: DiagnosticCode): string {
case 2651: return "A member initializer in a enum declaration cannot reference members declared after it, including members defined in other enums.";
case 2694: return "Namespace '{0}' has no exported member '{1}'.";
case 6054: return "File '{0}' not found.";
case 6188: return "Numeric separators are not allowed here.";
case 6189: return "Multiple consecutive numeric separators are not permitted.";
default: return "";
}
}

4
src/diagnosticMessages.json
View File

@ -106,5 +106,7 @@
"A member initializer in a enum declaration cannot reference members declared after it, including members defined in other enums.": 2651,
"Namespace '{0}' has no exported member '{1}'.": 2694,
"File '{0}' not found.": 6054
"File '{0}' not found.": 6054,
"Numeric separators are not allowed here.": 6188,
"Multiple consecutive numeric separators are not permitted.": 6189
}

86
src/tokenizer.ts
View File

@ -1222,7 +1222,8 @@ export class Tokenizer extends DiagnosticEmitter {
if (c == CharCode.DOT || c == CharCode.E || c == CharCode.e) {
return false;
}
if (c < CharCode._0 || c > CharCode._9) break;
if ((c < CharCode._0 || c > CharCode._9) && c != CharCode._) break;
// does not validate separator placement (this is done in readXYInteger)
pos++;
}
return true;
@ -1267,8 +1268,10 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_16 = i64_new(16, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._9) {
// value = value * 16 + c - CharCode._0;
value = i64_add(
@ -1287,16 +1290,31 @@ export class Tokenizer extends DiagnosticEmitter {
i64_mul(value, i64_16),
i64_new(10 + c - CharCode.a, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Hexadecimal_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1306,24 +1324,41 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_10 = i64_new(10, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._9) {
// value = value * 10 + c - CharCode._0;
value = i64_add(
i64_mul(value, i64_10),
i64_new(c - CharCode._0, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1333,14 +1368,26 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_8 = i64_new(8, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._7) {
// value = value * 8 + c - CharCode._0;
value = i64_add(
i64_mul(value, i64_8),
i64_new(c - CharCode._0, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
@ -1351,6 +1398,11 @@ export class Tokenizer extends DiagnosticEmitter {
DiagnosticCode.Octal_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1361,8 +1413,10 @@ export class Tokenizer extends DiagnosticEmitter {
var value = i64_new(0, 0);
var i64_2 = i64_new(2, 0);
var i64_1 = i64_new(1, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c == CharCode._0) {
// value = value * 2;
value = i64_mul(
@ -1375,16 +1429,31 @@ export class Tokenizer extends DiagnosticEmitter {
i64_mul(value, i64_2),
i64_1
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Binary_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1404,6 +1473,7 @@ export class Tokenizer extends DiagnosticEmitter {
}
readDecimalFloat(): f64 {
// TODO: numeric separators (parseFloat can't handle these)
var start = this.pos;
var text = this.source.text;
while (this.pos < this.end && isDecimalDigit(text.charCodeAt(this.pos))) {