fluencelabs/assemblyscript
1
0
Fork 0
mirror of https://github.com/fluencelabs/assemblyscript synced 2025-04-25 07:02:13 +00:00
Code Issues Projects Releases Wiki Activity

Handle numeric separators in integer literals while tokenizing

Browse Source
This commit is contained in:
dcodeIO 2018-06-02 13:39:24 +02:00
parent f69bccfe09
commit 0054eae491
7 changed files with 126 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dist/assemblyscript.js vendored
View File

File diff suppressed because one or more lines are too long

2
dist/assemblyscript.js.map vendored
View File

File diff suppressed because one or more lines are too long

6
src/diagnosticMessages.generated.ts
View File

@ -111,7 +111,9 @@ export enum DiagnosticCode {
Expected_0_type_arguments_but_got_1 = 2558,
A_member_initializer_in_a_enum_declaration_cannot_reference_members_declared_after_it_including_members_defined_in_other_enums = 2651,
Namespace_0_has_no_exported_member_1 = 2694,
File_0_not_found = 6054
File_0_not_found = 6054,
Numeric_separators_are_not_allowed_here = 6188,
Multiple_consecutive_numeric_separators_are_not_permitted = 6189
}
/** Translates a diagnostic code to its respective string. */
@ -222,6 +224,8 @@ export function diagnosticCodeToString(code: DiagnosticCode): string {
case 2651: return "A member initializer in a enum declaration cannot reference members declared after it, including members defined in other enums.";
case 2694: return "Namespace '{0}' has no exported member '{1}'.";
case 6054: return "File '{0}' not found.";
case 6188: return "Numeric separators are not allowed here.";
case 6189: return "Multiple consecutive numeric separators are not permitted.";
default: return "";
}
}

4
src/diagnosticMessages.json
View File

@ -106,5 +106,7 @@
"A member initializer in a enum declaration cannot reference members declared after it, including members defined in other enums.": 2651,
"Namespace '{0}' has no exported member '{1}'.": 2694,
"File '{0}' not found.": 6054
"File '{0}' not found.": 6054,
"Numeric separators are not allowed here.": 6188,
"Multiple consecutive numeric separators are not permitted.": 6189
}

86
src/tokenizer.ts
View File

@ -1222,7 +1222,8 @@ export class Tokenizer extends DiagnosticEmitter {
if (c == CharCode.DOT || c == CharCode.E || c == CharCode.e) {
return false;
}
if (c < CharCode._0 || c > CharCode._9) break;
if ((c < CharCode._0 || c > CharCode._9) && c != CharCode._) break;
// does not validate separator placement (this is done in readXYInteger)
pos++;
}
return true;
@ -1267,8 +1268,10 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_16 = i64_new(16, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._9) {
// value = value * 16 + c - CharCode._0;
value = i64_add(
@ -1287,16 +1290,31 @@ export class Tokenizer extends DiagnosticEmitter {
i64_mul(value, i64_16),
i64_new(10 + c - CharCode.a, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Hexadecimal_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1306,24 +1324,41 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_10 = i64_new(10, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._9) {
// value = value * 10 + c - CharCode._0;
value = i64_add(
i64_mul(value, i64_10),
i64_new(c - CharCode._0, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1333,14 +1368,26 @@ export class Tokenizer extends DiagnosticEmitter {
var start = this.pos;
var value = i64_new(0, 0);
var i64_8 = i64_new(8, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c >= CharCode._0 && c <= CharCode._7) {
// value = value * 8 + c - CharCode._0;
value = i64_add(
i64_mul(value, i64_8),
i64_new(c - CharCode._0, 0)
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
@ -1351,6 +1398,11 @@ export class Tokenizer extends DiagnosticEmitter {
DiagnosticCode.Octal_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1361,8 +1413,10 @@ export class Tokenizer extends DiagnosticEmitter {
var value = i64_new(0, 0);
var i64_2 = i64_new(2, 0);
var i64_1 = i64_new(1, 0);
var sepEnd = start;
while (this.pos < this.end) {
let c = text.charCodeAt(this.pos);
let pos = this.pos;
let c = text.charCodeAt(pos);
if (c == CharCode._0) {
// value = value * 2;
value = i64_mul(
@ -1375,16 +1429,31 @@ export class Tokenizer extends DiagnosticEmitter {
i64_mul(value, i64_2),
i64_1
);
} else if (c == CharCode._) {
if (sepEnd == pos) {
this.error(
sepEnd == start
? DiagnosticCode.Numeric_separators_are_not_allowed_here
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
this.range(pos)
);
}
sepEnd = pos + 1;
} else {
break;
}
++this.pos;
this.pos = pos + 1;
}
if (this.pos == start) {
this.error(
DiagnosticCode.Binary_digit_expected,
this.range(start)
);
} else if (sepEnd == this.pos) {
this.error(
DiagnosticCode.Numeric_separators_are_not_allowed_here,
this.range(sepEnd - 1)
);
}
return value;
}
@ -1404,6 +1473,7 @@ export class Tokenizer extends DiagnosticEmitter {
}
readDecimalFloat(): f64 {
// TODO: numeric separators (parseFloat can't handle these)
var start = this.pos;
var text = this.source.text;
while (this.pos < this.end && isDecimalDigit(text.charCodeAt(this.pos))) {

18
tests/parser/numeric-separators.ts Normal file
View File

@ -0,0 +1,18 @@
11_11_11;
0b01_01_01;
0o12_12_12;
0x23_23_23;
// error cases that should still continue parsing:
11_11_11_; // 6188
11__11_11; // 6189
0b01_01_01_; // 6188
0b01__01_01; // 6189
0o12_12_12_; // 6188
0o12__12_12; // 6189
0x23_23_23_; // 6188
0x23__23_23; // 6189

20
tests/parser/numeric-separators.ts.fixture.ts Normal file
View File

@ -0,0 +1,20 @@
111111;
21;
41610;
2302755;
111111;
111111;
21;
21;
41610;
41610;
2302755;
2302755;
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts:8:8
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts:9:3
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts:11:10
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts:12:5
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts:14:10
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts:15:5
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts:17:10
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts:18:5