Optional chaining (#546)

* Lexer support for optional chaining

* Parsing and transpile support

* Fix optional chaning indexed get and ternary

* re-enable failing tests.

* Addresses PR items

* Remove ?(

* Fixes for optional chaining tokens.

* Fixes

* another optional chain vs ternary test

* Add disclaimer to ternary docs

* add transpile tests for ?@ and @(

* Re-enable simple consequents tests

* Fix leading ? for print statements

Author: TwitchBronBron

docs/ternary-operator.md

@@ -1,5 +1,9 @@
 # Ternary (Conditional) Operator: ?
-The ternary (conditional) operator is the only BrighterScript operator that takes three operands: a condition followed by a question mark (?), then an expression to execute (consequent) if the condition is true followed by a colon (:), and finally the expression to execute (alternate) if the condition is false. This operator is frequently used as a shortcut for the if statement. It can be used in assignments, and in any other place where an expression is valid. Due to ambiguity in the brightscript syntax, ternary operators cannot be used as standalone statements. See the [No standalone statements](#no-standalone-statements) for more information.
+The ternary (conditional) operator is the only BrighterScript operator that takes three operands: a condition followed by a question mark (?), then an expression to execute (consequent) if the condition is true followed by a colon (:), and finally the expression to execute (alternate) if the condition is false. This operator is frequently used as a shortcut for the if statement. It can be used in assignments, and in any other place where an expression is valid. Due to ambiguity in the brightscript syntax, ternary operators cannot be used as standalone statements. See the [No standalone statements](#no-standalone-statements) section for more information.
+
+## Warning
+<p style="background-color: #fdf8e3; color: #333; padding: 20px">The <a href="https://developer.roku.com/docs/references/brightscript/language/expressions-variables-types.md#optional-chaining-operators">optional chaining operator</a> was added to the BrightScript runtime in <a href="https://developer.roku.com/docs/developer-program/release-notes/roku-os-release-notes.md#roku-os-110">Roku OS 11</a>, which introduced a slight limitation to the BrighterScript ternary operator. As such, all ternary expressions must have a space to the right of the question mark when followed by <b>[</b> or <b>(</b>. See the <a href="#">optional chaning</a> section for more information.
+</p>
 
 ## Basic usage
 
@@ -102,7 +106,7 @@ a = (function(__bsCondition, getNoNameMessage, m, user)
     end function)(user = invalid, getNoNameMessage, m, user)
 ```
 
-### nested scope protection
+### Nested Scope Protection
 The scope protection works for multiple levels as well
 ```BrighterScript
 m.count = 1
@@ -174,3 +178,45 @@ a = (myValue ? "a" : "b'")
 ```
 
 This ambiguity is why BrighterScript does not allow for standalone ternary statements.
+
+
+## Optional Chaining considerations
+The [optional chaining operator](https://developer.roku.com/docs/references/brightscript/language/expressions-variables-types.md#optional-chaining-operators) was added to the BrightScript runtime in <a href="https://developer.roku.com/docs/developer-program/release-notes/roku-os-release-notes.md#roku-os-110">Roku OS 11</a>, which introduced a slight limitation to the BrighterScript ternary operator. As such, all ternary expressions must have a space to the right of the question mark when followed by `[` or `(`. If there's no space, then it's optional chaining.
+
+For example:
+
+*Ternary:*
+```brightscript
+data = isTrue ? ["key"] : getFalseData()
+data = isTrue ? (1 + 2) : getFalseData()
+```
+*Optional chaining:*
+```brightscript
+data = isTrue ?["key"] : getFalseData()
+data = isTrue ?(1 + 2) : getFalseData()
+```
+
+The colon symbol `:` can be used in BrightScript to include multiple statements on a single line. So, let's look at the first ternary statement again.
+```brightscript
+data = isTrue ? ["key"] : getFalseData()
+```
+
+This can be logically rewritten as:
+```brightscript
+if isTrue then
+    data = ["key"]
+else
+    data = getFalseData()
+```
+
+Now consider the first optional chaining example:
+```brightscript
+data = isTrue ?["key"] : getFalseData()
+```
+This can be logically rewritten as:
+```brightscript
+data = isTrue ?["key"]
+getFalseData()
+```
+
+Both examples have valid use cases, so just remember that a single space could result in significantly different code output.

src/files/tests/optionalChaning.spec.ts

@@ -0,0 +1,94 @@
+import * as sinonImport from 'sinon';
+import * as fsExtra from 'fs-extra';
+import { Program } from '../../Program';
+import { standardizePath as s } from '../../util';
+import { getTestTranspile } from '../../testHelpers.spec';
+
+let sinon = sinonImport.createSandbox();
+let tmpPath = s`${process.cwd()}/.tmp`;
+let rootDir = s`${tmpPath}/rootDir`;
+let stagingFolderPath = s`${tmpPath}/staging`;
+
+describe('optional chaining', () => {
+    let program: Program;
+    const testTranspile = getTestTranspile(() => [program, rootDir]);
+
+    beforeEach(() => {
+        fsExtra.ensureDirSync(tmpPath);
+        fsExtra.emptyDirSync(tmpPath);
+        program = new Program({
+            rootDir: rootDir,
+            stagingFolderPath: stagingFolderPath
+        });
+    });
+    afterEach(() => {
+        sinon.restore();
+        fsExtra.ensureDirSync(tmpPath);
+        fsExtra.emptyDirSync(tmpPath);
+        program.dispose();
+    });
+
+    it('transpiles ?. properly', () => {
+        testTranspile(`
+            sub main()
+                print m?.value
+            end sub
+        `);
+    });
+
+    it('transpiles ?[ properly', () => {
+        testTranspile(`
+            sub main()
+                print m?["value"]
+            end sub
+        `);
+    });
+
+    it(`transpiles '?.[`, () => {
+        testTranspile(`
+            sub main()
+                print m?["value"]
+            end sub
+        `);
+    });
+
+    it(`transpiles '?@`, () => {
+        testTranspile(`
+            sub main()
+                print xmlThing?@someAttr
+            end sub
+        `);
+    });
+
+    it(`transpiles '?(`, () => {
+        testTranspile(`
+            sub main()
+                localFunc = sub()
+                end sub
+                print localFunc?()
+                print m.someFunc?()
+            end sub
+        `);
+    });
+
+    it('transpiles various use cases', () => {
+        testTranspile(`
+            print arr?.["0"]
+            print arr?.value
+            print assocArray?.[0]
+            print assocArray?.getName()?.first?.second
+            print createObject("roByteArray")?.value
+            print createObject("roByteArray")?["0"]
+            print createObject("roList")?.value
+            print createObject("roList")?["0"]
+            print createObject("roXmlList")?["0"]
+            print createObject("roDateTime")?.value
+            print createObject("roDateTime")?.GetTimeZoneOffset
+            print createObject("roSGNode", "Node")?[0]
+            print pi?.first?.second
+            print success?.first?.second
+            print a.b.xmlThing?@someAttr
+            print a.b.localFunc?()
+        `);
+    });
+});

src/lexer/Lexer.spec.ts

@@ -20,6 +20,67 @@ describe('lexer', () => {
         ]);
     });
 
+    it('recognizes the question mark operator in various contexts', () => {
+        expectKinds('? ?? ?. ?[ ?.[ ?( ?@', [
+            TokenKind.Question,
+            TokenKind.QuestionQuestion,
+            TokenKind.QuestionDot,
+            TokenKind.QuestionLeftSquare,
+            TokenKind.QuestionDot,
+            TokenKind.LeftSquareBracket,
+            TokenKind.QuestionLeftParen,
+            TokenKind.QuestionAt
+        ]);
+    });
+
+    it('separates optional chain characters and LeftSquare when found at beginning of statement locations', () => {
+        //a statement starting with a question mark is actually a print statement, so we need to keep the ? separate from [
+        expectKinds(`?[ ?[ : ?[ ?[`, [
+            TokenKind.Question,
+            TokenKind.LeftSquareBracket,
+            TokenKind.QuestionLeftSquare,
+            TokenKind.Colon,
+            TokenKind.Question,
+            TokenKind.LeftSquareBracket,
+            TokenKind.QuestionLeftSquare
+        ]);
+    });
+
+    it('separates optional chain characters and LeftParen when found at beginning of statement locations', () => {
+        //a statement starting with a question mark is actually a print statement, so we need to keep the ? separate from [
+        expectKinds(`?( ?( : ?( ?(`, [
+            TokenKind.Question,
+            TokenKind.LeftParen,
+            TokenKind.QuestionLeftParen,
+            TokenKind.Colon,
+            TokenKind.Question,
+            TokenKind.LeftParen,
+            TokenKind.QuestionLeftParen
+        ]);
+    });
+
+    it('handles QuestionDot and Square properly', () => {
+        expectKinds('?.[ ?. [', [
+            TokenKind.QuestionDot,
+            TokenKind.LeftSquareBracket,
+            TokenKind.QuestionDot,
+            TokenKind.LeftSquareBracket
+        ]);
+    });
+
+    it('does not make conditional chaining tokens with space between', () => {
+        expectKinds('? . ? [ ? ( ? @', [
+            TokenKind.Question,
+            TokenKind.Dot,
+            TokenKind.Question,
+            TokenKind.LeftSquareBracket,
+            TokenKind.Question,
+            TokenKind.LeftParen,
+            TokenKind.Question,
+            TokenKind.At
+        ]);
+    });
+
     it('recognizes the callfunc operator', () => {
         let { tokens } = Lexer.scan('@.');
         expect(tokens[0].kind).to.equal(TokenKind.Callfunc);
@@ -35,11 +96,6 @@ describe('lexer', () => {
         expect(tokens[0].kind).to.eql(TokenKind.Library);
     });
 
-    it('recognizes the question mark operator', () => {
-        let { tokens } = Lexer.scan('?');
-        expect(tokens[0].kind).to.equal(TokenKind.Question);
-    });
-
     it('produces an at symbol token', () => {
         let { tokens } = Lexer.scan('@');
         expect(tokens[0].kind).to.equal(TokenKind.At);
@@ -1306,3 +1362,10 @@ describe('lexer', () => {
         });
     });
 });
+
+function expectKinds(text: string, tokenKinds: TokenKind[]) {
+    let actual = Lexer.scan(text).tokens.map(x => x.kind);
+    //remove the EOF token
+    actual.pop();
+    expect(actual).to.eql(tokenKinds);
+}

src/lexer/Lexer.ts

@@ -277,12 +277,45 @@ export class Lexer {
             if (this.peek() === '?') {
                 this.advance();
                 this.addToken(TokenKind.QuestionQuestion);
+            } else if (this.peek() === '.') {
+                this.advance();
+                this.addToken(TokenKind.QuestionDot);
+            } else if (this.peek() === '[' && !this.isStartOfStatement()) {
+                this.advance();
+                this.addToken(TokenKind.QuestionLeftSquare);
+            } else if (this.peek() === '(' && !this.isStartOfStatement()) {
+                this.advance();
+                this.addToken(TokenKind.QuestionLeftParen);
+            } else if (this.peek() === '@') {
+                this.advance();
+                this.addToken(TokenKind.QuestionAt);
             } else {
                 this.addToken(TokenKind.Question);
             }
         }
     };
 
+    /**
+     * Determine if the current position is at the beginning of a statement.
+     * This means the token to the left, excluding whitespace, is either a newline or a colon
+     */
+    private isStartOfStatement() {
+        for (let i = this.tokens.length - 1; i >= 0; i--) {
+            const token = this.tokens[i];
+            //skip whitespace
+            if (token.kind === TokenKind.Whitespace) {
+                continue;
+            }
+            if (token.kind === TokenKind.Newline || token.kind === TokenKind.Colon) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+        //if we got here, there were no tokens or only whitespace, so it's the start of the file
+        return true;
+    }
+
     /**
      * Map for looking up token kinds based solely on a single character.
      * Should be used in conjunction with `tokenFunctionMap`

src/lexer/TokenKind.ts

@@ -77,7 +77,10 @@ export enum TokenKind {
     Question = 'Question', // ?
     QuestionQuestion = 'QuestionQuestion', // ??
     BackTick = 'BackTick', // `
-
+    QuestionDot = 'QuestionDot', // ?.
+    QuestionLeftSquare = 'QuestionLeftSquare', // ?[
+    QuestionLeftParen = 'QuestionLeftParen', // ?(
+    QuestionAt = 'QuestionAt', // ?@
 
     // conditional compilation
     HashIf = 'HashIf', // #if

src/parser/Expression.ts

@@ -69,6 +69,9 @@ export class CallExpression extends Expression {
 
     constructor(
         readonly callee: Expression,
+        /**
+         * Can either be `(`, or `?(` for optional chaining
+         */
         readonly openingParen: Token,
         readonly closingParen: Token,
         readonly args: Expression[],
@@ -368,6 +371,9 @@ export class DottedGetExpression extends Expression {
     constructor(
         readonly obj: Expression,
         readonly name: Identifier,
+        /**
+         * Can either be `.`, or `?.` for optional chaining
+         */
         readonly dot: Token
     ) {
         super();
@@ -383,7 +389,7 @@ export class DottedGetExpression extends Expression {
         } else {
             return [
                 ...this.obj.transpile(state),
-                '.',
+                state.transpileToken(this.dot),
                 state.transpileToken(this.name)
             ];
         }
@@ -400,6 +406,9 @@ export class XmlAttributeGetExpression extends Expression {
     constructor(
         readonly obj: Expression,
         readonly name: Identifier,
+        /**
+         * Can either be `@`, or `?@` for optional chaining
+         */
         readonly at: Token
     ) {
         super();
@@ -411,7 +420,7 @@ export class XmlAttributeGetExpression extends Expression {
     transpile(state: BrsTranspileState) {
         return [
             ...this.obj.transpile(state),
-            '@',
+            state.transpileToken(this.at),
             state.transpileToken(this.name)
         ];
     }
@@ -425,20 +434,25 @@ export class XmlAttributeGetExpression extends Expression {
 
 export class IndexedGetExpression extends Expression {
     constructor(
-        readonly obj: Expression,
-        readonly index: Expression,
-        readonly openingSquare: Token,
-        readonly closingSquare: Token
+        public obj: Expression,
+        public index: Expression,
+        /**
+         * Can either be `[` or `?[`. If `?.[` is used, this will be `[` and `optionalChainingToken` will be `?.`
+         */
+        public openingSquare: Token,
+        public closingSquare: Token,
+        public questionDotToken?: Token //  ? or ?.
     ) {
         super();
-        this.range = util.createRangeFromPositions(this.obj.range.start, this.closingSquare.range.end);
+        this.range = util.createBoundingRange(this.obj, this.openingSquare, this.questionDotToken, this.openingSquare, this.index, this.closingSquare);
     }
 
     public readonly range: Range;
 
     transpile(state: BrsTranspileState) {
         return [
             ...this.obj.transpile(state),
+            this.questionDotToken ? state.transpileToken(this.questionDotToken) : '',
             state.transpileToken(this.openingSquare),
             ...this.index.transpile(state),
             state.transpileToken(this.closingSquare)