[npm install] node weirdness, fix standalone expressions and more tests

Author: znorman-harris

apps/test-tokenizer/src/test-maker/cache/cache.json

@@ -5307,6 +5307,18 @@ export const AUTO_SYNTAX_TESTS: IAutoSyntaxValidatorTest[] = [
           `end`,
         ],
       },
+      {
+        name: `problems`,
+        code: [`compile_opt idl2`, `!quiet`, `end`],
+      },
+      {
+        name: `no problems (procedure)`,
+        code: [`compile_opt idl2`, `a = hash()`, `a`, `end`],
+      },
+      {
+        name: `problems`,
+        code: [`compile_opt idl2`, `a = hash()`, `a['key']`, `end`],
+      },
     ],
   },
   {

apps/test-tokenizer/src/test-maker/tests/auto-syntax-validator-tests.interface.ts

@@ -13,6 +13,7 @@ import {
   TOKEN_NAMES,
 } from '@idl/tokenizer';
 import { IDL_PROBLEM_CODES } from '@idl/types/problem-codes';
+import { PositionArray } from '@idl/types/tokenizer';
 
 /**
  * Tokens that can appear first
@@ -76,6 +77,18 @@ AFTER_VAR[TOKEN_NAMES.ASSIGNMENT] = true;
 AFTER_VAR[TOKEN_NAMES.OPERATOR_INCREMENT_DECREMENT] = true;
 AFTER_VAR[TOKEN_NAMES.CALL_PROCEDURE_METHOD] = true;
 
+/**
+ * Checks if two tokens are on the same line or not
+ *
+ * If nothing after, then returns true because we are on the "same" line
+ */
+function ValidateSameLine(ref: PositionArray, next?: PositionArray) {
+  if (!next) {
+    return false;
+  }
+  return ref[0] === next[0];
+}
+
 /**
  * Call back to make sure that, if we have children, there is a comma first
  */
@@ -129,20 +142,40 @@ function Callback(token: TreeBranchToken, parsed: IParsed) {
       /** Get the end of the chain */
       i = GetChainEnd(kids, i);
 
-      /** If the next token is not allowed, then we have problem */
-      if (!(kids[i + 1]?.name in AFTER_VAR) && kids[i + 1]) {
-        parsed.parseProblems.push(
-          SyntaxProblemWithTranslation(
-            inMain && parsed.type === 'notebook'
-              ? IDL_PROBLEM_CODES.IMPLIED_PRINT_NOTEBOOK
-              : IDL_PROBLEM_CODES.STANDALONE_EXPRESSION,
-            kids[orig].pos,
-            (kids[i] as TreeBranchToken)?.end?.pos || kids[i].pos
-          )
-        );
-      } else {
+      switch (true) {
+        /**
+         * Check last item in chain for valid skip condition
+         */
+        case kids[i].name === TOKEN_NAMES.OPERATOR_INCREMENT_DECREMENT:
+          break;
+        /**
+         * If there's a next token, but it is not on the same line, then
+         * we have a problem
+         */
+        case !(kids[i + 1] !== undefined) &&
+          !ValidateSameLine(
+            (kids[i] as TreeBranchToken)?.end?.pos || kids[i].pos,
+            kids[i + 1]?.pos
+          ):
+        /**
+         * If the next token can't be next, then we have a problem
+         */
+        // eslint-disable-next-line no-fallthrough
+        case !(kids[i + 1]?.name in AFTER_VAR):
+          parsed.parseProblems.push(
+            SyntaxProblemWithTranslation(
+              inMain && parsed.type === 'notebook'
+                ? IDL_PROBLEM_CODES.IMPLIED_PRINT_NOTEBOOK
+                : IDL_PROBLEM_CODES.STANDALONE_EXPRESSION,
+              kids[orig].pos,
+              (kids[i] as TreeBranchToken)?.end?.pos || kids[i].pos
+            )
+          );
+          break;
         // shift
-        i++;
+        default:
+          i++;
+          break;
       }
 
       continue;

libs/parsing/syntax-validators/src/lib/validators/standalone-expression.ts

@@ -44,6 +44,13 @@ describe(`[auto generated] Detects problems with statements not being closed`, (
         end: [0, 0, 1],
         canReport: true,
       },
+      {
+        code: 108,
+        info: 'Standalone expression detected. One or more statements need to be assigned to a variable or have a value assigned to them.',
+        start: [0, 0, 1],
+        end: [0, 0, 1],
+        canReport: true,
+      },
     ];
 
     // verify results
@@ -451,6 +458,13 @@ describe(`[auto generated] Detects problems with statements not being closed`, (
         end: [0, 0, 1],
         canReport: true,
       },
+      {
+        code: 108,
+        info: 'Standalone expression detected. One or more statements need to be assigned to a variable or have a value assigned to them.',
+        start: [0, 0, 1],
+        end: [0, 0, 1],
+        canReport: true,
+      },
     ];
 
     // verify results

libs/tests/syntax-validators/src/lib/code.0.not-closed.spec.ts

@@ -303,4 +303,121 @@ describe(`[auto generated] Detect standalone expressions`, () => {
       tokenized.parseProblems.concat(tokenized.postProcessProblems)
     ).toEqual(expected);
   });
+
+  it(`[auto generated] problems`, async () => {
+    // create index
+    const index = new IDLIndex(
+      new LogManager({
+        alert: () => {
+          // do nothing
+        },
+      }),
+      0
+    );
+
+    // test code to extract tokens from
+    const code = [`compile_opt idl2`, `!quiet`, `end`];
+
+    // extract tokens
+    const tokenized = await index.getParsedProCode(
+      'not-real',
+      code,
+      new CancellationToken(),
+      { postProcess: true }
+    );
+
+    // define expected tokens
+    const expected: SyntaxProblems = [
+      {
+        code: 108,
+        info: 'Standalone expression detected. One or more statements need to be assigned to a variable or have a value assigned to them.',
+        start: [1, 0, 6],
+        end: [1, 0, 6],
+        canReport: true,
+      },
+    ];
+
+    // verify results
+    expect(
+      tokenized.parseProblems.concat(tokenized.postProcessProblems)
+    ).toEqual(expected);
+  });
+
+  it(`[auto generated] no problems (procedure)`, async () => {
+    // create index
+    const index = new IDLIndex(
+      new LogManager({
+        alert: () => {
+          // do nothing
+        },
+      }),
+      0
+    );
+
+    // test code to extract tokens from
+    const code = [`compile_opt idl2`, `a = hash()`, `a`, `end`];
+
+    // extract tokens
+    const tokenized = await index.getParsedProCode(
+      'not-real',
+      code,
+      new CancellationToken(),
+      { postProcess: true }
+    );
+
+    // define expected tokens
+    const expected: SyntaxProblems = [
+      {
+        code: 104,
+        info: 'Unused variable "a"',
+        start: [1, 0, 1],
+        end: [1, 0, 1],
+        canReport: true,
+      },
+    ];
+
+    // verify results
+    expect(
+      tokenized.parseProblems.concat(tokenized.postProcessProblems)
+    ).toEqual(expected);
+  });
+
+  it(`[auto generated] problems`, async () => {
+    // create index
+    const index = new IDLIndex(
+      new LogManager({
+        alert: () => {
+          // do nothing
+        },
+      }),
+      0
+    );
+
+    // test code to extract tokens from
+    const code = [`compile_opt idl2`, `a = hash()`, `a['key']`, `end`];
+
+    // extract tokens
+    const tokenized = await index.getParsedProCode(
+      'not-real',
+      code,
+      new CancellationToken(),
+      { postProcess: true }
+    );
+
+    // define expected tokens
+    const expected: SyntaxProblems = [
+      {
+        code: 108,
+        info: 'Standalone expression detected. One or more statements need to be assigned to a variable or have a value assigned to them.',
+        start: [2, 0, 1],
+        end: [2, 7, 1],
+        canReport: true,
+      },
+    ];
+
+    // verify results
+    expect(
+      tokenized.parseProblems.concat(tokenized.postProcessProblems)
+    ).toEqual(expected);
+  });
 });