Merge branch 'develop'

Author: znorman-harris

CHANGELOG.md

@@ -18,6 +18,20 @@ Document some advanced types so users may try them out and provide feedback. The
 
 IDL 9.1 introduces new, command-line based progress bars. We have a first-pass of support for these progress bars inside IDL Notebooks.
 
+## 4.6.2 - September 2024
+
+Changed the language server startup process to remove all files from memory after initial startup. This reduces RAM by 0.25-0.5 GB of memory after startup, depending on the volume of code in workspaces and on your path.
+
+When we clean up the language server (happens every 5 minutes), we now check our in-memory cache and remove any files that haven't been accessed recently. Helps reduce overall RAM when VSCode is open for long periods at a time.
+
+Changed the way that the language server sends work to the threads that parse code. The main difference is that we no longer send the parsed version of code back to the main thread which we were doing.
+
+For large files, this had a significant impact on perceived performance as the worker threads could get locked serializing and de-serializing objects (also leads to more memory usage). Now, large files like slicer3.pro which are included in the IDL installation, provide auto-complete and hover help in about ~0.5 seconds instead of 5+ seconds.
+
+Fixed an issue where, from the JSON settings editor, a valid extension setting would be highlighted as being an incorrect value.
+
+Fixed an issue where the "don't ask again" options didn't honor your choice for using IDL as the formatter.
+
 ## 4.6.1 - September 2024
 
 Added layer controls to the Notebook Map.

apps/package-json/src/contributes/config/questions-config.ts

@@ -42,28 +42,32 @@ export function AddQuestionsConfig(nls: IPackageNLS) {
     properties: {
       forIDLDir: {
         type: 'boolean',
-        default: false,
         description: TranslationFromConfiguration(
           IDL_EXTENSION_CONFIG_KEYS.dontAskForIDLDir,
           nls
         ),
       },
       forIconChange: {
         type: 'boolean',
-        default: false,
         description: TranslationFromConfiguration(
           IDL_EXTENSION_CONFIG_KEYS.dontAskForIconChange,
           nls
         ),
       },
       forFormatterChange: {
         type: 'boolean',
-        default: false,
         description: TranslationFromConfiguration(
           IDL_EXTENSION_CONFIG_KEYS.dontAskForFormatterChange,
           nls
         ),
       },
+      toOpenDocs: {
+        type: 'boolean',
+        description: TranslationFromConfiguration(
+          IDL_EXTENSION_CONFIG_KEYS.dontAskToOpenDocs,
+          nls
+        ),
+      },
       // toInitConfig: {
       //   type: 'boolean',
       //   default: false,

apps/parsing-worker/src/main.ts

@@ -7,12 +7,16 @@ import {
   ChangeDetection,
   GetCompletionRecipes,
   GetHoverHelpLookup,
+  GetParsedPROCode,
   GetSyntaxProblems,
   IDL_INDEX_OPTIONS,
   IDLIndex,
   ReduceGlobals,
 } from '@idl/parsing/index';
-import { RemoveScopeDetail } from '@idl/parsing/syntax-tree';
+import {
+  IParsedLightWeight,
+  RemoveScopeDetail,
+} from '@idl/parsing/syntax-tree';
 import { IDL_TRANSLATION } from '@idl/translation';
 import {
   ChangeDetectionResponse,
@@ -107,7 +111,8 @@ client.on(
   LSP_WORKER_THREAD_MESSAGE_LOOKUP.MIGRATE_CODE,
   async (message, cancel) => {
     // index the file
-    const parsed = await WORKER_INDEX.getParsedProCode(
+    const parsed = await GetParsedPROCode(
+      WORKER_INDEX,
       message.file,
       message.code,
       cancel
@@ -179,7 +184,8 @@ client.on(
 /**
  * Clean up and return memory usage
  */
-client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, async () => {
+client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, async (message) => {
+  await WORKER_INDEX.clearParsedCache(message?.all);
   await WORKER_INDEX.cleanUp();
 });
 
@@ -247,18 +253,23 @@ client.on(
   LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_FILE,
   async (message, cancel) => {
     // index the file
-    const parsed = await WORKER_INDEX.getParsedProCode(
+    const parsed = await GetParsedPROCode(
+      WORKER_INDEX,
       message.file,
       WORKER_INDEX.getFileStrings(message.file),
       cancel,
       message
     );
 
-    // make non-circular
-    RemoveScopeDetail(parsed, cancel, true);
+    const lightParsed: IParsedLightWeight = {
+      disabledProblems: parsed.disabledProblems,
+      global: parsed.global,
+      parseProblems: parsed.parseProblems,
+      postProcessProblems: parsed.postProcessProblems,
+    };
 
     // return
-    return parsed;
+    return lightParsed;
   }
 );
 
@@ -269,7 +280,9 @@ client.on(
   LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE,
   async (message, cancel) => {
     // index the file
-    const parsed = await WORKER_INDEX.getParsedProCode(
+
+    const parsed = await GetParsedPROCode(
+      WORKER_INDEX,
       message.file,
       message.code,
       cancel,
@@ -279,8 +292,15 @@ client.on(
     // make non-circular
     RemoveScopeDetail(parsed, cancel, true);
 
+    const lightParsed: IParsedLightWeight = {
+      disabledProblems: parsed.disabledProblems,
+      global: parsed.global,
+      parseProblems: parsed.parseProblems,
+      postProcessProblems: parsed.postProcessProblems,
+    };
+
     // return
-    return parsed;
+    return lightParsed;
   }
 );
 
@@ -390,12 +410,12 @@ client.on(
       /**
        * Skip if we dont have a file. Could happen from parsing errors
        */
-      if (!WORKER_INDEX.tokensByFile.has(files[i])) {
+      if (!WORKER_INDEX.parsedCache.has(files[i])) {
         continue;
       }
 
       // save lines
-      resp.lines += WORKER_INDEX.tokensByFile.lines(files[i]);
+      resp.lines += WORKER_INDEX.parsedCache.lines(files[i]);
 
       // track globals
       resp.globals[files[i]] = WORKER_INDEX.getGlobalsForFile(files[i]);
@@ -457,18 +477,20 @@ client.on(
   LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_NOTEBOOK_CELL,
   async (message, cancel) => {
     // get parsed code and return
-    const parsed = await WORKER_INDEX.getParsedProCode(
+    const parsed = await GetParsedPROCode(
+      WORKER_INDEX,
       message.file,
       message.code,
       cancel
     );
 
-    // make non-circular
-    if (parsed !== undefined) {
-      RemoveScopeDetail(parsed, cancel, true);
-    }
-
-    return parsed;
+    const lightParsed: IParsedLightWeight = {
+      disabledProblems: parsed.disabledProblems,
+      global: parsed.global,
+      parseProblems: parsed.parseProblems,
+      postProcessProblems: parsed.postProcessProblems,
+    };
+    return lightParsed;
   }
 );
 
@@ -481,7 +503,7 @@ client.on(
     /** Get files */
     const files = Array.isArray(message.files)
       ? message.files
-      : WORKER_INDEX.tokensByFile.allFiles();
+      : WORKER_INDEX.parsedCache.allFiles();
 
     // post process, no change detection
     const missing = await WORKER_INDEX.postProcessProFiles(
@@ -511,12 +533,12 @@ client.on(
       /**
        * Skip if we dont have a file. Could happen from parsing errors
        */
-      if (!WORKER_INDEX.tokensByFile.has(files[i])) {
+      if (!WORKER_INDEX.parsedCache.has(files[i])) {
         continue;
       }
 
       // save lines
-      resp.lines += WORKER_INDEX.tokensByFile.lines(files[i]);
+      resp.lines += WORKER_INDEX.parsedCache.lines(files[i]);
 
       // populate problems
       resp.problems[files[i]] = problems[files[i]] || [];
@@ -559,7 +581,7 @@ client.on(
     await WORKER_INDEX.removeWorkspaceFiles(message.files, false);
 
     /** Get files that we manage */
-    const ourFiles = WORKER_INDEX.tokensByFile.allFiles();
+    const ourFiles = WORKER_INDEX.parsedCache.allFiles();
 
     // post process all of our files again
     const missing = await WORKER_INDEX.postProcessProFiles(
@@ -588,7 +610,7 @@ client.on(
       /**
        * Skip if we dont have a file. Could happen from parsing errors
        */
-      if (!WORKER_INDEX.tokensByFile.has(ourFiles[i])) {
+      if (!WORKER_INDEX.parsedCache.has(ourFiles[i])) {
         continue;
       }
 

libs/assembling/assembler/src/lib/assembler-with-index.ts

@@ -1,6 +1,6 @@
 import { FormatterType, IAssemblerInputOptions } from '@idl/assembling/config';
 import { CancellationToken } from '@idl/cancellation-tokens';
-import { IDLIndex } from '@idl/parsing/index';
+import { GetParsedPROCode, IDLIndex } from '@idl/parsing/index';
 
 import { Assembler } from './assembler';
 
@@ -15,7 +15,7 @@ export async function AssembleWithIndex<T extends FormatterType>(
   formatting?: Partial<IAssemblerInputOptions<T>>
 ): Promise<string | undefined> {
   // get the tokens for our file
-  const parsed = await index.getParsedProCode(file, code, cancel);
+  const parsed = await GetParsedPROCode(index, file, code, cancel);
 
   // format and return
   return Assembler(parsed, cancel, formatting);

libs/parsing/index/src/index.ts

@@ -1,5 +1,6 @@
 export * from './lib/auto-complete/get-completion-recipes';
 export * from './lib/change-detection/change-detection';
+export * from './lib/get-parsed/get-parsed-pro-code';
 export * from './lib/global-index.class';
 export * from './lib/global-index.interface';
 export * from './lib/helpers/get-routine';

libs/parsing/index/src/lib/auto-complete/get-completion-recipes.ts

@@ -33,6 +33,7 @@ import {
 } from '@idl/types/auto-complete';
 import { Position } from 'vscode-languageserver/node';
 
+import { GetParsedPROCode } from '../get-parsed/get-parsed-pro-code';
 import { GetTypeBefore } from '../helpers/get-type-before';
 import { IDLIndex } from '../idl-index.class';
 import { GetCompileOptCompletionOptions } from './completion-for/completion-compile-opts';
@@ -103,7 +104,8 @@ export async function GetCompletionRecipes(
   const recipes: AutoCompleteRecipe<AutoCompleteType>[] = [];
 
   // get the tokens for our file
-  const parsed = await index.getParsedProCode(
+  const parsed = await GetParsedPROCode(
+    index,
     file,
     code,
     new CancellationToken(),

libs/parsing/index/src/lib/change-detection/change-detection.ts

@@ -27,12 +27,12 @@ export function ChangeDetection(
   const missingFiles: string[] = [];
 
   /** Get the current indexed files */
-  const files = index.tokensByFile.allFiles();
+  const files = index.parsedCache.allFiles();
 
   // process each file
   for (let z = 0; z < files.length; z++) {
     // get our parsed file
-    const uses = index.tokensByFile.uses(files[z]);
+    const uses = index.parsedCache.uses(files[z]);
 
     // check for a match
     for (let i = 0; i < changed.length; i++) {
@@ -56,7 +56,7 @@ export function ChangeDetection(
       PostProcessParsed(
         index,
         postProcessThese[z],
-        index.tokensByFile.get(postProcessThese[z]),
+        index.parsedCache.get(postProcessThese[z]),
         token
       );
     } catch (err) {

libs/parsing/index/src/lib/get-parsed/get-parsed-notebook-cell.ts

@@ -37,9 +37,9 @@ export async function GetParsedNotebookCell(
   }
 
   // check if we have already parsed the file
-  if (index.tokensByFile.has(cellFile)) {
-    if (index.tokensByFile.checksumMatches(cellFile, checksum)) {
-      return index.tokensByFile.get(cellFile);
+  if (index.parsedCache.has(cellFile)) {
+    if (index.parsedCache.checksumMatches(cellFile, checksum)) {
+      return index.parsedCache.get(cellFile);
     }
   }
 

libs/parsing/index/src/lib/get-parsed/get-parsed-pro-code.ts

@@ -69,9 +69,9 @@ export async function GetParsedPROCode(
   }
 
   // check if we have the file in our cache and it matches our checksum
-  if (index.tokensByFile.has(file)) {
-    if (index.tokensByFile.checksumMatches(file, checksum)) {
-      return index.tokensByFile.get(file);
+  if (index.parsedCache.has(file)) {
+    if (index.parsedCache.checksumMatches(file, checksum)) {
+      return index.parsedCache.get(file);
     }
   }
 
@@ -107,7 +107,7 @@ export async function GetParsedPROCode(
         const newPending: IGetParsedPROCodePending = {
           checksum,
           token: resp.token,
-          promise: resp.response,
+          promise: resp.response as any,
         };
 
         // save new pending file
@@ -144,7 +144,7 @@ export async function GetParsedPROCode(
       const newPending: IGetParsedPROCodePending = {
         checksum,
         token: resp.token,
-        promise: resp.response,
+        promise: resp.response as any,
       };
 
       // save new pending file
@@ -176,7 +176,7 @@ export async function GetParsedPROCode(
       }
 
       // make sure that we never cache in memory if we are the main thread
-      index.tokensByFile.remove(file);
+      index.parsedCache.remove(file);
 
       // return the tokens from the worker
       return current;

libs/parsing/index/src/lib/helpers/get-code-semantic-tokens.ts

@@ -26,8 +26,8 @@ export async function GetCodeSemanticTokens(
     /**
      * Check if it is in our lookup
      */
-    case index.tokensByFile.has(file):
-      return index.tokensByFile.semantic(file);
+    case index.parsedCache.has(file):
+      return index.parsedCache.semantic(file);
 
     /**
      * Parse to get the outline