Merge remote-tracking branch 'origin/develop' into develop

Author: znorman-harris

libs/mcp/language-server-tools/src/lib/tasks/register-mcp-task-tools.ts

@@ -6,14 +6,9 @@ import {
   RegisterMCPTool_ListENVITools,
   RegisterMCPTool_RunENVITool,
 } from '@idl/mcp/server-tools';
-import { FilterMCPENVITasks, MCPTaskRegistry } from '@idl/mcp/tasks';
+import { MCPTaskRegistry } from '@idl/mcp/tasks';
 import { IDLIndex } from '@idl/parsing/index';
-import {
-  GLOBAL_TOKEN_TYPES,
-  GlobalFunctionToken,
-  GlobalStructureToken,
-  IGlobalIndexedToken,
-} from '@idl/types/idl-data-types';
+import { GLOBAL_TOKEN_TYPES } from '@idl/types/idl-data-types';
 
 /**
  * Registers MCP Task tools from parsed code on IDL's search path
@@ -46,30 +41,11 @@ export async function RegisterMCPTaskTools(server: MCPServer, index: IDLIndex) {
   RegisterMCPTool_RunENVITool(server, registry);
   RegisterMCPTool_CreateENVIModelerWorkflow(server, registry);
 
-  /** Get all functions that we know about */
-  const functions =
-    index.globalIndex.globalTokensByTypeByName[GLOBAL_TOKEN_TYPES.FUNCTION];
-
-  /** Get all structures that we know about */
-  const structures =
-    index.globalIndex.globalTokensByTypeByName[GLOBAL_TOKEN_TYPES.STRUCTURE];
-
-  /** Find names of ENVI Tasks and exclude those we dont need to expose  */
-  const keys = FilterMCPENVITasks(functions, Object.keys(structures)).sort();
-
-  server.logManager.log({
-    log: IDL_MCP_LOG,
-    type: 'info',
-    content: `Attempting to register ${keys.length} ENVI Tools`,
-  });
-
-  // add all ENVI Tasks
-  for (let i = 0; i < keys.length; i++) {
-    registry.registerTask(
-      functions[keys[i]][0] as IGlobalIndexedToken<GlobalFunctionToken>,
-      structures[keys[i]][0] as IGlobalIndexedToken<GlobalStructureToken>,
-    );
-  }
+  /** Register tasks that we have found */
+  registry.registerTasksFromGlobalTokens(
+    index.globalIndex.globalTokensByTypeByName[GLOBAL_TOKEN_TYPES.FUNCTION],
+    index.globalIndex.globalTokensByTypeByName[GLOBAL_TOKEN_TYPES.STRUCTURE],
+  );
 
   // server.logManager.log({
   //   log: IDL_MCP_LOG,

libs/mcp/server-tools/src/lib/tools/envi/register-mcp-tool-create-envi-modeler-workflow.ts

@@ -214,7 +214,7 @@ export function RegisterMCPTool_CreateENVIModelerWorkflow(
         edges as ENVIModelerEdge[],
         registry,
       );
-      const modelContent = JSON.stringify(modelJSON, null, 4);
+      const modelContent = JSON.stringify(modelJSON, null, 2);
 
       // ---- write to disk
       try {

libs/mcp/tasks/src/lib/mcp-task-registry.class.ts

@@ -12,6 +12,7 @@ import Ajv from 'ajv';
 import { z, ZodRawShape } from 'zod';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 
+import { FilterMCPENVITasks } from './helpers/filter-mcp-envi-tasks';
 import { GetCleanDescription } from './helpers/get-clean-description';
 import { StrictCheck } from './helpers/strict-check';
 import {
@@ -104,7 +105,7 @@ export class MCPTaskRegistry {
   /**
    * Gets information about a task
    */
-  getTaskDetail(taskName: string): ITaskInformation {
+  getTaskDetail(taskName: string): ITaskInformation | undefined {
     /** Get lower case name */
     const lc = taskName.toLowerCase();
 
@@ -138,8 +139,16 @@ export class MCPTaskRegistry {
     taskFunction: IGlobalIndexedToken<GlobalFunctionToken>,
     taskStructure: IGlobalIndexedToken<GlobalStructureToken>,
   ) {
+    // extract task name
+    const match = TASK_REGEX.exec(taskStructure.meta.display);
+
+    // return if no match - prevent errors when we don't really have a task
+    if (match === null) {
+      return;
+    }
+
     /** Get task display name */
-    const taskDisplay = TASK_REGEX.exec(taskStructure.meta.display)[1];
+    const taskDisplay = match[1];
 
     /** Determine the type of task */
     const toolType = taskStructure.name.startsWith('envi') ? 'ENVI' : 'IDL';
@@ -296,6 +305,32 @@ export class MCPTaskRegistry {
     };
   }
 
+  /**
+   * Loads tasks from global tokens (i.e. after we index a location)
+   */
+  registerTasksFromGlobalTokens(
+    functions: {
+      [key: string]: IGlobalIndexedToken<GlobalFunctionToken>[];
+    },
+    structures: {
+      [key: string]: IGlobalIndexedToken<GlobalStructureToken>[];
+    },
+  ) {
+    /** Find names of ENVI Tasks and exclude those we dont need to expose  */
+    const keys = FilterMCPENVITasks(functions, Object.keys(structures)).sort();
+
+    this.logger.log({
+      log: IDL_MCP_LOG,
+      type: 'info',
+      content: `Attempting to register ${keys.length} ENVI Tools`,
+    });
+
+    // add all ENVI Tasks
+    for (let i = 0; i < keys.length; i++) {
+      this.registerTask(functions[keys[i]][0], structures[keys[i]][0]);
+    }
+  }
+
   /**
    * Remove a task from the registry
    *
@@ -356,7 +391,7 @@ export class MCPTaskRegistry {
       // check if we have JSON schema problems
       if (!valid) {
         // Format validation errors for the response
-        const errors = validate.errors
+        const errors = (validate.errors || [])
           .map(
             (err) =>
               `- ${err.instancePath || 'root'}: ${err.message}${

libs/notebooks/shared/src/lib/parsed-idl-notebook.interface.ts

@@ -4,5 +4,5 @@ import { IParsed } from '@idl/types/syntax-tree';
  * Data structure for a parsed notebook file
  */
 export interface IParsedIDLNotebook {
-  [key: string]: IParsed;
+  [key: string]: IParsed | undefined;
 }

libs/parsing/index/src/lib/global-index.class.ts

@@ -56,7 +56,7 @@ export class GlobalIndex {
   /**
    * Tokens categorized by type to reduce number of checks for global conflicts
    */
-  globalTokensByTypeByName: GlobalTokensByTypeByName = {};
+  globalTokensByTypeByName!: GlobalTokensByTypeByName;
 
   /**
    * Constructor which initializes properties in our constants so that they
@@ -111,7 +111,7 @@ export class GlobalIndex {
         const byNameForType = this.globalTokensByTypeByName[types[i]];
         const names = Object.keys(byNameForType);
         for (let j = 0; j < names.length; j++) {
-          exported[types[i]].push(byNameForType[names[j]][0]);
+          exported[types[i]].push(byNameForType[names[j]][0] as any);
         }
       }
       return exported;
@@ -332,7 +332,9 @@ export class GlobalIndex {
                 break;
               case byName.length === 1:
                 // remove display name
-                delete IDL_DISPLAY_NAMES[tokens[i].type][tokens[i].name];
+                delete (IDL_DISPLAY_NAMES[tokens[i].type] || {})[
+                  tokens[i].name
+                ];
 
                 // remove structure/type names
                 if (tokens[i].type === GLOBAL_TOKEN_TYPES.STRUCTURE) {
@@ -355,7 +357,7 @@ export class GlobalIndex {
 
       // mark as changed if we had problems
       if (file in this.globalSyntaxProblemsByFile) {
-        this.changedFiles[file] = undefined;
+        this.changedFiles[file] = false;
       }
 
       // reset problems for our file as well
@@ -456,9 +458,11 @@ export class GlobalIndex {
 
       // save by type
       if (token.name in this.globalTokensByTypeByName[token.type]) {
-        this.globalTokensByTypeByName[token.type][token.name].push(token);
+        this.globalTokensByTypeByName[token.type][token.name].push(
+          token as any,
+        );
       } else {
-        this.globalTokensByTypeByName[token.type][token.name] = [token];
+        this.globalTokensByTypeByName[token.type][token.name] = [token as any];
       }
     }
   }
@@ -558,8 +562,8 @@ export class GlobalIndex {
 
     // check all files that we need to process
     const files = this.globalTokensByTypeByName[token.type][token.name]
-      .filter((item) => item.file !== undefined)
-      .map((item) => item.file);
+      .map((item) => item.file)
+      .filter((item) => item !== undefined);
 
     // process each file
     for (let z = 0; z < files.length; z++) {
@@ -577,6 +581,11 @@ export class GlobalIndex {
         // extract the problem
         const problem = problems[j];
 
+        // skip if no file
+        if (!problem.file) {
+          continue;
+        }
+
         // check if our file matches
         if (
           (problem.file === token.file || clear) &&

libs/parsing/index/src/lib/global-index.interface.ts

@@ -22,9 +22,9 @@ export type GlobalIndexedToken = IGlobalIndexedToken<GlobalTokenType>;
  * Lookup with all tokens by type. We store an array of the token types
  * which we can manually search through to find.
  */
-export interface GlobalTokensByTypeByName {
-  [key: string]: { [key: string]: GlobalIndexedToken[] };
-}
+export type GlobalTokensByTypeByName = {
+  [key in GlobalTokenType]: { [name: string]: IGlobalIndexedToken<key>[] };
+};
 
 /**
  * Lookup with tokens by type for easier access when we export
@@ -33,14 +33,14 @@ export type ExportedGlobalTokensByType = {
   [key in GlobalTokenType]: IGlobalIndexedToken<key>[];
 };
 
-/**
- * For each file we index tokens for, track which types of tokens they
- * contribute to filter the locations we need to clean up tokens from so
- * we don't need to process all tokens.
- */
-export interface ITokenTypesByFile {
-  [file: string]: { [key: string]: GlobalIndexedToken[] };
-}
+// /**
+//  * For each file we index tokens for, track which types of tokens they
+//  * contribute to filter the locations we need to clean up tokens from so
+//  * we don't need to process all tokens.
+//  */
+// export type ITokenTypesByFile = {
+//   [file: string]: { [key in GlobalTokenType]: GlobalIndexedToken[] };
+// };
 
 /**
  * Maps our token types to problem codes

libs/parsing/index/src/lib/idl-index.class.ts

@@ -445,7 +445,7 @@ export class IDLIndex {
     // check if we have the file in our lookup
     if (this.parsedCache.has(file)) {
       const text = this.parsedCache.text(file);
-      if (text.length > 0) {
+      if (Array.isArray(text)) {
         return text.join('\n');
       }
     }
@@ -473,7 +473,7 @@ export class IDLIndex {
        * Check if local
        */
       case this.parsedCache.has(file):
-        globals = this.parsedCache.get(file).global;
+        globals = (this.parsedCache.get(file) as IParsed).global;
         break;
 
       default:
@@ -554,7 +554,7 @@ export class IDLIndex {
         IDLFileHelper.isPRODef(file)
       )
     ) {
-      return undefined;
+      return [];
     }
 
     // if we are multi threaded, then
@@ -881,14 +881,14 @@ export class IDLIndex {
       // post process cell
       await this.postProcessProFile(
         files[i],
-        byCell[files[i]],
+        byCell[files[i]] as IParsed,
         token,
         [],
         false,
       );
 
       // update stored token
-      this.parsedCache.add(files[i], byCell[files[i]]);
+      this.parsedCache.add(files[i], byCell[files[i]] as IParsed);
     }
 
     return byCell;
@@ -1292,6 +1292,8 @@ export class IDLIndex {
         },
       });
     }
+
+    return false;
   }
 
   /**
@@ -1328,7 +1330,7 @@ export class IDLIndex {
 
       try {
         /** Get file from cache */
-        const parsed = this.parsedCache.get(files[i]);
+        const parsed = this.parsedCache.get(files[i]) as IParsed;
 
         /** Post-process and check for global changes */
         const didChange = await this.postProcessProFile(
@@ -1842,10 +1844,7 @@ export class IDLIndex {
     this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.ALL_FILES, {
       files,
     });
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
 
     // track that we have PRO files
     if (files.length > 0) {
@@ -1946,10 +1945,7 @@ export class IDLIndex {
     await Promise.all(parsing);
 
     // send message to clean up
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
 
     /**
      * Messages for global token synchronization
@@ -2008,10 +2004,7 @@ export class IDLIndex {
     await Promise.all(synchronize);
 
     // send message to clean up
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
 
     // save stats
     this.lastWorkspaceIndexStats.timePro = Math.floor(performance.now() - t0);
@@ -2084,10 +2077,7 @@ export class IDLIndex {
     await Promise.all(parsing);
 
     // send message to clean up
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
 
     /**
      * Messages for global token synchronization
@@ -2198,10 +2188,7 @@ export class IDLIndex {
     await Promise.all(postProcessing);
 
     // send message to clean up
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
 
     /**
      * Process our results
@@ -2233,10 +2220,7 @@ export class IDLIndex {
     this.lastWorkspaceIndexStats.linesPro = lines;
 
     // send message to clean up
-    this.indexerPool.postToAll(
-      LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP,
-      undefined,
-    );
+    this.indexerPool.postToAll(LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, {});
   }
 
   /**