feat: Allow including external tasks from plugins

EvolvingLMMs-Lab · Jun 20, 2024 · 03947e1 · 03947e1
1 parent b80a91f
commit 03947e1
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/lmms_eval/__main__.py b/lmms_eval/__main__.py
@@ -1,3 +1,4 @@
+import importlib
 import os
 import yaml
 import sys
@@ -245,6 +246,12 @@ def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
         eval_logger.info(f"Including path: {args.include_path}")
         include_path(args.include_path)
 
+    if os.environ.get("LMMS_EVAL_PLUGINS", None):
+        for plugin in os.environ["LMMS_EVAL_PLUGINS"].split(","):
+            package_tasks_location = importlib.util.find_spec(f"{plugin}.tasks").submodule_search_locations[0]
+            eval_logger.info(f"Including path: {args.include_path}")
+            include_path(package_tasks_location)
+
     if args.tasks is None:
         task_names = ALL_TASKS
     elif args.tasks == "list":