Workflows: Read workflow schemas with the taskmaestro CLI

Replace the rips.taskmaestro_helper introspect subcommand with
`python -m taskmaestro workflow describe --json`. RimWorkflowDescribeTools
converts the describe output (JSON schemas, depends_on, config_values)
into the workflow graph used by the Workflow UI. This also adds support
for collect dependencies and maps rips.Reservoir to EclipseCase pickers.
This commit is contained in:
Kristian Bendiksen authored and Magne Sjaastad committed 2026-10-05 19:56:36 +02:00
1 parent 4175bfdf7b
commit 65942d455d
15 files changed
+1027 -530

No files matched your search

@@ -1,6 +1,6 @@
"""Bridge between ResInsight and the taskmaestro workflow library.
ResInsight invokes this package as a subprocess to introspect workflow
schemas and to execute workflows. The Python side exists so ResInsight
itself does not need to parse YAML or know about Pydantic.
ResInsight invokes this package as a subprocess to execute workflows against
a running ResInsight. Workflow schemas are read with the taskmaestro CLI
(`python -m taskmaestro workflow describe`).
"""
@@ -8,7 +8,7 @@ import sys
def main() -> int:
if len(sys.argv) < 2:
print(
"usage: python -m rips.taskmaestro_helper <introspect|run> ...",
"usage: python -m rips.taskmaestro_helper run ...",
file=sys.stderr,
)
return 2
@@ -16,11 +16,6 @@ def main() -> int:
subcommand = sys.argv[1]
rest = sys.argv[2:]
if subcommand == "introspect":
from . import introspect
return introspect.main(rest)
if subcommand == "run":
from . import run
@@ -1,231 +0,0 @@
"""Introspect a taskmaestro workflow directory and emit a JSON schema.
The schema is consumed by ResInsight to auto-generate property-editor
fields for each task's config inputs. Only fields listed in the workflow's
`config_fields` are inspected, so models that also reference non-Pydantic
types (e.g. live `rips` objects passed via `depends_on`) do not break
introspection.
"""
from __future__ import annotations
import datetime
import enum
import json
import pathlib
import sys
from pathlib import Path
from types import UnionType
from typing import Annotated, Any, Union, get_args, get_origin
from pydantic import TypeAdapter
from pydantic.fields import FieldInfo
from .refs import annotation_to_resinsight_type
_SCALAR_TYPE_MAP: dict[type, str] = {
str: "string",
bool: "boolean",
int: "integer",
float: "number",
}
def _format_for(field_info: FieldInfo) -> str | None:
"""Ask Pydantic for the JSON-schema `format` of a field, if any.
Pydantic stores e.g. `pydantic.DirectoryPath` (which is
`Annotated[Path, PathType('dir')]`) as `annotation=Path` and
`metadata=[PathType('dir')]`. We reassemble both before asking
TypeAdapter, so format markers like `directory-path` survive.
"""
annotation = field_info.annotation
if field_info.metadata:
annotation = Annotated[tuple([annotation, *field_info.metadata])]
try:
return TypeAdapter(annotation).json_schema().get("format")
except Exception:
return None
def _annotation_type(annotation: object) -> str:
"""Map a Pydantic field annotation to a JSON-schema-style type label."""
if isinstance(annotation, type) and annotation in _SCALAR_TYPE_MAP:
return _SCALAR_TYPE_MAP[annotation]
# Optional[T] / T | None — peel the union and map the non-None arm
origin = get_origin(annotation)
if origin in (Union, UnionType):
non_none = [a for a in get_args(annotation) if a is not type(None)]
if len(non_none) == 1:
return _annotation_type(non_none[0])
if origin in (list, tuple, set, frozenset):
return "array"
if isinstance(annotation, type):
# Recognised rips object types are reported as "object" with a separate marker.
if annotation_to_resinsight_type(annotation) is not None:
return "object"
return "string"
_SENTINEL = object()
def _serialize_default(value: Any) -> Any:
"""Convert a Python value into a JSON-friendly form for the schema's `default`."""
if isinstance(value, (datetime.date, datetime.datetime)):
return value.isoformat()
if isinstance(value, pathlib.PurePath):
return str(value)
if isinstance(value, enum.Enum):
return _serialize_default(value.value)
if isinstance(value, dict):
return {key: _serialize_default(item) for key, item in value.items()}
if isinstance(value, (list, tuple, set, frozenset)):
return [_serialize_default(item) for item in value]
return value
def _field_schema(
field_name: str, field_info: FieldInfo, input_value: Any = _SENTINEL
) -> dict[str, Any]:
entry: dict[str, Any] = {
"name": field_name,
"type": _annotation_type(field_info.annotation),
"required": field_info.is_required(),
}
if field_info.description:
entry["description"] = field_info.description
# Prefer the value the user already has in input.yaml; fall back to the
# Pydantic-side default so the UI is pre-filled in either case.
if input_value is not _SENTINEL:
entry["default"] = _serialize_default(input_value)
elif not field_info.is_required():
default = field_info.get_default(call_default_factory=True)
entry["default"] = _serialize_default(default)
fmt = _format_for(field_info)
if fmt is not None:
entry["format"] = fmt
ri_type = annotation_to_resinsight_type(field_info.annotation)
if ri_type is not None:
entry["resinsight_type"] = ri_type
return entry
def _dependency_edges(task_name: str, deps: Any) -> list[dict[str, str]]:
"""Flatten taskmaestro's resolved dependency forms into directed edges."""
if deps is None:
return []
if isinstance(deps, str):
return [{"from": deps, "to": task_name}]
if isinstance(deps, tuple):
return [{"from": deps[0], "to": task_name, "output": deps[1]}]
edges = []
for input_name, upstream in sorted(deps.items()):
edge = {
"from": upstream[0] if isinstance(upstream, tuple) else upstream,
"to": task_name,
"input": input_name,
}
if isinstance(upstream, tuple):
edge["output"] = upstream[1]
edges.append(edge)
return edges
def _output_fields(output_type: type) -> list[str]:
"""Expose output fields, but not ObjectModel's internal wrapped value."""
from taskmaestro import ObjectModel
fields = set(output_type.model_fields)
if issubclass(output_type, ObjectModel):
fields.discard("value")
return sorted(fields)
def collect_schema(workflow_dir: Path) -> dict[str, Any]:
"""Load the workflow at `workflow_dir` and produce its UI schema."""
workflow_yaml = workflow_dir / "workflow.yaml"
input_yaml = workflow_dir / "input.yaml"
if not workflow_yaml.is_file():
raise FileNotFoundError(f"Missing workflow.yaml in {workflow_dir}")
# Workflow source files (e.g. pipeline.py) live next to workflow.yaml and
# are referenced as plain dotted paths in workflow.yaml — make them importable.
workflow_dir_str = str(workflow_dir)
if workflow_dir_str not in sys.path:
sys.path.insert(0, workflow_dir_str)
from taskmaestro.task import get_input_type, get_output_type
from taskmaestro.yaml_config import _load_workflow_only
wf, jc = _load_workflow_only(
workflow_yaml,
input_yaml if input_yaml.is_file() else None,
)
tasks: list[dict[str, Any]] = []
edges: list[dict[str, str]] = []
for task_name, task_cls in wf.topological_order():
edges.extend(_dependency_edges(task_name, wf.get_dependencies(task_name)))
config_field_names = wf.get_config_fields(task_name)
task_config = jc.get_config_for_task(task_name) if jc is not None else {}
input_type = get_input_type(task_cls)
config_fields: list[dict[str, Any]] = []
for field_name in sorted(config_field_names):
field_info = input_type.model_fields.get(field_name)
if field_info is None:
config_fields.append(
{
"name": field_name,
"type": "string",
"required": True,
"error": "field not found in input model",
}
)
continue
input_value = task_config.get(field_name, _SENTINEL)
config_fields.append(
_field_schema(field_name, field_info, input_value=input_value)
)
tasks.append(
{
"name": task_name,
"inputs": sorted(input_type.model_fields),
"outputs": _output_fields(get_output_type(task_cls)),
"config_fields": config_fields,
}
)
return {
"name": wf.name,
"description": "",
"tasks": tasks,
"edges": edges,
}
def main(argv: list[str]) -> int:
if len(argv) != 1:
print("usage: introspect <workflow_dir>", file=sys.stderr)
return 2
workflow_dir = Path(argv[0]).resolve()
try:
schema = collect_schema(workflow_dir)
except Exception as exc:
print(
json.dumps({"error": str(exc), "type": type(exc).__name__}), file=sys.stderr
)
return 1
json.dump(schema, sys.stdout, indent=2)
sys.stdout.write("\n")
return 0
@@ -1,56 +1,12 @@
"""Mapping between Pydantic field annotations and ResInsight object types.
"""ResInsight object references in taskmaestro input files.
The whitelist below is the single source of truth for which `rips` types
the helper recognises as object pickers. It is referenced both by
`introspect` (to add the `resinsight_type` marker on field schemas) and
by `run` (to resolve `__resinsight_ref__` values back to live `rips`
objects).
The Workflow UI writes a `{__resinsight_ref__: <type>, ...id...}` map into
input.yaml for fields that refer to a ResInsight object. `run` resolves these
maps back to live `rips` objects. The C++ side maps Python types to the
`<type>` labels in RimWorkflowDescribeTools.
"""
from __future__ import annotations
# Maps rips class name -> resinsight_type label sent to the C++ side.
# The label is also the C++ binding subclass selector (RimWorkflow{Label}Binding).
# Rips classes live in `rips.generated.generated_classes` but are re-exported as
# `rips.<Name>`; we match by class name only since the module is generated.
RIPS_TYPE_WHITELIST: dict[str, str] = {
"Case": "EclipseCase",
"EclipseCase": "EclipseCase",
"WellPath": "WellPath",
"View": "View",
"EclipseView": "View",
}
_RIPS_MODULE_PREFIXES = ("rips.", "rips")
# Marker used inside input.yaml to denote a ResInsight object reference.
REF_MARKER = "__resinsight_ref__"
def annotation_to_resinsight_type(annotation: object) -> str | None:
"""Return the resinsight_type label for a Pydantic field annotation, or None.
Handles both bare rips classes and ObjectModel[rips.X] wrappers used by
the example workflow (e.g. `class GridCase(ObjectModel[rips.EclipseCase])`).
"""
candidates: list[type] = []
if isinstance(annotation, type):
candidates.append(annotation)
# Walk the MRO for ObjectModel[T] parameterizations: pydantic stores the
# type argument on the intermediate class in __pydantic_generic_metadata__.
for cls in annotation.__mro__:
md = getattr(cls, "__pydantic_generic_metadata__", None)
if not md:
continue
for arg in md.get("args", ()) or ():
if isinstance(arg, type):
candidates.append(arg)
for cls in candidates:
module = getattr(cls, "__module__", "") or ""
if not module.startswith(_RIPS_MODULE_PREFIXES):
continue
label = RIPS_TYPE_WHITELIST.get(cls.__name__)
if label is not None:
return label
return None
@@ -1,15 +1,12 @@
"""Unit tests for rips.taskmaestro_helper.
These tests do not require a running ResInsight; they exercise the
Pydantic introspection logic in isolation against a synthetic workflow
written into a tmp_path so they stay self-contained.
These tests do not require a running ResInsight; they exercise reference
resolution and progress reporting for the `run` subcommand in isolation.
"""
from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
import pytest
@@ -17,203 +14,6 @@ import pytest
pytest.importorskip("taskmaestro")
SYNTHETIC_PIPELINE = '''
"""Synthetic workflow for taskmaestro_helper tests."""
from __future__ import annotations
import datetime
import pathlib
import pydantic
from pydantic import BaseModel, Field
from taskmaestro import EmptyConfig, ExecutionContext, Task
class GreetInput(BaseModel):
name: str = Field(description="Person to greet", default="world")
times: int = Field(description="How many times", ge=1)
when: datetime.date = Field(description="Greet date", default=datetime.date(2024, 1, 1))
out_file: pathlib.Path = Field(description="Where to write greeting", default=pathlib.Path("/tmp/hi.txt"))
out_dir: pydantic.DirectoryPath = Field(description="Where to write logs")
class GreetOutput(BaseModel):
message: str
class Greet(Task[GreetInput, GreetOutput]):
name = "greet"
def run(self, input: GreetInput, ctx: ExecutionContext) -> GreetOutput: # pragma: no cover
return GreetOutput(message=f"hi {input.name}" * input.times)
class StartInput(BaseModel):
pass
class StartOutput(BaseModel):
pass
class Start(Task[StartInput, StartOutput]):
name = "start"
def run(self, input: StartInput, ctx: ExecutionContext) -> StartOutput: # pragma: no cover
return StartOutput()
'''
SYNTHETIC_WORKFLOW_YAML = """workflow:
name: synthetic
tasks:
- task: pipeline.Start
- task: pipeline.Greet
depends_on: pipeline.Start
config_fields: [name, times, when, out_file, out_dir]
"""
SYNTHETIC_INPUT_YAML = """greet:
name: alice
times: 3
out_dir: /tmp
"""
@pytest.fixture
def workflow_dir(tmp_path: Path) -> Path:
(tmp_path / "pipeline.py").write_text(SYNTHETIC_PIPELINE)
(tmp_path / "workflow.yaml").write_text(SYNTHETIC_WORKFLOW_YAML)
(tmp_path / "input.yaml").write_text(SYNTHETIC_INPUT_YAML)
return tmp_path
def test_collect_schema_returns_workflow_name_and_tasks(workflow_dir: Path) -> None:
from rips.taskmaestro_helper.introspect import collect_schema
schema = collect_schema(workflow_dir)
assert schema["name"] == "synthetic"
task_names = [t["name"] for t in schema["tasks"]]
assert task_names == ["start", "greet"]
assert schema["tasks"][0]["config_fields"] == []
assert schema["tasks"][0]["inputs"] == []
assert schema["tasks"][0]["outputs"] == []
assert schema["tasks"][1]["inputs"] == [
"name",
"out_dir",
"out_file",
"times",
"when",
]
assert schema["tasks"][1]["outputs"] == ["message"]
assert schema["edges"] == [{"from": "start", "to": "greet"}]
def test_collect_schema_extracts_field_types_and_metadata(workflow_dir: Path) -> None:
from rips.taskmaestro_helper.introspect import collect_schema
schema = collect_schema(workflow_dir)
greet = schema["tasks"][1]
fields_by_name = {f["name"]: f for f in greet["config_fields"]}
# input.yaml has `name: alice`, so that wins over the Pydantic default "world"
assert fields_by_name["name"]["type"] == "string"
assert fields_by_name["name"]["default"] == "alice"
assert fields_by_name["name"]["description"] == "Person to greet"
assert fields_by_name["name"]["required"] is False
# input.yaml has `times: 3`; field has no Pydantic default but is required
assert fields_by_name["times"]["type"] == "integer"
assert fields_by_name["times"]["required"] is True
assert fields_by_name["times"]["default"] == 3
assert "resinsight_type" not in fields_by_name["times"]
# input.yaml has no `when:` block, so Pydantic default kicks in
assert fields_by_name["when"]["type"] == "string"
assert fields_by_name["when"]["format"] == "date"
assert fields_by_name["when"]["default"] == "2024-01-01"
# input.yaml has no out_file; Pydantic default flows through
assert fields_by_name["out_file"]["type"] == "string"
assert fields_by_name["out_file"]["format"] == "path"
assert fields_by_name["out_file"]["default"] == "/tmp/hi.txt"
# input.yaml has `out_dir: /tmp`; pre-fills even though field is required
assert fields_by_name["out_dir"]["type"] == "string"
assert fields_by_name["out_dir"]["format"] == "directory-path"
assert fields_by_name["out_dir"]["required"] is True
assert fields_by_name["out_dir"]["default"] == "/tmp"
def test_field_schema_preserves_arrays_and_null_defaults() -> None:
from pydantic import BaseModel, Field
from rips.taskmaestro_helper.introspect import _field_schema
class Config(BaseModel):
values: list[int] = Field(default_factory=lambda: [1, 2])
optional_count: int | None = None
values = _field_schema("values", Config.model_fields["values"])
optional_count = _field_schema(
"optional_count", Config.model_fields["optional_count"]
)
assert values["type"] == "array"
assert values["default"] == [1, 2]
assert optional_count["type"] == "integer"
assert "default" in optional_count
assert optional_count["default"] is None
def test_field_schema_preserves_explicit_null_input() -> None:
from pydantic import BaseModel
from rips.taskmaestro_helper.introspect import _field_schema
class Config(BaseModel):
optional_name: str | None = "fallback"
schema = _field_schema(
"optional_name", Config.model_fields["optional_name"], input_value=None
)
assert "default" in schema
assert schema["default"] is None
def test_object_model_value_is_not_an_output_port() -> None:
from pydantic import BaseModel
from taskmaestro import ObjectModel
from rips.taskmaestro_helper.introspect import _output_fields
class WrappedOutput(ObjectModel[str]):
other: int
class PlainOutput(BaseModel):
value: str
other: int
assert _output_fields(WrappedOutput) == ["other"]
assert _output_fields(PlainOutput) == ["other", "value"]
def test_dependency_edges_cover_fan_in_and_field_routing() -> None:
from rips.taskmaestro_helper.introspect import _dependency_edges
assert _dependency_edges("root", None) == []
assert _dependency_edges("next", ("producer", "value")) == [
{"from": "producer", "to": "next", "output": "value"}
]
assert _dependency_edges("merge", {"b": ("second", "value"), "a": "first"}) == [
{"from": "first", "to": "merge", "input": "a"},
{"from": "second", "to": "merge", "input": "b", "output": "value"},
]
def test_resolve_refs_substitutes_object_model_value() -> None:
from rips.taskmaestro_helper.run import resolve_refs
@@ -374,24 +174,3 @@ def test_run_helper_emits_progress_without_grpc(
("done", "completed"),
]
assert all(event["run_id"] == "run-456" for event in events)
def test_main_emits_json_to_stdout(workflow_dir: Path) -> None:
rips_root = Path(__file__).resolve().parents[2]
proc = subprocess.run(
[
sys.executable,
"-m",
"rips.taskmaestro_helper",
"introspect",
str(workflow_dir),
],
cwd=rips_root,
capture_output=True,
text=True,
check=True,
)
payload = json.loads(proc.stdout)
assert payload["name"] == "synthetic"
assert {t["name"] for t in payload["tasks"]} == {"start", "greet"}
assert payload["edges"] == [{"from": "start", "to": "greet"}]