Files
ResInsight/GrpcInterface/Python/rips/tests/test_error_reporting.py
T
Kristian Bendiksen b0496f82d6 #14683 Python: Do not treat a busy ResInsight as a dead one
ResInsight serves gRPC requests from its main thread, so a long running call
leaves the heartbeat pings unanswered for as long as that call runs. The
heartbeat counted those timeouts as connection failures and closed the channel,
which aborted the very call the user was waiting for. The client reported
"RipsError: Channel closed!" while the ResInsight process was alive and well.

Classify a ping that fails with DEADLINE_EXCEEDED or RESOURCE_EXHAUSTED as a
busy server and never count it towards the failure threshold. Only failures
that indicate a broken connection declare the connection lost. A new
busy_warning_sec argument controls when a warning is logged for a server that
has been unresponsive for a while, so long operations are visible without being
killed.

Keep the process handle from Instance.launch so the heartbeat can check whether
ResInsight actually exited. A dead process is now detected immediately and
reported with its exit code instead of a bare channel error, and check_alive()
repeats that diagnosis.
2026-09-07 18:58:05 +02:00

175 lines
5.2 KiB
Python

import sys
import os
import time
import grpc
import pytest
sys.path.insert(1, os.path.join(sys.path[0], "../../"))
import rips # noqa: E402
from rips.exception import RipsError # noqa: E402
import dataroot # noqa: E402
class _FakeRpcError(grpc.RpcError):
def __init__(self, code, details):
self._code = code
self._details = details
def code(self):
return self._code
def details(self):
return self._details
def test_rips_error_is_constructible_from_message_only():
err = RipsError("boom")
assert str(err) == "boom"
assert err.code is None
assert err.details is None
assert err.location is None
def test_rips_error_carries_structured_fields():
err = RipsError("boom", code=grpc.StatusCode.NOT_FOUND, details="x", location="h:1")
assert err.code == grpc.StatusCode.NOT_FOUND
assert err.details == "x"
assert err.location == "h:1"
def test_rips_error_from_rpc_error_extracts_code_and_details():
rpc = _FakeRpcError(grpc.StatusCode.INVALID_ARGUMENT, "bad arg")
err = RipsError.from_rpc_error(rpc, location="localhost:50051")
assert err.code == grpc.StatusCode.INVALID_ARGUMENT
assert err.details == "bad arg"
assert err.location == "localhost:50051"
text = str(err)
assert "INVALID_ARGUMENT" in text
assert "bad arg" in text
assert "localhost:50051" in text
def test_pdm_method_error_includes_code_and_details(rips_instance, initialize_test):
# Trigger a server-side INVALID_ARGUMENT on a real gRPC round-trip and
# confirm the new structured fields on RipsError are populated.
case_path = dataroot.PATH + "/Case_with_10_timesteps/Real0/BRUGGE_0000.EGRID"
rips_instance.project.load_case(path=case_path)
surface_collection = rips_instance.project.descendants(rips.SurfaceCollection)[0]
with pytest.raises(RipsError) as excinfo:
surface_collection.new_regular_surface(increment_x=-1.0)
err = excinfo.value
assert err.code is not None and err.code != grpc.StatusCode.OK
assert err.details and "Invalid increment X" in err.details
def test_heartbeat_starts_and_stops(rips_instance, initialize_test):
rips_instance.start_heartbeat(interval_sec=0.5, deadline_sec=1.0)
rips_instance.stop_heartbeat()
rips_instance.check_alive()
class _FakeApp:
"""Stub App stub whose GetVersion always fails with a given status code."""
def __init__(self, code, details=""):
self.calls = 0
self._code = code
self._details = details
def GetVersion(self, _request, timeout=None):
self.calls += 1
raise _FakeRpcError(self._code, self._details)
class _FakeChannel:
def __init__(self):
self.closed = False
def close(self):
self.closed = True
class _FakeInstance:
"""Duck-typed stand-in that exercises Instance.start_heartbeat without a
running ResInsight."""
def __init__(self, code, exit_code=None):
self.location = "localhost:50051"
self.app = _FakeApp(code)
self.channel = _FakeChannel()
self._connection_lost = False
self._connection_lost_message = None
self._heartbeat_thread = None
self._heartbeat_stop = None
self._exit_code = exit_code
def _process_exit_code(self):
return self._exit_code
start_heartbeat = rips.Instance.start_heartbeat
stop_heartbeat = rips.Instance.stop_heartbeat
check_alive = rips.Instance.check_alive
def _run_heartbeat(instance, seconds=0.5):
failures = []
instance.start_heartbeat(
interval_sec=0.02,
deadline_sec=0.02,
failure_threshold=2,
busy_warning_sec=0.05,
on_failure=failures.append,
)
time.sleep(seconds)
instance.stop_heartbeat()
return failures
def test_heartbeat_tolerates_busy_server():
"""A busy-but-alive server leaves pings unanswered (DEADLINE_EXCEEDED).
The connection must not be declared lost: closing the channel would abort
the long-running call the user is waiting for (issue #14683)."""
instance = _FakeInstance(grpc.StatusCode.DEADLINE_EXCEEDED)
failures = _run_heartbeat(instance)
assert instance.app.calls > 2, "heartbeat should keep pinging a busy server"
assert not instance._connection_lost
assert not instance.channel.closed
assert failures == []
instance.check_alive()
def test_heartbeat_detects_dead_server():
"""A gone server fails pings with UNAVAILABLE, which must close the
channel so pending calls unblock instead of hanging."""
instance = _FakeInstance(grpc.StatusCode.UNAVAILABLE)
failures = _run_heartbeat(instance)
assert instance._connection_lost
assert instance.channel.closed
assert len(failures) == 1
with pytest.raises(RipsError):
instance.check_alive()
def test_heartbeat_reports_exited_process_with_diagnostic():
"""When the launched process has exited, say so explicitly instead of
surfacing a bare 'Channel closed!'."""
instance = _FakeInstance(grpc.StatusCode.DEADLINE_EXCEEDED, exit_code=-11)
_run_heartbeat(instance)
assert instance._connection_lost
with pytest.raises(RipsError) as excinfo:
instance.check_alive()
message = str(excinfo.value)
assert "exited unexpectedly" in message
assert "-11" in message