mirror of
https://github.com/OPM/ResInsight.git
synced 2026-10-25 14:40:26 -05:00
ResInsight serves gRPC requests from its main thread, so a long running call leaves the heartbeat pings unanswered for as long as that call runs. The heartbeat counted those timeouts as connection failures and closed the channel, which aborted the very call the user was waiting for. The client reported "RipsError: Channel closed!" while the ResInsight process was alive and well. Classify a ping that fails with DEADLINE_EXCEEDED or RESOURCE_EXHAUSTED as a busy server and never count it towards the failure threshold. Only failures that indicate a broken connection declare the connection lost. A new busy_warning_sec argument controls when a warning is logged for a server that has been unresponsive for a while, so long operations are visible without being killed. Keep the process handle from Instance.launch so the heartbeat can check whether ResInsight actually exited. A dead process is now detected immediately and reported with its exit code instead of a bare channel error, and check_alive() repeats that diagnosis.
175 lines
5.2 KiB
Python
175 lines
5.2 KiB
Python
import sys
|
|
import os
|
|
import time
|
|
|
|
import grpc
|
|
import pytest
|
|
|
|
sys.path.insert(1, os.path.join(sys.path[0], "../../"))
|
|
import rips # noqa: E402
|
|
from rips.exception import RipsError # noqa: E402
|
|
|
|
import dataroot # noqa: E402
|
|
|
|
|
|
class _FakeRpcError(grpc.RpcError):
|
|
def __init__(self, code, details):
|
|
self._code = code
|
|
self._details = details
|
|
|
|
def code(self):
|
|
return self._code
|
|
|
|
def details(self):
|
|
return self._details
|
|
|
|
|
|
def test_rips_error_is_constructible_from_message_only():
|
|
err = RipsError("boom")
|
|
assert str(err) == "boom"
|
|
assert err.code is None
|
|
assert err.details is None
|
|
assert err.location is None
|
|
|
|
|
|
def test_rips_error_carries_structured_fields():
|
|
err = RipsError("boom", code=grpc.StatusCode.NOT_FOUND, details="x", location="h:1")
|
|
assert err.code == grpc.StatusCode.NOT_FOUND
|
|
assert err.details == "x"
|
|
assert err.location == "h:1"
|
|
|
|
|
|
def test_rips_error_from_rpc_error_extracts_code_and_details():
|
|
rpc = _FakeRpcError(grpc.StatusCode.INVALID_ARGUMENT, "bad arg")
|
|
err = RipsError.from_rpc_error(rpc, location="localhost:50051")
|
|
assert err.code == grpc.StatusCode.INVALID_ARGUMENT
|
|
assert err.details == "bad arg"
|
|
assert err.location == "localhost:50051"
|
|
text = str(err)
|
|
assert "INVALID_ARGUMENT" in text
|
|
assert "bad arg" in text
|
|
assert "localhost:50051" in text
|
|
|
|
|
|
def test_pdm_method_error_includes_code_and_details(rips_instance, initialize_test):
|
|
# Trigger a server-side INVALID_ARGUMENT on a real gRPC round-trip and
|
|
# confirm the new structured fields on RipsError are populated.
|
|
case_path = dataroot.PATH + "/Case_with_10_timesteps/Real0/BRUGGE_0000.EGRID"
|
|
rips_instance.project.load_case(path=case_path)
|
|
|
|
surface_collection = rips_instance.project.descendants(rips.SurfaceCollection)[0]
|
|
|
|
with pytest.raises(RipsError) as excinfo:
|
|
surface_collection.new_regular_surface(increment_x=-1.0)
|
|
|
|
err = excinfo.value
|
|
assert err.code is not None and err.code != grpc.StatusCode.OK
|
|
assert err.details and "Invalid increment X" in err.details
|
|
|
|
|
|
def test_heartbeat_starts_and_stops(rips_instance, initialize_test):
|
|
rips_instance.start_heartbeat(interval_sec=0.5, deadline_sec=1.0)
|
|
rips_instance.stop_heartbeat()
|
|
rips_instance.check_alive()
|
|
|
|
|
|
class _FakeApp:
|
|
"""Stub App stub whose GetVersion always fails with a given status code."""
|
|
|
|
def __init__(self, code, details=""):
|
|
self.calls = 0
|
|
self._code = code
|
|
self._details = details
|
|
|
|
def GetVersion(self, _request, timeout=None):
|
|
self.calls += 1
|
|
raise _FakeRpcError(self._code, self._details)
|
|
|
|
|
|
class _FakeChannel:
|
|
def __init__(self):
|
|
self.closed = False
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class _FakeInstance:
|
|
"""Duck-typed stand-in that exercises Instance.start_heartbeat without a
|
|
running ResInsight."""
|
|
|
|
def __init__(self, code, exit_code=None):
|
|
self.location = "localhost:50051"
|
|
self.app = _FakeApp(code)
|
|
self.channel = _FakeChannel()
|
|
self._connection_lost = False
|
|
self._connection_lost_message = None
|
|
self._heartbeat_thread = None
|
|
self._heartbeat_stop = None
|
|
self._exit_code = exit_code
|
|
|
|
def _process_exit_code(self):
|
|
return self._exit_code
|
|
|
|
start_heartbeat = rips.Instance.start_heartbeat
|
|
stop_heartbeat = rips.Instance.stop_heartbeat
|
|
check_alive = rips.Instance.check_alive
|
|
|
|
|
|
def _run_heartbeat(instance, seconds=0.5):
|
|
failures = []
|
|
instance.start_heartbeat(
|
|
interval_sec=0.02,
|
|
deadline_sec=0.02,
|
|
failure_threshold=2,
|
|
busy_warning_sec=0.05,
|
|
on_failure=failures.append,
|
|
)
|
|
time.sleep(seconds)
|
|
instance.stop_heartbeat()
|
|
return failures
|
|
|
|
|
|
def test_heartbeat_tolerates_busy_server():
|
|
"""A busy-but-alive server leaves pings unanswered (DEADLINE_EXCEEDED).
|
|
The connection must not be declared lost: closing the channel would abort
|
|
the long-running call the user is waiting for (issue #14683)."""
|
|
instance = _FakeInstance(grpc.StatusCode.DEADLINE_EXCEEDED)
|
|
|
|
failures = _run_heartbeat(instance)
|
|
|
|
assert instance.app.calls > 2, "heartbeat should keep pinging a busy server"
|
|
assert not instance._connection_lost
|
|
assert not instance.channel.closed
|
|
assert failures == []
|
|
instance.check_alive()
|
|
|
|
|
|
def test_heartbeat_detects_dead_server():
|
|
"""A gone server fails pings with UNAVAILABLE, which must close the
|
|
channel so pending calls unblock instead of hanging."""
|
|
instance = _FakeInstance(grpc.StatusCode.UNAVAILABLE)
|
|
|
|
failures = _run_heartbeat(instance)
|
|
|
|
assert instance._connection_lost
|
|
assert instance.channel.closed
|
|
assert len(failures) == 1
|
|
with pytest.raises(RipsError):
|
|
instance.check_alive()
|
|
|
|
|
|
def test_heartbeat_reports_exited_process_with_diagnostic():
|
|
"""When the launched process has exited, say so explicitly instead of
|
|
surfacing a bare 'Channel closed!'."""
|
|
instance = _FakeInstance(grpc.StatusCode.DEADLINE_EXCEEDED, exit_code=-11)
|
|
|
|
_run_heartbeat(instance)
|
|
|
|
assert instance._connection_lost
|
|
with pytest.raises(RipsError) as excinfo:
|
|
instance.check_alive()
|
|
message = str(excinfo.value)
|
|
assert "exited unexpectedly" in message
|
|
assert "-11" in message
|