From cfce0a35b456451b705fd400ca9e27545f3146ff Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Wed, 13 Mar 2024 22:34:44 -0400 Subject: [PATCH] add inspect to python api --- veilid-core/src/routing_table/mod.rs | 9 ++ .../src/veilid_api/json_api/process.rs | 12 +++ .../veilid_api/json_api/routing_context.rs | 11 +++ veilid-python/tests/test_dht.py | 25 ++++++ veilid-python/veilid/api.py | 10 +++ veilid-python/veilid/json_api.py | 24 +++++ veilid-python/veilid/operations.py | 1 + veilid-python/veilid/schema/RecvMessage.json | 88 +++++++++++++++++++ veilid-python/veilid/schema/Request.json | 83 +++++++++++++++++ veilid-python/veilid/types.py | 41 ++++++++- 10 files changed, 303 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 0d1601e5..915679f5 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -390,6 +390,15 @@ impl RoutingTable { for b in &c.network.routing_table.bootstrap { cache_validity_key.append(&mut b.as_bytes().to_vec()); } + cache_validity_key.append( + &mut c + .network + .network_key_password + .clone() + .unwrap_or_default() + .as_bytes() + .to_vec(), + ); }; // Deserialize bucket map and all entries from the table store diff --git a/veilid-core/src/veilid_api/json_api/process.rs b/veilid-core/src/veilid_api/json_api/process.rs index 7a4beb02..100698e6 100644 --- a/veilid-core/src/veilid_api/json_api/process.rs +++ b/veilid-core/src/veilid_api/json_api/process.rs @@ -354,6 +354,18 @@ impl JsonRequestProcessor { ), } } + RoutingContextRequestOp::InspectDhtRecord { + key, + subkeys, + scope, + } => RoutingContextResponseOp::InspectDhtRecord { + result: to_json_api_result( + routing_context + .inspect_dht_record(key, subkeys, scope) + .await + .map(Box::new), + ), + }, }; RoutingContextResponse { rc_id: rcr.rc_id, diff --git a/veilid-core/src/veilid_api/json_api/routing_context.rs b/veilid-core/src/veilid_api/json_api/routing_context.rs index a2643f6e..b4ef2a9a 100644 --- a/veilid-core/src/veilid_api/json_api/routing_context.rs +++ b/veilid-core/src/veilid_api/json_api/routing_context.rs @@ -85,7 +85,14 @@ pub enum RoutingContextRequestOp { key: TypedKey, subkeys: ValueSubkeyRangeSet, }, + InspectDhtRecord { + #[schemars(with = "String")] + key: TypedKey, + subkeys: ValueSubkeyRangeSet, + scope: DHTReportScope, + }, } + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "rc_op")] pub enum RoutingContextResponseOp { @@ -146,4 +153,8 @@ pub enum RoutingContextResponseOp { #[serde(flatten)] result: ApiResult, }, + InspectDhtRecord { + #[serde(flatten)] + result: ApiResult>, + }, } diff --git a/veilid-python/tests/test_dht.py b/veilid-python/tests/test_dht.py index de013b1e..426367e8 100644 --- a/veilid-python/tests/test_dht.py +++ b/veilid-python/tests/test_dht.py @@ -202,3 +202,28 @@ async def test_open_writer_dht_value(api_connection: veilid.VeilidAPI): # Clean up await rc.close_dht_record(key) await rc.delete_dht_record(key) + + +@pytest.mark.asyncio +async def test_inspect_dht_record(api_connection: veilid.VeilidAPI): + rc = await api_connection.new_routing_context() + async with rc: + rec = await rc.create_dht_record(veilid.DHTSchema.dflt(2)) + + vd = await rc.set_dht_value(rec.key, 0, b"BLAH BLAH BLAH") + assert vd == None + + rr = await rc.inspect_dht_record(rec.key, [], veilid.DHTReportScope.LOCAL) + print("rr: {}", rr.__dict__) + assert rr.subkeys == [[0,1]] + assert rr.local_seqs == [0, 0xFFFFFFFF] + assert rr.network_seqs == [] + + rr2 = await rc.inspect_dht_record(rec.key, [], veilid.DHTReportScope.SYNC_GET) + print("rr2: {}", rr2.__dict__) + assert rr2.subkeys == [[0,1]] + assert rr2.local_seqs == [0, 0xFFFFFFFF] + assert rr2.network_seqs == [0, 0xFFFFFFFF] + + await rc.close_dht_record(rec.key) + await rc.delete_dht_record(rec.key) \ No newline at end of file diff --git a/veilid-python/veilid/api.py b/veilid-python/veilid/api.py index cc7e5baa..f04e83bf 100644 --- a/veilid-python/veilid/api.py +++ b/veilid-python/veilid/api.py @@ -97,6 +97,16 @@ class RoutingContext(ABC): ) -> bool: pass + @abstractmethod + async def inspect_dht_record( + self, + key: types.TypedKey, + subkeys: list[tuple[types.ValueSubkey, types.ValueSubkey]], + scope: types.DHTReportScope, + ) -> types.DHTRecordReport: + pass + + class TableDbTransaction(ABC): async def __aenter__(self) -> Self: diff --git a/veilid-python/veilid/json_api.py b/veilid-python/veilid/json_api.py index a5337978..300f2286 100644 --- a/veilid-python/veilid/json_api.py +++ b/veilid-python/veilid/json_api.py @@ -23,6 +23,8 @@ from .types import ( CryptoKeyDistance, CryptoKind, DHTRecordDescriptor, + DHTRecordReport, + DHTReportScope, DHTSchema, HashDigest, KeyPair, @@ -681,6 +683,28 @@ class _JsonRoutingContext(RoutingContext): ) ) + async def inspect_dht_record( + self, + key: TypedKey, + subkeys: list[tuple[ValueSubkey, ValueSubkey]], + scope: DHTReportScope, + ) -> DHTRecordReport: + return DHTRecordReport.from_json( + raise_api_result( + await self.api.send_ndjson_request( + Operation.ROUTING_CONTEXT, + validate=validate_rc_op, + rc_id=self.rc_id, + rc_op=RoutingContextOperation.INSPECT_DHT_RECORD, + key=key, + subkeys=subkeys, + scope=scope, + ) + ) + ) + + + ###################################################### diff --git a/veilid-python/veilid/operations.py b/veilid-python/veilid/operations.py index 3cd498a3..ccd44f21 100644 --- a/veilid-python/veilid/operations.py +++ b/veilid-python/veilid/operations.py @@ -48,6 +48,7 @@ class RoutingContextOperation(StrEnum): SET_DHT_VALUE = "SetDhtValue" WATCH_DHT_VALUES = "WatchDhtValues" CANCEL_DHT_WATCH = "CancelDhtWatch" + INSPECT_DHT_RECORD = "InspectDhtRecord" class TableDbOperation(StrEnum): diff --git a/veilid-python/veilid/schema/RecvMessage.json b/veilid-python/veilid/schema/RecvMessage.json index 49deabce..45999517 100644 --- a/veilid-python/veilid/schema/RecvMessage.json +++ b/veilid-python/veilid/schema/RecvMessage.json @@ -923,6 +923,44 @@ ] } } + }, + { + "type": "object", + "anyOf": [ + { + "type": "object", + "required": [ + "value" + ], + "properties": { + "value": { + "$ref": "#/definitions/DHTRecordReport" + } + } + }, + { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "$ref": "#/definitions/VeilidAPIError" + } + } + } + ], + "required": [ + "rc_op" + ], + "properties": { + "rc_op": { + "type": "string", + "enum": [ + "InspectDhtRecord" + ] + } + } } ], "required": [ @@ -2717,6 +2755,56 @@ } } }, + "DHTRecordReport": { + "description": "DHT Record Report", + "type": "object", + "required": [ + "local_seqs", + "network_seqs", + "subkeys" + ], + "properties": { + "local_seqs": { + "description": "The sequence numbers of each subkey requested from a locally stored DHT Record", + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + }, + "network_seqs": { + "description": "The sequence numbers of each subkey requested from the DHT over the network", + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + }, + "subkeys": { + "description": "The actual subkey range within the schema being reported on This may be a subset of the requested range if it exceeds the schema limits or has more than 512 subkeys", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + ], + "maxItems": 2, + "minItems": 2 + } + } + } + }, "DHTSchema": { "description": "Enum over all the supported DHT Schemas", "oneOf": [ diff --git a/veilid-python/veilid/schema/Request.json b/veilid-python/veilid/schema/Request.json index 1c9c4b9f..077b8f90 100644 --- a/veilid-python/veilid/schema/Request.json +++ b/veilid-python/veilid/schema/Request.json @@ -547,6 +547,49 @@ } } } + }, + { + "type": "object", + "required": [ + "key", + "rc_op", + "scope", + "subkeys" + ], + "properties": { + "key": { + "type": "string" + }, + "rc_op": { + "type": "string", + "enum": [ + "InspectDhtRecord" + ] + }, + "scope": { + "$ref": "#/definitions/DHTReportScope" + }, + "subkeys": { + "type": "array", + "items": { + "type": "array", + "items": [ + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + ], + "maxItems": 2, + "minItems": 2 + } + } + } } ], "required": [ @@ -1525,6 +1568,46 @@ } }, "definitions": { + "DHTReportScope": { + "description": "DHT Record Report Scope", + "oneOf": [ + { + "description": "Return only the local copy sequence numbers Useful for seeing what subkeys you have locally and which ones have not been retrieved", + "type": "string", + "enum": [ + "Local" + ] + }, + { + "description": "Return the local sequence numbers and the network sequence numbers with GetValue fanout parameters Provides an independent view of both the local sequence numbers and the network sequence numbers for nodes that would be reached as if the local copy did not exist locally. Useful for determining if the current local copy should be updated from the network.", + "type": "string", + "enum": [ + "SyncGet" + ] + }, + { + "description": "Return the local sequence numbers and the network sequence numbers with SetValue fanout parameters Provides an independent view of both the local sequence numbers and the network sequence numbers for nodes that would be reached as if the local copy did not exist locally. Useful for determining if the unchanged local copy should be pushed to the network.", + "type": "string", + "enum": [ + "SyncSet" + ] + }, + { + "description": "Return the local sequence numbers and the network sequence numbers with GetValue fanout parameters Provides an view of both the local sequence numbers and the network sequence numbers for nodes that would be reached as if a GetValue operation were being performed, including accepting newer values from the network. Useful for determining which subkeys would change with a GetValue operation", + "type": "string", + "enum": [ + "UpdateGet" + ] + }, + { + "description": "Return the local sequence numbers and the network sequence numbers with SetValue fanout parameters Provides an view of both the local sequence numbers and the network sequence numbers for nodes that would be reached as if a SetValue operation were being performed, including accepting newer values from the network. This simulates a SetValue with the initial sequence number incremented by 1, like a real SetValue would when updating. Useful for determine which subkeys would change on an SetValue operation", + "type": "string", + "enum": [ + "UpdateSet" + ] + } + ] + }, "DHTSchema": { "description": "Enum over all the supported DHT Schemas", "oneOf": [ diff --git a/veilid-python/veilid/types.py b/veilid-python/veilid/types.py index c47d7590..c2c2313e 100644 --- a/veilid-python/veilid/types.py +++ b/veilid-python/veilid/types.py @@ -85,6 +85,14 @@ class SafetySelectionKind(StrEnum): SAFE = "Safe" +class DHTReportScope(StrEnum): + LOCAL = "Local" + SYNC_GET = "SyncGet" + SYNC_SET = "SyncSet" + UPDATE_GET = "UpdateGet" + UPDATE_SET = "UpdateSet" + + #################################################################### @@ -353,7 +361,7 @@ class DHTRecordDescriptor: self.schema = schema def __repr__(self) -> str: - return f"<{self.__class__.__name__}(key={self.key!r})>" + return f"<{self.__class__.__name__}(key={self.key!r}, owner={self.owner!r}, owner_secret={self.owner_secret!r}, schema={self.schema!r})>" @classmethod def from_json(cls, j: dict) -> Self: @@ -368,6 +376,37 @@ class DHTRecordDescriptor: return self.__dict__ + +class DHTRecordReport: + subkeys: list[tuple[ValueSubkey, ValueSubkey]] + local_seqs: list[ValueSeqNum] + network_seqs: list[ValueSeqNum] + + def __init__( + self, + subkeys: list[tuple[ValueSubkey, ValueSubkey]], + local_seqs: list[ValueSeqNum], + network_seqs: list[ValueSeqNum], + ): + self.subkeys = subkeys + self.local_seqs = local_seqs + self.network_seqs = network_seqs + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}(subkeys={self.subkeys!r}, local_seqs={self.local_seqs!r}, network_seqs={self.network_seqs!r})>" + + @classmethod + def from_json(cls, j: dict) -> Self: + return cls( + [[p[0], p[1]] for p in j["subkeys"]], + [ValueSeqNum(s) for s in j["local_seqs"]], + [ValueSeqNum(s) for s in j["network_seqs"]], + ) + + def to_json(self) -> dict: + return self.__dict__ + + @total_ordering class ValueData: seq: ValueSeqNum