@daviesthomas, thank you for flagging this issue.
Thank you also to @Inspector_POI for providing some very useful input, it is much appreciated.
We have checked our logs from 4th December (the epoch in question): we did not submit a manual POI in order to close the allocation on subgraph with Deployment ID QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR. As @Inspector_POI has suggested, the stale POI was provided by the Indexer Agent. Excerpts from our logs with comments are included below for clarity. Please let us know if you require any further details.
It is our understanding that the agent should only provide a stale POI for a subgraph where the error is deterministic β and it was also our understanding that a deterministic error should be experienced simultaneously by all indexers on that subgraph and should not be recoverable from. Given that some indexers have managed to overcome the issue we presume that there must be a discrepancy between the different indexer stacks (as already mentioned by @Inspector_POI).
Our subgraph deployment shows the error:
A determinstic fatal error occured at block 20243414: "transaction ff13ada1d989429b4ae0eb06523f2dc30a023fed64fcc28f852b48e6c49e0130: Mapping aborted at src/common/transactions.ts, line 63, column 30, with message: unexpected null\twasm backtrace:\t 0: 0xd2a7 - <unknown>!src/common/transactions/createWithdrawTransaction\t 1: 0xd739 - <unknown>!src/mappings/helpers/createWithdraw\t 2: 0xd82a - <unknown>!src/mappings/handlers/handleWithdrawn\t in handler 'handleWithdrawn' at block #20243414 (10e950f41b6e65faf3647fc842c9d6abe21c47cb9092160d2502fd50ffaba4a3)"
To clarify generally - since the agent has been able to generate POIs for deterministic failures, we do not manually generate or provide POIs when closing allocations.
Logs:
# unallocate action queued via the CLI
Dec 04 12:36:21 lunanova-agent graph-indexer-agent[2210627]:
{
"level": 20,
"time": 1733315781891,
"pid": 2210627,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "IndexerManagementClient",
"actions": [
{
"status": "queued",
"type": "unallocate",
"deploymentID": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"allocationID": "0x6e4cb16d9b519056ded7271330f44bff3b21d82f",
"force": false,
"source": "indexerCLI",
"reason": "manual",
"priority": 0,
"protocolNetwork": "eip155:42161"
}
],
"msg": "Execute 'queueActions' mutation"
}
# unallocate action approved via the CLI - note null value for poi
Dec 04 13:02:40 lunanova-agent graph-indexer-agent[2210627]:
{
"level": 30,
"time": 1733317360074,
"pid": 2210627,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "QueueMonitor",
"protocolNetwork": "eip155:42161",
"indexer": "0xE13840A2E92e0Cb17A246609b432D0fA2e418774",
"operator": "0xF6Bad1DCa165c492006317B4f6E03a1eF5414E23",
"approvedActions": [
{
"id": 10610,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"allocationID": "0x6e4cb16d9b519056ded7271330f44bff3b21d82f",
"amount": null,
"poi": null,
"force": false,
"source": "indexerCLI",
"reason": "manual",
"createdAt": "2024-12-04T12:36:22.521Z",
"updatedAt": "2024-12-04T13:02:22.618Z",
"transaction": null,
"failureReason": null,
"protocolNetwork": "eip155:42161"
},
{
"id": 10609,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmaAqQHCmACkd8HyreiMboMtPNCmtvVps9Ju5E4kNh6KAo",
"allocationID": "0x95009482032ae25f776ae496158b6d08dedbb65e",
"amount": null,
"poi": null,
"force": false,
"source": "indexerCLI",
"reason": "manual",
"createdAt": "2024-12-04T12:36:20.678Z",
"updatedAt": "2024-12-04T13:02:22.618Z",
"transaction": null,
"failureReason": null,
"protocolNetwork": "eip155:42161"
},
...
# approved actions executed via the CLI (note POI at this point is still null)
Dec 04 13:57:53 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 20,
"time": 1733320673880,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "ActionManager",
"function": "executeApprovedActions",
"protocolNetwork": "eip155:42161",
"approvedActions": [
{
"id": 10605,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmaD9zWMc6kQrBVbtjXNjNjbhpGWUHQFQdxABBEaVJqK5s",
...
},
{
"id": 10610,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"allocationID": "0x6e4cb16d9b519056ded7271330f44bff3b21d82f",
"amount": null,
"poi": null,
"force": false,
"source": "indexerCLI",
"reason": "manual",
"createdAt": "2024-12-04T12:36:22.521Z",
"updatedAt": "2024-12-04T13:57:50.504Z",
"transaction": null,
"failureReason": null,
"protocolNetwork": "eip155:42161"
},
{
"id": 10600,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmXcxzZioHXV5ts2UcG6gNNEayoaZ9ip7D9JvPS88K2HXe",
...
}
],
"msg": "Found 11 approved actions for this network "
}
# batch validated
Dec 04 13:57:53 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 20,
"time": 1733320673881,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"function": "validateActionBatch",
"size": 11,
"msg": "Validating action batch"
}
# batch ready (POI still null)
Dec 04 13:57:55 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 30,
"time": 1733320675057,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "QueueMonitor",
"protocolNetwork": "eip155:42161",
"indexer": "0xE13840A2E92e0Cb17A246609b432D0fA2e418774",
"operator": "0xF6Bad1DCa165c492006317B4f6E03a1eF5414E23",
"approvedActions": [
{
"id": 10610,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"allocationID": "0x6e4cb16d9b519056ded7271330f44bff3b21d82f",
"amount": null,
"poi": null,
"force": false,
"source": "indexerCLI",
"reason": "manual",
"createdAt": "2024-12-04T12:36:22.521Z",
"updatedAt": "2024-12-04T13:57:50.504Z",
"transaction": null,
"failureReason": null,
"protocolNetwork": "eip155:42161"
},
{
"id": 10609,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmaAqQHCmACkd8HyreiMboMtPNCmtvVps9Ju5E4kNh6KAo",
"allocationID": "0x95009482032ae25f776ae496158b6d08dedbb65e",
"amount": null,
"poi": null,
...
}
],
"msg": "Batch ready?"
}
# batch being executed (poi still null)
{
"level": 30,
"time": 1733320675057,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "QueueMonitor",
"protocolNetwork": "eip155:42161",
"indexer": "0xE13840A2E92e0Cb17A246609b432D0fA2e418774",
"operator": "0xF6Bad1DCa165c492006317B4f6E03a1eF5414E23",
"actions": [
{
"id": 10610,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"allocationID": "0x6e4cb16d9b519056ded7271330f44bff3b21d82f",
"amount": null,
"poi": null,
"force": false,
"source": "indexerCLI",
"reason": "manual",
"createdAt": "2024-12-04T12:36:22.521Z",
"updatedAt": "2024-12-04T13:57:50.504Z",
"transaction": null,
"failureReason": null,
"protocolNetwork": "eip155:42161"
},
{
"id": 10609,
"type": "unallocate",
"status": "approved",
"priority": 0,
"deploymentID": "QmaAqQHCmACkd8HyreiMboMtPNCmtvVps9Ju5E4kNh6KAo",
"allocationID": "0x95009482032ae25f776ae496158b6d08dedbb65e",
"amount": null,
"poi": null,
...
],
"note": "If actions were approved very recently they may be missing from this batch",
"msg": "Executing batch of approved actions"
}
# allocation preparing to close (poi = "none provided")
Dec 04 13:57:55 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 30,
"time": 1733320675100,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"action": 10602,
"allocationID": "0x26834a0a8a9aa19af100cf82d1ab4f45c6d3a41e",
"poi": "none provided",
"msg": "Preparing to close allocation (unallocate)"
}
# allocation closed (POI provided by the agent at this point)
Dec 04 13:58:01 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 30,
"time": 1733320681533,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"action": 10609,
"deployment": {
"bytes32": "0xafc88676236f1704a1bd94d7536f4ef79fcd4aa1dd12ad15fdb6aeb5d9b83b68",
"ipfsHash": "QmaAqQHCmACkd8HyreiMboMtPNCmtvVps9Ju5E4kNh6KAo"
},
"allocation": "0x95009482032Ae25F776AE496158b6d08deDbb65e",
"indexer": "0xE13840A2E92e0Cb17A246609b432D0fA2e418774",
"amountGRT": "1117.69",
"poi": "0xf841a068d0e47f12ab4d7f191e7d7bbc35fb41b03ed5c09a7bc79c7e48ab7012",
"transaction": "0xb97a4b13c330e834bfe19c9aa39a9a8a4097ac7c219799e1f79d71bc68f418c5",
"indexingRewards": {
"type": "BigNumber",
"hex": "0x350370c878bb49c0"
},
"msg": "Successfully closed allocation"
}
# transaction confirmed as closed
Dec 04 13:58:01 lunanova-agent graph-indexer-agent[2211217]: {
{
"level": 30,
"time": 1733320681533,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"action": 10610,
"msg": "Confirming 'closeAllocation' transaction"
}
# allocation closed with POI provided by the agent
Dec 04 13:58:01 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 30,
"time": 1733320681537,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"action": 10610,
"deployment": {
"bytes32": "0x2cad565522817104e42bb2f6c2200079d76827a2f60476ff1460ff492e71b5f6",
"ipfsHash": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR"
},
"allocation": "0x6e4CB16D9B519056deD7271330F44bFF3b21d82F",
"indexer": "0xE13840A2E92e0Cb17A246609b432D0fA2e418774",
"amountGRT": "29051.1",
"poi": "0xa5130d48a714d22733f90426f3503c84deb22a1132f395a49125efa5081c4252",
"transaction": "0xb97a4b13c330e834bfe19c9aa39a9a8a4097ac7c219799e1f79d71bc68f418c5",
"indexingRewards": {
"type": "BigNumber",
"hex": "0x0481791183db7201e4"
},
"msg": "Successfully closed allocation"
}
# rules updated by the agent after allocation closed
Dec 04 13:58:01 lunanova-agent graph-indexer-agent[2211217]:
{
"level": 20,
"time": 1733320681778,
"pid": 2211217,
"hostname": "lunanova-agent",
"name": "IndexerAgent",
"component": "AllocationManager",
"protocolNetwork": "eip155:42161",
"action": 10610,
"rule": {
"allocationLifetime": null,
"autoRenewal": true,
"requireSupported": true,
"safety": true,
"identifier": "QmRM3wyF3d76L6PP79GxCcSSdFAKEv6AtivfFr4c7RELSR",
"protocolNetwork": "eip155:42161",
"identifierType": "deployment",
"decisionBasis": "offchain",
"id": 367,
"allocationAmount": "141648350000000000000000",
"parallelAllocations": null,
"maxAllocationPercentage": null,
"minSignal": null,
"minStake": null,
"maxSignal": null,
"minAverageQueryFees": null,
"custom": null,
"createdAt": "2023-07-21T05:46:22.360Z",
"updatedAt": "2024-12-04T13:58:01.769Z"
},
"msg": "DecisionBasis.offchain rule merged into indexing rules"
}