[Observability] Add claim relays counter (#644)

Co-authored-by: Daniel Olshansky <[email protected]>

telemetry/event_counters.go

@@ -94,6 +94,33 @@ func ClaimComputeUnitsCounter(
 	)
 }
 
+// ClaimRelaysCounter increments a counter which tracks the number of relays
+// represented by on-chain claims at the given ClaimProofStage.
+// If err is not nil, the counter is not incremented and an "error" label is added
+// with the error's message. I.e., Prometheus will ingest this event.
+func ClaimRelaysCounter(
+	claimProofStage prooftypes.ClaimProofStage,
+	numRelays uint64,
+	err error,
+) {
+	incrementAmount := numRelays
+	labels := []metrics.Label{
+		{Name: "unit", Value: "relays"},
+		{Name: "claim_proof_stage", Value: claimProofStage.String()},
+	}
+
+	// Ensure the counter is not incremented if there was an error.
+	if err != nil {
+		incrementAmount = 0
+	}
+
+	telemetry.IncrCounterWithLabels(
+		[]string{eventTypeMetricKey},
+		float32(incrementAmount),
+		labels,
+	)
+}
+
 // ClaimCounter increments a counter which tracks the number of claims at the given
 // ClaimProofStage.
 // If err is not nil, the counter is not incremented but Prometheus will ingest this event.

x/proof/keeper/msg_server_create_claim.go

@@ -30,13 +30,9 @@ func (k msgServer) CreateClaim(
 	defer func() {
 		// Only increment these metrics counters if handling a new claim.
 		if !isExistingClaim {
-			// TODO_IMPROVE: We could track on-chain relays here with claim.GetNumRelays().
 			telemetry.ClaimCounter(types.ClaimProofStage_CLAIMED, 1, err)
-			telemetry.ClaimComputeUnitsCounter(
-				types.ClaimProofStage_CLAIMED,
-				numComputeUnits,
-				err,
-			)
+			telemetry.ClaimRelaysCounter(types.ClaimProofStage_CLAIMED, numRelays, err)
+			telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_CLAIMED, numComputeUnits, err)
 		}
 	}()
 

x/proof/keeper/msg_server_submit_proof.go

@@ -74,13 +74,9 @@ func (k msgServer) SubmitProof(
 	defer func() {
 		// Only increment these metrics counters if handling a new claim.
 		if !isExistingProof {
-			// TODO_IMPROVE: We could track on-chain relays here with claim.GetNumRelays().
 			telemetry.ClaimCounter(types.ClaimProofStage_PROVEN, 1, err)
-			telemetry.ClaimComputeUnitsCounter(
-				types.ClaimProofStage_PROVEN,
-				numComputeUnits,
-				err,
-			)
+			telemetry.ClaimRelaysCounter(types.ClaimProofStage_PROVEN, numRelays, err)
+			telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_PROVEN, numComputeUnits, err)
 		}
 	}()
 

x/tokenomics/keeper/keeper_settle_pending_claims_test.go

@@ -119,12 +119,12 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimPendingBeforeSettlement() {
 	// Expectations: No claims should be settled because the session is still ongoing
 	blockHeight := claim.SessionHeader.SessionEndBlockHeight - 2 // session is still active
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that no claims were settled or expired.
-	require.Equal(t, uint64(0), numClaimsSettled)
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), settledResult.NumClaims)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that one claim still remains.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -142,12 +142,12 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimPendingBeforeSettlement() {
 	// 2. Settle pending claims just after the session ended.
 	// Expectations: Claims should not be settled because the proof window hasn't closed yet.
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err = s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err = s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that no claims were settled or expired.
-	require.Equal(t, uint64(0), numClaimsSettled)
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), settledResult.NumClaims)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that the claim still exists
 	claims = s.keepers.GetAllClaims(ctx)
@@ -175,13 +175,13 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimExpired_ProofRequiredAndNotProv
 	sessionEndHeight := claim.SessionHeader.SessionEndBlockHeight
 	blockHeight := shared.GetProofWindowCloseHeight(&sharedParams, sessionEndHeight)
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that no claims were settled.
-	require.Equal(t, uint64(0), numClaimsSettled)
+	require.Equal(t, uint64(0), settledResult.NumClaims)
 	// Validate that exactly one claims expired
-	require.Equal(t, uint64(1), numClaimsExpired)
+	require.Equal(t, uint64(1), expiredResult.NumClaims)
 
 	// Validate that no claims remain.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -223,14 +223,14 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimSettled_ProofRequiredAndProvide
 	sessionEndHeight := claim.SessionHeader.SessionEndBlockHeight
 	blockHeight := shared.GetProofWindowCloseHeight(&sharedParams, sessionEndHeight)
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that one claim was settled.
-	require.Equal(t, uint64(1), numClaimsSettled)
+	require.Equal(t, uint64(1), settledResult.NumClaims)
 
 	// Validate that no claims expired.
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that no claims remain.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -283,13 +283,13 @@ func (s *TestSuite) TestClaimSettlement_ClaimSettled_ProofRequiredAndProvided_Vi
 	sessionEndHeight := claim.SessionHeader.SessionEndBlockHeight
 	blockHeight := shared.GetProofWindowCloseHeight(&sharedParams, sessionEndHeight)
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that one claim was settled.
-	require.Equal(t, uint64(1), numClaimsSettled)
+	require.Equal(t, uint64(1), settledResult.NumClaims)
 	// Validate that no claims expired.
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that no claims remain.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -336,13 +336,13 @@ func (s *TestSuite) TestSettlePendingClaims_Settles_WhenAProofIsNotRequired() {
 	sessionEndHeight := claim.SessionHeader.SessionEndBlockHeight
 	blockHeight := shared.GetProofWindowCloseHeight(&sharedParams, sessionEndHeight)
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that one claim was settled.
-	require.Equal(t, uint64(1), numClaimsSettled)
+	require.Equal(t, uint64(1), settledResult.NumClaims)
 	// Validate that no claims expired.
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that no claims remain.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -426,14 +426,14 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimPendingAfterSettlement() {
 	// Expectations: No claims should be settled because the session is still ongoing
 	blockHeight := shared.GetProofWindowCloseHeight(&sharedParams, sessionOneEndHeight)
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err := s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err := s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that one claim was settled.
-	require.Equal(t, uint64(1), numClaimsSettled)
+	require.Equal(t, uint64(1), settledResult.NumClaims)
 
 	// Validate that no claims expired.
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that one claim still remains.
 	claims = s.keepers.GetAllClaims(ctx)
@@ -445,12 +445,12 @@ func (s *TestSuite) TestSettlePendingClaims_ClaimPendingAfterSettlement() {
 	// 2. Settle pending claims just after the session ended.
 	// Expectations: Claims should not be settled because the proof window hasn't closed yet.
 	sdkCtx = sdkCtx.WithBlockHeight(blockHeight)
-	numClaimsSettled, numClaimsExpired, _, _, err = s.keepers.SettlePendingClaims(sdkCtx)
+	settledResult, expiredResult, err = s.keepers.SettlePendingClaims(sdkCtx)
 	require.NoError(t, err)
 
 	// Check that no claims were settled or expired.
-	require.Equal(t, uint64(0), numClaimsSettled)
-	require.Equal(t, uint64(0), numClaimsExpired)
+	require.Equal(t, uint64(0), settledResult.NumClaims)
+	require.Equal(t, uint64(0), expiredResult.NumClaims)
 
 	// Validate that the claim still exists
 	claims = s.keepers.GetAllClaims(ctx)

x/tokenomics/keeper/settle_pending_claims.go

@@ -20,9 +20,8 @@ import (
 //
 // TODO_TECHDEBT: Refactor this function to return a struct instead of multiple return values.
 func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
-	numClaimsSettled, numClaimsExpired uint64,
-	relaysPerServiceMap map[string]uint64,
-	computeUnitsPerServiceMap map[string]uint64,
+	settledResult types.PendingClaimsResult,
+	expiredResult types.PendingClaimsResult,
 	err error,
 ) {
 	logger := k.Logger().With("method", "SettlePendingClaims")
@@ -36,8 +35,9 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 
 	logger.Info(fmt.Sprintf("found %d expiring claims at block height %d", len(expiringClaims), blockHeight))
 
-	relaysPerServiceMap = make(map[string]uint64)
-	computeUnitsPerServiceMap = make(map[string]uint64)
+	// Initialize results structs.
+	settledResult = types.NewClaimSettlementResult()
+	expiredResult = types.NewClaimSettlementResult()
 
 	logger.Debug("settling expiring claims")
 	for _, claim := range expiringClaims {
@@ -53,12 +53,12 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 		// of the total number of relays serviced and work done.
 		numClaimComputeUnits, err = claim.GetNumComputeUnits()
 		if err != nil {
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		numRelaysInSessionTree, err = claim.GetNumRelays()
 		if err != nil {
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		sessionId := claim.SessionHeader.SessionId
@@ -68,7 +68,7 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 		// claim required an on-chain proof
 		proofRequirement, err = k.proofRequirementForClaim(ctx, &claim)
 		if err != nil {
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		logger := k.logger.With(
@@ -89,7 +89,7 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 					NumRelays:       numRelaysInSessionTree,
 				}
 				if err = ctx.EventManager().EmitTypedEvent(&claimExpiredEvent); err != nil {
-					return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+					return settledResult, expiredResult, err
 				}
 
 				logger.Info("claim expired; required proof not found")
@@ -98,7 +98,9 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 				// to take up on-chain space.
 				k.proofKeeper.RemoveClaim(ctx, sessionId, claim.SupplierAddress)
 
-				numClaimsExpired++
+				expiredResult.NumClaims++
+				expiredResult.NumRelays += numRelaysInSessionTree
+				expiredResult.NumComputeUnits += numClaimComputeUnits
 				continue
 			}
 			// NB: If a proof is found, it is valid because verification is done
@@ -108,7 +110,7 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 		// Manage the mint & burn accounting for the claim.
 		if err = k.SettleSessionAccounting(ctx, &claim); err != nil {
 			logger.Error(fmt.Sprintf("error settling session accounting for claim %q: %v", claim.SessionHeader.SessionId, err))
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		claimSettledEvent := types.EventClaimSettled{
@@ -119,7 +121,7 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 		}
 
 		if err = ctx.EventManager().EmitTypedEvent(&claimSettledEvent); err != nil {
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		if err = ctx.EventManager().EmitTypedEvent(&prooftypes.EventProofUpdated{
@@ -128,7 +130,7 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 			NumRelays:       0,
 			NumComputeUnits: 0,
 		}); err != nil {
-			return 0, 0, relaysPerServiceMap, computeUnitsPerServiceMap, err
+			return settledResult, expiredResult, err
 		}
 
 		logger.Info("claim settled")
@@ -144,16 +146,22 @@ func (k Keeper) SettlePendingClaims(ctx sdk.Context) (
 			k.proofKeeper.RemoveProof(ctx, sessionId, claim.SupplierAddress)
 		}
 
-		relaysPerServiceMap[claim.SessionHeader.Service.Id] += numRelaysInSessionTree
-		computeUnitsPerServiceMap[claim.SessionHeader.Service.Id] += numClaimComputeUnits
+		settledResult.NumClaims++
+		settledResult.NumRelays += numRelaysInSessionTree
+		settledResult.NumComputeUnits += numClaimComputeUnits
+		settledResult.RelaysPerServiceMap[claim.SessionHeader.Service.Id] += numRelaysInSessionTree
 
-		numClaimsSettled++
 		logger.Info(fmt.Sprintf("Successfully settled claim for session ID %q at block height %d", claim.SessionHeader.SessionId, blockHeight))
 	}
 
-	logger.Info(fmt.Sprintf("settled %d and expired %d claims at block height %d", numClaimsSettled, numClaimsExpired, blockHeight))
+	logger.Info(fmt.Sprintf(
+		"settled %d and expired %d claims at block height %d",
+		settledResult.NumClaims,
+		expiredResult.NumClaims,
+		blockHeight,
+	))
 
-	return numClaimsSettled, numClaimsExpired, relaysPerServiceMap, computeUnitsPerServiceMap, nil
+	return settledResult, expiredResult, nil
 }
 
 // getExpiringClaims returns all claims that are expiring at the current block height.