Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Observability] Add claim relays counter #644

Merged
merged 15 commits into from
Jul 12, 2024
Merged

[Observability] Add claim relays counter #644

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
fa2c875
chore: add claim relays telemetry counter
bryanchriswhite Jul 2, 2024
7210376
telemetry: increment claim relays counter in claim/proof handlers
bryanchriswhite Jul 2, 2024
b8a2edc
refactor: pending claims result & increment claim relays counter
bryanchriswhite Jul 2, 2024
ae41701
fix: pending results tests post-refactor
bryanchriswhite Jul 2, 2024
154716e
docs: add godoc comments
bryanchriswhite Jul 2, 2024
beedf18
chore: fix & update stress test dashboard
bryanchriswhite Jul 2, 2024
9182b0c
Merge remote-tracking branch 'pokt/main' into issues/643/telemetry/re…
bryanchriswhite Jul 4, 2024
5850eb2
fix: relays stress test
bryanchriswhite Jul 4, 2024
df546c6
chore: restore dashboard updates
bryanchriswhite Jul 4, 2024
54b1c4c
chore: remove done todo
bryanchriswhite Jul 4, 2024
5eb79cc
chore: review feedback improvements
bryanchriswhite Jul 6, 2024
86ee63a
Merge remote-tracking branch 'pokt/main' into issues/643/telemetry/re…
bryanchriswhite Jul 6, 2024
15dd400
Merge branch 'main' into issues/643/telemetry/relays
bryanchriswhite Jul 8, 2024
b74238e
Merge branch 'main' into issues/643/telemetry/relays
bryanchriswhite Jul 11, 2024
6e0fd69
Merge branch 'main' into issues/643/telemetry/relays
bryanchriswhite Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 184 additions & 15 deletions localnet/grafana-dashboards/stress-test-dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"proven\", unit=\"claims\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"PROVEN\", unit=\"claims\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -933,7 +933,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"claimed\", unit=\"claims\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"CLAIMED\", unit=\"claims\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand Down Expand Up @@ -1034,7 +1034,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"not_required\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"NOT_REQUIRED\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1052,7 +1052,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"probabilistic_selection\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"PROBABILISTIC\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1069,7 +1069,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"threshold_selection\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", proof_required_reason=\"THRESHOLD\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand Down Expand Up @@ -1170,7 +1170,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"claimed\", unit=\"compute_units\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"CLAIMED\", unit=\"compute_units\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1188,7 +1188,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"proven\", unit=\"compute_units\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"PROVEN\", unit=\"compute_units\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1205,7 +1205,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"settled\", unit=\"compute_units\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"SETTLED\", unit=\"compute_units\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1214,6 +1214,23 @@
"range": true,
"refId": "C",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"EXPIRED\", unit=\"compute_units\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "expired",
"range": true,
"refId": "D",
"useBackend": false
}
],
"title": "Compute Units Lifecycle",
Expand Down Expand Up @@ -1306,7 +1323,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"settled\", unit=\"claims\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"SETTLED\", unit=\"claims\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand All @@ -1324,7 +1341,7 @@
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"expired\", unit=\"claims\"})",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"EXPIRED\", unit=\"claims\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
Expand Down Expand Up @@ -1399,7 +1416,7 @@
},
"gridPos": {
"h": 8,
"w": 12,
"w": 8,
"x": 0,
"y": 32
},
Expand Down Expand Up @@ -1499,8 +1516,161 @@
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"w": 8,
"x": 8,
"y": 32
},
"id": 16,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"maxHeight": 600,
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"CLAIMED\", unit=\"relays\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"instant": false,
"interval": "",
"legendFormat": "claimed",
"range": true,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"PROVEN\", unit=\"relays\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "proven",
"range": true,
"refId": "B",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"SETTLED\", unit=\"relays\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "settled",
"range": true,
"refId": "C",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "avg by(job) (event_type{container=\"poktrolld-validator\", claim_proof_stage=\"EXPIRED\", unit=\"relays\"})",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "expired",
"range": true,
"refId": "D",
"useBackend": false
}
],
"title": "Relays Lifecycle",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 32
},
"id": 15,
Expand Down Expand Up @@ -1548,10 +1718,9 @@
"list": []
},
"time": {
"from": "now-5m",
"from": "now-15m",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
"timezone": "browser",
"title": "Protocol / Stress test",
Expand Down
28 changes: 28 additions & 0 deletions telemetry/event_counters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,34 @@ func ClaimComputeUnitsCounter(
)
}

// ClaimRelaysCounter increments a counter which tracks the number of relays which
// are represented by on-chain claims at the given ClaimProofStage.
bryanchriswhite marked this conversation as resolved.
Show resolved Hide resolved
// If err is not nil, the counter is not incremented and an "error" label is added
// with the error's message. I.e., Prometheus will ingest this event.
func ClaimRelaysCounter(
claimProofStage prooftypes.ClaimProofStage,
numRelays uint64,
err error,
) {
incrementAmount := numRelays
labels := []metrics.Label{
{Name: "unit", Value: "relays"},
{Name: "claim_proof_stage", Value: claimProofStage.String()},
}

// Ensure the counter is not incremented if there was an error.
if err != nil {
incrementAmount = 0
labels = AppendErrLabel(err, labels...)
bryanchriswhite marked this conversation as resolved.
Show resolved Hide resolved
}

telemetry.IncrCounterWithLabels(
[]string{eventTypeMetricKey},
float32(incrementAmount),
labels,
)
}

// ClaimCounter increments a counter which tracks the number of claims at the given
// ClaimProofStage.
// If err is not nil, the counter is not incremented and an "error" label is added
Expand Down
7 changes: 2 additions & 5 deletions x/proof/keeper/msg_server_create_claim.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,8 @@ func (k msgServer) CreateClaim(
if !isExistingClaim {
// TODO_IMPROVE: We could track on-chain relays here with claim.GetNumRelays().
bryanchriswhite marked this conversation as resolved.
Show resolved Hide resolved
telemetry.ClaimCounter(types.ClaimProofStage_CLAIMED, 1, err)
telemetry.ClaimComputeUnitsCounter(
types.ClaimProofStage_CLAIMED,
numComputeUnits,
err,
)
telemetry.ClaimRelaysCounter(types.ClaimProofStage_CLAIMED, numRelays, err)
telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_CLAIMED, numComputeUnits, err)
}
}()

Expand Down
8 changes: 2 additions & 6 deletions x/proof/keeper/msg_server_submit_proof.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,9 @@ func (k msgServer) SubmitProof(
defer func() {
// Only increment these metrics counters if handling a new claim.
if !isExistingProof {
// TODO_IMPROVE: We could track on-chain relays here with claim.GetNumRelays().
telemetry.ClaimCounter(types.ClaimProofStage_PROVEN, 1, err)
telemetry.ClaimComputeUnitsCounter(
types.ClaimProofStage_PROVEN,
numComputeUnits,
err,
)
telemetry.ClaimRelaysCounter(types.ClaimProofStage_PROVEN, numRelays, err)
telemetry.ClaimComputeUnitsCounter(types.ClaimProofStage_PROVEN, numComputeUnits, err)
}
}()

Expand Down
Loading
Loading