feat(cu): add option for multiple gql urls for loadblockmeta

servers/cu/src/bootstrap.js

@@ -303,6 +303,8 @@ export const createApis = async (ctx) => {
   //   labelNames: ['processId', 'cron', 'dryRun', 'error']
   // })
 
+  const BLOCK_GRAPHQL_ARRAY = ctx.GRAPHQL_URLS.length > 0 ? ctx.GRAPHQL_URLS : [ctx.GRAPHQL_URL]
+
   const sharedDeps = (logger) => ({
     loadTransactionMeta: ArweaveClient.loadTransactionMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, logger }),
     loadTransactionData: ArweaveClient.loadTransactionDataWith({ fetch: ctx.fetch, ARWEAVE_URL: ctx.ARWEAVE_URL, logger }),
@@ -336,7 +338,7 @@ export const createApis = async (ctx) => {
     saveEvaluation: AoEvaluationClient.saveEvaluationWith({ db, logger }),
     findBlocks: AoBlockClient.findBlocksWith({ db, logger }),
     saveBlocks: AoBlockClient.saveBlocksWith({ db, logger }),
-    loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: ctx.GRAPHQL_URL, pageSize: 90, logger }),
+    loadBlocksMeta: AoBlockClient.loadBlocksMetaWith({ fetch: ctx.fetch, GRAPHQL_URL: BLOCK_GRAPHQL_ARRAY, pageSize: 90, logger }),
     findModule: AoModuleClient.findModuleWith({ db, logger }),
     saveModule: AoModuleClient.saveModuleWith({ db, logger }),
     loadEvaluator: AoModuleClient.evaluatorWith({

servers/cu/src/config.js

@@ -132,6 +132,7 @@ const CONFIG_ENVS = {
     ENABLE_METRICS_ENDPOINT: process.env.ENABLE_METRICS_ENDPOINT,
     GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net',
     GRAPHQL_URL: process.env.GRAPHQL_URL,
+    GRAPHQL_URLS: process.env.GRAPHQL_URLS?.split(',') || ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql'],
     CHECKPOINT_GRAPHQL_URL: process.env.CHECKPOINT_GRAPHQL_URL,
     ARWEAVE_URL: process.env.ARWEAVE_URL,
     UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net',
@@ -183,6 +184,7 @@ const CONFIG_ENVS = {
     ENABLE_METRICS_ENDPOINT: process.env.ENABLE_METRICS_ENDPOINT,
     GATEWAY_URL: process.env.GATEWAY_URL || 'https://arweave.net',
     GRAPHQL_URL: process.env.GRAPHQL_URL,
+    GRAPHQL_URLS: process.env.GRAPHQL_URLS?.split(',') || ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql'],
     CHECKPOINT_GRAPHQL_URL: process.env.CHECKPOINT_GRAPHQL_URL,
     ARWEAVE_URL: process.env.ARWEAVE_URL,
     UPLOADER_URL: process.env.UPLOADER_URL || 'https://up.arweave.net',

servers/cu/src/domain/model.js

@@ -49,6 +49,13 @@ export const domainConfigSchema = z.object({
    * ie. https://arweave.net/graphql
    */
   GRAPHQL_URL: z.string().url('GRAPHQL_URL must be a valid URL'),
+  /**
+   * An array of urls for the graphql server to be used by the CU
+   * to query for block metadata from an Arweave Gateway. On retries, the urls will be cycled through.
+   *
+   * ie. ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql']
+   */
+  GRAPHQL_URLS: z.array(z.string().url('GraphQL_URLS must be a valid URL')),
   /**
    * The url for the graphql server to be used by the CU
    * to query for process Checkpoints.

servers/cu/src/domain/utils.js

@@ -376,7 +376,7 @@ export const backoff = (
    * Recursive function that recurses with exponential backoff
    */
   const action = (retry, delay) => {
-    return Promise.resolve()
+    return Promise.resolve({ retry })
       .then(fn)
       .catch((err) => {
         // Reached max number of retries

servers/cu/src/effects/ao-block.js

@@ -141,6 +141,11 @@ export function loadBlocksMetaWith ({
       }
     `
 
+  function customFetch (urls, options, retry = 0) {
+    const urlLength = urls.length
+    const url = urls[retry % urlLength]
+    return fetch(url, options)
+  }
   async function fetchPage ({ min, maxTimestamp }) {
     return Promise.resolve({ min, limit: pageSize })
       .then(variables => {
@@ -154,21 +159,29 @@ export function loadBlocksMetaWith ({
       })
       .then((variables) => {
         return backoff(
-          () => fetch(GRAPHQL_URL, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({
-              query: GET_BLOCKS_QUERY,
-              variables
-            })
-          }).then(okRes).catch(async (e) => {
-            logger(
-              'Error Encountered when fetching page of block metadata from gateway with minBlock \'%s\' and maxTimestamp \'%s\'',
-              min,
-              maxTimestamp
+          ({ retry }) => {
+            return customFetch(
+              GRAPHQL_URL,
+              {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                  query: GET_BLOCKS_QUERY,
+                  variables
+                })
+              },
+              retry
             )
-            throw new Error(`Can not communicate with gateway to retrieve block metadata: ${await strFromFetchError(e)}`)
-          }),
+              .then(okRes)
+              .catch(async (e) => {
+                logger(
+                  'Error Encountered when fetching page of block metadata from gateway with minBlock \'%s\' and maxTimestamp \'%s\'',
+                  min,
+                  maxTimestamp
+                )
+                throw new Error(`Can not communicate with gateway to retrieve block metadata: ${await strFromFetchError(e)}`)
+              })
+          },
           { maxRetries: 2, delay: 300, log: logger, name: `loadBlockMeta(${JSON.stringify({ newMin: min, maxTimestamp })})` }
         )
       })

servers/cu/src/effects/ao-block.test.js

@@ -8,7 +8,7 @@ import { createTestLogger } from '../domain/logger.js'
 import { findBlocksSchema, loadBlocksMetaSchema, saveBlocksSchema } from '../domain/dal.js'
 import { findBlocksWith, loadBlocksMetaWith, saveBlocksWith } from './ao-block.js'
 
-const GRAPHQL_URL = globalThis.GRAPHQL_URL || 'https://arweave.net/graphql'
+const GRAPHQL_URL = globalThis.GRAPHQL_URL || ['https://arweave.net/graphql', 'https://arweave-search.goldsky.com/graphql']
 const logger = createTestLogger({ name: 'ao-cu' })
 
 describe('ao-block', () => {
@@ -132,7 +132,8 @@ describe('ao-block', () => {
       let errorCount = 0
       let errorCaught = false
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
-        fetch: () => {
+        fetch: (url) => {
+          assert.equal(url, GRAPHQL_URL[errorCount % GRAPHQL_URL.length])
           errorCount++
           throw Error('Fetch error!')
         },
@@ -151,7 +152,8 @@ describe('ao-block', () => {
     test('should circuit break on failure', async () => {
       let errorsCount = 0
       const loadBlocksMeta = loadBlocksMetaSchema.implement(loadBlocksMetaWith({
-        fetch: () => {
+        fetch: (url) => {
+          assert.equal(url, GRAPHQL_URL[errorsCount % GRAPHQL_URL.length])
           throw Error('Fetch error!')
         },
         GRAPHQL_URL,
@@ -160,7 +162,7 @@ describe('ao-block', () => {
         breakerOptions: {
           timeout: 5000, // 5 seconds timeout
           errorThresholdPercentage: 50, // open circuit after 50% failures
-          resetTimeout: 1000 // attempt to close circuit after 15 seconds
+          resetTimeout: 1000 // attempt to close circuit after 1 second
         }
       }))