Add rate limiting in development

README.md

@@ -42,6 +42,14 @@ Run it!
 
     $ yarn run dev
 
+To slow? Run it with your own ratelimit config
+
+    $ MAX_CONCURRENT=20; MIN_TIME=100; yarn run dev
+
+For running in production, turn off the ratelimiter
+
+    $ NO_RATELIMIT=true && yarn run build
+
 ### License
 
 This project is licensed under the MIT license. Please see [`LICENSE.md`](LICENSE.md) for the full text.

gatsby-node.js

@@ -1,40 +1,57 @@
-const { writeFile, existsSync, mkdirSync } = require('fs')
+const { writeFile, mkdir } = require('fs').promises
+const { existsSync } = require('fs')
 const axios = require('axios')
 const path = require('path')
 const readline = require('readline')
+const Bottleneck = require('bottleneck')
+
 const regions = require('./src/regions.js')
 
+const ratelimitConfig = {}
+if (!process.env.NO_RATELIMIT) {
+  ratelimitConfig.maxConcurrent = process.env.MAX_CONCURRENT || 20,
+  ratelimitConfig.minTime = process.env.MIN_TIME || 100
+}
+
+const limiter = new Bottleneck(ratelimitConfig)
+
+const axiosGet = limiter.wrap(axios.get)
+
+let startTime = Date.now()
+const logMessage = (...args) => {
+  readline.clearLine(process.stdout)
+  readline.cursorTo(process.stdout, 0)
+  const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000)
+  console.log('    ', ...args, `– ${elapsedTime} s`)
+  startTime = Date.now()
+}
+
 const imageFolder = 'static/images/'
+const dataFolder = 'data/'
 
-const downloadImage = (event, image, type = 'event') =>
-  new Promise((resolve, reject) => {
-    if (!image) resolve(null)
-    axios
-      .get(`https://api.hackclub.com${image.file_path}`, {
-        responseType: 'arraybuffer',
-      })
-      .then(res => {
-        let extension = ''
-        switch (res.headers['content-type']) {
-          case 'image/jpeg':
-            extension = '.jpg'
-            break
-          case 'image/png':
-            extension = '.png'
-            break
-          default:
-            throw `Invalid content-type: ${res.headers['content-type']}`
-        }
-        let updatedAt = Date.parse(image.updated_at)
-        const fileName = `${type}_${image.type}_${event.id}.${updatedAt}${extension}`
-        writeFile(imageFolder + fileName, res.data, 'binary', err => {
-          if (err) throw err
-          resolve(`images/${fileName}`)
-        })
-      })
-      .catch(err => reject(err))
+const downloadImage = async (event, image, type = 'event') => {
+  if (!image) {return null}
+  const res = await axiosGet(`https://api.hackclub.com${image.file_path}`, {
+    responseType: 'arraybuffer'
   })
 
+  let extension
+  switch(res.headers['content-type']) {
+    case 'image/jpeg':
+      extension = '.jpg'
+      break
+    case 'image/png':
+      extension = '.png'
+      break
+    default:
+      throw `Invalid content-type: ${res.headers['content-type']}`
+  }
+  const updatedAt = Date.parse(image.updated_at)
+  const filename = `${type}_${image.type}_${event.id}.${updatedAt}${extension}`
+  await writeFile(imageFolder + filename, res.data, 'binary')
+  return 'images/' + filename
+}
+
 const processEvent = async event => ({
   ...event,
   id: event.id.toString(),
@@ -49,139 +66,114 @@ const processGroup = async group => ({
   logo: await downloadImage(group, group.logo, 'group')
 })
 
-exports.onPreBootstrap = () => {
-  let startTime = Date.now()
-  const logMessage = (msg) => {
-    readline.clearLine(process.stdout)
-    readline.cursorTo(process.stdout, 0)
-    const elapsedTime = ((Date.now() - startTime).toFixed(2) / 1000)
-    console.log(`    ${msg} – ${elapsedTime} s`)
-    startTime = Date.now()
+const getSubscriberInfo = async () => {
+  const statsJson = await axiosGet('https://api.hackclub.com/v1/event_email_subscribers/stats')
+  logMessage('Fetched subscriber stats')
+
+  await writeFile(dataFolder + 'stats.json', JSON.stringify(statsJson.data))
+  logMessage('Subscriber stats written to file')
+}
+
+const getEventInfo = async () => {
+  const eventsJson = await axiosGet('https://api.hackclub.com/v1/events')
+  logMessage('Fetched events json')
+
+  const eventsData = await Promise.all(eventsJson.data.map(processEvent))
+  logMessage('Fetched event images')
+
+  await writeFile(dataFolder + 'events.json', JSON.stringify(eventsData))
+  logMessage('Event data written to file')
+}
+
+const getGroupInfo = async () => {
+  const groupsJson = await axiosGet('https://api.hackclub.com/v1/events/groups')
+  logMessage('Fetched groups json')
+
+  const groupsData = await Promise.all(groupsJson.data.map(processGroup))
+  logMessage('Fetched group images')
+
+  await writeFile(dataFolder + 'groups.json', JSON.stringify(groupsData))
+  logMessage('Group data written to file')
+}
+
+const ensureFolderExists = async foldername => {
+  if (!existsSync(foldername)) {
+    await mkdir(foldername)
+    logMessage('Created folder at', foldername)
   }
+}
 
-  // Download & process events
-  return axios
-    .get('https://api.hackclub.com/v1/events')
-    .then(eventsRes => {
-      logMessage('Fetched events data')
-      return axios.get('https://api.hackclub.com/v1/events/groups').then(groupsRes => {
-        logMessage('Fetched groups data')
-
-        if (!existsSync(imageFolder)) {
-          mkdirSync(imageFolder)
-          logMessage('Created image folder')
-        }
-        const groupsPromiseArray = groupsRes.data.map(group => processGroup(group))
-        return Promise.all(groupsPromiseArray).then(groupsData => {
-          const eventsPromiseArray = eventsRes.data.map(event => processEvent(event))
-          return Promise.all(eventsPromiseArray)
-            .then(eventsData => {
-              logMessage('Mapped through event data')
-              const writeGroups = new Promise((resolve, reject) => {
-                writeFile('data/groups.json', JSON.stringify(groupsData), err => {
-                  if (err) return reject(err)
-
-                  logMessage('Group data written to file')
-                  resolve()
-                })
-              })
-
-              const writeEvents = new Promise((resolve, reject) => {
-                writeFile('data/events.json', JSON.stringify(eventsData), err => {
-                  if (err) return reject(err)
-
-                  logMessage('Event data written to file')
-                  resolve()
-                })
-              })
-              return Promise.all([writeEvents, writeGroups])
-            })
-        })
-      })
-    })
-    // Download & process event stats
-    .then(() => (
-      axios.get('https://api.hackclub.com/v1/event_email_subscribers/stats')
-    ))
-    .then(res => (
-      new Promise((resolve, reject) => {
-        writeFile('data/stats.json', JSON.stringify(res.data), err => {
-          if (err) return reject(err)
-
-          logMessage('Event stats written to file')
-          resolve()
-        })
-      })
-    ))
+exports.onPreBootstrap = async () => {
+  await Promise.all([
+    ensureFolderExists(dataFolder),
+    ensureFolderExists(imageFolder)
+  ])
+
+  await Promise.all([
+    getSubscriberInfo(),
+    getEventInfo(),
+    getGroupInfo(),
+  ])
 }
 
-exports.createPages = ({ graphql, actions }) => {
+exports.createPages = async ({ graphql, actions }) => {
+  logMessage('Creating pages')
   const { createPage } = actions
 
-  return new Promise((resolve, reject) => {
-    const component = path.resolve('src/templates/region.js')
-    resolve(
-      graphql(`
-        {
-          allEventsJson {
-            edges {
-              node {
-                id
-                startHumanized: start(formatString: "MMMM D")
-                endHumanized: end(formatString: "D")
-                start
-                end
-                startYear: start(formatString: "YYYY")
-                parsed_city
-                parsed_state
-                parsed_state_code
-                parsed_country
-                parsed_country_code
-                name
-                website: website_redirect
-                latitude
-                longitude
-                banner
-                logo
-                mlh: mlh_associated
-              }
+  const component = path.resolve('src/templates/region.js')
+  const query = await graphql(`
+      {
+        allEventsJson {
+          edges {
+            node {
+              id
+              startHumanized: start(formatString: "MMMM D")
+              endHumanized: end(formatString: "D")
+              start
+              end
+              startYear: start(formatString: "YYYY")
+              parsed_city
+              parsed_state
+              parsed_state_code
+              parsed_country
+              parsed_country_code
+              name
+              website: website_redirect
+              latitude
+              longitude
+              banner
+              logo
+              mlh: mlh_associated
             }
           }
-          dataJson {
-            cities
-            countries
-          }
         }
-      `).then(result => {
-        if (result.errors) {
-          console.error(result.errors)
-          reject(result.errors)
+        dataJson {
+          cities
+          countries
         }
+      }
+    `)
+  logMessage('Ran graphql query')
+  if (query.errors) {
+    console.error(query.errors)
+    throw query.errors
+  }
 
-        regions.map(region => {
-          const events = result.data.allEventsJson.edges.filter(edge =>
-            region.filter(edge.node)
-          )
-          const emailStats = result.data.dataJson
-          if (events.length > 3) {
-            createPage({
-              path: region.path,
-              component,
-              context: {
-                region,
-                events,
-                emailStats
-              },
-            })
-          }
-        })
-      })
+  await Promise.all(regions.map(async region => {
+    const events = query.data.allEventsJson.edges.filter(edge =>
+      region.filter(edge.node)
     )
-    locations.map(location => {
-      createPage({
-        path: location,
+    const emailStats = query.data.dataJson
+    if (events.length > 3) {
+      await createPage({
+        path: region.path,
         component,
+        context: {
+          region,
+          events,
+          emailStats
+        },
       })
-    })
-  })
+    }
+  }))
 }

package.json

@@ -7,6 +7,7 @@
     "@hackclub/design-system": "0.0.1-10",
     "axios": "^0.19.0",
     "babel-plugin-styled-components": "^1.10.6",
+    "bottleneck": "^2.19.5",
     "formik": "^2.0.8",
     "gatsby": "^2.18.16",
     "gatsby-plugin-canonical-urls": "^2.1.18",

yarn.lock

@@ -2595,6 +2595,11 @@ boolbase@^1.0.0, boolbase@~1.0.0:
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
 
+bottleneck@^2.19.5:
+  version "2.19.5"
+  resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
+  integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==
+
 [email protected]:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b"