Initial Commit

.gitignore

@@ -0,0 +1,104 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and *not* Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port

README.md

@@ -0,0 +1,16 @@
+```
+usage: index.js [-h] [-v] -u USERNAME -p PASSWORD id
+
+The Tech Game DL
+
+Positional arguments:
+  id                    The ID of the category to download
+
+Optional arguments:
+  -h, --help            Show this help message and exit.
+  -v, --version         Show program's version number and exit.
+  -u USERNAME, --username USERNAME
+                        Your username
+  -p PASSWORD, --password PASSWORD
+                        Your password
+```

index.js

@@ -0,0 +1,125 @@
+const { ArgumentParser } = require('argparse')
+const sanitize = require('sanitize-filename')
+const mkdirp = require('mkdirp2').promise
+const puppeteer = require('puppeteer')
+const fs = require('fs').promises
+const path = require('path')
+
+const parser = new ArgumentParser({
+  version: require('./package.json').version,
+  description: 'The Tech Game DL',
+  addHelp: true
+})
+parser.addArgument('id', {
+  help: 'The ID of the category to download',
+})
+parser.addArgument([ '-u', '--username' ], {
+  help: 'Your username',
+  required: true
+})
+parser.addArgument([ '-p', '--password' ], {
+  help: 'Your password',
+  required: true
+})
+
+const args = parser.parseArgs()
+
+async function main() {
+  const browser = await puppeteer.launch({
+    headless: true,
+    args: [
+      '--no-sandbox'
+    ]
+  })
+  const page = await browser.newPage()
+
+  await signIn(page, {
+    username: args.username,
+    password: args.password
+  })
+
+  await downloadCategory(browser, page, args.id)
+
+}
+
+async function downloadCategory(browser, page, id) {
+  await page.goto(`https://www.thetechgame.com/Downloads/cid=${id}.html`, {
+    waitUntil: 'domcontentloaded'
+  })
+  while (true) {
+    const links = await page.$$('a.forumlink[title=Download]')
+    for (const link of links) {
+      await downloadItem(browser, await link.evaluate((node) => node.href))
+    }
+    const nextPage = await page.$('a[title="Next page"]')
+    if (!nextPage) break
+    await nextPage.click()
+    await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
+  }
+  await browser.close()
+
+  console.log(`Everything is downloaded! Saved to directory '${args.id}'.`)
+}
+
+/**
+ * 
+ * @param {import('puppeteer').Page} page 
+ * @param {Object} credentials 
+ */
+async function signIn(page, { username, password }) {
+  await page.goto('https://www.thetechgame.com/Account.html', { waitUntil: 'domcontentloaded' })
+  await page.type('#username', username)
+  await page.type('#password', password)
+  try {
+    await Promise.all([
+      page.click('#buttons > button[type="submit"]'),
+      page.waitForNavigation({ timeout: 1000 * 120 })
+    ])
+  } catch (error) {
+    if (error.name === 'TimeoutError') {
+      console.error('Could not log in! Are your credentials correct?')
+      process.exit(1)
+    }
+    throw error
+  }
+}
+
+async function downloadItem(browser, url) {
+  const id = /\/id=(\d*)\//.exec(url)[1]
+  if (!id) throw new Error('no id') // should never happen, but will make debugging easier if it does
+
+  const page = await browser.newPage()
+
+  await page.goto(url, { waitUntil: 'domcontentloaded' })
+
+  await page.$eval('a[title="Manage your profile"]', el => el.outerHTML = '')
+
+  const pageTitle = await page.title()
+
+  const itemName = pageTitle.substr(0, pageTitle.length - ' - The Tech Game'.length)
+
+  const itemPath = path.join(
+    path.resolve('./'),
+    args.id,
+    sanitize(`${itemName} [${id}]`)
+  )
+
+  await mkdirp(itemPath)
+
+  await fs.writeFile(path.join(itemPath, 'page.html'), await page.content())
+  try {
+    await page.pdf({ path: path.join(itemPath, 'page.pdf') })
+  } catch {} // when not headless, pdf fails
+
+  await page._client.send('Page.setDownloadBehavior', {
+    behavior: 'allow',
+    downloadPath: itemPath
+  })
+  await page.click('#buttons > button[title=Download]')
+
+  await page.close()
+
+  console.log(`Downloaded: ${itemName}`)
+}
+
+main()