From 5bc50a54565c1c814da1f1a1b07b4fc7c9604dca Mon Sep 17 00:00:00 2001 From: Jeff Steward Date: Fri, 31 Jul 2020 23:33:38 -0400 Subject: [PATCH] refactor and add more stats --- modules/data.js | 209 +++++++++++++++++++++++++++++++++---------- routes/index.js | 110 ++++++++++------------- views/production.hbs | 88 ++++++++++++------ 3 files changed, 271 insertions(+), 136 deletions(-) diff --git a/modules/data.js b/modules/data.js index c0b8915..3dc8cb6 100644 --- a/modules/data.js +++ b/modules/data.js @@ -1,58 +1,171 @@ const fetch = require('node-fetch'); -const async = require('async'); +const querystring = require('querystring'); +const { response } = require('express'); const API_KEY = process.env['API_KEY']; -let data = { - objects: { - recordcount: 0, - onview: 0 - }, - exhibitions: { - current: [] +function makeURL(endpoint, parameters, aggregations) { + const HAM_API_URL = 'https://api.harvardartmuseums.org'; + + let qs = { + apikey: API_KEY + }; + + if (parameters) { + qs = {...qs, ...parameters}; + }; + + if (aggregations) { + qs.aggregation = JSON.stringify(aggregations); + } + + return `${HAM_API_URL}/${endpoint}?${querystring.encode(qs)}`; +} + +function getObjectStats(callback) { + const params = { + size: 1, + }; + const aggs = { + "by_accesslevel": { + "terms": { + "field": "accesslevel" + } } }; + const url = makeURL('object', params, aggs); -module.exports = { - getAPIStats: function() { - - async.series([ - function(callback) { - const url = `https://api.harvardartmuseums.org/object?apikey=${API_KEY}&size=0&q=accesslevel:1`; - fetch(url) - .then(response => response.json()) - .then(results => { - callback(null, results['info']['totalrecords']); - }); - + fetch(url) + .then(response => response.json()) + .then(results => { + let e = new Date(results['records'][0]['lastupdate']); + let d = new Date(results['records'][0]['lastupdate']); + d.setHours(d.getHours() + 2); + + let output = { + lastexport: e.toLocaleString('en-US', {timeZone: "America/New_York"}), + lastrefresh: d.toLocaleString('en-US', {timeZone: "America/New_York"}), + recordcount: results['info']['totalrecords'], + recordcount_public: results['aggregations']['by_accesslevel']['buckets'][0]['doc_count'] + }; + + callback(null, output); + }); +} + +function getObjectsInGalleryStats(callback) { + const params = { + size: 0, + gallery: 'any' + }; + const url = makeURL('object', params); + + fetch(url) + .then(response => response.json()) + .then(results => { + callback(null, results['info']['totalrecords']); + }); +} + +function getCurrentExhibitions(callback) { + const params = { + venue: 'HAM', + status: 'current' + }; + const url = makeURL('exhibition', params); + + fetch(url) + .then(response => response.json()) + .then(results => { + callback(null, results['records']); + }); +} + +function getAltTextStats(callback) { + const params = { + size: 0, + q: 'images.alttext:* AND accesslevel:1' + }; + const url = makeURL('object', params); + + fetch(url) + .then(response => response.json()) + .then(results => { + callback(null, results['info']['totalrecords']); + }); +} + +function getActivityStats(callback) { + const params = { + size: 1, + sort: 'activitycount', + sortorder: 'desc' + }; + const aggs = { + "by_type": { + "terms": { + "field": "activitytype" + }, + "aggs": { + "objects_touched": { + "cardinality": { + "field": "objectid" + } + }, + "activity_stats": { + "extended_stats": { + "field": "activitycount" + } + }, + "date_stats": { + "extended_stats": { + "field": "date" + } + } + } + } + }; + const url = makeURL('activity', params, aggs); + + fetch(url) + .then(response => response.json()) + .then(results => { + let object = results['records'][0]; + let databuckets = results['aggregations']['by_type']['buckets']; + let pageviews = databuckets.find(bucket => bucket.key === 'pageviews'); + + let output = { + pageviews: { + objects: { + count: pageviews['objects_touched']['value'] }, - function(callback) { - const url = `https://api.harvardartmuseums.org/object?apikey=${API_KEY}&size=0&gallery=any`; - fetch(url) - .then(response => response.json()) - .then(results => { - callback(null, results['info']['totalrecords']); - }); - + statsdates: { + start: pageviews['date_stats']['min_as_string'].substr(0, 10), + end: pageviews['date_stats']['max_as_string'].substr(0, 10) }, - function(callback) { - const url = `https://api.harvardartmuseums.org/exhibition?apikey=${API_KEY}&venue=HAM&status=current`; - fetch(url) - .then(response => response.json()) - .then(results => { - callback(null, results['records']); - }); - - } - ], - function(err, results) { - data.objects.recordcount = results[0]; - data.objects.onview = results[1]; - data.exhibitions.current = results[2]; - - return data; + singledaymostviews: { + date: object['date'], + activitycount: object['activitycount'] } - ); - - } -} \ No newline at end of file + } + }; + + let objectUrl = makeURL(`object/${object['objectid']}`); + fetch(objectUrl) + .then(response => response.json()) + .then(results => { + output.pageviews.singledaymostviews.object = results; + + callback(null, output); + }); + + }); +} + +module.exports = { + getObjectStats: getObjectStats, + getObjectsInGalleryStats: getObjectsInGalleryStats, + getCurrentExhibitions: getCurrentExhibitions, + getAltTextStats: getAltTextStats, + getActivityStats: getActivityStats +}; \ No newline at end of file diff --git a/routes/index.js b/routes/index.js index 3b4506b..ac90727 100644 --- a/routes/index.js +++ b/routes/index.js @@ -1,10 +1,9 @@ var express = require('express'); var router = express.Router(); -var fetch = require('node-fetch'); var async = require('async'); -var stats = require('../modules/data') +var stats = require('../modules/data'); +const { stat } = require('fs'); -const API_KEY = process.env['API_KEY']; const APP_TITLE = 'HAM Dashboard'; let data = { @@ -12,15 +11,26 @@ let data = { dateoflastrefresh: "2000-01-01", dateoflastexport: "2000-01-01", objects: { - recordcount: 0, - onview: 0 + count: 0, + public: { + count: 0, + count_as_percent: 0 + }, + onview: { + count: 0, + count_as_percent: 0 + }, + alttext: { + count: 0, + count_as_percent: 0 + } }, + pageviews: {}, exhibitions: { current: [] } }; - /* GET environment specific page. */ router.get('/:env', function(req, res, next) { res.render(req.params.env, { title: APP_TITLE }); @@ -28,68 +38,42 @@ router.get('/:env', function(req, res, next) { /* GET home page. */ router.get('/', function(req, res, next) { - - async.series([ - function(callback) { - const url = `https://api.harvardartmuseums.org/object?apikey=${API_KEY}&size=1&q=accesslevel:1`; - fetch(url) - .then(response => response.json()) - .then(results => { - let e = new Date(results['records'][0]['lastupdate']); - let d = new Date(results['records'][0]['lastupdate']); - d.setHours(d.getHours() + 2); - - let output = { - lastexport: e.toLocaleString('en-US', {timeZone: "America/New_York"}), - lastrefresh: d.toLocaleString('en-US', {timeZone: "America/New_York"}), - recordcount: results['info']['totalrecords'] - }; - - callback(null, output); - }); - + + async.parallel({ + objectStats: stats.getObjectStats, + currentExhibitions: stats.getCurrentExhibitions, + alttextStats: stats.getAltTextStats, + objectsOnViewStats: stats.getObjectsInGalleryStats, + activityStats: stats.getActivityStats }, - function(callback) { - const url = `https://api.harvardartmuseums.org/object?apikey=${API_KEY}&size=0&gallery=any`; - fetch(url) - .then(response => response.json()) - .then(results => { - callback(null, results['info']['totalrecords']); - }); - - }, - function(callback) { - const url = `https://api.harvardartmuseums.org/exhibition?apikey=${API_KEY}&venue=HAM&status=current`; - fetch(url) - .then(response => response.json()) - .then(results => { - callback(null, results['records']); - }); - - } - ], - function(err, results) { - data.dateoflastrefresh = results[0]['lastrefresh']; - data.dateoflastexport = results[0]['lastexport']; - data.objects.recordcount = results[0]['recordcount']; - data.objects.onview = results[1]; - data.exhibitions.current = results[2]; + function(err, results) { + data.dateoflastrefresh = results['objectStats']['lastrefresh']; + data.dateoflastexport = results['objectStats']['lastexport']; + data.objects.count = results['objectStats']['recordcount']; + data.objects.public.count = results['objectStats']['recordcount_public']; + data.objects.public.count_as_percent = ((results['objectStats']['recordcount_public']/results['objectStats']['recordcount'])*100).toFixed(2); + data.objects.onview.count = results['objectsOnViewStats']; + data.objects.onview.count_as_percent = ((results['objectsOnViewStats']/results['objectStats']['recordcount'])*100).toFixed(2); + data.exhibitions.current = results['currentExhibitions']; + data.objects.alttext.count = results['alttextStats']; + data.objects.alttext.count_as_percent = ((data.objects.alttext.count/data.objects.count)*100).toFixed(2); + data.pageviews = results['activityStats']['pageviews']; + data.pageviews.objects.count_as_percent = ((data.pageviews.objects.count/data.objects.public.count)*100).toFixed(2) - // calculate the age of the data - // freshness = number of hours old - let now = new Date(); - now = new Date(now.toUTCString()); + // calculate the age of the data + // freshness = number of hours old + let now = new Date(); + now = new Date(now.toUTCString()); - let exportdate = new Date(data.dateoflastexport); - exportdate = new Date(exportdate.toUTCString()); - - const freshness = Math.round((now - exportdate)/3600000); - data.datafreshness = freshness; + let exportdate = new Date(data.dateoflastexport); + exportdate = new Date(exportdate.toUTCString()); + + const freshness = Math.round((now - exportdate)/3600000); + data.datafreshness = freshness; - res.render('production', { title: APP_TITLE, apistats: data }); + res.render('production', { title: APP_TITLE, apistats: data }); } - ); - + ); }); module.exports = router; diff --git a/views/production.hbs b/views/production.hbs index 1d97205..f33d893 100644 --- a/views/production.hbs +++ b/views/production.hbs @@ -1,35 +1,73 @@ -
-
-

Data Freshness

+
+
+
+
+

Data Freshness

-

The data on our website is about {{apistats.datafreshness}} hour(s) old.

- -

Collections data was last exported around
{{apistats.dateoflastexport}}.

-

Collections online was last refreshed around
{{apistats.dateoflastrefresh}}.

+

The data on our website is about {{apistats.datafreshness}} hour(s) old.

+ +

Collections data was last exported around
{{apistats.dateoflastexport}}.

+

Collections online was last refreshed around
{{apistats.dateoflastrefresh}}.

-
-
+
+
+ +
+
+

Collection Statistics

-
-
-

Collection Statistics

+

{{apistats.objects.public.count_as_percent}}% of the collection is viewable online.

+

{{apistats.objects.onview.count_as_percent}}% of the collection is on view in galleries.

+ +
    +
  • # of objects: {{apistats.objects.count}}
    • +
  • # of objects online: {{apistats.objects.public.count}}
    • +
  • # of objects installed: {{apistats.objects.onview.count}}
    • +
+
+
-
    -
  • # of objects online: {{apistats.objects.recordcount}}
    • -
  • # of objects installed: {{apistats.objects.onview}}
    • -
-
+
+
+
+

Alt Text Statistics

+ +

{{apistats.objects.alttext.count_as_percent}}% of collection images have alt text.

+

# of images online with alttext: {{apistats.objects.alttext.count}}

+ +
+
+ +
+
+

Website Statistics*

+ +

{{apistats.pageviews.objects.count}} individual objects have been viewed on our website to date. That's {{apistats.pageviews.objects.count_as_percent}}% of the collection.

+ +

The object with the most views in a single day:

+ +

{{apistats.pageviews.singledaymostviews.object.title}} was viewed {{apistats.pageviews.singledaymostviews.activitycount}} times on {{apistats.pageviews.singledaymostviews.date}}.

+
+ +
+ * Website statistics are based on data collected from Google Analytics from {{apistats.pageviews.statsdates.start}} to {{apistats.pageviews.statsdates.end}} +
+
+
-
-
-

Current Exhibitions

+
+
+
+

Current Exhibitions

-
    - {{#each apistats.exhibitions.current}} -
  • {{title}}
    • - {{/each}} -
+
    + {{#each apistats.exhibitions.current}} +
  • {{title}}
    • + {{/each}} +
+
+
\ No newline at end of file