feat(-interface): Add support for measure dimension when returning da…

…ta to Tableau

Closes #59

src/util/wdc-interface.coffee

@@ -4,12 +4,16 @@ chunkSize = 1000
 fieldNames = []
 fieldTypes = []
 dataToReturn = []
+measureDimensionIndex = null
 
 #-------------------------------------------------------------------------------
 
-submit = (url, dimension) ->
-  tableau.connectionData = JSON.stringify { url: url, dimension: dimension }
-  tableau.connectionName = 'SDMX-REST Connection'
+submit = (url, index) ->
+  connectionData = { url: url, measureDimIndex: -1 }
+  connectionData.measureDimIndex = index unless isNaN(index) or
+    index is null or index is undefined
+  tableau.connectionData = JSON.stringify connectionData
+  tableau.connectionName = "SDMX-REST Connection"
   tableau.submit()
 
 #-------------------------------------------------------------------------------
@@ -25,15 +29,15 @@ getComponents = (structure) ->
           if type is 'attributes'
             prefix = "M"
           else
+            if not c.keyPosition? then c.keyPosition = dimPos
             dimPos += 1
-            if c.keyPosition?
-              keyPos = c.keyPosition + 1 # keyPosition is zero-based
-            else
-              keyPos = dimPos
+            keyPos = c.keyPosition + 1 # keyPosition is zero-based
             # assumes there will always be less than 99 dimensions
             prefix = if keyPos < 10 then "0#{keyPos}" else "#{keyPos}"
           c.name = "#{prefix} - #{c.name}"
           c.type = type
+          c.isDimension = (type is 'dimensions')
+          c.isAttribute = (type is 'attributes')
           c.level = level
           c
 
@@ -46,60 +50,129 @@ formatDT = (d) -> (new Date(d)).toISOString()[0..18].replace('T',' ')
 processResponse = (response) ->
   msg = JSON.parse response
   components = getComponents msg.structure
+  mDimPos = -1
 
-  columnNames = for c in components
-    switch
+  for c, i in components
+    c.index = i
+    if 0 <= measureDimensionIndex
+      mDimPos = i if +c.keyPosition is measureDimensionIndex
+    c.columnNames = switch
       when c.values[0]?.start? then [c.name, "#{c.name} start", "#{c.name} end"]
       when c.values[0]?.id? then [c.name, "#{c.name} ID"]
       else c.name
-
-  columnTypes = for c in components
-    switch
+    c.columnTypes = switch
       when c.values[0]?.start? then ['string', 'datetime', 'datetime']
       when c.values[0]?.id? then ['string', 'string']
       else 'string'
-
-  columnValues = for c in components
-    switch
+    c.columnValues = switch
       when c.values[0]?.start?
         ([v.name, formatDT(v.start), formatDT(v.end)] for v in c.values)
       when c.values[0]?.id?
         ([v.name, v.id] for v in c.values)
       else
         ([v.name] for v in c.values)
 
-  columnNames.push 'Observation Value'
-  columnTypes.push 'float'
-
-  fieldNames = [].concat.apply [], columnNames
-  fieldTypes = [].concat.apply [], columnTypes
-  dataToReturn = getTableData msg, columnValues
-
-
-getTableData = (msg, columnValues) ->
+  fieldNames = [].concat.apply [],
+    (c.columnNames for c, i in components when i isnt mDimPos)
+  fieldTypes = [].concat.apply [],
+    (c.columnTypes for c, i in components when i isnt mDimPos)
+
+  if mDimPos < 0
+    fieldNames.push 'Observation Value'
+    fieldTypes.push 'float'
+    dataToReturn = getTableData msg, components
+  else
+    for v in components[mDimPos].values
+      fieldNames.push v.name
+      fieldTypes.push 'float'
+    dataToReturn = getTableDataPivoted msg, components, mDimPos
+
+# returns observations in a flattened array:
+# [ds dims, ser dims, obs dims, ds attrs, ser attrs, obs attrs, obs value]
+flattenObservations = (msg) ->
   dsDims = msg.structure.dimensions.dataSet?.map (d) -> 0
   dsAttrs = msg.structure.attributes?.dataSet?.map (d) -> 0
   dsDims ?= []
   dsAttrs ?= []
   results = []
 
-  mapToValue = (i, j) -> if i? then columnValues[j][i] else i
-
   for dataset in msg.dataSets
     for seriesKey, series of dataset.series
       seriesDims = dsDims.concat seriesKey.split(':').map( (d) -> +d )
       seriesAttrs = dsAttrs.concat series.attributes
       for obsKey, obs of series.observations
         obsRow = seriesDims.concat(+obsKey, seriesAttrs, obs[1..])
-        obsRow = [].concat.apply([], obsRow.map(mapToValue))
         obsRow.push obs[0]
         results.push obsRow
 
   return results
 
+
+getTableData = (msg, components) ->
+  obsArray = flattenObservations msg
+  columnValues = (c.columnValues for c in components)
+  results = []
+
+  mapToValue = (i, j) ->
+    return i unless i?
+    return i if (columnValues.length - 1) < j
+    return columnValues[j][i]
+
+  for obsRow in obsArray
+    results.push [].concat.apply([], obsRow.map(mapToValue))
+
+  return results
+
+
+getTableDataPivoted = (msg, components, mDimPos) ->
+  obsArray = flattenObservations msg
+  obsMap = {}
+  results = []
+  dimCount = 0
+  dimCount += 1 for c in components when c.isDimension
+  lastDim = dimCount - 1
+  firstAttr = lastDim + 1
+  attrCount = 0
+  attrCount += 1 for c in components when c.isAttribute
+  lastAttr = firstAttr + attrCount - 1
+  lastValue = components[mDimPos].values.length - 1
+  columnValues = (c.columnValues for c, i in components when i isnt mDimPos)
+
+  # group observations by the key
+  for obs in obsArray
+    key = obs[0..lastDim]
+    key.splice(mDimPos, 1)  # Remove the measure dimension from the key
+    key = key.join(':')     # Join dimensions for the groupping key
+    obsMap[key] ?= (null for [0..lastValue])  # Reserve a slot for each measure
+    obsMap[key][obs[mDimPos]] = obs # Allocate the slot for the measure
+
+  obsArray = []
+  for key, value of obsMap
+    obs = key.split ':'
+
+    obsValues = (null for [0..lastValue])
+    for obs2, i in value when obs2?
+      attributes = obs2[firstAttr..lastAttr] # Assume attributes are same
+      obsValues[i] = obs2[obs2.length - 1] # Take the obs value and allocate
+
+    obs = obs.concat attributes
+    obsArray.push obs.concat obsValues
+
+  mapToValue = (i, j) ->
+    return i unless i?
+    return i if (columnValues.length - 1) < j
+    return columnValues[j][i]
+
+  for obsRow in obsArray
+    results.push [].concat.apply([], obsRow.map(mapToValue))
+
+  return results
+
 #-------------------------------------------------------------------------------
 
-makeRequest = (url, callback) ->
+makeRequest = (url, measureDimIndex, callback) ->
+  measureDimensionIndex = measureDimIndex
+
   errorHandler = (error) ->
     console.log error
     tableau.abortWithError "#{error}"
@@ -123,9 +196,9 @@ registerConnector = () ->
   connector = tableau.makeConnector()
 
   connector.getColumnHeaders = ->
-    connectionData = JSON.parse tableau.connectionData
+    connData = JSON.parse tableau.connectionData
     callback = () -> tableau.headersCallback fieldNames, fieldTypes
-    makeRequest connectionData.url, callback
+    makeRequest connData.url, connData.measureDimIndex, callback
 
   connector.getTableData = (lastRecordToken) ->
     if lastRecordToken.length is 0

test/fixtures/ECB_EXR1_USD_GBP.json

@@ -0,0 +1 @@
+{"header":{"id":"1ae59482-9b8d-48b1-b85c-0bab5f4da022","test":false,"prepared":"2016-10-28T22:49:33.000+02:00","sender":{"id":"ECB"}},"dataSets":[{"action":"Replace","validFrom":"2016-10-28T22:49:33.000+02:00","series":{"0:0:0:0:0":{"attributes":[null,null,0,null,null,null,null,null,null,null,0,null,0,null,0,0,0,0],"observations":{"0":[0.790493636363636,0,null,null,null],"1":[0.841057619047619,0,null,null,null],"2":[0.855208695652174,0,null,null,null],"3":[0.852276818181818,0,null,null,null]}},"0:1:0:0:0":{"attributes":[null,null,0,null,null,null,null,null,null,null,1,null,0,null,1,1,1,0],"observations":{"0":[1.122890909090909,0,null,null,null],"1":[1.106852380952381,0,null,null,null],"2":[1.121173913043479,0,null,null,null],"3":[1.121209090909091,0,null,null,null]}}}}],"structure":{"links":[{"title":"Exchange Rates","rel":"dataflow","href":"http://a-sdw-wsrest.ecb.europa.eu:80null/service/dataflow/ECB/EXR/1.0"}],"name":"Exchange Rates","dimensions":{"series":[{"id":"FREQ","name":"Frequency","values":[{"id":"M","name":"Monthly"}]},{"id":"CURRENCY","name":"Currency","values":[{"id":"GBP","name":"UK pound sterling"},{"id":"USD","name":"US dollar"}]},{"id":"CURRENCY_DENOM","name":"Currency denominator","values":[{"id":"EUR","name":"Euro"}]},{"id":"EXR_TYPE","name":"Exchange rate type","values":[{"id":"SP00","name":"Spot"}]},{"id":"EXR_SUFFIX","name":"Series variation - EXR context","values":[{"id":"A","name":"Average"}]}],"observation":[{"id":"TIME_PERIOD","name":"Time period or range","role":"time","values":[{"id":"2016-06","name":"2016-06","start":"2016-06-01T00:00:00.000+02:00","end":"2016-06-30T23:59:59.999+02:00"},{"id":"2016-07","name":"2016-07","start":"2016-07-01T00:00:00.000+02:00","end":"2016-07-31T23:59:59.999+02:00"},{"id":"2016-08","name":"2016-08","start":"2016-08-01T00:00:00.000+02:00","end":"2016-08-31T23:59:59.999+02:00"},{"id":"2016-09","name":"2016-09","start":"2016-09-01T00:00:00.000+02:00","end":"2016-09-30T23:59:59.999+02:00"}]}]},"attributes":{"series":[{"id":"TIME_FORMAT","name":"Time format code","values":[]},{"id":"BREAKS","name":"Breaks","values":[]},{"id":"COLLECTION","name":"Collection indicator","values":[{"id":"A","name":"Average of observations through period"}]},{"id":"DOM_SER_IDS","name":"Domestic series ids","values":[]},{"id":"PUBL_ECB","name":"Source publication (ECB only)","values":[]},{"id":"PUBL_MU","name":"Source publication (Euro area only)","values":[]},{"id":"PUBL_PUBLIC","name":"Source publication (public)","values":[]},{"id":"UNIT_INDEX_BASE","name":"Unit index base","values":[]},{"id":"COMPILATION","name":"Compilation","values":[]},{"id":"COVERAGE","name":"Coverage","values":[]},{"id":"DECIMALS","name":"Decimals","values":[{"id":"5","name":"Five"},{"id":"4","name":"Four"}]},{"id":"NAT_TITLE","name":"National language title","values":[]},{"id":"SOURCE_AGENCY","name":"Source agency","values":[{"id":"4F0","name":"European Central Bank (ECB)"}]},{"id":"SOURCE_PUB","name":"Publication source","values":[]},{"id":"TITLE","name":"Title","values":[{"name":"UK pound sterling/Euro"},{"name":"US dollar/Euro"}]},{"id":"TITLE_COMPL","name":"Title complement","values":[{"name":"ECB reference exchange rate, UK pound sterling/Euro, 2:15 pm (C.E.T.)"},{"name":"ECB reference exchange rate, US dollar/Euro, 2:15 pm (C.E.T.)"}]},{"id":"UNIT","name":"Unit","values":[{"id":"GBP","name":"UK pound sterling"},{"id":"USD","name":"US dollar"}]},{"id":"UNIT_MULT","name":"Unit multiplier","values":[{"id":"0","name":"Units"}]}],"observation":[{"id":"OBS_STATUS","name":"Observation status","values":[{"id":"A","name":"Normal value"}]},{"id":"OBS_CONF","name":"Observation confidentiality","values":[]},{"id":"OBS_PRE_BREAK","name":"Pre-break observation value","values":[]},{"id":"OBS_COM","name":"Observation comment","values":[]}]}}}

test/util/wdc-interface.test.coffee

@@ -14,12 +14,21 @@ describe 'Web Data Connector Interface', ->
       response = wdcInterface.response()
       response.fieldNames.should.be.an('array').with.lengthOf(42)
       response.fieldNames.should.include('02 - Currency')
+      response.fieldNames[0].should.equal('01 - Frequency')
+      response.fieldNames[1].should.equal('01 - Frequency ID')
+      response.fieldNames[41].should.equal('Observation Value')
       response.fieldTypes.should.be.an('array').with.lengthOf(42)
       response.fieldTypes.should.have.members(['string', 'datetime', 'float'])
+      response.fieldTypes[12].should.equal('datetime')
+      response.fieldTypes[41].should.equal('float')
       response.dataToReturn.should.be.an('array').with.lengthOf(4)
+      response.dataToReturn[0].should.be.an('array').with.lengthOf(42)
+      response.dataToReturn[0][0].should.equal('Monthly')
+      response.dataToReturn[0][41].should.equal(1.085965)
       done()
 
     wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR',
+      undefined,
       checkResult
 
     return
@@ -35,6 +44,62 @@ describe 'Web Data Connector Interface', ->
       done()
 
     wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/MNA',
+      undefined,
+      checkResult
+
+    return
+
+  it 'it supports setting measure dimension', (done) ->
+    query = nock('http://sdw-wsrest.ecb.europa.eu')
+      .get((uri) -> uri.indexOf('EXR') > -1)
+      .replyWithFile(200, __dirname + '/../fixtures/ECB_EXR.json')
+
+    checkResult = () ->
+      response = wdcInterface.response()
+      response.fieldNames.should.be.an('array').with.lengthOf(40)
+      response.fieldNames.should.not.include('02 - Currency')
+      response.fieldNames[0].should.equal('01 - Frequency')
+      response.fieldNames[39].should.equal('US dollar')
+      response.fieldTypes.should.be.an('array').with.lengthOf(40)
+      response.fieldTypes.should.have.members(['string', 'datetime', 'float'])
+      response.fieldTypes[10].should.equal('datetime')
+      response.fieldTypes[39].should.equal('float')
+      response.dataToReturn.should.be.an('array').with.lengthOf(4)
+      response.dataToReturn[0].should.be.an('array').with.lengthOf(40)
+      response.dataToReturn[0][0].should.equal('Monthly')
+      response.dataToReturn[0][39].should.equal(1.085965)
+      done()
+
+    wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR', 1,
+      checkResult
+
+    return
+
+  it 'it supports measure dimension with multiple values', (done) ->
+    query = nock('http://sdw-wsrest.ecb.europa.eu')
+      .get((uri) -> uri.indexOf('EXR') > -1)
+      .replyWithFile(200, __dirname + '/../fixtures/ECB_EXR1_USD_GBP.json')
+
+    checkResult = () ->
+      response = wdcInterface.response()
+      response.fieldNames.should.be.an('array').with.lengthOf(41)
+      response.fieldNames.should.not.include('02 - Currency')
+      response.fieldNames[0].should.equal('01 - Frequency')
+      response.fieldNames[39].should.equal('UK pound sterling')
+      response.fieldNames[40].should.equal('US dollar')
+      response.fieldTypes.should.be.an('array').with.lengthOf(41)
+      response.fieldTypes.should.have.members(['string', 'datetime', 'float'])
+      response.fieldTypes[10].should.equal('datetime')
+      response.fieldTypes[39].should.equal('float')
+      response.fieldTypes[40].should.equal('float')
+      response.dataToReturn.should.be.an('array').with.lengthOf(4)
+      response.dataToReturn[0].should.be.an('array').with.lengthOf(41)
+      response.dataToReturn[0][0].should.equal('Monthly')
+      response.dataToReturn[0][39].should.equal(0.790493636363636)
+      response.dataToReturn[0][40].should.equal(1.122890909090909)
+      done()
+
+    wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR', 1,
       checkResult
 
     return