Add RDFa datasource (#142)

* Add RDFa datasource

* Polish RDFa datasource

Co-authored-by: Ruben Taelman <[email protected]>

.gitignore

@@ -13,3 +13,4 @@ config/*.json
 
 # Ignore dev environment files
 .idea
+.devcontainer

README.md

@@ -114,6 +114,7 @@ The following modules are available:
 * [`@ldf/feature-webid`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/feature-webid): Feature that enables authenticated requests from clients with WebID.
 * [`@ldf/datasource-hdt`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-hdt): Datasource that allows HDT files to be loaded.
 * [`@ldf/datasource-jsonld`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-jsonld): Datasource that allows JSON-LD files to be loaded.
+* [`@ldf/datasource-rdfa`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-rdfa): Datasource that allows RDFa files to be loaded.
 * [`@ldf/datasource-n3`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-n3): Datasource that allows [N-Quads](https://www.w3.org/TR/n-quads/), [N-Triples](https://www.w3.org/TR/n-triples/), [Trig](https://www.w3.org/TR/trig/) and [Turtle](https://www.w3.org/TR/turtle/) files to be loaded.
 * [`@ldf/datasource-sparql`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-sparql): Datasource that allows SPARQL endpoints to be used as a data proxy.
 * [`@ldf/datasource-composite`](https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-composite): Datasource that delegates queries to an sequence of other datasources.

greenkeeper.json

@@ -17,6 +17,7 @@
         "packages/datasource-composite/package.json",
         "packages/preset-qpf/package.json",
         "packages/datasource-jsonld/package.json",
+        "packages/datasource-rdfa/package.json",
         "packages/feature-qpf/package.json",
         "packages/feature-summary/package.json"
       ]

packages/datasource-rdfa/README.md

@@ -0,0 +1,57 @@
+# Linked Data Fragments Server - RDFa Datasource
+<img src="http://linkeddatafragments.org/images/logo.svg" width="200" align="right" alt="" />
+
+[![npm version](https://badge.fury.io/js/%40ldf%2Fdatasource-rdfa.svg)](https://www.npmjs.com/package/@ldf/datasource-rdfa)
+
+This module contains a RDFa datasource for the [Linked Data Fragments server](https://github.com/LinkedDataFragments/Server.js).
+It allows HTML files containing RDFa to be loaded.
+
+_This package is a [Linked Data Fragments Server module](https://github.com/LinkedDataFragments/Server.js/)._
+
+## Usage in `@ldf/server`
+
+This package exposes the following config entries:
+* `RdfaDatasource`: A RDFa datasource that requires at least one `file` field. _Should be used as `@type` value._
+
+Example:
+```json
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/server/^3.0.0/components/context.jsonld",
+  "@id": "urn:ldf-server:my",
+  "import": "preset-qpf:config-defaults.json",
+
+  "datasources": [
+    {
+      "@id": "urn:ldf-server:myRdfaDatasource",
+      "@type": "RdfaDatasource",
+      "datasourceTitle": "My RDFa HTML file",
+      "description": "My dataset with a RDFa back-end",
+      "datasourcePath": "myrdfa",
+      "file": "path/to/file.html"
+    }
+  ]
+}
+```
+
+## Usage in other packages
+
+When this module is used in a package other than `@ldf/server`,
+then the JSON-LD context `https://linkedsoftwaredependencies.org/contexts/@ldf/datasource-rdfa.jsonld` must be imported.
+
+For example:
+```
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/core/^3.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/preset-qpf/^3.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-rdfa/^3.0.0/components/context.jsonld",
+  ],
+  // Same as above...
+}
+```
+
+## License
+The Linked Data Fragments server is written by [Ruben Verborgh](https://ruben.verborgh.org/), Miel Vander Sande, [Ruben Taelman](https://www.rubensworks.net/) and colleagues.
+
+This code is copyrighted by [Ghent University – imec](http://idlab.ugent.be/)
+and released under the [MIT license](http://opensource.org/licenses/MIT).

packages/datasource-rdfa/components/Datasource/Rdfa.jsonld

@@ -0,0 +1,19 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/core/^3.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-rdfa/^3.0.0/components/context.jsonld"
+  ],
+  "@id": "npmd:@ldf/datasource-rdfa",
+  "components": [
+    {
+      "@id": "ldfdr:Datasource/Rdfa",
+      "@type": "Class",
+      "extends": "ldfc:Datasource/Memory",
+      "requireElement": "datasources.RdfaDatasource",
+      "comment": "An RdfaDatasource fetches data from an RDFa HTML document",
+      "constructorArguments": {
+        "extends": "ldfc:Datasource/Memory#constructorArgumentsObject"
+      }
+    }
+  ]
+}

packages/datasource-rdfa/components/components.jsonld

@@ -0,0 +1,9 @@
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-rdfa/^3.0.0/components/context.jsonld",
+  "@id": "npmd:@ldf/datasource-rdfa",
+  "@type": "Module",
+  "requireName": "@ldf/datasource-rdfa",
+  "import": [
+    "files-ldfdr:components/Datasource/Rdfa.jsonld"
+  ]
+}

packages/datasource-rdfa/components/context.jsonld

@@ -0,0 +1,11 @@
+{
+    "@context": [
+        "https://linkedsoftwaredependencies.org/bundles/npm/componentsjs/^4.0.0/components/context.jsonld",
+        {
+            "npmd": "https://linkedsoftwaredependencies.org/bundles/npm/",
+            "ldfdr": "npmd:@ldf/datasource-rdfa/",
+            "files-ldfdr": "ldfdr:^3.0.0/",
+            "RdfaDatasource": "ldfdr:Datasource/Rdfa"
+        }
+    ]
+}

packages/datasource-rdfa/config/config-example.json

@@ -0,0 +1,16 @@
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/server/^3.0.0/components/context.jsonld",
+  "@id": "urn:ldf-server:my",
+  "import": "preset-qpf:config-defaults.json",
+
+  "datasources": [
+    {
+      "@id": "urn:ldf-server:myRdfaDatasource",
+      "@type": "RdfaDatasource",
+      "datasourceTitle": "My RDFa HTML file",
+      "description": "My dataset with a RDFa back-end",
+      "datasourcePath": "myrdfa",
+      "file": "path/to/file.html"
+    }
+  ]
+}

packages/datasource-rdfa/index.js

@@ -0,0 +1,8 @@
+/*! @license MIT ©2015-2016 Ruben Verborgh, Ghent University - imec */
+/* Exports of the components of this package */
+
+module.exports = {
+  datasources: {
+    RdfaDatasource: require('./lib/datasources/RdfaDatasource'),
+  },
+};

packages/datasource-rdfa/lib/datasources/RdfaDatasource.js

@@ -0,0 +1,27 @@
+/*! @license MIT ©2014-2016 Ruben Verborgh, Ghent University - imec */
+/* An RdfaDatasource fetches data from a JSON-LD document. */
+
+let MemoryDatasource = require('@ldf/core').datasources.MemoryDatasource,
+    RdfaParser = require('rdfa-streaming-parser').RdfaParser;
+
+let ACCEPT = 'text/html;q=1.0,application/xhtml+xml;q=0.7';
+
+// Creates a new RdfaDatasource
+class RdfaDatasource extends MemoryDatasource {
+  constructor(options) {
+    super(options);
+    this._url = options && (options.url || options.file);
+  }
+
+  // Retrieves all quads from the document
+  _getAllQuads(addQuad, done) {
+    let document = this._fetch({ url: this._url, headers: { accept: ACCEPT } });
+    new RdfaParser({ baseIRI: this._url, dataFactory: this.dataFactory })
+      .import(document)
+      .on('error', done)
+      .on('data', addQuad)
+      .on('end', done);
+  }
+}
+
+module.exports = RdfaDatasource;

packages/datasource-rdfa/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@ldf/datasource-rdfa",
+  "description": "Linked Data Fragments Server - RDFa Datasource",
+  "version": "3.2.1",
+  "lsd:module": true,
+  "license": "MIT",
+  "main": "index.js",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "components",
+    "lib/**/*.js",
+    "index.js"
+  ],
+  "repository": "https://github.com/LinkedDataFragments/Server.js/tree/master/packages/datasource-rdfa",
+  "bugs": {
+    "url": "https://github.com/LinkedDataFragments/Server.js/issues"
+  },
+  "scripts": {
+    "test": "nyc mocha",
+    "lint": "eslint bin/* lib test"
+  },
+  "dependencies": {
+    "rdf-string": "^1.3.1",
+    "rdfa-streaming-parser": "^1.5.0"
+  },
+  "peerDependencies": {
+    "@ldf/core": "^3.0.0"
+  },
+  "devDependencies": {
+    "@ldf/core": "^3.2.1"
+  }
+}

packages/datasource-rdfa/test/.eslintrc

@@ -0,0 +1,18 @@
+{
+  globals: {
+    describe: true,
+    it: true,
+    before: true,
+    after: true,
+    beforeEach: true,
+    afterEach: true,
+    expect: true,
+    sinon: true,
+    test: true,
+  },
+
+  rules: {
+    new-cap: 0, // test constructors as regular functions
+    max-nested-callbacks: 0, // Mocha works with deeply nested callbacks
+  },
+}

packages/datasource-rdfa/test/datasources/RdfaDatasource-test.js

@@ -0,0 +1,99 @@
+/*! @license MIT ©2014-2016 Ruben Verborgh, Ghent University - imec */
+let RdfaDatasource = require('../../').datasources.RdfaDatasource;
+
+let Datasource = require('@ldf/core').datasources.Datasource,
+    path = require('path'),
+    dataFactory = require('n3').DataFactory;
+
+let exampleRdfaUrl = 'file://' + path.join(__dirname, '../../../../test/assets/test.html');
+
+describe('RdfaDatasource', () => {
+  describe('The RdfaDatasource module', () => {
+    it('should be a function', () => {
+      RdfaDatasource.should.be.a('function');
+    });
+
+    it('should be a RdfaDatasource constructor', (done) => {
+      let instance = new RdfaDatasource({ dataFactory, url: exampleRdfaUrl });
+      instance.should.be.an.instanceof(RdfaDatasource);
+      instance.close(done);
+    });
+
+    it('should create Datasource objects', (done) => {
+      let instance = new RdfaDatasource({ dataFactory, url: exampleRdfaUrl });
+      instance.should.be.an.instanceof(Datasource);
+      instance.close(done);
+    });
+  });
+
+  describe('A RdfaDatasource instance for an example RDFa HTML file', () => {
+    let datasource = new RdfaDatasource({ dataFactory, url: exampleRdfaUrl });
+    datasource.initialize();
+    after((done) => { datasource.close(done); });
+
+    itShouldExecute(datasource,
+      'the empty query',
+      { features: { triplePattern: true } },
+      129, 129);
+
+    itShouldExecute(datasource,
+      'the empty query with a limit',
+      { limit: 10, features: { triplePattern: true, limit: true } },
+      10, 129);
+
+    itShouldExecute(datasource,
+      'the empty query with an offset',
+      { offset: 10, features: { triplePattern: true, offset: true } },
+      119, 129);
+
+    itShouldExecute(datasource,
+      'a query for an existing subject',
+      { subject: dataFactory.namedNode('http://example.org/s1'),   limit: 10, features: { triplePattern: true, limit: true } },
+      10, 100);
+
+    itShouldExecute(datasource,
+      'a query for a non-existing subject',
+      { subject: dataFactory.namedNode('http://example.org/p1'),   limit: 10, features: { triplePattern: true, limit: true } },
+      0, 0);
+
+    itShouldExecute(datasource,
+      'a query for an existing predicate',
+      { predicate: dataFactory.namedNode('http://example.org/p1'), limit: 10, features: { triplePattern: true, limit: true } },
+      10, 110);
+
+    itShouldExecute(datasource,
+      'a query for a non-existing predicate',
+      { predicate: dataFactory.namedNode('http://example.org/s1'), limit: 10, features: { triplePattern: true, limit: true } },
+      0, 0);
+
+    itShouldExecute(datasource,
+      'a query for an existing object',
+      { object: dataFactory.namedNode('http://example.org/o001'),  limit: 10, features: { triplePattern: true, limit: true } },
+      3, 3);
+
+    itShouldExecute(datasource,
+      'a query for a non-existing object',
+      { object: dataFactory.namedNode('http://example.org/s1'),    limit: 10, features: { triplePattern: true, limit: true } },
+      0, 0);
+  });
+});
+
+function itShouldExecute(datasource, name, query, expectedResultsCount, expectedTotalCount) {
+  describe('executing ' + name, () => {
+    let resultsCount = 0, totalCount;
+    before((done) => {
+      let result = datasource.select(query);
+      result.getProperty('metadata', (metadata) => { totalCount = metadata.totalCount; });
+      result.on('data', (triple) => { resultsCount++; });
+      result.on('end', done);
+    });
+
+    it('should return the expected number of triples', () => {
+      expect(resultsCount).to.equal(expectedResultsCount);
+    });
+
+    it('should emit the expected total number of triples', () => {
+      expect(totalCount).to.equal(expectedTotalCount);
+    });
+  });
+}

packages/datasource-rdfa/test/mocha.opts

@@ -0,0 +1,3 @@
+--require ../../test/test-setup
+--recursive
+--timeout 500

packages/server/components/context.jsonld

@@ -5,6 +5,7 @@
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-hdt/^3.0.0/components/context.jsonld",
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-jsonld/^3.0.0/components/context.jsonld",
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-n3/^3.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-rdfa/^3.0.0/components/context.jsonld",
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/datasource-sparql/^3.0.0/components/context.jsonld",
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/feature-memento/^3.0.0/components/context.jsonld",
     "https://linkedsoftwaredependencies.org/bundles/npm/@ldf/feature-qpf/^3.0.0/components/context.jsonld",

packages/server/package.json

@@ -31,6 +31,7 @@
     "@ldf/datasource-composite": "^3.2.1",
     "@ldf/datasource-hdt": "^3.2.1",
     "@ldf/datasource-jsonld": "^3.2.1",
+    "@ldf/datasource-rdfa": "^3.0.0",
     "@ldf/datasource-n3": "^3.2.1",
     "@ldf/datasource-sparql": "^3.2.1",
     "@ldf/feature-memento": "^3.2.1",