Merge pull request #15 from houdiniproject/update-to-1.1.0-pre1

Update to 1.1.0-pre1

NOTICE

@@ -2522,6 +2522,62 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
 
+------
+
+** lodash; version 4.17.20 -- https://lodash.com/
+Copyright OpenJS Foundation and other contributors <https://openjsf.org/>
+Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
+
+Copyright OpenJS Foundation and other contributors <https://openjsf.org/>
+
+Based on Underscore.js, copyright Jeremy Ashkenas,
+DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
+
+This software consists of voluntary contributions made by many
+individuals. For exact contribution history, see the revision history
+available at https://github.com/lodash/lodash
+
+The following license applies to all parts of this software except as
+documented below:
+
+====
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+====
+
+Copyright and related rights for sample code are waived via CC0. Sample
+code is defined as all source code displayed within the prose of the
+documentation.
+
+CC0: http://creativecommons.org/publicdomain/zero/1.0/
+
+====
+
+Files located in the node_modules and vendor directories are externally
+maintained libraries used by this software which have their own
+licenses; we recommend you read them, as their terms may differ from the
+terms above.
+
+
 ------
 
 ** yargs; version 15.4.0 -- https://yargs.js.org/

cmd.js

@@ -29,10 +29,18 @@ const argv = yargs
     default: null,
     description: 'additional file containing packages where you copied in code from but which aren\'t dependencies'
   })
+  .option('chunksize', {
+    alias: 's',
+    type: 'number',
+    default: null,
+    description: "Creating a notice with more than 250 packages can cause a timeout of ClearlyDefined.io. To avoid this,"+
+        "you can pass the number of packages to put into a single chunk here. In the end, they'll all get combined into a single" +
+        "NOTICE file."
+  })
   .help()
   .argv;
 
-noticeme({path:'.', includedFile: argv.included}).then(notice => {
+noticeme({path:'.', includedFile: argv.included, chunkSize: argv.chunksize}).then(notice => {
 
   if (!argv.update) {
     if (!fs.existsSync(argv.filename)){
@@ -67,4 +75,7 @@ noticeme({path:'.', includedFile: argv.included}).then(notice => {
     process.stdout.write(`The notice file ${argv.filename} is now up to date\n`)
     process.exit(0)
   }
+}).catch(error => {
+  process.stderr.write(`noticeme had the following error: ${error.toString()}`)
+  process.exit(3)
 });

noticeService.js

@@ -0,0 +1,86 @@
+// Copyright (c) Houdini Project
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+const chunk = require('lodash/chunk');
+const sum = require('lodash/sum');
+const assign = require('lodash/assign');
+const concat = require('lodash/concat');
+
+const NOTICE_SIZE = 250;
+
+class NoticeService {
+
+  constructor(url, http){
+    this.url = url
+    this.http = http
+  }
+
+  generateNotice(coordinates, chunkSize = 0) {
+    var noticePromise = null;
+    if (chunkSize && chunkSize > 0) {
+      noticePromise = this.getSequentialPromises(
+        chunk(coordinates, NOTICE_SIZE)
+        .map(i => () => this.noticePromise(i))
+        )
+    }
+    else {
+      noticePromise = this.noticePromise(coordinates)
+    }
+    return noticePromise.then((results) => 
+
+      {
+        return {
+          content: results instanceof Array ? results.map((i) => i.content).join('\n') : results.content,
+          summary: {
+            total: results instanceof Array ? sum(results.map(i => i.summary.total)) : results.summary.total,
+            warnings: {
+              noDefinition: results instanceof Array ? concat(results.map(i => i.summary.warnings.noDefinition)) : results.summary.warnings.noDefinition,
+              noLicense: results instanceof Array ? concat(results.map(i => i.summary.warnings.noLicense)) : results.summary.warnings.noLicense,
+              noCopyright: results instanceof Array ? concat(results.map(i => i.summary.warnings.noCopyright)): results.summary.warnings.noCopyright,
+            }
+          }
+        }
+      });
+
+  }
+
+  // from: https://hackernoon.com/functional-javascript-resolving-promises-sequentially-7aac18c4431e
+  getSequentialPromises(funcs) {
+    return funcs.reduce((promise, func) =>
+      promise.then(result =>
+        func().then(Array.prototype.concat.bind(result))),
+        Promise.resolve([]))
+  }
+
+  noticePromise(coordinates) {
+    return this.http(this.url, {
+        method: 'post',
+        body: JSON.stringify({ coordinates }),
+        headers: { 'Content-Type': 'application/json' },
+      }).then(
+        (res) => {
+          if (res.status < 400) {
+            return res.json()
+          }
+          else {
+            throw new NoticeError(`Call to notice API returns status of ${res.status}`)
+          }
+
+        })
+        .catch(error => {
+          if (error instanceof NoticeError)
+            throw error;
+          else
+            throw new NoticeError(error.toString());
+        });
+  }
+}
+
+class NoticeError extends Error {
+
+}
+
+
+
+
+module.exports = NoticeService;

noticeme.js

@@ -4,8 +4,9 @@
 
 const readPkgTree = require('read-package-tree');
 const fetch = require('node-fetch');
-const fs = require('fs')
-const pathMod = require('path')
+const fs = require('fs');
+const pathMod = require('path');
+const NoticeService = require('./noticeService');
 
 const npmjsCoordinates = ({ name, version }) =>
   'npm/npmjs/' + (name.includes('/') ? name : `-/${name}`) + `/${version}`;
@@ -14,7 +15,7 @@ function retrieveIncludedJson(path) {
   return fs.existsSync(path) ? JSON.parse(fs.readFileSync(path, 'utf8')).packages : [];
 }
 
-module.exports = function noticeme({path, includedFile, rpt=readPkgTree, http = fetch}) {
+module.exports = function noticeme({path, includedFile, chunkSize=0, rpt=readPkgTree, http = fetch}) {
   return new Promise((resolve, reject) => {
     rpt(path, function (err, { children }) {
       if (err) reject(err);
@@ -36,13 +37,13 @@ module.exports = function noticeme({path, includedFile, rpt=readPkgTree, http =
             map((package) => npmjsCoordinates(package)));
       }
 
-      http('https://api.clearlydefined.io/notices', {
-        method: 'post',
-        body: JSON.stringify({ coordinates }),
-        headers: { 'Content-Type': 'application/json' },
+      const service = new NoticeService('https://api.clearlydefined.io/notices', http);
+      service.generateNotice(coordinates, chunkSize).then(json => 
+        resolve(json.content)
+      ).catch(error => {
+        reject(error);
       })
-        .then(res => res.json())
-        .then(json => resolve(json.content));
+
     });
   });
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@houdiniproject/noticeme",
-  "version": "1.0.0",
+  "version": "1.1.0-pre1",
   "description": "A NOTICE.txt generator for npm",
   "main": "noticeme.js",
   "bin": {
@@ -23,6 +23,7 @@
   "dependencies": {
     "colors": "^1.4.0",
     "diff-match-patch": "^1.0.5",
+    "lodash": "^4.17.20",
     "node-fetch": "^2.6.0",
     "read-package-tree": "^5.2.2",
     "yargs": "^15.4.0"

yarn.lock

@@ -266,6 +266,11 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+lodash@^4.17.20:
+  version "4.17.20"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
+  integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
+
 minimatch@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"