From 493060a9b6f1770de9918a2e399fba373a4ec13e Mon Sep 17 00:00:00 2001 From: erikbarke Date: Sun, 13 Aug 2017 00:06:32 +0200 Subject: [PATCH] Major source map overhaul, #144 --- README.md | 11 +++++- dist/bundler/bundler.js | 41 +++++++++++++++++--- dist/bundler/resolve/source-reader.js | 4 +- dist/bundler/source-map.js | 20 ---------- dist/istanbul/coverage.js | 7 ++-- dist/shared/configuration.js | 1 + package.json | 4 ++ src/api/configuration.ts | 1 + src/bundler/bundler.ts | 52 ++++++++++++++++++++++---- src/bundler/resolve/source-reader.ts | 4 +- src/bundler/source-map.ts | 35 ----------------- src/istanbul/coverage.ts | 7 ++-- src/shared/configuration.ts | 1 + tests/integration-latest/karma.conf.js | 3 +- 14 files changed, 108 insertions(+), 83 deletions(-) delete mode 100644 src/bundler/source-map.ts diff --git a/README.md b/README.md index a330f57c..01a41a0d 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,10 @@ If the defaults aren't enough, the settings can be configured from `karma.conf.j * **karmaTypescriptConfig.bundlerOptions.resolve.directories** - An array of directories where modules will be recursively looked up.
Defaults to `["node_modules"]`. +* **karmaTypescriptConfig.bundlerOptions.sourceMap** - A boolean indicating whether source maps should be generated for imported modules in the bundle, useful for debugging in a browser. + For more debugging options, please see `karmaTypescriptConfig.coverageOptions.instrumentation`.
+ Defaults to `false`. + * **karmaTypescriptConfig.bundlerOptions.transforms** - An array of functions altering or replacing compiled Typescript code/Javascript code loaded from `node_modules` before bundling it. For more detailed documentation on transforms, please refer to the [Transforms API section](#transforms-api) in this document.
@@ -181,8 +185,10 @@ If the defaults aren't enough, the settings can be configured from `karma.conf.j the karma process will exit with `ts.ExitStatus.DiagnosticsPresent_OutputsSkipped` if any compilation errors occur. * **karmaTypescriptConfig.coverageOptions.instrumentation** - A boolean indicating whether the code should be instrumented, - set to `false` to see the original Typescript code when debugging.
- Defaults to true. + set this property to `false` to see the original Typescript code when debugging. + Please note that setting this property to `true` requires the Typescript compiler option `sourceMap` to also be set to `true`. + For more debugging options, please see `karmaTypescriptConfig.coverageOptions.sourceMap`.
+ Defaults to `true`. * **karmaTypescriptConfig.coverageOptions.exclude** - A `RegExp` object or an array of `RegExp` objects for filtering which files should be excluded from coverage instrumentation.
Defaults to `/\.(d|spec|test)\.ts$/i` which excludes *.d.ts, *.spec.ts and *.test.ts (case insensitive). @@ -319,6 +325,7 @@ karmaTypescriptConfig: { extensions: [".js", ".json"], directories: ["node_modules"] }, + sourceMap: false, transforms: [require("karma-typescript-es6-transform")()], validateSyntax: true }, diff --git a/dist/bundler/bundler.js b/dist/bundler/bundler.js index 308c9302..d6e123bd 100644 --- a/dist/bundler/bundler.js +++ b/dist/bundler/bundler.js @@ -1,6 +1,8 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var async = require("async"); +var combineSourceMap = require("combine-source-map"); +var convertSourceMap = require("convert-source-map"); var fs = require("fs"); var lodash = require("lodash"); var os = require("os"); @@ -9,7 +11,6 @@ var tmp = require("tmp"); var benchmark_1 = require("../shared/benchmark"); var PathTool = require("../shared/path-tool"); var bundle_item_1 = require("./bundle-item"); -var SourceMap = require("./source-map"); var Bundler = (function () { function Bundler(config, dependencyWalker, globals, log, project, resolver, transformer, validator) { this.config = config; @@ -54,7 +55,7 @@ var Bundler = (function () { var benchmark = new benchmark_1.Benchmark(); this.transformer.applyTsTransforms(this.bundleQueue, function () { _this.bundleQueue.forEach(function (queued) { - queued.item = new bundle_item_1.BundleItem(queued.file.path, queued.file.originalPath, SourceMap.create(queued.file, queued.emitOutput.sourceFile.text, queued.emitOutput)); + queued.item = new bundle_item_1.BundleItem(queued.file.path, queued.file.originalPath, _this.createInlineSourceMap(queued)); }); var dependencyCount = _this.dependencyWalker.collectTypescriptDependencies(_this.bundleQueue); if (_this.shouldBundle(dependencyCount)) { @@ -65,6 +66,18 @@ var Bundler = (function () { } }); }; + Bundler.prototype.createInlineSourceMap = function (queued) { + var inlined = queued.emitOutput.outputText; + if (queued.emitOutput.sourceMapText) { + var map = convertSourceMap + .fromJSON(queued.emitOutput.sourceMapText) + .addProperty("sourcesContent", [queued.emitOutput.sourceFile.text]); + inlined = convertSourceMap.removeMapFileComments(queued.emitOutput.outputText) + map.toComment(); + // used by Karma to log errors with original source code line numbers + queued.file.sourceMap = map.toObject(); + } + return inlined; + }; Bundler.prototype.shouldBundle = function (dependencyCount) { if (this.config.hasPreprocessor("commonjs")) { this.log.debug("Preprocessor 'commonjs' detected, code will NOT be bundled"); @@ -169,12 +182,24 @@ var Bundler = (function () { }; Bundler.prototype.writeMainBundleFile = function (onMainBundleFileWritten) { var _this = this; - var bundle = "(function(global){" + os.EOL + - "global.wrappers={};" + os.EOL; + var bundle = "(function(global){" + os.EOL + "global.wrappers={};" + os.EOL; + var sourcemap = combineSourceMap.create(); + var line = this.getNumberOfNewlines(bundle); this.bundleBuffer.forEach(function (bundleItem) { - bundle += _this.addLoaderFunction(bundleItem, false); + if (_this.config.bundlerOptions.sourceMap) { + var sourceFile = path.relative(_this.config.karma.basePath, bundleItem.filename); + sourcemap.addFile({ sourceFile: path.join("/base", sourceFile), source: bundleItem.source }, { line: line }); + } + var wrapped = _this.addLoaderFunction(bundleItem, false); + bundle += wrapped; + if (_this.config.bundlerOptions.sourceMap) { + line += _this.getNumberOfNewlines(wrapped); + } }); - bundle += this.createEntrypointFilenames() + "})(this);"; + bundle += this.createEntrypointFilenames() + "})(this);" + os.EOL; + if (this.config.bundlerOptions.sourceMap) { + bundle += sourcemap.comment(); + } fs.writeFile(this.bundleFile.name, bundle, function (error) { if (error) { throw error; @@ -183,6 +208,10 @@ var Bundler = (function () { onMainBundleFileWritten(); }); }; + Bundler.prototype.getNumberOfNewlines = function (source) { + var newlines = source.match(/\n/g); + return newlines ? newlines.length : 0; + }; return Bundler; }()); exports.Bundler = Bundler; diff --git a/dist/bundler/resolve/source-reader.js b/dist/bundler/resolve/source-reader.js index 415b777a..1e9282be 100644 --- a/dist/bundler/resolve/source-reader.js +++ b/dist/bundler/resolve/source-reader.js @@ -1,9 +1,9 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var acorn = require("acorn"); +var combineSourceMap = require("combine-source-map"); var fs = require("fs"); var os = require("os"); -var SourceMap = require("../source-map"); var SourceReader = (function () { function SourceReader(config, log, transformer) { this.config = config; @@ -13,7 +13,7 @@ var SourceReader = (function () { SourceReader.prototype.read = function (bundleItem, onSourceRead) { var _this = this; this.readFile(bundleItem, function (source) { - bundleItem.source = SourceMap.deleteComment(source); + bundleItem.source = combineSourceMap.removeComments(source); bundleItem.ast = _this.createAbstractSyntaxTree(bundleItem); _this.transformer.applyTransforms(bundleItem, function () { _this.assertValidNonScriptSource(bundleItem); diff --git a/dist/bundler/source-map.js b/dist/bundler/source-map.js index 9fcfee64..f775a5d5 100644 --- a/dist/bundler/source-map.js +++ b/dist/bundler/source-map.js @@ -1,27 +1,7 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var path = require("path"); -function create(file, source, emitOutput) { - var result = emitOutput.outputText; - var map; - var datauri; - if (emitOutput.sourceMapText) { - map = JSON.parse(emitOutput.sourceMapText); - map.sources[0] = path.basename(file.originalPath); - map.sourcesContent = [source]; - map.file = path.basename(file.path); - file.sourceMap = map; - datauri = "data:application/json;charset=utf-8;base64," + new Buffer(JSON.stringify(map)).toString("base64"); - result = result.replace(createComment(file), "//# sourceMappingURL=" + datauri); - } - return result; -} -exports.create = create; function createComment(file) { return "//# sourceMappingURL=" + path.basename(file.path) + ".map"; } exports.createComment = createComment; -function deleteComment(source) { - return source.replace(/\/\/#\s?sourceMappingURL\s?=\s?.*\.map/g, ""); -} -exports.deleteComment = deleteComment; diff --git a/dist/istanbul/coverage.js b/dist/istanbul/coverage.js index d79e708c..bc39da65 100644 --- a/dist/istanbul/coverage.js +++ b/dist/istanbul/coverage.js @@ -1,6 +1,5 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -var SourceMap = require("../bundler/source-map"); var Coverage = (function () { function Coverage(config) { this.config = config; @@ -28,15 +27,15 @@ var Coverage = (function () { } if (!this.config.coverageOptions.instrumentation || this.isExcluded(this.config.coverageOptions.exclude, file.originalPath) || - this.hasNoOutput(file, emitOutput)) { + this.hasNoOutput(emitOutput)) { this.log.debug("Excluding file %s from instrumentation", file.originalPath); callback(bundled); return; } this.coveragePreprocessor(bundled, file, callback); }; - Coverage.prototype.hasNoOutput = function (file, emitOutput) { - return emitOutput.outputText === SourceMap.createComment(file); + Coverage.prototype.hasNoOutput = function (emitOutput) { + return emitOutput.outputText.startsWith("//# sourceMappingURL="); }; Coverage.prototype.isExcluded = function (regex, path) { if (Array.isArray(regex)) { diff --git a/dist/shared/configuration.js b/dist/shared/configuration.js index 8a4d01fe..e0685a37 100644 --- a/dist/shared/configuration.js +++ b/dist/shared/configuration.js @@ -66,6 +66,7 @@ var Configuration = (function () { directories: ["node_modules"], extensions: [".js", ".json", ".ts", ".tsx"] }, + sourceMap: false, transforms: [], validateSyntax: true }; diff --git a/package.json b/package.json index a0a661f6..938de22d 100644 --- a/package.json +++ b/package.json @@ -64,8 +64,10 @@ "browser-resolve": "^1.11.0", "browserify-zlib": "^0.2.0", "buffer": "^5.0.6", + "combine-source-map": "^0.8.0", "console-browserify": "^1.1.0", "constants-browserify": "^1.0.0", + "convert-source-map": "^1.5.0", "crypto-browserify": "^3.11.1", "diff": "^3.2.0", "domain-browser": "^1.1.7", @@ -107,6 +109,8 @@ "@types/acorn": "^4.0.0", "@types/async": "^2.0.38", "@types/browser-resolve": "0.0.4", + "@types/combine-source-map": "^0.8.0", + "@types/convert-source-map": "^1.3.33", "@types/diff": "0.0.31", "@types/glob": "^5.0.30", "@types/istanbul": "^0.4.29", diff --git a/src/api/configuration.ts b/src/api/configuration.ts index ad5248ee..b820976a 100644 --- a/src/api/configuration.ts +++ b/src/api/configuration.ts @@ -26,6 +26,7 @@ export interface BundlerOptions { ignore?: string[]; noParse?: string[]; resolve?: Resolve; + sourceMap: boolean; transforms?: Transform[]; validateSyntax?: boolean; } diff --git a/src/bundler/bundler.ts b/src/bundler/bundler.ts index 998627bd..69658c9d 100644 --- a/src/bundler/bundler.ts +++ b/src/bundler/bundler.ts @@ -1,4 +1,6 @@ import * as async from "async"; +import * as combineSourceMap from "combine-source-map"; +import * as convertSourceMap from "convert-source-map"; import * as fs from "fs"; import * as lodash from "lodash"; import * as os from "os"; @@ -20,7 +22,6 @@ import { BundleItem } from "./bundle-item"; import { DependencyWalker } from "./dependency-walker"; import { Queued } from "./queued"; import { Resolver } from "./resolve/resolver"; -import SourceMap = require("./source-map"); import { Transformer } from "./transformer"; import { Validator } from "./validator"; @@ -76,8 +77,8 @@ export class Bundler { this.transformer.applyTsTransforms(this.bundleQueue, () => { this.bundleQueue.forEach((queued) => { - queued.item = new BundleItem(queued.file.path, queued.file.originalPath, - SourceMap.create(queued.file, queued.emitOutput.sourceFile.text, queued.emitOutput)); + queued.item = new BundleItem( + queued.file.path, queued.file.originalPath, this.createInlineSourceMap(queued)); }); let dependencyCount = this.dependencyWalker.collectTypescriptDependencies(this.bundleQueue); @@ -91,6 +92,21 @@ export class Bundler { }); } + private createInlineSourceMap(queued: Queued): string { + let inlined = queued.emitOutput.outputText; + if (queued.emitOutput.sourceMapText) { + + let map = convertSourceMap + .fromJSON(queued.emitOutput.sourceMapText) + .addProperty("sourcesContent", [queued.emitOutput.sourceFile.text]); + inlined = convertSourceMap.removeMapFileComments(queued.emitOutput.outputText) + map.toComment(); + + // used by Karma to log errors with original source code line numbers + queued.file.sourceMap = map.toObject(); + } + return inlined; + } + private shouldBundle(dependencyCount: number): boolean { if (this.config.hasPreprocessor("commonjs")) { this.log.debug("Preprocessor 'commonjs' detected, code will NOT be bundled"); @@ -213,14 +229,31 @@ export class Bundler { private writeMainBundleFile(onMainBundleFileWritten: { (): void } ) { - let bundle = "(function(global){" + os.EOL + - "global.wrappers={};" + os.EOL; + let bundle = "(function(global){" + os.EOL + "global.wrappers={};" + os.EOL; + let sourcemap = combineSourceMap.create(); + let line = this.getNumberOfNewlines(bundle); this.bundleBuffer.forEach((bundleItem) => { - bundle += this.addLoaderFunction(bundleItem, false); + + if (this.config.bundlerOptions.sourceMap) { + let sourceFile = path.relative(this.config.karma.basePath, bundleItem.filename); + sourcemap.addFile( + { sourceFile: path.join("/base", sourceFile), source: bundleItem.source }, + { line } + ); + } + + let wrapped = this.addLoaderFunction(bundleItem, false); + bundle += wrapped; + if (this.config.bundlerOptions.sourceMap) { + line += this.getNumberOfNewlines(wrapped); + } }); - bundle += this.createEntrypointFilenames() + "})(this);"; + bundle += this.createEntrypointFilenames() + "})(this);" + os.EOL; + if (this.config.bundlerOptions.sourceMap) { + bundle += sourcemap.comment(); + } fs.writeFile(this.bundleFile.name, bundle, (error) => { if (error) { @@ -230,4 +263,9 @@ export class Bundler { onMainBundleFileWritten(); }); } + + private getNumberOfNewlines(source: any) { + let newlines = source.match(/\n/g); + return newlines ? newlines.length : 0; + } } diff --git a/src/bundler/resolve/source-reader.ts b/src/bundler/resolve/source-reader.ts index f58f7a7b..13bb3b52 100644 --- a/src/bundler/resolve/source-reader.ts +++ b/src/bundler/resolve/source-reader.ts @@ -1,4 +1,5 @@ import * as acorn from "acorn"; +import * as combineSourceMap from "combine-source-map"; import * as ESTree from "estree"; import * as fs from "fs"; import * as os from "os"; @@ -7,7 +8,6 @@ import { Logger } from "log4js"; import { Configuration } from "../../shared/configuration"; import { BundleItem } from "../bundle-item"; -import SourceMap = require("../source-map"); import { Transformer } from "../transformer"; export class SourceReader { @@ -20,7 +20,7 @@ export class SourceReader { this.readFile(bundleItem, (source: string) => { - bundleItem.source = SourceMap.deleteComment(source); + bundleItem.source = combineSourceMap.removeComments(source); bundleItem.ast = this.createAbstractSyntaxTree(bundleItem); this.transformer.applyTransforms(bundleItem, () => { diff --git a/src/bundler/source-map.ts b/src/bundler/source-map.ts deleted file mode 100644 index 34dfe489..00000000 --- a/src/bundler/source-map.ts +++ /dev/null @@ -1,35 +0,0 @@ -import * as path from "path"; -import { EmitOutput } from "../compiler/emit-output"; -import { File } from "../shared/file"; - -export function create(file: File, source: string, emitOutput: EmitOutput) { - - let result: string = emitOutput.outputText; - let map: any; - let datauri: string; - - if (emitOutput.sourceMapText) { - - map = JSON.parse(emitOutput.sourceMapText); - map.sources[0] = path.basename(file.originalPath); - map.sourcesContent = [source]; - map.file = path.basename(file.path); - file.sourceMap = map; - datauri = "data:application/json;charset=utf-8;base64," + new Buffer(JSON.stringify(map)).toString("base64"); - - result = result.replace( - createComment(file), - "//# sourceMappingURL=" + datauri - ); - } - - return result; -} - -export function createComment(file: File) { - return "//# sourceMappingURL=" + path.basename(file.path) + ".map"; -} - -export function deleteComment(source: string) { - return source.replace(/\/\/#\s?sourceMappingURL\s?=\s?.*\.map/g, ""); -} diff --git a/src/istanbul/coverage.ts b/src/istanbul/coverage.ts index 41b1b87e..17ad7361 100644 --- a/src/istanbul/coverage.ts +++ b/src/istanbul/coverage.ts @@ -4,7 +4,6 @@ import { EmitOutput } from "../compiler/emit-output"; import { Configuration } from "../shared/configuration"; import { File } from "../shared/file"; import { CoverageCallback } from "./coverage-callback"; -import SourceMap = require("../bundler/source-map"); export class Coverage { @@ -47,7 +46,7 @@ export class Coverage { if (!this.config.coverageOptions.instrumentation || this.isExcluded(this.config.coverageOptions.exclude, file.originalPath) || - this.hasNoOutput(file, emitOutput)) { + this.hasNoOutput(emitOutput)) { this.log.debug("Excluding file %s from instrumentation", file.originalPath); callback(bundled); @@ -57,8 +56,8 @@ export class Coverage { this.coveragePreprocessor(bundled, file, callback); } - private hasNoOutput(file: File, emitOutput: EmitOutput): boolean { - return emitOutput.outputText === SourceMap.createComment(file); + private hasNoOutput(emitOutput: EmitOutput): boolean { + return emitOutput.outputText.startsWith("//# sourceMappingURL="); } private isExcluded(regex: RegExp | RegExp[], path: string): boolean { diff --git a/src/shared/configuration.ts b/src/shared/configuration.ts index e93cb439..d48aa05c 100644 --- a/src/shared/configuration.ts +++ b/src/shared/configuration.ts @@ -114,6 +114,7 @@ export class Configuration implements KarmaTypescriptConfig { directories: ["node_modules"], extensions: [".js", ".json", ".ts", ".tsx"] }, + sourceMap: false, transforms: [], validateSyntax: true }; diff --git a/tests/integration-latest/karma.conf.js b/tests/integration-latest/karma.conf.js index 07298b7e..ae82447c 100644 --- a/tests/integration-latest/karma.conf.js +++ b/tests/integration-latest/karma.conf.js @@ -46,6 +46,7 @@ module.exports = function(config) { extensions: [".js", ".json", ".ts"], directories: ["node_modules"] }, + sourceMap: true, transforms: [ require("karma-typescript-cssmodules-transform")({}, {}, /style-import-tester\.css$/), require("karma-typescript-es6-transform")({presets: ["es2015"]}), @@ -76,7 +77,7 @@ module.exports = function(config) { lib: ["DOM", "ES2015"] }, coverageOptions: { - instrumentation: true, + instrumentation: false, exclude: [/\.(d|spec|test)\.ts$/i], threshold: { global: {