diff --git a/front_end/ndb.json b/front_end/ndb.json index bccce14b..79b509ab 100644 --- a/front_end/ndb.json +++ b/front_end/ndb.json @@ -1,15 +1,21 @@ { - "modules" : [ - { "name": "ndb_sdk", "type": "autostart" }, - { "name": "ndb", "type": "autostart" }, - { "name": "layer_viewer" }, - { "name": "timeline_model" }, - { "name": "timeline" }, - { "name": "product_registry" }, - { "name": "mobile_throttling" }, - { "name": "ndb_ui" }, - { "name": "xterm" } - ], + "modules": [ + { "name": "ndb_sdk", "type": "autostart" }, + { "name": "ndb", "type": "autostart" }, + { "name": "layer_viewer" }, + { "name": "timeline_model" }, + { "name": "timeline" }, + { "name": "product_registry" }, + { "name": "mobile_throttling" }, + { "name": "ndb_ui" }, + { "name": "xterm" }, + { "name": "emulation", "type": "autostart" }, + { "name": "inspector_main", "type": "autostart" }, + { "name": "mobile_throttling", "type": "autostart" }, + { "name": "cookie_table" }, + { "name": "har_importer" }, + { "name": "network" } + ], "extends": "shell", "has_html": true } diff --git a/front_end/ndb/Connection.js b/front_end/ndb/Connection.js index a4865f57..88dff0f3 100644 --- a/front_end/ndb/Connection.js +++ b/front_end/ndb/Connection.js @@ -1,8 +1,35 @@ +Ndb.ConnectionInterceptor = class { + constructor() { + this._onMessage = null; + } + + /** + * @param {string} message + * @return {boolean} + */ + sendRawMessage(message) { + throw new Error('Not implemented'); + } + + setOnMessage(onMessage) { + this._onMessage = onMessage; + } + + dispatchMessage(message) { + if (this._onMessage) + this._onMessage(message); + } + + disconnect() { + } +}; + Ndb.Connection = class { constructor(channel) { this._onMessage = null; this._onDisconnect = null; this._channel = channel; + this._interceptors = []; } static async create(channel) { @@ -16,6 +43,8 @@ Ndb.Connection = class { */ setOnMessage(onMessage) { this._onMessage = onMessage; + for (const interceptor of this._interceptors) + interceptor.setOnMessage(this._onMessage); } /** @@ -29,14 +58,25 @@ Ndb.Connection = class { * @param {string} message */ sendRawMessage(message) { + for (const interceptor of this._interceptors) { + if (interceptor.sendRawMessage(message)) + return; + } this._channel.send(message); } + addInterceptor(interceptor) { + this._interceptors.push(interceptor); + interceptor.setOnMessage(this._onMessage); + } + /** * @return {!Promise} */ disconnect() { this._channel.close(); + for (const interceptor of this._interceptors) + interceptor.disconnect(); } /** diff --git a/front_end/ndb/FileSystem.js b/front_end/ndb/FileSystem.js index f43df7a8..55a99d94 100644 --- a/front_end/ndb/FileSystem.js +++ b/front_end/ndb/FileSystem.js @@ -77,7 +77,8 @@ Ndb.FileSystem = class extends Persistence.PlatformFileSystem { */ getMetadata(path) { // This method should never be called as long as we are matching using file urls. - throw new Error('not implemented'); + // throw new Error('not implemented'); + return Promise.resolve(''); } /** diff --git a/front_end/ndb/InspectorFrontendHostOverrides.js b/front_end/ndb/InspectorFrontendHostOverrides.js index 70158e63..b96fd0cc 100644 --- a/front_end/ndb/InspectorFrontendHostOverrides.js +++ b/front_end/ndb/InspectorFrontendHostOverrides.js @@ -4,7 +4,7 @@ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -(function(){ +(async function(){ InspectorFrontendHost.getPreferences = async function(callback) { [Ndb.backend] = await carlo.loadParams(); const prefs = { @@ -30,4 +30,53 @@ callback({statusCode: 404}); } }; + + // InspectorFrontendHost.sendMessageToBackend = rawMessage => { + // const parsedMes = JSON.parse(rawMessage); + // if (parsedMes.method !== 'Network.getResponseBody') + // return; + // + // const mes = await target.runtimeAgent().invoke_evaluate({ + // expression: `process._sendMessage(${JSON.stringify(JSON.parse(rawMessage))})`, + // awaitPromise: true + // }); + // + // if (!mes.result) return; + // try { + // const [id, result] = mes.result.value; + // if (result) { + // InspectorFrontendHost.events.dispatchEventToListeners( + // InspectorFrontendHostAPI.Events.DispatchMessage, + // { + // id, + // result + // } + // ); + // } + // } catch (err) { + // console.log(err); + // } + // }; + // + // while (true) { + // const message = await target.runtimeAgent().invoke_evaluate({ + // expression: 'process._getNetworkMessages()', + // awaitPromise: true + // }); + // + // if (!message.result) return; + // const arrMessages = JSON.parse(message.result.value); + // + // for (const mes of arrMessages) { + // const { type, payload } = mes; + // + // if (type) { + // SDK._mainConnection._onMessage(JSON.stringify({ + // method: type, + // params: payload + // })); + // } + // } + // } + })(); diff --git a/front_end/ndb/NdbMain.js b/front_end/ndb/NdbMain.js index f8204af0..de29ea1b 100644 --- a/front_end/ndb/NdbMain.js +++ b/front_end/ndb/NdbMain.js @@ -207,7 +207,17 @@ Ndb.NodeProcessManager = class extends Common.Object { const target = this._targetManager.createTarget( info.id, userFriendlyName(info), SDK.Target.Type.Node, this._targetManager.targetById(info.ppid) || this._targetManager.mainTarget(), undefined, false, connection); + + target.runtimeAgent().invoke_evaluate({ + expression: await Ndb.backend.httpMonkeyPatchingSource(), + includeCommandLineAPI: true + }); + + const networkInterceptor = new Ndb.NetworkInterceptor(); + connection.addInterceptor(networkInterceptor); + networkInterceptor.setTarget(target); target[NdbSdk.connectionSymbol] = connection; + await this.addFileSystem(info.cwd, info.scriptName); if (info.scriptName) { const scriptURL = Common.ParsedURL.platformPathToURL(info.scriptName); diff --git a/front_end/ndb/NetworkInterceptor.js b/front_end/ndb/NetworkInterceptor.js new file mode 100644 index 00000000..57abb7a7 --- /dev/null +++ b/front_end/ndb/NetworkInterceptor.js @@ -0,0 +1,99 @@ +Ndb.NetworkInterceptor = class extends Ndb.ConnectionInterceptor { + constructor() { + super(); + this._buffer = []; + this._cacheRequests = []; + } + + setTarget(target) { + this._target = target; + for (const message of this._buffer.splice(0)) + this._sendRawMessage(message); + this._listen(); + } + + sendRawMessage(message) { + const parsed = JSON.parse(message); + if (parsed.method.startsWith('Network.')) { + this._sendRawMessage(message); + return true; + } + return false; + } + + setOnMessage(onMessage) { + this._onMessage = onMessage; + } + + dispatchMessage(message) { + if (this._onMessage) this._onMessage(message); + } + + disconnect() { + this._target = null; + } + + _sendRawMessage(rawMessage) {} + + async _listen() { + InspectorFrontendHost.sendMessageToBackend = rawMessage => { + const message = JSON.parse(rawMessage); + + const request = this._cacheRequests.filter(res => { + if ( + res.type === 'Network.getResponseBody' && + res.payload.requestId === message.params.requestId + ) + return res; + })[0]; + + if (request) { + InspectorFrontendHost.events.dispatchEventToListeners( + InspectorFrontendHostAPI.Events.DispatchMessage, + { + id: message.id, + result: { + base64Encoded: true, + body: request.payload.data + } + } + ); + } + }; + + while (this._target) { + const rawResponse = await this._target + .runtimeAgent() + .invoke_evaluate({ + expression: `process._fetchNetworkMessages()`, + awaitPromise: true, + returnByValue: true + }); + + if (!rawResponse || !rawResponse.result) return; + + const { + result: { value: messages } + } = rawResponse; + + if (!messages) return; + + // messages is array-like + const messagesArr = Array.from(JSON.parse(messages)); + + for (const message of messagesArr) { + const { type, payload } = message; + this._cacheRequests.push(message); + + // this is on the way back, this way doesn't work + if (type !== 'Network.getResponseBody') { + // but this does + SDK._mainConnection._onMessage(JSON.stringify({ + method: type, + params: payload + })); + } + } + } + } +}; diff --git a/front_end/ndb/module.json b/front_end/ndb/module.json index 4113c13c..83175367 100644 --- a/front_end/ndb/module.json +++ b/front_end/ndb/module.json @@ -56,6 +56,7 @@ "scripts": [ "InspectorFrontendHostOverrides.js", "Connection.js", + "NetworkInterceptor.js", "FileSystem.js", "NdbMain.js" ] diff --git a/lib/backend.js b/lib/backend.js index 53e4011b..68552db5 100644 --- a/lib/backend.js +++ b/lib/backend.js @@ -48,6 +48,12 @@ class Backend { opn(url); } + async httpMonkeyPatchingSource() { + const pathToHttpPatch = path.resolve(__dirname, '..', './lib/preload/ndb/httpMonkeyPatching.js'); + const content = await fsReadFile(pathToHttpPatch, 'utf8'); + return content; + } + pkg() { // TODO(ak239spb): implement it as decorations over package.json file. try { diff --git a/lib/preload/ndb/httpMonkeyPatching.js b/lib/preload/ndb/httpMonkeyPatching.js new file mode 100644 index 00000000..84ba7051 --- /dev/null +++ b/lib/preload/ndb/httpMonkeyPatching.js @@ -0,0 +1,192 @@ +const zlib = require('zlib'); +const http = require('http'); +const https = require('https'); + +const initTime = process.hrtime(); + +// DT requires us to use relative time in a strange format (xxx.xxx) +const getTime = () => { + const diff = process.hrtime(initTime); + + return diff[0] + diff[1] / 1e9; +}; + +const formatRequestHeaders = req => { + if (!req.headers) return {}; + return Object.keys(req.headers).reduce((acc, k) => { + if (typeof req.headers[k] === 'string') acc[k] = req.headers[k]; + return acc; + }, {}); +}; + +const formatResponseHeaders = res => { + if (!res.headers) return {}; + return Object.keys(res.headers).reduce((acc, k) => { + if (typeof res.headers[k] === 'string') acc[k] = res.headers[k]; + return acc; + }, {}); +}; + +const getMineType = mimeType => { + // nasty hack for ASF + if (mimeType === 'OPENJSON') + return 'application/json;charset=UTF-8'; + + + return mimeType; +}; + +const cacheRequests = {}; +let id = 1; +const getId = () => id++; + +const messages = []; +let messageAdded = null; + +function reportMessage(message) { + messages.push(message); + if (messageAdded) { + setTimeout(messageAdded, 0); + messageAdded = null; + } +} + +process._fetchNetworkMessages = async function() { + if (!messages.length) + await new Promise(resolve => messageAdded = resolve); + return JSON.stringify(messages.splice(0)); +}; + +process._sendNetworkCommand = async function(rawMessage) { + return new Promise(resolve => { + const message = JSON.parse(rawMessage); + console.log({ cacheRequests }); + console.log({ cacheRequests: cacheRequests[message.params.requestId] }); + if (!cacheRequests[message.params.requestId]) { + resolve(JSON.stringify({})); + } else { + if (message.method === 'Network.getResponseBody') { + console.log({ message }); + const { base64Encoded, data } = cacheRequests[message.params.requestId]; + + console.log({ cacheRequests }); + console.log({ data }); + resolve(JSON.stringify([message.id, { base64Encoded, body: data }])); + } + } + }); +}; + +const callbackWrapper = (callback, req) => res => { + const requestId = getId(); + res.req.__requestId = requestId; + + reportMessage({ + payload: { + requestId: requestId, + loaderId: requestId, + documentURL: req.href, + request: { + url: req.href, + method: req.method, + headers: formatRequestHeaders(req), + mixedContentType: 'none', + initialPriority: 'VeryHigh', + referrerPolicy: 'no-referrer-when-downgrade', + postData: req.body + }, + timestamp: getTime(), + wallTime: Date.now(), + initiator: { + type: 'other' + }, + type: 'Document' + }, + type: 'Network.requestWillBeSent' + }); + + const encoding = res.headers['content-encoding']; + let rawData = []; + + const onEnd = function() { + rawData = Buffer.concat(rawData); + rawData = rawData.toString('base64'); + + cacheRequests[res.req.__requestId] = { + ...res, + __rawData: rawData, + base64Encoded: true + }; + + const payload = { + id: res.req.__requestId, + requestId: res.req.__requestId, + loaderId: res.req.__requestId, + base64Encoded: true, + data: cacheRequests[res.req.__requestId].__rawData, + timestamp: getTime(), + type: 'XHR', + encodedDataLength: 100, + response: { + url: req.href, + status: res.statusCode, + statusText: res.statusText, + // set-cookie prop in the header has value as an array + // for example: ["__cfduid=dbfe006ef71658bf4dba321343c227f9a15449556…20:29 GMT; path=/; domain=.typicode.com; HttpOnly"] + headers: formatResponseHeaders(res), + mimeType: getMineType( + res.headers['content-encoding'] || + res.headers['content-type'] + ), + requestHeaders: formatRequestHeaders(req) + } + }; + + // Send the response back. + reportMessage({ payload: payload, type: 'Network.responseReceived' }); + reportMessage({ payload: payload, type: 'Network.loadingFinished' }); + reportMessage({ payload: payload, type: 'Network.getResponseBody' }); + }; + + if (encoding === 'gzip' || encoding === 'x-gzip') { + const gunzip = zlib.createGunzip(); + res.pipe(gunzip); + + gunzip.on('data', function(data) { + rawData.push(data); + }); + gunzip.on('end', onEnd); + } else { + res.on('data', chunk => { + rawData.push(chunk); + }); + res.on('end', onEnd); + } + + callback && callback(res); +}; + +const originHTTPRequest = http.request; +http.request = function wrapMethodRequest(req, callback) { + const request = originHTTPRequest.call( + this, + req, + callbackWrapper(callback, req) + ); + return request; +}; + +const originHTTPSRequest = https.request; +https.request = function wrapMethodRequest(req, callback) { + const request = originHTTPSRequest.call( + this, + req, + callbackWrapper(callback, req) + ); + const originWrite = request.write.bind(request); + request.write = data => { + req.body = data.toString(); + originWrite(data); + }; + return request; +};