From 7f0547d4e02d927e766de340152a2f75b659d889 Mon Sep 17 00:00:00 2001 From: eric thul Date: Sun, 23 Apr 2017 14:00:35 -0400 Subject: [PATCH] Refactoring cache usage --- package.json | 2 +- src/bundle.js | 21 ++- src/compile.js | 19 ++- src/ide.js | 324 ++++++++++++++++++++++--------------------- src/index.js | 226 +++++++++++++++++++++--------- src/to-javascript.js | 12 +- 6 files changed, 357 insertions(+), 247 deletions(-) diff --git a/package.json b/package.json index ca5e5c8..1a1cc7e 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "chalk": "^1.1.3", "cross-spawn": "^3.0.1", "dargs": "^5.1.0", - "debug": "^2.2.0", + "debug": "^2.6.0", "globby": "^4.0.0", "js-string-escape": "^1.0.1", "lodash.difference": "^4.5.0", diff --git a/src/bundle.js b/src/bundle.js index 6627ffe..3f55f01 100644 --- a/src/bundle.js +++ b/src/bundle.js @@ -12,12 +12,10 @@ const debug = require('debug')('purs-loader'); const dargs = require('./dargs'); -module.exports = function bundle(options, cache) { - if (cache.bundle) return Promise.resolve(cache.bundle) - +module.exports = function bundle(options, bundleModules) { const stdout = [] - const stderr = cache.bundle = [] + const stderr = [] const bundleCommand = options.pscBundle || 'purs'; @@ -27,9 +25,9 @@ module.exports = function bundle(options, cache) { namespace: options.bundleNamespace, }, options.pscBundleArgs))); - cache.bundleModules.forEach(name => bundleArgs.push('--module', name)) + bundleModules.forEach(name => bundleArgs.push('--module', name)) - debug('spawning bundler %s %o', bundleCommand, bundleArgs); + debug('bundle: %s %o', bundleCommand, bundleArgs); return (new Promise((resolve, reject) => { debug('bundling PureScript...') @@ -45,15 +43,16 @@ module.exports = function bundle(options, cache) { if (code !== 0) { const errorMessage = stderr.join(''); + if (errorMessage.length) { psModule.emitError(errorMessage); } - return reject(new Error('bundling failed')) - } - cache.bundle = stderr - - resolve(fs.appendFileAsync(options.bundleOutput, `module.exports = ${options.bundleNamespace}`)) + reject(new Error('bundling failed')) + } + else { + resolve(fs.appendFileAsync(options.bundleOutput, `module.exports = ${options.bundleNamespace}`)) + } }) })) }; diff --git a/src/compile.js b/src/compile.js index 8b5d87f..707605c 100644 --- a/src/compile.js +++ b/src/compile.js @@ -4,17 +4,17 @@ const Promise = require('bluebird'); const spawn = require('cross-spawn'); -const debug = require('debug')('purs-loader'); +const debug_ = require('debug'); + +const debug = debug_('purs-loader'); + +const debugVerbose = debug_('purs-loader:verbose'); const dargs = require('./dargs'); module.exports = function compile(psModule) { const options = psModule.options - const cache = psModule.cache - - const stderr = [] - const compileCommand = options.psc || 'purs'; const compileArgs = (options.psc ? [] : [ 'compile' ]).concat(dargs(Object.assign({ @@ -22,7 +22,9 @@ module.exports = function compile(psModule) { output: options.output, }, options.pscArgs))) - debug('spawning compiler %s %o', compileCommand, compileArgs) + const stderr = []; + + debug('compile %s %o', compileCommand, compileArgs) return new Promise((resolve, reject) => { debug('compiling PureScript...') @@ -33,8 +35,13 @@ module.exports = function compile(psModule) { stderr.push(data.toString()); }); + compilation.stdout.on('data', data => { + debugVerbose(data.toString()); + }); + compilation.on('close', code => { debug('finished compiling PureScript.') + if (code !== 0) { const errorMessage = stderr.join(''); if (errorMessage.length) { diff --git a/src/ide.js b/src/ide.js index f839fd5..ac80789 100644 --- a/src/ide.js +++ b/src/ide.js @@ -12,193 +12,71 @@ const spawn = require('cross-spawn'); const colors = require('chalk'); -const debug = require('debug')('purs-loader'); +const debug_ = require('debug'); -const dargs = require('./dargs'); - -const Psc = require('./Psc'); - -const PsModuleMap = require('./PsModuleMap'); - -module.exports.connect = function connect(psModule) { - const options = psModule.options - const cache = psModule.cache - - if (cache.ideServer) return Promise.resolve(psModule) +const debug = debug_('purs-loader'); - cache.ideServer = true +const debugVerbose = debug_('purs-loader:verbose'); - const connect_ = () => new Promise((resolve, reject) => { - const args = dargs(options.pscIdeArgs) - - debug('attempting to run purs ide client: %o', args) +const dargs = require('./dargs'); - const ideClient = spawn('purs', ['ide', 'client'].concat(args)) +const compile = require('./compile'); - ideClient.stderr.on('data', data => { - debug(data.toString()) - cache.ideServer = false - reject(new Error('purs ide client failed')) - }) - ideClient.stdout.once('data', data => { - debug(data.toString()) - if (data.toString()[0] === '{') { - const res = JSON.parse(data.toString()) - if (res.resultType === 'success') { - cache.ideServer = ideServer - resolve(psModule) - } else { - cache.ideServer = ideServer - reject(new Error('purs ide client failed')) - } - } else { - cache.ideServer = false - reject(new Error('purs ide client failed')) - } - }) - ideClient.stdin.resume() - ideClient.stdin.write(JSON.stringify({ command: 'load' })) - ideClient.stdin.write('\n') - }) +const PsModuleMap = require('./PsModuleMap'); - const serverArgs = dargs(Object.assign({ - outputDirectory: options.output, - '_': options.src - }, options.pscIdeServerArgs)) +function UnknownModuleError() { + this.name = 'UnknownModuleError'; + this.stack = (new Error()).stack; +} - debug('attempting to start purs ide server: %o', serverArgs) +UnknownModuleError.prototype = Object.create(Error.prototype); - const ideServer = cache.ideServer = spawn('purs', ['ide', 'server'].concat(serverArgs)) +UnknownModuleError.prototype.constructor = UnknownModuleError; - ideServer.stdout.on('data', data => { - debug('purs ide server stdout: %s', data.toString()); - }); +module.exports.UnknownModuleError = UnknownModuleError; - ideServer.stderr.on('data', data => { - debug('purs ide server stderr: %s', data.toString()); - }); +function spawnIdeClient(body, options) { + const ideClientCommand = 'purs'; - ideServer.on('error', error => { - debug('purs ide server error: %o', error); - }); + const ideClientArgs = ['ide', 'client'].concat(dargs(options.pscIdeArgs)); - ideServer.on('close', (code, signal) => { - debug('purs ide server close: %s %s', code, signal); - }); + const stderr = []; - return retryPromise((retry, number) => { - return connect_().catch(error => { - if (!cache.ideServer && number === 9) { - debug(error) + const stdout = []; - console.warn('Failed to connect to or start purs ide server. A full compilation will occur on rebuild'); + debug('ide client %s %o %o', ideClientCommand, ideClientArgs, body); - return Promise.resolve(psModule) - } + return new Promise((resolve, reject) => { + const ideClient = spawn(ideClientCommand, ideClientArgs); - return retry(error) + ideClient.stderr.on('data', data => { + stderr.push(data.toString()); }) - }, { - retries: 9, - factor: 1, - minTimeout: 333, - maxTimeout: 333, - }) -}; - -module.exports.rebuild = function rebuild(psModule) { - const options = psModule.options - const cache = psModule.cache - - debug('attempting rebuild with purs ide client %s', psModule.srcPath) - - const request = (body) => new Promise((resolve, reject) => { - const args = dargs(options.pscIdeArgs) - const ideClient = spawn('purs', ['ide', 'client'].concat(args)) - - var stdout = '' - var stderr = '' ideClient.stdout.on('data', data => { - stdout = stdout + data.toString() - }) - - ideClient.stderr.on('data', data => { - stderr = stderr + data.toString() + stdout.push(data.toString()); }) ideClient.on('close', code => { if (code !== 0) { - const error = stderr === '' ? 'Failed to spawn purs ide client' : stderr - return reject(new Error(error)) - } - - let res = null + const errorMessage = stderr.join(''); - try { - res = JSON.parse(stdout.toString()) - debug(res) - } catch (err) { - return reject(err) + reject(new Error(`ide client failed: ${errorMessage}`)); } + else { + const result = stdout.join(''); - if (res && !Array.isArray(res.result)) { - return resolve(psModule); + resolve(result); } - - Promise.map(res.result, (item, i) => { - debug(item) - return formatIdeResult(item, options, i, res.result.length) - }) - .then(compileMessages => { - if (res.resultType === 'error') { - if (res.result.some(item => { - const isModuleNotFound = item.errorCode === 'ModuleNotFound'; - - const isUnknownModule = item.errorCode === 'UnknownModule'; - - const isUnknownModuleImport = item.errorCode === 'UnknownName' && /Unknown module/.test(item.message); - - return isModuleNotFound || isUnknownModule || isUnknownModuleImport; - })) { - debug('unknown module, attempting full recompile') - return Psc.compile(psModule) - .then(() => PsModuleMap.makeMap(options.src).then(map => { - debug('rebuilt module map after unknown module forced a recompile'); - cache.psModuleMap = map; - })) - .then(() => request({ command: 'load' })) - .then(resolve) - .catch(() => resolve(psModule)) - } - const errorMessage = compileMessages.join('\n'); - if (errorMessage.length) { - psModule.emitError(errorMessage); - } - resolve(psModule); - } else { - const warningMessage = compileMessages.join('\n'); - if (options.warnings && warningMessage.length) { - psModule.emitWarning(warningMessage); - } - resolve(psModule); - } - }) }) - debug('purs ide client stdin: %o', body); + ideClient.stdin.resume(); - ideClient.stdin.write(JSON.stringify(body)) - ideClient.stdin.write('\n') - }) + ideClient.stdin.write(JSON.stringify(body)); - return request({ - command: 'rebuild', - params: { - file: psModule.srcPath, - } - }) -}; + ideClient.stdin.write('\n'); + }); +} function formatIdeResult(result, options, index, length) { let numAndErr = `[${index+1}/${length} ${result.errorCode}]` @@ -253,8 +131,140 @@ function formatIdeResult(result, options, index, length) { } return Promise.resolve(`\n${numAndErr} ${fileAndPos}\n\n${snippet}\n\n${result.message}`) - }) + }).catch(error => { + debug('failed to format ide result: %o', error); + + return Promise.resolve(''); + }); } return result.filename && result.position ? makeResultSnippet(result.filename, result.position) : makeResult(); } + +module.exports.connect = function connect(psModule) { + const options = psModule.options + + const serverCommand = 'purs'; + + const serverArgs = ['ide', 'server'].concat(dargs(Object.assign({ + outputDirectory: options.output, + '_': options.src + }, options.pscIdeServerArgs))); + + debug('ide server: %s %o', serverCommand, serverArgs); + + const ideServer = spawn(serverCommand, serverArgs); + + ideServer.stdout.on('data', data => { + debugVerbose('ide server stdout: %s', data.toString()); + }); + + ideServer.stderr.on('data', data => { + debugVerbose('ide server stderr: %s', data.toString()); + }); + + ideServer.on('error', error => { + debugVerbose('ide server error: %o', error); + }); + + ideServer.on('close', (code, signal) => { + debugVerbose('ide server close: %s %s', code, signal); + }); + + return Promise.resolve(ideServer); +}; + +module.exports.load = function load(psModule) { + const options = psModule.options + + const body = {command: 'load'}; + + return spawnIdeClient(body, options); +}; + +module.exports.loadWithRetry = function loadWithRetry(psModule) { + const retries = 9; + + return retryPromise((retry, number) => { + debugVerbose('attempting to load modules (%d out of %d attempts)', number, retries); + + return module.exports.load(psModule).catch(retry); + }, { + retries: retries, + factor: 1, + minTimeout: 333, + maxTimeout: 333, + }).then(() => psModule); +}; + +module.exports.rebuild = function rebuild(psModule) { + const options = psModule.options; + + const body = { + command: 'rebuild', + params: { + file: psModule.srcPath, + } + }; + + const parseResponse = response => { + try { + const parsed = JSON.parse(response); + + debugVerbose('parsed JSON response: %o', parsed); + + return Promise.resolve(parsed); + } + catch (error) { + return Promise.reject(error); + } + }; + + const formatResponse = parsed => { + const result = Array.isArray(parsed.result) ? parsed.result : []; + + return Promise.map(result, (item, i) => { + debugVerbose('formatting result %o', item); + + return formatIdeResult(item, options, i, result.length); + }).then(formatted => ({ + parsed: parsed, + formatted: formatted, + formattedMessage: formatted.join('\n') + })); + }; + + return spawnIdeClient(body, options) + .then(parseResponse) + .then(formatResponse) + .then(({ parsed, formatted, formattedMessage }) => { + if (parsed.resultType === 'success') { + if (options.warnings && formattedMessage.length) { + psModule.emitWarning(formattedMessage); + } + + return psModule; + } + else if ((parsed.result || []).some(item => { + const isModuleNotFound = item.errorCode === 'ModuleNotFound'; + + const isUnknownModule = item.errorCode === 'UnknownModule'; + + const isUnknownModuleImport = item.errorCode === 'UnknownName' && /Unknown module/.test(item.message); + + return isModuleNotFound || isUnknownModule || isUnknownModuleImport; + })) { + debug('failed to rebuild because the module is unknown') + + return Promise.reject(new UnknownModuleError()); + } + else { + if (formattedMessage.length) { + psModule.emitError(formattedMessage); + } + + return psModule; + } + }) + ; +}; diff --git a/src/index.js b/src/index.js index 799f8f9..7cf942c 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,10 @@ 'use strict' -const debug = require('debug')('purs-loader') +const debug_ = require('debug'); + +const debug = debug_('purs-loader'); + +const debugVerbose = debug_('purs-loader:verbose'); const loaderUtils = require('loader-utils') @@ -25,33 +29,44 @@ const spawn = require('cross-spawn').sync const eol = require('os').EOL module.exports = function purescriptLoader(source, map) { - const callback = this.async() - const config = this.options - const query = loaderUtils.getOptions(this) || {} - const webpackOptions = this.options.purescriptLoader || {} + this.cacheable && this.cacheable(); + + const callback = this.async(); + + const webpackConfig = this.options; + + const loaderOptions = loaderUtils.getOptions(this) || {}; - const depsPaths = (pscPackage => { + const srcOption = (pscPackage => { if (pscPackage) { - debug('calling psc-package...') + const pscPackageCommand = 'psc-package'; - return spawn('psc-package', ['sources']).stdout.toString().split(eol).filter(v => v != '') + const pscPackageArgs = ['sources']; + + debug('psc-package %s %o', pscPackageCommand, pscPackageArgs); + + return spawn(pscPackageCommand, pscPackageArgs).stdout.toString().split(eol).filter(v => v != '').concat( + loaderOptions.src || [ + path.join('src', '**', '*.purs'), + ] + ) } else { - return [ path.join('bower_components', 'purescript-*', 'src', '**', '*.purs') ] + return loaderOptions.src || [ + path.join('bower_components', 'purescript-*', 'src', '**', '*.purs'), + path.join('src', '**', '*.purs'), + ]; } - }) - - let options = Object.assign(webpackOptions, query) + })(loaderOptions.pscPackage); - const defaultDeps = depsPaths(options.pscPackage) - const defaultOptions = { - context: config.context, + const options = Object.assign({ + context: webpackConfig.context, psc: null, pscArgs: {}, pscBundle: null, pscBundleArgs: {}, pscIde: false, - pscIdeColors: options.psc === 'psa', + pscIdeColors: loaderOptions.psc === 'psa', pscIdeArgs: {}, pscPackage: false, bundleOutput: 'output/bundle.js', @@ -60,36 +75,29 @@ module.exports = function purescriptLoader(source, map) { warnings: true, watch: false, output: 'output', - src: [ - path.join('src', '**', '*.purs'), - ...defaultDeps - ] - } - - this.cacheable && this.cacheable() + src: [] + }, loaderOptions, { + src: srcOption + }); - let cache = config.purescriptLoaderCache = config.purescriptLoaderCache || { + var cache = webpackConfig.purescriptLoaderCache = webpackConfig.purescriptLoaderCache || { rebuild: false, deferred: [], bundleModules: [], warnings: [], errors: [] - } - - if (options.pscPackage && options.src) { - options.src = options.src.concat(defaultDeps) // append psc-package-provided source paths with users' - } + }; - options = Object.assign(defaultOptions, options) + if (!webpackConfig.purescriptLoaderInstalled) { + debugVerbose('installing purs-loader with options: %O', options); - if (!config.purescriptLoaderInstalled) { - config.purescriptLoaderInstalled = true + webpackConfig.purescriptLoaderInstalled = true // invalidate loader cache when bundle is marked as invalid (in watch mode) this._compiler.plugin('invalid', () => { - debug('invalidating loader cache'); + debugVerbose('invalidating loader cache'); - cache = config.purescriptLoaderCache = { + cache = webpackConfig.purescriptLoaderCache = { rebuild: options.pscIde, deferred: [], bundleModules: [], @@ -97,7 +105,7 @@ module.exports = function purescriptLoader(source, map) { psModuleMap: cache.psModuleMap, warnings: [], errors: [] - } + }; }); // add psc warnings to webpack compilation warnings @@ -114,7 +122,8 @@ module.exports = function purescriptLoader(source, map) { }); } - const psModuleName = PsModuleMap.matchModule(source) + const psModuleName = PsModuleMap.matchModule(source); + const psModule = { name: psModuleName, load: js => callback(null, js), @@ -136,51 +145,132 @@ module.exports = function purescriptLoader(source, map) { } } - debug('loader called', psModule.name) + debug('loading %s', psModule.name); if (options.bundle) { - cache.bundleModules.push(psModule.name) + cache.bundleModules.push(psModule.name); } if (cache.rebuild) { - return ide.connect(psModule) - .then(ide.rebuild) + const connect = () => { + if (!cache.ideServer) { + cache.ideServer = true; + + return ide.connect(psModule) + .then(ideServer => { + cache.ideServer = ideServer; + return psModule; + }) + .then(ide.loadWithRetry) + .catch(error => { + if (cache.ideServer.kill) { + debug('ide failed to initially load modules, stopping the ide server process'); + + cache.ideServer.kill(); + } + + cache.ideServer = null; + + return Promise.reject(error); + }) + ; + } + else { + return Promise.resolve(psModule); + } + }; + + const rebuild = () => + ide.rebuild(psModule).catch(error => { + if (error instanceof ide.UnknownModuleError) { + if (!cache.compilationStarted) { + cache.compilationStarted = true; + + return compile(psModule) + .then(() => { + cache.compilationFinished = true; + }) + .then(() => + PsModuleMap.makeMap(options.src).then(map => { + debug('rebuilt module map after unknown module forced a recompilation'); + + cache.psModuleMap = map; + }) + ) + .then(() => ide.load(psModule)) + .then(() => psModule) + ; + } + else { + return Promise.resolve(psModule); + } + } + else { + debug('ide rebuild failed due to an unhandled error: %o', error); + + return Promise.reject(error); + } + }) + ; + + connect() + .then(rebuild) .then(toJavaScript) .then(psModule.load) .catch(psModule.reject) + ; } + else if (cache.compilationFinished) { + debugVerbose('compilation is already finished, loading module %s', psModule.name); - if (cache.compilationFinished) { - return toJavaScript(psModule).then(psModule.load).catch(psModule.reject) + toJavaScript(psModule) + .then(psModule.load) + .catch(psModule.reject); } - - // We need to wait for compilation to finish before the loaders run so that - // references to compiled output are valid. - cache.deferred.push(psModule) - - if (!cache.compilationStarted) { - cache.compilationStarted = true; - - return compile(psModule) - .then(() => { - cache.compilationFinished = true; - - const bundlePromise = options.bundle ? bundle(options, cache) : Promise.resolve(); - - return bundlePromise.then(() => + else { + // The compilation has not finished yet. We need to wait for + // compilation to finish before the loaders run so that references + // to compiled output are valid. Push the modules into the cache to + // be loaded once the complation is complete. + + cache.deferred.push(psModule); + + if (!cache.compilationStarted) { + cache.compilationStarted = true; + + compile(psModule) + .then(() => { + cache.compilationFinished = true; + }) + .then(() => { + if (options.bundle) { + return bundle(options, cache.bundleModules); + } + }) + .then(() => PsModuleMap.makeMap(options.src).then(map => { - debug('rebuilt module map after compile'); + debug('rebuilt module map after compilation'); + cache.psModuleMap = map; }) - ); - }) - .then(() => Promise.map(cache.deferred, psModule => { - if (typeof cache.ideServer === 'object') cache.ideServer.kill() - return toJavaScript(psModule).then(psModule.load) - })) - .catch(error => { - cache.deferred[0].reject(error) - cache.deferred.slice(1).forEach(psModule => psModule.reject(new Error('purs-loader failed'))) - }) + ) + .then(() => + Promise.map(cache.deferred, psModule => + toJavaScript(psModule).then(psModule.load) + ) + ) + .catch(error => { + cache.deferred[0].reject(error); + + cache.deferred.slice(1).forEach(psModule => { + psModule.reject(new Error('purs-loader failed')); + }) + }) + ; + } + else { + // The complation has started. Nothing to do but wait until it is + // done before loading all of the modules. + } } } diff --git a/src/to-javascript.js b/src/to-javascript.js index b402ad4..d0934d5 100644 --- a/src/to-javascript.js +++ b/src/to-javascript.js @@ -10,7 +10,11 @@ const jsStringEscape = require('js-string-escape'); const difference = require('lodash.difference'); -const debug = require('debug')('purs-loader'); +const debug_ = require('debug'); + +const debug = debug_('purs-loader'); + +const debugVerbose = debug_('purs-loader:verbose'); const PsModuleMap = require('./PsModuleMap'); @@ -99,7 +103,7 @@ function makeJS(psModule, psModuleMap, js) { const additionalImports = difference(imports, replacedImports); if (additionalImports.length) { - debug('additional imports for %s: %o', name, additionalImports); + debugVerbose('additional imports for %s: %o', name, additionalImports); } const additionalImportsResult = additionalImports.map(import_ => { @@ -129,13 +133,13 @@ module.exports = function toJavaScript(psModule) { const bundlePath = path.resolve(options.bundleOutput); - const jsPath = cache.bundle ? bundlePath : psModule.jsPath; + const jsPath = options.bundle ? bundlePath : psModule.jsPath; const js = fs.readFileAsync(jsPath, 'utf8').catch(() => ''); const psModuleMap = updatePsModuleMap(psModule); - debug('loading JavaScript for %s', psModule.name); + debugVerbose('loading JavaScript for %s', psModule.name); return Promise.props({js: js, psModuleMap: psModuleMap}).then(result => options.bundle ? -- 2.41.0