Merge pull request #577 from ivogabe/declaration

Source maps for declaration files (continuation of 575)

Author: ivogabe

.gitmodules

@@ -5,3 +5,9 @@
 [submodule "typescript/2.3"]
 	path = typescript/2.3
 	url = https://github.com/Microsoft/TypeScript.git
+[submodule "2.9"]
+	path = 2.9
+	url = https://github.com/Microsoft/TypeScript
+[submodule "typescript/2.9"]
+	path = typescript/2.9
+	url = https://github.com/Microsoft/TypeScript

.vscode/tasks.json

@@ -11,6 +11,7 @@ const diff = require('gulp-diff');
 const tsVersions = {
 	dev: './typescript/dev',
 	release23: './typescript/2.3',
+	release29: './typescript/2.9'
 };
 
 function findTSDefinition(location) {
@@ -35,9 +36,9 @@ const tests = fs.readdirSync(path.join(__dirname, 'test')).filter(function(dir)
 });
 
 // Clean
-gulp.task('clean', function(cb) {
+function clean(cb) {
 	rimraf(paths.releaseBeta, cb);
-});
+}
 gulp.task('clean-test', function(cb) {
 	rimraf('test/output', cb);
 });
@@ -46,35 +47,36 @@ gulp.task('clean-release', function(cb) {
 });
 
 // Compile sources
-gulp.task('scripts', ['clean'], function() {
+const compile = gulp.series(clean, function compile() {
 	return gulp.src(paths.scripts.concat(paths.definitionTypeScript))
 		.pipe(tsProject())
 		.pipe(gulp.dest(paths.releaseBeta));
 });
 
+
 // Type checking against multiple versions of TypeScript
-gulp.task('typecheck-dev', function() {
+function typecheckDev() {
 	return gulp.src(paths.scripts.concat([
 		'!definitions/typescript.d.ts',
 		findTSDefinition(tsVersions.dev)
 	])).pipe(createProject({ noEmit: true })());
-});
-
-gulp.task('typecheck-2.3', function() {
+}
+function typecheck2_3() {
 	return gulp.src(paths.scripts.concat([
 		'!definitions/typescript.d.ts',
 		findTSDefinition(tsVersions.release23)
 	])).pipe(createProject({ noEmit: true })());
-});
+}
 
-gulp.task('typecheck', ['typecheck-dev', 'typecheck-2.3']);
+const typecheck = gulp.parallel(typecheckDev, typecheck2_3);
 
 // Tests
 
 // We run every test on multiple typescript versions:
 const libs = [
 	['2.7', undefined],
 	['2.3', require(tsVersions.release23)],
+	['2.9', require(tsVersions.release29)],
 	['dev', require(tsVersions.dev)]
 ];
 
@@ -128,18 +130,18 @@ async function runTest(name) {
 	}));
 }
 
-gulp.task('test-run', ['clean-test', 'scripts'], async function() {
+gulp.task('test-run', gulp.series('clean-test', async function testRun() {
 	fs.mkdirSync('test/output/');
 	for (const testName of tests) {
 		await runTest(testName);
 	}
-});
+}));
 
 /**
  * Executes all the test tasks and then compares their output against the expected output (defined in
  * `test/baseline`).
  */
-gulp.task('test', ['test-run'], function() {
+gulp.task('test', gulp.series('test-run', function testVerify() {
 	let failed = false;
 	function onError(error) {
 		failed = true;
@@ -155,21 +157,23 @@ gulp.task('test', ['test-run'], function() {
 				throw new Error('Tests failed');
 			}
 		});
-});
+}));
 
 // Accept new baselines
-gulp.task('test-baselines-accept', function(cb) {
+gulp.task('test-baselines-accept', function testBaselinesAccept(cb) {
 	rimraf('test/baselines', function() {
 		gulp.src('test/output/**').pipe(gulp.dest('test/baselines')).on('finish', cb);
 	});
 });
 
-gulp.task('release', function() {
+gulp.task('release', function release() {
 	return gulp.src(paths.releaseBeta + '/**').pipe(gulp.dest(paths.release));
 });
 
-gulp.task('watch', ['scripts'], function() {
-	gulp.watch(paths.scripts, ['scripts']);
-});
+// Expose main tasks
+gulp.task('scripts', compile);
+gulp.task('default', gulp.series(compile, gulp.parallel(typecheck, 'test')));
 
-gulp.task('default', ['scripts', 'typecheck', 'test']);
+gulp.task('watch', gulp.series('scripts', function watch() {
+	gulp.watch(paths.scripts, compile);
+}));

gulpfile.js

@@ -18,9 +18,11 @@ interface OutputFile {
 
 	jsFileName?: string;
 	dtsFileName?: string;
+	dtsMapFileName?: string;
 	jsContent?: string;
 	jsMapContent?: string;
 	dtsContent?: string;
+	dtsMapContent?: string;
 }
 
 /**
@@ -125,13 +127,17 @@ export class ProjectCompiler implements ICompiler {
 	}
 
 	private attachContentToFile(file: OutputFile, fileName: string, content: string) {
-		const [, extension] = utils.splitExtension(fileName, ['d.ts']);
+		const [, extension] = utils.splitExtension(fileName, ['d.ts', 'd.ts.map']);
 		switch (extension) {
 			case 'js':
 			case 'jsx':
 				file.jsFileName = fileName;
 				file.jsContent = content;
 				break;
+			case 'd.ts.map':
+				file.dtsMapFileName = fileName;
+				file.dtsMapContent = content;
+				break;
 			case 'd.ts':
 				file.dtsFileName = fileName;
 				file.dtsContent = content;
@@ -149,7 +155,7 @@ export class ProjectCompiler implements ICompiler {
 
 		result.emitSkipped = emitOutput.emitSkipped;
 	}
-	private emitFile({ file, jsFileName, dtsFileName, jsContent, dtsContent, jsMapContent }: OutputFile, currentDirectory: string) {
+	private emitFile({ file, jsFileName, dtsFileName, dtsMapFileName, jsContent, dtsContent, dtsMapContent, jsMapContent }: OutputFile, currentDirectory: string) {
 		if (!jsFileName) return;
 
 		let base: string;
@@ -184,7 +190,7 @@ export class ProjectCompiler implements ICompiler {
 			this.project.output.writeJs(base, jsFileName, jsContent, jsMapContent, file ? file.gulp.cwd : currentDirectory, file);
 		}
 		if (dtsContent !== undefined) {
-			this.project.output.writeDts(baseDeclarations, dtsFileName, dtsContent, file ? file.gulp.cwd : currentDirectory);
+			this.project.output.writeDts(baseDeclarations, dtsFileName, dtsContent, dtsMapContent, file ? file.gulp.cwd : currentDirectory, file);
 		}
 	}
 
@@ -270,7 +276,7 @@ export class FileCompiler implements ICompiler {
 
 		mapString = mapString.substring(start.length);
 
-		let map: RawSourceMap = JSON.parse(new Buffer(mapString, 'base64').toString());
+		let map: RawSourceMap = JSON.parse(Buffer.from(mapString, 'base64').toString());
 		// TODO: Set paths correctly
 		// map.sourceRoot = path.resolve(file.gulp.cwd, file.gulp.base);
 		// map.sources[0] = path.relative(map.sourceRoot, file.gulp.path);

lib/compiler.ts

@@ -70,12 +70,18 @@ function checkAndNormalizeSettings(settings: compile.Settings): compile.Settings
 	return standardSettings;
 }
 
-function normalizeCompilerOptions(options: ts.CompilerOptions): void {
+function normalizeCompilerOptions(options: ts.CompilerOptions, typescript: typeof ts): void {
 	options.sourceMap = true;
 	(options as any).suppressOutputPathCheck = true;
 	options.inlineSourceMap = false;
 	options.sourceRoot = undefined;
 	options.inlineSources = false;
+
+	// For TS >=2.9, we set `declarationMap` to true, if `declaration` is set.
+	// We check for this version by checking whether `createFileLevelUniqueName` exists.
+	if ("createFileLevelUniqueName" in typescript && options.declaration && !options.isolatedModules) {
+		options.declarationMap = true;
+	}
 }
 
 function reportErrors(errors: ts.Diagnostic[], typescript: typeof ts, ignore: number[] = []): void {
@@ -196,7 +202,7 @@ module compile {
 			}
 		}
 
-		normalizeCompilerOptions(compilerOptions);
+		normalizeCompilerOptions(compilerOptions, typescript);
 		const project = _project.setupProject(projectDirectory, tsConfigFileName, rawConfig, tsConfigContent, compilerOptions, typescript);
 
 		return project;

lib/main.ts

@@ -25,31 +25,50 @@ export class Output {
 	// .d.ts files
 	streamDts: stream.Readable;
 
+	// Number of pending IO operatrions
+	private pendingIO = 0;
+
 	writeJs(base: string, fileName: string, content: string, sourceMapContent: string, cwd: string, original: input.File) {
 		const file = new VinylFile({
 			path: fileName,
-			contents: new Buffer(content),
+			contents: Buffer.from(content),
 			cwd,
 			base
 		});
-		const appliedSourceMap = this.applySourceMap(sourceMapContent, original, file);
-		if (appliedSourceMap) file.sourceMap = JSON.parse(appliedSourceMap);
-		this.streamFull.push(file);
-		this.streamJs.push(file);
+
+		this.pendingIO++;
+
+		this.applySourceMap(sourceMapContent, original, file).then(appliedSourceMap => {
+			if (appliedSourceMap) file.sourceMap = JSON.parse(appliedSourceMap);
+			this.streamFull.push(file);
+			this.streamJs.push(file);
+
+			this.pendingIO--;
+			this.mightFinish();
+		});
 	}
 
-	writeDts(base: string, fileName: string, content: string, cwd: string) {
+	async writeDts(base: string, fileName: string, content: string, declarationMapContent: string, cwd: string, original: input.File) {
 		const file = new VinylFile({
 			path: fileName,
-			contents: new Buffer(content),
+			contents: Buffer.from(content),
 			cwd,
 			base
 		});
-		this.streamFull.push(file);
-		this.streamDts.push(file);
+
+		this.pendingIO++;
+
+		this.applySourceMap(declarationMapContent, original, file).then(appliedSourceMap => {
+			if (appliedSourceMap) file.sourceMap = JSON.parse(appliedSourceMap);
+			this.streamFull.push(file);
+			this.streamDts.push(file);
+
+			this.pendingIO--;
+			this.mightFinish();
+		});
 	}
 
-	private applySourceMap(sourceMapContent: string, original: input.File, output: VinylFile) {
+	private async applySourceMap(sourceMapContent: string, original: input.File, output: VinylFile) {
 		if (sourceMapContent === undefined) return undefined;
 
 		const map = JSON.parse(sourceMapContent);
@@ -62,13 +81,13 @@ export class Output {
 
 		delete map.sourceRoot;
 
-		const generator = sourceMap.SourceMapGenerator.fromSourceMap(new sourceMap.SourceMapConsumer(map));
+		const consumer = await new sourceMap.SourceMapConsumer(map);
+		const generator = sourceMap.SourceMapGenerator.fromSourceMap(consumer);
 
 		const sourceMapOrigins = this.project.singleOutput
 			? this.project.input.getFileNames(true).map(fName => this.project.input.getFile(fName))
 			: [original];
 
-
 		for (const sourceFile of sourceMapOrigins) {
 			if (!sourceFile || !sourceFile.gulp || !sourceFile.gulp.sourceMap) continue;
 
@@ -78,8 +97,9 @@ export class Output {
 			// We should only apply the input mappings if the input mapping isn't empty,
 			// since `generator.applySourceMap` has a really bad performance on big inputs.
 			if (inputMap.mappings !== '') {
-				const consumer = new sourceMap.SourceMapConsumer(inputMap);
-				generator.applySourceMap(consumer);
+				const inputConsumer = await new sourceMap.SourceMapConsumer(inputMap);
+				generator.applySourceMap(inputConsumer);
+				inputConsumer.destroy();
 			}
 
 			if (!inputMap.sources || !inputMap.sourcesContent) continue;
@@ -89,6 +109,7 @@ export class Output {
 				generator.setSourceContent(utils.forwardSlashes(relative), inputMap.sourcesContent[i]);
 			}
 		}
+		consumer.destroy();
 		return generator.toString();
 
 		function relativeToOutput(fileName: string) {
@@ -99,7 +120,18 @@ export class Output {
 
 	finish(result: reporter.CompilationResult) {
 		this.result = result;
-		if (this.project.reporter.finish) this.project.reporter.finish(result);
+
+		this.mightFinish();
+	}
+	
+	private mightFinish() {
+		if (this.result === undefined || this.pendingIO !== 0) return;
+
+		if (this.project.reporter.finish) this.project.reporter.finish(this.result);
+
+		if (reporter.countErrors(this.result) !== 0) {
+			this.streamFull.emit('error', new Error("TypeScript: Compilation failed"));
+		}
 
 		this.streamFull.emit('finish');
 		this.streamFull.push(null);
@@ -122,6 +154,5 @@ export class Output {
 		// call reporter callback
 		if (this.project.reporter.error) this.project.reporter.error(<reporter.TypeScriptError> error, this.project.typescript);
 		// & emit the error on the stream.
-		this.streamFull.emit('error', error);
 	}
 }

lib/output.ts

@@ -144,9 +144,6 @@ class CompileStream extends stream.Duplex implements ICompileStream {
 		super({objectMode: true});
 
 		this.project = project;
-
-		// Prevent "Unhandled stream error in pipe" when a compilation error occurs.
-		this.on('error', () => {});
 	}
 
 	private project: ProjectInfo;

lib/project.ts

@@ -55,6 +55,16 @@ export interface Reporter {
 	finish?: (results: CompilationResult) => void;
 }
 
+export function countErrors(results: CompilationResult) {
+	return results.transpileErrors
+		+ results.optionsErrors
+		+ results.syntaxErrors
+		+ results.globalErrors
+		+ results.semanticErrors
+		+ results.declarationErrors
+		+ results.emitErrors;
+}
+
 function defaultFinishHandler(results: CompilationResult) {
 	let hasError = false;
 	const showErrorCount = (count: number, type: string) => {

lib/reporter.ts

@@ -1,6 +1,6 @@
 {
 	"compilerOptions": {
-		"target": "ES5",
+		"target": "es2015",
 		"module": "commonjs",
 		"noUnusedLocals": true,
 		"noImplicitAny": true,