Fix bug where obsolete strings were not removed when --clean was used. Closes #29

README.md

@@ -17,7 +17,7 @@ Add an `extract` script to your project's `package.json`:
 ```
 You can now run `npm run extract` to extract strings.
 
-## Commandline examples
+## Extract examples
 
 **Extract from dir and save to file**
 
@@ -39,6 +39,29 @@ You can now run `npm run extract` to extract strings.
 
 `ngx-translate-extract -i ./src -o ./src/i18n/*.json`
 
+**or (update only)**
+
+## Custom indentation
+By default, tabs are used for indentation when writing extracted strings to json formats:
+
+`ngx-translate-extract -i ./src -o ./src/i18n/en.json --format-indentation $'\t'`
+
+If you want to use spaces instead, you can do the following:
+
+`ngx-translate-extract -i ./src -o ./src/i18n/en.json --format-indentation '  '`
+
+## Mark strings for extraction using a marker function
+If, for some reason, you want to extract strings not passed directly to TranslateService, you can wrap them in a custom marker function.
+
+```ts
+import { _ } from '@biesbjerg/ngx-translate-extract';
+
+_('Extract me');
+```
+
+Add the `marker` argument when running the extract script:
+
+`ngx-translate-extract ... -m _`
 
 Modify the scripts arguments as required.
 
@@ -48,20 +71,25 @@ Usage:
 ngx-translate-extract [options]
 
 Options:
-  --version, -v   Show version number                                  [boolean]
-  --help, -h      Show help                                            [boolean]
-  --input, -i     Paths you would like to extract strings from. You can use path
-                  expansion, glob patterns and multiple paths
-                     [array] [default: "/Users/kim/ionic/ngx-translate-extract"]
-  --patterns, -p  Extract strings from the following file patterns
+  --version, -v               Show version number                      [boolean]
+  --help, -h                  Show help                                [boolean]
+  --input, -i                 Paths you would like to extract strings from. You
+                              can use path expansion, glob patterns and multiple
+                              paths
+                     [array] [default: current working path]
+  --patterns, -p              Extract strings from the following file patterns
                                     [array] [default: ["/**/*.html","/**/*.ts"]]
-  --output, -o    Paths where you would like to save extracted strings. You can
-                  use path expansion, glob patterns and multiple paths
-                                                              [array] [required]
-  --format, -f    Output format
+  --output, -o                Paths where you would like to save extracted
+                              strings. You can use path expansion, glob patterns
+                              and multiple paths              [array] [required]
+  --marker, -m                Extract strings passed to a marker function
+                                                       [string] [default: false]
+  --format, -f                Output format
           [string] [choices: "json", "namespaced-json", "pot"] [default: "json"]
-  --replace, -r   Replace the contents of output file if it exists (Merges by
-                  default)                            [boolean] [default: false]
-  --sort, -s      Sort strings in alphabetical order when saving
+  --format-indentation, --fi  Output format indentation [string] [default: "\t"]
+  --replace, -r               Replace the contents of output file if it exists
+                              (Merges by default)     [boolean] [default: false]
+  --sort, -s                  Sort strings in alphabetical order when saving
+                                                      [boolean] [default: false]
+  --clean, -c                 Remove obsolete strings when merging
                                                       [boolean] [default: false]
-  --clean, -c     Remove obsolete strings when merging[boolean] [default: false]

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@biesbjerg/ngx-translate-extract",
-  "version": "2.0.0",
+  "version": "2.2.1",
   "description": "Extract strings from projects using ngx-translate",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -50,17 +50,17 @@
     "@types/chai": "3.4.35",
     "@types/glob": "5.0.30",
     "@types/mocha": "2.2.40",
-    "@types/cheerio": "0.22.0",
+    "@types/cheerio": "0.22.1",
     "@types/chalk": "0.4.31",
     "@types/flat": "0.0.28",
     "@types/yargs": "6.6.0",
     "@types/mkdirp": "0.3.29",
     "chai": "3.5.0",
     "mocha": "3.2.0",
-    "ts-node": "2.1.0",
+    "ts-node": "3.0.2",
     "tslint": "4.5.1",
     "tslint-eslint-rules": "3.5.1",
-    "typescript": "2.2.1"
+    "typescript": "2.2.2"
   },
   "dependencies": {
     "chalk": "1.1.3",

src/cli/cli.ts

@@ -1,7 +1,11 @@
 import { ExtractTask } from './tasks/extract.task';
+import { ParserInterface } from '../parsers/parser.interface';
 import { PipeParser } from '../parsers/pipe.parser';
 import { DirectiveParser } from '../parsers/directive.parser';
 import { ServiceParser } from '../parsers/service.parser';
+import { FunctionParser } from '../parsers/function.parser';
+import { CompilerInterface } from '../compilers/compiler.interface';
+import { CompilerFactory } from '../compilers/compiler.factory';
 
 import * as fs from 'fs';
 import * as yargs from 'yargs';
@@ -22,7 +26,7 @@ export const cli = yargs
 	.check(options => {
 		options.input.forEach((dir: string) => {
 			if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
-				throw new Error(`The path you supplied was not found: '${dir}'`)
+				throw new Error(`The path you supplied was not found: '${dir}'`);
 			}
 
 		});
@@ -41,6 +45,12 @@ export const cli = yargs
 		normalize: true,
 		required: true
 	})
+	.option('marker', {
+		alias: 'm',
+		describe: 'Extract strings passed to a marker function',
+		default: false,
+		type: 'string'
+	})
 	.option('format', {
 		alias: 'f',
 		describe: 'Output format',
@@ -48,6 +58,12 @@ export const cli = yargs
 		type: 'string',
 		choices: ['json', 'namespaced-json', 'pot']
 	})
+	.option('format-indentation', {
+		alias: 'fi',
+		describe: 'Output format indentation',
+		default: '\t',
+		type: 'string'
+	})
 	.option('replace', {
 		alias: 'r',
 		describe: 'Replace the contents of output file if it exists (Merges by default)',
@@ -69,19 +85,28 @@ export const cli = yargs
 	.exitProcess(true)
 	.parse(process.argv);
 
-const extractTask = new ExtractTask(cli.input, cli.output, {
+const extract = new ExtractTask(cli.input, cli.output, {
 	replace: cli.replace,
 	sort: cli.sort,
 	clean: cli.clean,
 	patterns: cli.patterns
 });
 
-extractTask
-	.setParsers([
-		new ServiceParser(),
-		new PipeParser(),
-		new DirectiveParser()
-	])
-	.setCompiler(cli.format)
-	.execute();
+const compiler: CompilerInterface = CompilerFactory.create(cli.format, {
+	indentation: cli.formatIndentation
+});
+extract.setCompiler(compiler);
 
+const parsers: ParserInterface[] = [
+	new PipeParser(),
+	new DirectiveParser(),
+	new ServiceParser()
+];
+if (cli.marker) {
+	parsers.push(new FunctionParser({
+		identifier: cli.marker
+	}));
+}
+extract.setParsers(parsers);
+
+extract.execute();

src/cli/tasks/extract.task.ts

@@ -2,7 +2,6 @@ import { TranslationCollection } from '../../utils/translation.collection';
 import { TaskInterface } from './task.interface';
 import { ParserInterface } from '../../parsers/parser.interface';
 import { CompilerInterface } from '../../compilers/compiler.interface';
-import { CompilerFactory } from '../../compilers/compiler.factory';
 
 import * as chalk from 'chalk';
 import * as glob from 'glob';
@@ -42,11 +41,6 @@ export class ExtractTask implements TaskInterface {
 		}
 
 		const collection = this._extract();
-		if (collection.isEmpty()) {
-			this._out(chalk.yellow('Did not find any extractable strings\n'));
-			return;
-		}
-
 		this._out(chalk.green('Extracted %d strings\n'), collection.count());
 		this._save(collection);
 	}
@@ -56,13 +50,8 @@ export class ExtractTask implements TaskInterface {
 		return this;
 	}
 
-	public setCompiler(compiler: CompilerInterface | string): this {
-		if (typeof compiler === 'string') {
-			this._compiler = CompilerFactory.create(compiler)
-		} else {
-			this._compiler = compiler;
-		}
-
+	public setCompiler(compiler: CompilerInterface): this {
+		this._compiler = compiler;
 		return this;
 	}
 
@@ -70,10 +59,9 @@ export class ExtractTask implements TaskInterface {
 	 * Extract strings from input dirs using configured parsers
 	 */
 	protected _extract(): TranslationCollection {
-		let collection: TranslationCollection = new TranslationCollection();
-
 		this._out(chalk.bold('Extracting strings...'));
 
+		let collection: TranslationCollection = new TranslationCollection();
 		this._input.forEach(dir => {
 			this._readDir(dir, this._options.patterns).forEach(path => {
 				this._out(chalk.gray('- %s'), path);
@@ -83,6 +71,7 @@ export class ExtractTask implements TaskInterface {
 				});
 			});
 		});
+
 		return collection;
 	}
 
@@ -115,7 +104,7 @@ export class ExtractTask implements TaskInterface {
 
 				if (this._options.clean) {
 					const collectionCount = processedCollection.count();
-					processedCollection = processedCollection.intersect(processedCollection);
+					processedCollection = processedCollection.intersect(collection);
 					const removeCount = collectionCount - processedCollection.count();
 					if (removeCount > 0) {
 						this._out(chalk.dim('- removed %d obsolete strings'), removeCount);

src/compilers/compiler.factory.ts

@@ -5,11 +5,11 @@ import { PoCompiler } from '../compilers/po.compiler';
 
 export class CompilerFactory {
 
-	public static create(format: string): CompilerInterface {
+	public static create(format: string, options?: {}): CompilerInterface {
 		switch (format) {
-			case 'pot': return new PoCompiler();
-			case 'json': return new JsonCompiler();
-			case 'namespaced-json': return new NamespacedJsonCompiler();
+			case 'pot': return new PoCompiler(options);
+			case 'json': return new JsonCompiler(options);
+			case 'namespaced-json': return new NamespacedJsonCompiler(options);
 			default: throw new Error(`Unknown format: ${format}`);
 		}
 	}

src/compilers/json.compiler.ts

@@ -3,10 +3,18 @@ import { TranslationCollection } from '../utils/translation.collection';
 
 export class JsonCompiler implements CompilerInterface {
 
+	public indentation: string = '\t';
+
 	public extension = 'json';
 
+	public constructor(options?: any) {
+		if (options && typeof options.indentation !== 'undefined') {
+			this.indentation = options.indentation;
+		}
+	}
+
 	public compile(collection: TranslationCollection): string {
-		return JSON.stringify(collection.values, null, '\t');
+		return JSON.stringify(collection.values, null, this.indentation);
 	}
 
 	public parse(contents: string): TranslationCollection {

src/compilers/namespaced-json.compiler.ts

@@ -5,11 +5,21 @@ import * as flat from 'flat';
 
 export class NamespacedJsonCompiler implements CompilerInterface {
 
+	public indentation: string = '\t';
+
 	public extension = 'json';
 
+	public constructor(options?: any) {
+		if (options && typeof options.indentation !== 'undefined') {
+			this.indentation = options.indentation;
+		}
+	}
+
 	public compile(collection: TranslationCollection): string {
-		const values: {} = flat.unflatten(collection.values);
-		return JSON.stringify(values, null, '\t');
+		const values: {} = flat.unflatten(collection.values, {
+			object: true
+		});
+		return JSON.stringify(values, null, this.indentation);
 	}
 
 	public parse(contents: string): TranslationCollection {

src/compilers/po.compiler.ts

@@ -12,6 +12,8 @@ export class PoCompiler implements CompilerInterface {
 	 */
 	public domain = '';
 
+	public constructor(options?: any) { }
+
 	public compile(collection: TranslationCollection): string {
 		const data = {
 			charset: 'utf-8',

src/index.ts

@@ -1,5 +1,5 @@
 export * from './utils/translation.collection';
-export * from './utils/ast-utils';
+export * from './utils/utils';
 
 export * from './cli/cli';
 export * from './cli/tasks/task.interface';
@@ -7,9 +7,11 @@ export * from './cli/tasks/extract.task';
 
 export * from './parsers/parser.interface';
 export * from './parsers/abstract-template.parser';
+export * from './parsers/abstract-ast.parser';
 export * from './parsers/directive.parser';
 export * from './parsers/pipe.parser';
 export * from './parsers/service.parser';
+export * from './parsers/function.parser';
 
 export * from './compilers/compiler.interface';
 export * from './compilers/compiler.factory';

src/parsers/abstract-ast.parser.ts

@@ -0,0 +1,69 @@
+import * as ts from 'typescript';
+
+export abstract class AbstractAstParser {
+
+	protected _sourceFile: ts.SourceFile;
+
+	protected _createSourceFile(path: string, contents: string): ts.SourceFile {
+		return ts.createSourceFile(path, contents, null, /*setParentNodes */ false);
+	}
+
+	/**
+	 * Get strings from function call's first argument
+	 */
+	protected _getCallArgStrings(callNode: ts.CallExpression): string[] {
+		if (!callNode.arguments.length) {
+			return;
+		}
+
+		const firstArg = callNode.arguments[0];
+		switch (firstArg.kind) {
+			case ts.SyntaxKind.StringLiteral:
+			case ts.SyntaxKind.FirstTemplateToken:
+				return [(firstArg as ts.StringLiteral).text];
+			case ts.SyntaxKind.ArrayLiteralExpression:
+				return (firstArg as ts.ArrayLiteralExpression).elements
+					.map((element: ts.StringLiteral) => element.text);
+			case ts.SyntaxKind.Identifier:
+				console.log('WARNING: We cannot extract variable values passed to TranslateService (yet)');
+				break;
+			default:
+				console.log(`SKIP: Unknown argument type: '${this._syntaxKindToName(firstArg.kind)}'`, firstArg);
+		}
+	}
+
+	/**
+	 * Find all child nodes of a kind
+	 */
+	protected _findNodes(node: ts.Node, kind: ts.SyntaxKind, onlyOne: boolean = false): ts.Node[] {
+		if (node.kind === kind && onlyOne) {
+			return [node];
+		}
+
+		const childrenNodes: ts.Node[] = node.getChildren(this._sourceFile);
+		const initialValue: ts.Node[] = node.kind === kind ? [node] : [];
+
+		return childrenNodes.reduce((result: ts.Node[], childNode: ts.Node) => {
+			return result.concat(this._findNodes(childNode, kind));
+		}, initialValue);
+	}
+
+	protected _syntaxKindToName(kind: ts.SyntaxKind): string {
+		return ts.SyntaxKind[kind];
+	}
+
+	protected _printAllChildren(sourceFile: ts.SourceFile, node: ts.Node, depth = 0): void {
+		console.log(
+			new Array(depth + 1).join('----'),
+			`[${node.kind}]`,
+			this._syntaxKindToName(node.kind),
+			`[pos: ${node.pos}-${node.end}]`,
+			':\t\t\t',
+			node.getFullText(sourceFile).trim()
+		);
+
+		depth++;
+		node.getChildren(sourceFile).forEach(childNode => this._printAllChildren(sourceFile, childNode, depth));
+	}
+
+}

src/parsers/function.parser.ts

@@ -0,0 +1,61 @@
+import { ParserInterface } from './parser.interface';
+import { AbstractAstParser } from './abstract-ast.parser';
+import { TranslationCollection } from '../utils/translation.collection';
+
+import * as ts from 'typescript';
+
+export class FunctionParser extends AbstractAstParser implements ParserInterface {
+
+	protected _functionIdentifier: string = '_';
+
+	public constructor(options?: any) {
+		super();
+		if (options && typeof options.identifier !== 'undefined') {
+			this._functionIdentifier = options.identifier;
+		}
+	}
+
+	public extract(contents: string, path?: string): TranslationCollection {
+		let collection: TranslationCollection = new TranslationCollection();
+
+		this._sourceFile = this._createSourceFile(path, contents);
+
+		const callNodes = this._findCallNodes();
+		callNodes.forEach(callNode => {
+			const keys: string[] = this._getCallArgStrings(callNode);
+			if (keys && keys.length) {
+				collection = collection.addKeys(keys);
+			}
+		});
+
+		return collection;
+	}
+
+	/**
+	 * Find all calls to marker function
+	 */
+	protected _findCallNodes(node?: ts.Node): ts.CallExpression[] {
+		if (!node) {
+			node = this._sourceFile;
+		}
+
+		let callNodes = this._findNodes(node, ts.SyntaxKind.CallExpression) as ts.CallExpression[];
+		callNodes = callNodes
+			.filter(callNode => {
+				// Only call expressions with arguments
+				if (callNode.arguments.length < 1) {
+					return false;
+				}
+
+				const identifier = (callNode.getChildAt(0) as ts.Identifier).text;
+				if (identifier !== this._functionIdentifier) {
+					return false;
+				}
+
+				return true;
+			});
+
+		return callNodes;
+	}
+
+}

src/parsers/service.parser.ts

@@ -1,10 +1,10 @@
 import { ParserInterface } from './parser.interface';
+import { AbstractAstParser } from './abstract-ast.parser';
 import { TranslationCollection } from '../utils/translation.collection';
-import { syntaxKindToName } from '../utils/ast-utils';
 
 import * as ts from 'typescript';
 
-export class ServiceParser implements ParserInterface {
+export class ServiceParser extends AbstractAstParser implements ParserInterface {
 
 	protected _sourceFile: ts.SourceFile;
 
@@ -33,10 +33,6 @@ export class ServiceParser implements ParserInterface {
 		return collection;
 	}
 
-	protected _createSourceFile(path: string, contents: string): ts.SourceFile {
-		return ts.createSourceFile(path, contents, null, /*setParentNodes */ false);
-	}
-
 	/**
 	 * Detect what the TranslateService instance property
 	 * is called by inspecting constructor params
@@ -91,10 +87,12 @@ export class ServiceParser implements ParserInterface {
 
 		let callNodes = this._findNodes(node, ts.SyntaxKind.CallExpression) as ts.CallExpression[];
 		callNodes = callNodes
-			// Only call expressions with arguments
-			.filter(callNode => callNode.arguments.length > 0)
-			// More filters
 			.filter(callNode => {
+				// Only call expressions with arguments
+				if (callNode.arguments.length < 1) {
+					return false;
+				}
+
 				const propAccess = callNode.getChildAt(0).getChildAt(0) as ts.PropertyAccessExpression;
 				if (!propAccess || propAccess.kind !== ts.SyntaxKind.PropertyAccessExpression) {
 					return false;
@@ -120,44 +118,4 @@ export class ServiceParser implements ParserInterface {
 		return callNodes;
 	}
 
-	/**
-	 * Get strings from function call's first argument
-	 */
-	protected _getCallArgStrings(callNode: ts.CallExpression): string[] {
-		if (!callNode.arguments.length) {
-			return;
-		}
-
-		const firstArg = callNode.arguments[0];
-		switch (firstArg.kind) {
-			case ts.SyntaxKind.StringLiteral:
-			case ts.SyntaxKind.FirstTemplateToken:
-				return [(firstArg as ts.StringLiteral).text];
-			case ts.SyntaxKind.ArrayLiteralExpression:
-				return (firstArg as ts.ArrayLiteralExpression).elements
-					.map((element: ts.StringLiteral) => element.text);
-			case ts.SyntaxKind.Identifier:
-				console.log('WARNING: We cannot extract variable values passed to TranslateService (yet)');
-				break;
-			default:
-				console.log(`SKIP: Unknown argument type: '${syntaxKindToName(firstArg.kind)}'`, firstArg);
-		}
-	}
-
-	/**
-	 * Find all child nodes of a kind
-	 */
-	protected _findNodes(node: ts.Node, kind: ts.SyntaxKind, onlyOne: boolean = false): ts.Node[] {
-		if (node.kind === kind && onlyOne) {
-			return [node];
-		}
-
-		const childrenNodes: ts.Node[] = node.getChildren(this._sourceFile);
-		const initialValue: ts.Node[] = node.kind === kind ? [node] : [];
-
-		return childrenNodes.reduce((result: ts.Node[], childNode: ts.Node) => {
-			return result.concat(this._findNodes(childNode, kind));
-		}, initialValue);
-	}
-
 }

src/utils/ast-utils.ts

@@ -1,19 +0,0 @@
-import * as ts from 'typescript';
-
-export function printAllChildren(sourceFile: ts.SourceFile, node: ts.Node, depth = 0) {
-	console.log(
-		new Array(depth + 1).join('----'),
-		`[${node.kind}]`,
-		syntaxKindToName(node.kind),
-		`[pos: ${node.pos}-${node.end}]`,
-		':\t\t\t',
-		node.getFullText(sourceFile).trim()
-	);
-
-	depth++;
-	node.getChildren(sourceFile).forEach(childNode => printAllChildren(sourceFile, childNode, depth));
-}
-
-export function syntaxKindToName(kind: ts.SyntaxKind) {
-	return ts.SyntaxKind[kind];
-}

src/utils/utils.ts

@@ -0,0 +1,3 @@
+export function _(key: string | string[]): string | string[] {
+	return key;
+}

tests/compilers/namespaced-json.compiler.spec.ts

@@ -35,4 +35,26 @@ describe('NamespacedJsonCompiler', () => {
 		expect(result).to.equal('{\n\t"NAMESPACE": {\n\t\t"KEY": {\n\t\t\t"FIRST_KEY": "",\n\t\t\t"SECOND_KEY": "VALUE"\n\t\t}\n\t}\n}');
 	});
 
+	it('should preserve numeric values on compile', () => {
+		const collection = new TranslationCollection({
+			"option.0": '',
+			"option.1": '',
+			"option.2": ''
+		});
+		const result: string = compiler.compile(collection);
+		expect(result).to.equal('{\n\t"option": {\n\t\t"0": "",\n\t\t"1": "",\n\t\t"2": ""\n\t}\n}');
+	});
+
+	it('should use custom indentation chars', () => {
+		const collection = new TranslationCollection({
+			'NAMESPACE.KEY.FIRST_KEY': '',
+			'NAMESPACE.KEY.SECOND_KEY': 'VALUE'
+		});
+		const customCompiler = new NamespacedJsonCompiler({
+			indentation: '  '
+		});
+		const result: string = customCompiler.compile(collection);
+		expect(result).to.equal('{\n  "NAMESPACE": {\n    "KEY": {\n      "FIRST_KEY": "",\n      "SECOND_KEY": "VALUE"\n    }\n  }\n}');
+	});
+
 });

tests/parsers/function.parser.spec.ts

@@ -0,0 +1,27 @@
+import { expect } from 'chai';
+
+import { FunctionParser } from '../../src/parsers/function.parser';
+
+describe('FunctionParser', () => {
+
+	const componentFilename: string = 'test.component.ts';
+
+	let parser: FunctionParser;
+
+	beforeEach(() => {
+		parser = new FunctionParser();
+	});
+
+
+	it('should extract strings using marker function', () => {
+		const contents = `
+			import { _ } from '@biesbjerg/ngx-translate-extract';
+			_('Hello world');
+			_(['I', 'am', 'extracted']);
+			otherFunction('But I am not');
+		`;
+		const keys = parser.extract(contents, componentFilename).keys();
+		expect(keys).to.deep.equal(['Hello world', 'I', 'am', 'extracted']);
+	});
+
+});