fix(pipe-parser): add support for more sophisticated expressions that worked in version 4.2.0 with the regex based parser (#185)

* fix(pipe-parser): add support for more sophisticated expressions

.github/FUNDING.yml

@@ -1,2 +1,2 @@
-github: #biesbjerg
+github: biesbjerg
 custom: https://donate.biesbjerg.com

README.md

@@ -15,7 +15,8 @@ Add a script to your project's `package.json`:
 ```json
 ...
 "scripts": {
-  "extract-i18n": "ngx-translate-extract --input ./src --output ./src/assets/i18n/ --clean --sort --format namespaced-json"
+  "i18n:init": "ngx-translate-extract --input ./src --output ./src/assets/i18n/template.json --key-as-default-value --replace --format json",
+  "i18n:extract": "ngx-translate-extract --input ./src --output ./src/assets/i18n/{en,da,de,fi,nb,nl,sv}.json --clean --format json"
 }
 ...
 ```
@@ -25,11 +26,11 @@ You can now run `npm run extract-i18n` and it will extract strings from your pro
 
 **Extract from dir and save to file**
 
-`ngx-translate-extract --input ./src --output ./src/assets/i18n/template.json`
+`ngx-translate-extract --input ./src --output ./src/assets/i18n/strings.json`
 
 **Extract from multiple dirs**
 
-`ngx-translate-extract --input ./src-a ./src-b --output ./src/assets/i18n/template.json`
+`ngx-translate-extract --input ./src-a ./src-b --output ./src/assets/i18n/strings.json`
 
 
 **Extract and save to multiple files using path expansion**
@@ -99,4 +100,8 @@ Examples:
                                                            using brace expansion
   ngx-translate-extract -i './src/**/*.{ts,tsx,html}' -o strings.json     Extract from ts, tsx and html
   ngx-translate-extract -i './src/**/!(*.spec).{ts,html}' -o              Extract from ts, html, excluding files with ".spec"
-  strings.json
+  strings.json
+
+## Note for GetText users
+
+Please pay attention of which version of `gettext-parser` you actually use in your project. For instance, `gettext-parser:1.2.2` does not support HTML tags in translation keys.

package-lock.json

@@ -1,13 +1,14 @@
 {
   "name": "@biesbjerg/ngx-translate-extract",
-  "version": "6.0.0",
+  "version": "6.0.3",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
     "@angular/compiler": {
-      "version": "9.0.7",
-      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-9.0.7.tgz",
-      "integrity": "sha512-hFpkuGpzxpK5h59LHHAjTFWsY6DCXZwgJFqvCuTPxWi/srvLGZRXrpC6Z1SlgHI9xxXaPfoa4uWw2VfA3BnqEg=="
+      "version": "9.1.1",
+      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-9.1.1.tgz",
+      "integrity": "sha512-u1IP6IzUgK6lIzrG1cxp96umXgtThyhuFn/KPoyVt7wPxZ6vVR0ZxjM7zycEcrMGzk0nf0nyOKaksJk9sTXTbg==",
+      "dev": true
     },
     "@babel/code-frame": {
       "version": "7.8.3",
@@ -136,6 +137,15 @@
       "integrity": "sha512-GcgAp7RXXGmA61spVEKZYpIy3/iV6GHbTW9f9kaKwHVQgnWitt6X026e+3N6j8ep1bkIWj83qPHQ3Y9Ft8FqiQ==",
       "dev": true
     },
+    "@types/gettext-parser": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/gettext-parser/-/gettext-parser-4.0.0.tgz",
+      "integrity": "sha512-I/wvhr+l5M7IwBF1ADBfNQ6qGfUg85UTjj/AZWn09Y+POqyLe1cfbdJSMWzCobiJ3EJNY23MQCbP6jxQT81OTQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/glob": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz",
@@ -2437,7 +2447,8 @@
     "typescript": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz",
-      "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w=="
+      "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==",
+      "dev": true
     },
     "util": {
       "version": "0.10.4",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@biesbjerg/ngx-translate-extract",
-  "version": "6.0.1",
+  "version": "6.0.4",
   "description": "Extract strings from projects using ngx-translate",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -61,9 +61,11 @@
   },
   "config": {},
   "devDependencies": {
+    "@angular/compiler": "^9.1.1",
     "@types/braces": "^3.0.0",
     "@types/chai": "^4.2.11",
     "@types/flat": "^5.0.0",
+    "@types/gettext-parser": "4.0.0",
     "@types/glob": "^7.1.1",
     "@types/mkdirp": "^1.0.0",
     "@types/mocha": "^7.0.2",
@@ -80,10 +82,14 @@
     "tslint": "^6.1.0",
     "tslint-config-prettier": "^1.18.0",
     "tslint-eslint-rules": "^5.4.0",
-    "tslint-etc": "^1.10.1"
+    "tslint-etc": "^1.10.1",
+    "typescript": "^3.8.3"
+  },
+  "peerDependencies": {
+    "@angular/compiler": "^8.0.0 || ^9.0.0",
+    "typescript": "^3.0.0"
   },
   "dependencies": {
-    "@angular/compiler": "^9.0.7",
     "@phenomnomnominal/tsquery": "^4.0.0",
     "boxen": "^4.2.0",
     "colorette": "^1.1.0",
@@ -93,7 +99,6 @@
     "mkdirp": "^1.0.3",
     "path": "^0.12.7",
     "terminal-link": "^2.1.1",
-    "typescript": "^3.8.3",
     "yargs": "^15.3.1"
   }
 }

src/cli/cli.ts

@@ -1,4 +1,5 @@
 import * as yargs from 'yargs';
+import { red, green } from 'colorette';
 
 import { ExtractTask } from './tasks/extract.task';
 import { ParserInterface } from '../parsers/parser.interface';
@@ -148,6 +149,13 @@ const compiler: CompilerInterface = CompilerFactory.create(cli.format, {
 });
 extractTask.setCompiler(compiler);
 
-extractTask.execute();
-
-console.log(donateMessage);
+// Run task
+try {
+	extractTask.execute();
+	console.log(green('\nDone.\n'));
+	console.log(donateMessage);
+	process.exit(0);
+} catch (e) {
+	console.log(red(`\nAn error occurred: ${e}\n`));
+	process.exit(1);
+}

src/cli/tasks/extract.task.ts

@@ -4,7 +4,7 @@ import { ParserInterface } from '../../parsers/parser.interface';
 import { PostProcessorInterface } from '../../post-processors/post-processor.interface';
 import { CompilerInterface } from '../../compilers/compiler.interface';
 
-import { cyan, green, bold, dim } from 'colorette';
+import { cyan, green, bold, dim, red } from 'colorette';
 import * as glob from 'glob';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -56,26 +56,33 @@ export class ExtractTask implements TaskInterface {
 
 			let existing: TranslationCollection = new TranslationCollection();
 			if (!this.options.replace && fs.existsSync(outputPath)) {
-				existing = this.compiler.parse(fs.readFileSync(outputPath, 'utf-8'));
+				try {
+					existing = this.compiler.parse(fs.readFileSync(outputPath, 'utf-8'));
+				} catch (e) {
+					this.out(`%s %s`, dim(`- ${outputPath}`), red(`[ERROR]`));
+					throw e;
+				}
 			}
 
 			// merge extracted strings with existing
 			const draft = extracted.union(existing);
 
-			if (existing.isEmpty()) {
-				this.out(dim(`- ${outputPath}`));
-			} else {
-				this.out(dim(`- ${outputPath} (merged)`));
-			}
-
 			// Run collection through post processors
 			const final = this.process(draft, extracted, existing);
 
-			// Save to file
-			this.save(outputPath, final);
+			// Save
+			try {
+				let event = 'CREATED';
+				if (fs.existsSync(outputPath)) {
+					this.options.replace ? event = 'REPLACED' : event = 'MERGED';
+				}
+				this.save(outputPath, final);
+				this.out(`%s %s`, dim(`- ${outputPath}`), green(`[${event}]`));
+			} catch (e) {
+				this.out(`%s %s`, dim(`- ${outputPath}`), red(`[ERROR]`));
+				throw e;
+			}
 		});
-
-		this.out(green('\nDone.\n'));
 	}
 
 	public setParsers(parsers: ParserInterface[]): this {

src/compilers/po.compiler.ts

@@ -1,7 +1,7 @@
 import { CompilerInterface } from './compiler.interface';
 import { TranslationCollection, TranslationType } from '../utils/translation.collection';
 
-import * as gettext from 'gettext-parser';
+import { po } from 'gettext-parser';
 
 export class PoCompiler implements CompilerInterface {
 	public extension: string = 'po';
@@ -34,23 +34,24 @@ export class PoCompiler implements CompilerInterface {
 			}
 		};
 
-		return gettext.po.compile(data);
+		return po.compile(data).toString('utf8');
 	}
 
 	public parse(contents: string): TranslationCollection {
 		const collection = new TranslationCollection();
 
-		const po = gettext.po.parse(contents, 'utf8');
-		if (!po.translations.hasOwnProperty(this.domain)) {
+		const parsedPo = po.parse(contents, 'utf8');
+
+		if (!parsedPo.translations.hasOwnProperty(this.domain)) {
 			return collection;
 		}
 
-		const values = Object.keys(po.translations[this.domain])
+		const values = Object.keys(parsedPo.translations[this.domain])
 			.filter((key) => key.length > 0)
 			.reduce((result, key) => {
 				return {
 					...result,
-					[key]: po.translations[this.domain][key].msgstr.pop()
+					[key]: parsedPo.translations[this.domain][key].msgstr.pop()
 				};
 			}, {} as TranslationType);
 

src/declarations.d.ts

@@ -1 +0,0 @@
-declare module 'gettext-parser';

src/parsers/directive.parser.ts

@@ -82,6 +82,6 @@ export class DirectiveParser implements ParserInterface {
 	}
 
 	protected cleanKey(val: string): string {
-		return val.replace(/\r?\n|\r|\t/g, '');
+		return val.replace(/\r?\n|\r|\t/g, '').trim();
 	}
 }

src/parsers/pipe.parser.ts

@@ -1,4 +1,16 @@
-import { TmplAstNode, parseTemplate, BindingPipe, LiteralPrimitive, Conditional, TmplAstTextAttribute } from '@angular/compiler';
+import {
+	AST,
+	TmplAstNode,
+	parseTemplate,
+	BindingPipe,
+	LiteralPrimitive,
+	Conditional,
+	TmplAstTextAttribute,
+	Binary,
+	LiteralMap,
+	LiteralArray,
+	Interpolation
+} from '@angular/compiler';
 
 import { ParserInterface } from './parser.interface';
 import { TranslationCollection } from '../utils/translation.collection';
@@ -36,9 +48,8 @@ export class PipeParser implements ParserInterface {
 			);
 		}
 
-		if (node?.value?.ast?.expressions) {
-			const translateables = node.value.ast.expressions.filter((exp: any) => this.expressionIsOrHasBindingPipe(exp));
-			ret.push(...translateables);
+		if (node?.value?.ast) {
+			ret.push(...this.getTranslatablesFromAst(node.value.ast));
 		}
 
 		if (node?.attributes) {
@@ -51,17 +62,8 @@ export class PipeParser implements ParserInterface {
 		if (node?.inputs) {
 			node.inputs.forEach((input: any) => {
 				// <element [attrib]="'identifier' | translate">
-				if (input?.value?.ast && this.expressionIsOrHasBindingPipe(input.value.ast)) {
-					ret.push(input.value.ast);
-				}
-
-				// <element attrib="{{'identifier' | translate}}>"
-				if (input?.value?.ast?.expressions) {
-					input.value.ast.expressions.forEach((exp: BindingPipe) => {
-						if (this.expressionIsOrHasBindingPipe(exp)) {
-							ret.push(exp);
-						}
-					});
+				if (input?.value?.ast) {
+					ret.push(...this.getTranslatablesFromAst(input.value.ast));
 				}
 			});
 		}
@@ -84,7 +86,63 @@ export class PipeParser implements ParserInterface {
 		return ret;
 	}
 
-	protected expressionIsOrHasBindingPipe(exp: any): boolean {
+	protected getTranslatablesFromAst(ast: AST): BindingPipe[] {
+		// the entire expression is the translate pipe, e.g.:
+		// - 'foo' | translate
+		// - (condition ? 'foo' : 'bar') | translate
+		if (this.expressionIsOrHasBindingPipe(ast)) {
+			return [ast];
+		}
+
+		// angular double curly bracket interpolation, e.g.:
+		// - {{ expressions }}
+		if (ast instanceof Interpolation) {
+			return this.getTranslatablesFromAsts(ast.expressions);
+		}
+
+		// ternary operator, e.g.:
+		// - condition ? null : ('foo' | translate)
+		// - condition ? ('foo' | translate) : null
+		if (ast instanceof Conditional) {
+			return this.getTranslatablesFromAsts([ast.trueExp, ast.falseExp]);
+		}
+
+		// string concatenation, e.g.:
+		// - 'foo' + 'bar' + ('baz' | translate)
+		if (ast instanceof Binary) {
+			return this.getTranslatablesFromAsts([ast.left, ast.right]);
+		}
+
+		// a pipe on the outer expression, but not the translate pipe - ignore the pipe, visit the expression, e.g.:
+		// - { foo: 'Hello' | translate } | json
+		if (ast instanceof BindingPipe) {
+			return this.getTranslatablesFromAst(ast.exp);
+		}
+
+		// object - ignore the keys, visit all values, e.g.:
+		// - { key1: 'value1' | translate, key2: 'value2' | translate }
+		if (ast instanceof LiteralMap) {
+			return this.getTranslatablesFromAsts(ast.values);
+		}
+
+		// array - visit all its values, e.g.:
+		// - [ 'value1' | translate, 'value2' | translate ]
+		if (ast instanceof LiteralArray) {
+			return this.getTranslatablesFromAsts(ast.expressions);
+		}
+
+		return [];
+	}
+
+	protected getTranslatablesFromAsts(asts: AST[]): BindingPipe[] {
+		return this.flatten(asts.map(ast => this.getTranslatablesFromAst(ast)));
+	}
+
+	protected flatten<T extends AST>(array: T[][]): T[] {
+		return [].concat(...array);
+	}
+
+	protected expressionIsOrHasBindingPipe(exp: any): exp is BindingPipe {
 		if (exp.name && exp.name === TRANSLATE_PIPE_NAME) {
 			return true;
 		}

tests/compilers/po.compiler.spec.ts

@@ -0,0 +1,24 @@
+import { expect } from 'chai';
+
+import { TranslationCollection } from '../../src/utils/translation.collection';
+import { PoCompiler } from '../../src/compilers/po.compiler';
+
+describe('PoCompiler', () => {
+	let compiler: PoCompiler;
+
+	beforeEach(() => {
+		compiler = new PoCompiler();
+	});
+
+	it('should still include html ', () => {
+		const collection = new TranslationCollection({
+			'A <strong>test</strong>': 'Un <strong>test</strong>',
+			'With a lot of <em>html</em> included': 'Avec beaucoup d\'<em>html</em> inclus'
+		});
+		const result: Buffer = Buffer.from(compiler.compile(collection));
+		expect(result.toString('utf8')).to.equal('msgid ""\nmsgstr ""\n"mime-version: 1.0\\n"\n"Content-Type: text/plain; charset=utf-8\\n"\n"Content-Transfer-Encoding: 8bit\\n"\n\nmsgid "A <strong>test</strong>"\nmsgstr "Un <strong>test</strong>"\n\nmsgid "With a lot of <em>html</em> included"\nmsgstr "Avec beaucoup d\'<em>html</em> inclus"');
+	});
+});
+
+
+

tests/parsers/directive.parser.spec.ts

@@ -36,6 +36,12 @@ describe('DirectiveParser', () => {
 		expect(keys).to.deep.equal(['Hello <strong translate>World</strong>']);
 	});
 
+	it('should not exclude html tags in children', () => {
+		const contents = `<div translate>Hello <strong>World</strong></div>`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['Hello <strong>World</strong>']);
+	});
+
 	it('should extract and parse inline template', () => {
 		const contents = `
 			@Component({
@@ -117,4 +123,22 @@ describe('DirectiveParser', () => {
 		const keys = parser.extract(contents, templateFilename).keys();
 		expect(keys).to.deep.equal(['client.search.searchBtn']);
 	});
+
+	it('should extract contents without indent spaces and trim leading/trailing whitespace', () => {
+		const contents = `
+			<div translate>
+				this is an example 
+				of a long label 
+			</div>
+
+			<div>
+				<p translate>
+					this is an example 
+					of a long label 
+				</p>
+			</div>
+		`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['this is an example of a long label']);
+	});
 });

tests/parsers/marker.parser.spec.ts

@@ -36,6 +36,17 @@ describe('MarkerParser', () => {
 		expect(keys).to.deep.equal(['Hello world', 'This is a very very very very long line.', 'Mix of different types']);
 	});
 
+	it('should extract split strings while keeping html tags', () => {
+		const contents = `
+			import { marker as _ } from '@biesbjerg/ngx-translate-extract-marker';
+			_('Hello ' + 'world');
+			_('This <em>is</em> a ' + 'very ' + 'very ' + 'very ' + 'very ' + 'long line.');
+			_('Mix ' + \`of \` + 'different ' + \`types\`);
+		`;
+		const keys = parser.extract(contents, componentFilename).keys();
+		expect(keys).to.deep.equal(['Hello world', 'This <em>is</em> a very very very very long line.', 'Mix of different types']);
+	});
+
 	it('should extract the strings', () => {
 		const contents = `
 		import { marker } from '@biesbjerg/ngx-translate-extract-marker';

tests/parsers/pipe.parser.spec.ts

@@ -47,12 +47,48 @@ describe('PipeParser', () => {
 		expect(keys).to.deep.equal(['Hello', 'World']);
 	});
 
+	it('should extract strings from ternary operators right expression', () => {
+		const contents = `{{ condition ? null : ('World' | translate) }}`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['World']);
+	});
+
+	it('should extract strings from ternary operators inside attribute bindings', () => {
+		const contents = `<span [attr]="condition ? null : ('World' | translate)"></span>`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['World']);
+	});
+
+	it('should extract strings from ternary operators left expression', () => {
+		const contents = `{{ condition ? ('World' | translate) : null }}`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['World']);
+	});
+
+	it('should extract strings inside string concatenation', () => {
+		const contents = `{{ 'a' + ('Hello' | translate) + 'b' + 'c' + ('World' | translate) + 'd' }}`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['Hello', 'World']);
+	});
+
+	it('should extract strings from object', () => {
+		const contents = `{{ { foo: 'Hello' | translate, bar: ['World' | translate], deep: { nested: { baz: 'Yes' | translate } } } | json }}`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['Hello', 'World', 'Yes']);
+	});
+
 	it('should extract strings from ternary operators inside attribute bindings', () => {
 		const contents = `<span [attr]="(condition ? 'Hello' : 'World') | translate"></span>`;
 		const keys = parser.extract(contents, templateFilename).keys();
 		expect(keys).to.deep.equal(['Hello', 'World']);
 	});
 
+	it('should extract strings from nested expressions', () => {
+		const contents = `<span [attr]="{ foo: ['a' + ((condition ? 'Hello' : 'World') | translate) + 'b'] }"></span>`;
+		const keys = parser.extract(contents, templateFilename).keys();
+		expect(keys).to.deep.equal(['Hello', 'World']);
+	});
+
 	it('should extract strings from nested ternary operators ', () => {
 		const contents = `<h3>{{ (condition ? 'Hello' : anotherCondition ? 'Nested' : 'World' ) | translate }}</h3>`;
 		const keys = parser.extract(contents, templateFilename).keys();

tsconfig.json

@@ -1,7 +1,6 @@
 {
     "compilerOptions": {
         "allowSyntheticDefaultImports": true,
-        "noUnusedLocals": true,
         "noImplicitAny": true,
         "noImplicitReturns": true,
         "noImplicitThis": true,

tslint.json

@@ -112,6 +112,7 @@
       "check-operator",
       "check-separator",
       "check-type"
-    ]
+    ],
+    "quotemark": [true, "single"]
   }
 }