Add option to merge extracted strings with existing translations (Thanks to @ocombe)

README.md

@@ -25,16 +25,19 @@ You can also install the package globally:
 
 Now you can execute the script from everywhere:
 
-`ng2-translate-extract --dir /extract/from/this/dir --output /save/to/this/dir --format pot`
+`ng2-translate-extract --dir /extract/from/this/dir --output /save/to/this/dir --format json --merge --clean`
 ## Commandline arguments
 ```
 Usage:
   ng2-translate-extract [OPTIONS] [ARGS]
 
 Options:
-  -d, --dir [DIR]        Directory path you would like to extract strings from  (Default is /Users/kim/ionic/ng2-translate-extract/bin)
+  -d, --dir [DIR]        Directory path you would like to extract strings from  (Default is current directory)
   -o, --output [DIR]     Directory path you would like to save extracted
-                         strings  (Default is /Users/kim/ionic/ng2-translate-extract/bin)
+                         strings  (Default is Default is current directory)
   -f, --format [VALUE]   Output format. VALUE must be either [json|pot]  (Default is json)
-  -h, --help             Display help and usage details
+  -m, --merge [BOOLEAN]  Merge extracted strings with existing file if it
+                         exists  (Default is true)
+  -c, --clean BOOLEAN    Remove unused keys when merging
+  -h, --help             Display help and usage details
 ```

bin/extract.js

@@ -5,13 +5,15 @@ var fs = require('fs');
 var path = require('path');
 
 var Extractor = require('../dist/extractor').Extractor;
-var JsonSerializer = require('../dist/serializers/json.serializer').JsonSerializer;
+var JsonCompiler = require('../dist/compilers/json.compiler').JsonCompiler;
-var PotSerializer = require('../dist/serializers/pot.serializer').PotSerializer;
+var PoCompiler = require('../dist/compilers/po.compiler').PoCompiler
 
 var options = cli.parse({
 	dir: ['d', 'Directory path you would like to extract strings from', 'dir', process.env.PWD],
 	output: ['o', 'Directory path you would like to save extracted strings', 'dir', process.env.PWD],
-	format: ['f', 'Output format', ['json', 'pot'], 'json']
+	format: ['f', 'Output format', ['json', 'pot'], 'json'],
+	merge: ['m', 'Merge extracted strings with existing file if it exists', 'boolean', true],
+	clean: ['c', 'Remove unused keys when merging', 'boolean', false]
 });
 
 [options.dir, options.output].forEach(dir => {
@@ -22,25 +24,41 @@ var options = cli.parse({
 
 switch (options.format) {
 	case 'pot':
-		var serializer = new PotSerializer();
+		var compiler = new PoCompiler();
 		break;
 	case 'json':
-		var serializer = new JsonSerializer();
+		var compiler = new JsonCompiler();
 		break;
 }
 
 var filename = 'template.' + options.format;
-var destination = path.join(options.output, filename);
+var dest = path.join(options.output, filename);
 
 try {
-	var extractor = new Extractor(serializer);
+	var extracted = new Extractor().process(options.dir);
-	var collection = extractor.process(options.dir);
+	cli.info(`* Extracted ${extracted.count()} strings`);
-	if (collection.count() > 0) {
+
-		extractor.save(destination);
+	if (extracted.isEmpty()) {
-		cli.ok(`Extracted ${collection.count()} strings: '${destination}'`);
+		process.exit();
-	} else {
-		cli.info(`Found no extractable strings in the supplied directory path: '${options.dir}'`);
 	}
+
+	let collection = extracted;
+
+	if (options.merge && fs.existsSync(dest)) {
+		const existing = compiler.parse(fs.readFileSync(dest, 'utf-8'));
+
+		collection = extracted.union(existing);
+		cli.info(`* Merged with existing strings`);
+
+		if (options.clean) {
+			const stringCount = collection.count();
+			collection = collection.intersect(extracted);
+			cli.info(`* Removed ${stringCount - collection.count()} unused strings`);
+		}
+	}
+
+	fs.writeFileSync(dest, compiler.compile(collection));
+	cli.ok(`Saved to: '${dest}'`);
 } catch (e) {
 	cli.fatal(e.toString());
 }

package.json

@@ -58,6 +58,7 @@
     "cheerio": "~0.22.0",
     "cli": "^1.0.1",
     "fs": "0.0.1-security",
+    "gettext-parser": "^1.2.1",
     "glob": "^7.1.1",
     "path": "^0.12.7"
   }

src/compilers/compiler.interface.ts

@@ -0,0 +1,9 @@
+import { TranslationCollection } from '../utils/translation.collection';
+
+export interface CompilerInterface {
+
+	compile(collection: TranslationCollection): string;
+
+	parse(contents: string): TranslationCollection;
+
+}

src/compilers/json.compiler.ts

@@ -0,0 +1,14 @@
+import { CompilerInterface } from './compiler.interface';
+import { TranslationCollection } from '../utils/translation.collection';
+
+export class JsonCompiler implements CompilerInterface {
+
+	public compile(collection: TranslationCollection): string {
+		return JSON.stringify(collection.values, null, '\t');
+	}
+
+	public parse(contents: string): TranslationCollection {
+		return new TranslationCollection(JSON.parse(contents));
+	}
+
+}

src/compilers/po.compiler.ts

@@ -0,0 +1,53 @@
+import { CompilerInterface } from './compiler.interface';
+import { TranslationCollection } from '../utils/translation.collection';
+
+import * as gettext from 'gettext-parser';
+
+export class PoCompiler implements CompilerInterface {
+
+	/**
+	 * Translation domain
+	 */
+	public domain = '';
+
+	public compile(collection: TranslationCollection): string {
+		const data = {
+			charset: 'utf-8',
+			headers: {
+				'mime-version': '1.0',
+				'content-type': 'text/plain; charset=utf-8',
+				'content-transfer-encoding': '8bit'
+			},
+			translations: {
+				'default': Object.keys(collection.values).reduce((translations, key) => {
+					translations[key] = {
+						msgid: key,
+						msgstr: collection.get(key)
+					};
+					return translations;
+				}, {})
+			}
+		};
+
+		return gettext.po.compile(data, 'utf-8');
+	}
+
+	public parse(contents: string): TranslationCollection {
+		const collection = new TranslationCollection();
+
+		const po = gettext.po.parse(contents, 'utf-8');
+		if (!po.translations.hasOwnProperty(this.domain)) {
+			return collection;
+		}
+
+		const values = Object.keys(po.translations[this.domain])
+			.filter(key => key.length > 0)
+			.reduce((values, key) => {
+				values[key] = po.translations[this.domain][key].msgstr.pop();
+				return values;
+			}, {});
+
+		return new TranslationCollection(values);
+	}
+
+}

src/declarations.d.ts

@@ -0,0 +1 @@
+declare module '*';

src/example.ts

@@ -1,17 +1,16 @@
 import { Extractor } from './extractor';
-import { JsonSerializer } from './serializers/json.serializer';
+import { JsonCompiler } from './compilers/json.compiler';
-import { StringCollection } from './utils/string.collection';
+import { TranslationCollection } from './utils/translation.collection';
 
-const serializer = new JsonSerializer();
+const compiler = new JsonCompiler();
-const extractor = new Extractor(serializer);
+const extractor = new Extractor();
 
-const src = '/your/project';
+const dirPath = '/your/project';
-const dest = '/your/project/template.json';
 
 try {
-	const collection: StringCollection = extractor.process(src);
+	const collection: TranslationCollection = extractor.process(dirPath);
-	const serialized: string = extractor.save(dest);
+	const result: string = compiler.compile(collection);
-	console.log({ strings: collection.keys(), serialized });
+	console.log(result);
 } catch (e) {
 	console.log(`Something went wrong: ${e.toString()}`);
 }

src/extractor.ts

@@ -2,8 +2,7 @@ import { ParserInterface } from './parsers/parser.interface';
 import { PipeParser } from './parsers/pipe.parser';
 import { DirectiveParser } from './parsers/directive.parser';
 import { ServiceParser } from './parsers/service.parser';
-import { SerializerInterface } from './serializers/serializer.interface';
+import { TranslationCollection } from './utils/translation.collection';
-import { StringCollection } from './utils/string.collection';
 
 import * as glob from 'glob';
 import * as fs from 'fs';
@@ -22,42 +21,24 @@ export class Extractor {
 		new ServiceParser()
 	];
 
-	public collection: StringCollection = new StringCollection();
-
-	public constructor(public serializer: SerializerInterface) { }
-
 	/**
-	 * Process dir
+	 * Extract strings from dir
 	 */
-	public process(dir: string): StringCollection {
+	public process(dir: string): TranslationCollection {
+		let collection: TranslationCollection = new TranslationCollection();
+
 		this._readDir(dir, this.patterns).forEach(path => {
 			const contents: string = fs.readFileSync(path, 'utf-8');
 			this.parsers.forEach((parser: ParserInterface) => {
-				this.collection.merge(parser.extract(contents, path));
+				collection = collection.union(parser.extract(contents, path));
 			});
 		});
 
-		return this.collection;
+		return collection;
 	}
 
 	/**
-	 * Serialize and return output
+	 * Get all files in dir matching patterns
-	 */
-	public serialize(): string {
-		return this.serializer.serialize(this.collection);
-	}
-
-	/**
-	 * Serialize and save to destination
-	 */
-	public save(destination: string): string {
-		const data = this.serialize();
-		fs.writeFileSync(destination, data);
-		return data;
-	}
-
-	/**
-	 * Get all files in dir matching find patterns
 	 */
 	protected _readDir(dir: string, patterns: string[]): string[] {
 		return patterns.reduce((results, pattern) => {

src/parsers/directive.parser.ts

@@ -1,12 +1,12 @@
 import { ParserInterface } from './parser.interface';
 import { AbstractTemplateParser } from './abstract-template.parser';
-import { StringCollection } from '../utils/string.collection';
+import { TranslationCollection } from '../utils/translation.collection';
 
 import * as $ from 'cheerio';
 
 export class DirectiveParser extends AbstractTemplateParser implements ParserInterface {
 
-	public extract(contents: string, path?: string): StringCollection {
+	public extract(contents: string, path?: string): TranslationCollection {
 		if (path && this._isAngularComponent(path)) {
 			contents = this._extractInlineTemplate(contents);
 		}
@@ -14,8 +14,8 @@ export class DirectiveParser extends AbstractTemplateParser implements ParserInt
 		return this._parseTemplate(contents);
 	}
 
-	protected _parseTemplate(template: string): StringCollection {
+	protected _parseTemplate(template: string): TranslationCollection {
-		const collection = new StringCollection();
+		let collection: TranslationCollection = new TranslationCollection();
 
 		template = this._normalizeTemplateAttributes(template);
 		$(template)
@@ -26,7 +26,7 @@ export class DirectiveParser extends AbstractTemplateParser implements ParserInt
 				const attr = $element.attr('translate') || $element.attr('ng2-translate');
 
 				if (attr) {
-					collection.add(attr);
+					collection = collection.add(attr);
 				} else {
 					$element
 						.contents()
@@ -34,7 +34,7 @@ export class DirectiveParser extends AbstractTemplateParser implements ParserInt
 						.filter(node => node.type === 'text')
 						.map(node => node.nodeValue.trim())
 						.filter(text => text.length > 0)
-						.forEach(text => collection.add(text));
+						.forEach(text => collection = collection.add(text));
 				}
 			});
 

src/parsers/parser.interface.ts

@@ -1,7 +1,7 @@
-import { StringCollection } from '../utils/string.collection';
+import { TranslationCollection } from '../utils/translation.collection';
 
 export interface ParserInterface {
 
-	extract(contents: string, path?: string): StringCollection;
+	extract(contents: string, path?: string): TranslationCollection;
 
 }

src/parsers/pipe.parser.ts

@@ -1,10 +1,10 @@
 import { ParserInterface } from './parser.interface';
 import { AbstractTemplateParser } from './abstract-template.parser';
-import { StringCollection } from '../utils/string.collection';
+import { TranslationCollection } from '../utils/translation.collection';
 
 export class PipeParser extends AbstractTemplateParser implements ParserInterface {
 
-	public extract(contents: string, path?: string): StringCollection {
+	public extract(contents: string, path?: string): TranslationCollection {
 		if (path && this._isAngularComponent(path)) {
 			contents = this._extractInlineTemplate(contents);
 		}
@@ -12,14 +12,14 @@ export class PipeParser extends AbstractTemplateParser implements ParserInterfac
 		return this._parseTemplate(contents);
 	}
 
-	protected _parseTemplate(template: string): StringCollection {
+	protected _parseTemplate(template: string): TranslationCollection {
-		const collection = new StringCollection();
+		let collection: TranslationCollection = new TranslationCollection();
 
 		const regExp = new RegExp(/(['"`])([^\1\r\n]*)\1\s*\|\s*translate(:.*?)?/, 'g');
 
 		let matches;
 		while (matches = regExp.exec(template)) {
-			collection.add(matches[2]);
+			collection = collection.add(matches[2]);
 		}
 
 		return collection;

src/parsers/service.parser.ts

@@ -1,10 +1,10 @@
 import { ParserInterface } from './parser.interface';
-import { StringCollection } from '../utils/string.collection';
+import { TranslationCollection } from '../utils/translation.collection';
 
 export class ServiceParser implements ParserInterface {
 
-	public extract(contents: string, path?: string): StringCollection {
+	public extract(contents: string, path?: string): TranslationCollection {
-		const collection = new StringCollection();
+		let collection: TranslationCollection = new TranslationCollection();
 
 		const translateServiceVar = this._extractTranslateServiceVar(contents);
 		if (!translateServiceVar) {
@@ -17,10 +17,9 @@ export class ServiceParser implements ParserInterface {
 		let matches;
 		while (matches = regExp.exec(contents)) {
 			if (this._stringContainsArray(matches[1])) {
-				const matchCollection = StringCollection.fromArray(this._stringToArray(matches[1]));
+				collection = collection.addKeys(this._stringToArray(matches[1]));
-				collection.merge(matchCollection);
 			} else {
-				collection.add(matches[3]);
+				collection = collection.add(matches[3]);
 			}
 		}
 

src/serializers/json.serializer.ts

@@ -1,10 +0,0 @@
-import { SerializerInterface } from './serializer.interface';
-import { StringCollection } from '../utils/string.collection';
-
-export class JsonSerializer implements SerializerInterface {
-
-	public serialize(collection: StringCollection): string {
-		return JSON.stringify(collection.values, null, '\t');
-	}
-
-}

src/serializers/pot.serializer.ts

@@ -1,48 +0,0 @@
-import { SerializerInterface } from './serializer.interface';
-import { StringCollection } from '../utils/string.collection';
-
-export class PotSerializer implements SerializerInterface {
-
-	protected _headers = {
-		'Content-Type': 'text/plain; charset=utf-8',
-		'Content-Transfer-Encoding': '8bit'
-	};
-
-	protected _buffer: string[] = [];
-
-	public serialize(collection: StringCollection): string {
-		this._reset();
-		this._addHeader(this._headers);
-		this._addMessages(collection);
-
-		return this._buffer.join('\n');
-	}
-
-	protected _addHeader(headers: {}): void {
-		this._add('msgid', '');
-		this._add('msgstr', '');
-		Object.keys(headers).forEach(key => {
-			this._buffer.push(`"${key}: ${headers[key]}\\n"`);
-		});
-	}
-
-	protected _addMessages(collection: StringCollection): void {
-		Object.keys(collection.values).forEach(key => {
-			this._add('msgid', key);
-			this._add('msgstr', collection.get(key));
-		});
-	}
-
-	protected _add(key: string, val: string): void {
-		this._buffer.push(`${key} "${this._escape(val)}"`);
-	}
-
-	protected _reset(): void {
-		this._buffer = [];
-	}
-
-	protected _escape(message: string): string {
-		return message.replace(/"([^"\\]+)*"/g, '\\"$1\\"');
-	}
-
-}

src/serializers/serializer.interface.ts

@@ -1,7 +0,0 @@
-import { StringCollection } from '../utils/string.collection';
-
-export interface SerializerInterface {
-
-	serialize(collection: StringCollection): string;
-
-}

src/utils/string.collection.ts

@@ -1,51 +0,0 @@
-export interface StringType {
-	[key: string]: string
-};
-
-export class StringCollection {
-
-	public values: StringType = {};
-
-	public static fromObject(values: StringType): StringCollection {
-		const collection = new StringCollection();
-		Object.keys(values).forEach(key => collection.add(key, values[key]));
-		return collection;
-	}
-
-	public static fromArray(keys: string[]): StringCollection {
-		const collection = new StringCollection();
-		keys.forEach(key => collection.add(key));
-		return collection;
-	}
-
-	public add(keys: string | string[], val: string = ''): StringCollection {
-		if (!Array.isArray(keys)) {
-			keys = [keys];
-		}
-		keys.forEach(key => this.values[key] = val);
-		return this;
-	}
-
-	public remove(key: string): StringCollection {
-		delete this.values[key];
-		return this;
-	}
-
-	public merge(collection: StringCollection): StringCollection {
-		this.values = Object.assign({}, this.values, collection.values);
-		return this;
-	}
-
-	public get(key: string): string {
-		return this.values[key];
-	}
-
-	public keys(): string[] {
-		return Object.keys(this.values);
-	}
-
-	public count(): number {
-		return Object.keys(this.values).length;
-	}
-
-}

src/utils/translation.collection.ts

@@ -0,0 +1,69 @@
+export interface TranslationType {
+	[key: string]: string
+};
+
+export class TranslationCollection {
+
+	public values: TranslationType = {};
+
+	public constructor(values: TranslationType = {}) {
+		this.values = values;
+	}
+
+	public add(key: string, val: string = ''): TranslationCollection {
+		return new TranslationCollection(Object.assign({}, this.values, { [key]: val }));
+	}
+
+	public addKeys(keys: string[]): TranslationCollection {
+		const values = keys.reduce((results, key) => {
+			results[key] = '';
+			return results;
+		}, {});
+		return new TranslationCollection(Object.assign({}, this.values, values));
+	}
+
+	public remove(key: string): TranslationCollection {
+		let newCollection = new TranslationCollection();
+		Object.keys(this.values).forEach(collectionKey => {
+			if (key !== collectionKey) {
+				newCollection = newCollection.add(key, this.values[key]);
+			}
+		});
+		return newCollection;
+	}
+
+	public union(collection: TranslationCollection): TranslationCollection {
+		return new TranslationCollection(Object.assign({}, this.values, collection.values));
+	}
+
+	public intersect(collection: TranslationCollection): TranslationCollection {
+		let newCollection = new TranslationCollection();
+		Object.keys(this.values).forEach(key => {
+			if (collection.has(key)) {
+				newCollection = newCollection.add(key, this.values[key]);
+			}
+		});
+		return newCollection;
+	}
+
+	public has(key: string): boolean {
+		return this.values.hasOwnProperty(key);
+	}
+
+	public get(key: string): string {
+		return this.values[key];
+	}
+
+	public keys(): string[] {
+		return Object.keys(this.values);
+	}
+
+	public count(): number {
+		return Object.keys(this.values).length;
+	}
+
+	public isEmpty(): boolean {
+		return Object.keys(this.values).length === 0;
+	}
+
+}

tests/utils/string.collection.spec.ts

@@ -1,54 +0,0 @@
-import { expect } from 'chai';
-
-import { StringCollection } from '../../src/utils/string.collection';
-
-describe('StringCollection', () => {
-
-	let collection: StringCollection;
-
-	beforeEach(() => {
-		collection = new StringCollection();
-	});
-
-	it('should add item with value', () => {
-		collection.add('key', 'translation');
-		expect(collection.get('key')).to.equal('translation');
-	});
-
-	it('should add item with default value', () => {
-		collection.add('key');
-		expect(collection.get('key')).to.equal('');
-	});
-
-	it('should add array of items with default values', () => {
-		collection.add(['key', 'key2']);
-		expect(collection.count()).to.equal(2);
-	});
-
-	it('should remove item', () => {
-		collection.add('key1').add('key2').remove('key1');
-		expect(collection.count()).to.equal(1);
-	});
-
-	it('should return number of items', () => {
-		collection.add('key1').add('key2');
-		expect(collection.count()).to.equal(2);
-	});
-
-	it('should initialize with array of keys', () => {
-		const newCollection = StringCollection.fromArray(['Hello', 'World']);
-		expect(newCollection.count()).to.equal(2);
-	});
-
-	it('should initialize with key/value pairs', () => {
-		const newCollection = StringCollection.fromObject({'key': 'translation'});
-		expect(newCollection.get('key')).to.equal('translation');
-	});
-
-	it('should merge with other collection', () => {
-		collection.add('Hello');
-		const newCollection = StringCollection.fromArray(['World']);
-		expect(collection.merge(newCollection).count()).to.equal(2);
-	});
-
-});

tests/utils/translation.collection.spec.ts

@@ -0,0 +1,82 @@
+import { expect } from 'chai';
+
+import { TranslationCollection } from '../../src/utils/translation.collection';
+
+describe('StringCollection', () => {
+
+	let collection: TranslationCollection;
+
+	beforeEach(() => {
+		collection = new TranslationCollection();
+	});
+
+	it('should initialize with key/value pairs', () => {
+		collection = new TranslationCollection({ key1: 'val1', key2: 'val2' });
+		expect(collection.values).to.deep.equal({ key1: 'val1', key2: 'val2' });
+	});
+
+	it('should add key with value', () => {
+		const newCollection = collection.add('theKey', 'theVal');
+		expect(newCollection.get('theKey')).to.equal('theVal');
+	});
+
+	it('should add key with default value', () => {
+		collection = collection.add('theKey');
+		expect(collection.get('theKey')).to.equal('');
+	});
+
+	it('should not mutate collection when adding key', () => {
+		collection.add('theKey', 'theVal');
+		expect(collection.has('theKey')).to.equal(false);
+	});
+
+	it('should add array of keys with default value', () => {
+		collection = collection.addKeys(['key1', 'key2']);
+		expect(collection.values).to.deep.equal({ key1: '', key2: '' });
+	});
+
+	it('should return true when collection has key', () => {
+		collection = collection.add('key');
+		expect(collection.has('key')).to.equal(true);
+	});
+
+	it('should return false when collection does not have key', () => {
+		expect(collection.has('key')).to.equal(false);
+	});
+
+	it('should remove key', () => {
+		collection = new TranslationCollection({ removeThisKey: '' });
+		collection = collection.remove('removeThisKey');
+		expect(collection.has('removeThisKey')).to.equal(false);
+	});
+
+	it('should not mutate collection when removing key', () => {
+		collection = new TranslationCollection({ removeThisKey: '' });
+		collection.remove('removeThisKey');
+		expect(collection.has('removeThisKey')).to.equal(true);
+	});
+
+	it('should return number of keys', () => {
+		collection = collection.addKeys(['key1', 'key2', 'key3']);
+		expect(collection.count()).to.equal(3);
+	});
+
+	it('should merge with other collection', () => {
+		collection = collection.add('oldKey', 'oldVal');
+		const newCollection = new TranslationCollection({ newKey: 'newVal' });
+		expect(collection.union(newCollection).values).to.deep.equal({ oldKey: 'oldVal', newKey: 'newVal' });
+	});
+
+	it('should intersect with passed collection', () => {
+		collection = collection.addKeys(['red', 'green', 'blue']);
+		const newCollection = new TranslationCollection( { red: '', blue: '' });
+		expect(collection.intersect(newCollection).values).to.deep.equal({ red: '', blue: '' });
+	});
+
+	it('should intersect with passed collection and keep original values', () => {
+		collection = new TranslationCollection({ red: 'rød', blue: 'blå' });
+		const newCollection = new TranslationCollection({ red: 'no value', blue: 'also no value' });
+		expect(collection.intersect(newCollection).values).to.deep.equal({ red: 'rød', blue: 'blå' });
+	});
+
+});