- Added typings for packages where typings existed

- Remove regexp ServiceParser and make AstServiceParser the default. #23
- Replaced CLI parser to add support for multiple input/output paths (supports file expansion, glob patterns and multiple values/parameters) #7
This commit is contained in:
Kim Biesbjerg
2017-03-20 15:29:41 +01:00
parent c5d68cfcaa
commit 3facc0c287
10 changed files with 317 additions and 475 deletions

View File

@@ -0,0 +1,9 @@
export interface CliOptionsInterface {
dir: string[];
output: string[];
format: 'json' | 'namespaced-json' | 'pot';
replace: boolean;
sort: boolean;
clean: boolean;
help: boolean;
}

148
src/cli/cli.ts Executable file
View File

@@ -0,0 +1,148 @@
import { Extractor } from '../utils/extractor';
import { CliOptionsInterface } from './cli-options.interface';
import { TranslationCollection } from '../utils/translation.collection';
import { ParserInterface } from '../parsers/parser.interface';
import { PipeParser } from '../parsers/pipe.parser';
import { DirectiveParser } from '../parsers/directive.parser';
import { ServiceParser } from '../parsers/service.parser';
import { CompilerInterface } from '../compilers/compiler.interface';
import { JsonCompiler } from '../compilers/json.compiler';
import { NamespacedJsonCompiler } from '../compilers/namespaced-json.compiler';
import { PoCompiler } from '../compilers/po.compiler';
import * as fs from 'fs';
import * as path from 'path';
import * as mkdirp from 'mkdirp';
import * as chalk from 'chalk';
import * as yargs from 'yargs';
const options: CliOptionsInterface = yargs
.usage('Extract strings from files for translation.\nUsage: $0 [options]')
.help('help')
.option('dir', {
alias: 'd',
describe: 'Paths you would like to extract strings from. Multiple paths can be specified',
default: process.env.PWD,
type: 'array'
})
.option('output', {
alias: 'o',
describe: 'Path you would like to save extracted strings to. Multiple paths can be specified',
default: process.env.PWD,
type: 'array'
})
.option('format', {
alias: 'f',
describe: 'Output format',
default: 'json',
type: 'string',
choices: ['json', 'namespaced-json', 'pot']
})
.option('replace', {
alias: 'r',
describe: 'Replace the contents of output file if it exists (Merges by default)',
default: false,
type: 'boolean'
})
.option('sort', {
alias: 's',
describe: 'Sort translations in the output file in alphabetical order',
default: false,
type: 'boolean'
})
.option('clean', {
alias: 'c',
describe: 'Remove obsolete strings when merging',
default: false,
type: 'boolean'
})
.argv;
const patterns: string[] = [
'/**/*.html',
'/**/*.ts'
];
const parsers: ParserInterface[] = [
new ServiceParser(),
new PipeParser(),
new DirectiveParser()
];
let compiler: CompilerInterface;
let ext: string;
switch (options.format) {
case 'pot':
compiler = new PoCompiler();
ext = 'pot';
break;
case 'json':
compiler = new JsonCompiler();
ext = 'json';
break;
case 'namespaced-json':
compiler = new NamespacedJsonCompiler();
ext = 'json';
break;
}
const extractor: Extractor = new Extractor(parsers, patterns);
let extractedStrings: TranslationCollection = new TranslationCollection();
// Extract strings from paths
console.log(chalk.bold('Extracting strings from...'));
options.dir.forEach(dir => {
const normalizedDir: string = path.resolve(dir);
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
console.log(`The path you supplied was not found: '${dir}'`);
process.exit(1);
}
console.log(chalk.gray('- %s'), normalizedDir);
extractedStrings = extractedStrings.union(extractor.process(normalizedDir));
});
console.log(chalk.green('Extracted %d strings\n'), extractedStrings.count());
// Save extracted strings to output paths
options.output.forEach(output => {
const normalizedOutput: string = path.resolve(output);
let outputDir: string = normalizedOutput;
let outputFilename: string = `template.${ext}`;
if (!fs.existsSync(normalizedOutput) || !fs.statSync(normalizedOutput).isDirectory()) {
outputDir = path.dirname(normalizedOutput);
outputFilename = path.basename(normalizedOutput);
}
const outputPath: string = path.join(outputDir, outputFilename);
console.log(chalk.bold('Saving to: %s'), outputPath);
if (!fs.existsSync(outputDir)) {
console.log(chalk.dim('- Created output dir: %s'), outputDir);
mkdirp.sync(outputDir);
}
let processedStrings: TranslationCollection = extractedStrings;
if (fs.existsSync(outputPath) && !options.replace) {
const existingStrings: TranslationCollection = compiler.parse(fs.readFileSync(outputPath, 'utf-8'));
if (existingStrings.count() > 0) {
processedStrings = processedStrings.union(existingStrings);
console.log(chalk.dim('- Merged with %d existing strings'), existingStrings.count());
}
if (options.clean) {
const collectionCount = processedStrings.count();
processedStrings = processedStrings.intersect(processedStrings);
const removeCount = collectionCount - processedStrings.count();
console.log(chalk.dim('- Removed %d obsolete strings'), removeCount);
}
}
if (options.sort) {
processedStrings = processedStrings.sort();
console.log(chalk.dim('- Sorted strings'));
}
fs.writeFileSync(outputPath, compiler.compile(processedStrings));
console.log(chalk.green('OK!\n'));
});

View File

@@ -1,105 +0,0 @@
import { Extractor } from '../utils/extractor';
import { TranslationCollection } from '../utils/translation.collection';
import { ParserInterface } from '../parsers/parser.interface';
import { PipeParser } from '../parsers/pipe.parser';
import { DirectiveParser } from '../parsers/directive.parser';
import { ServiceParser } from '../parsers/service.parser';
import { AstServiceParser } from '../parsers/ast-service.parser';
import { CompilerInterface } from '../compilers/compiler.interface';
import { JsonCompiler } from '../compilers/json.compiler';
import { NamespacedJsonCompiler } from '../compilers/namespaced-json.compiler';
import { PoCompiler } from '../compilers/po.compiler';
import * as fs from 'fs';
import * as path from 'path';
import * as cli from 'cli';
const options = cli.parse({
dir: ['d', 'Path you would like to extract strings from', 'dir', process.env.PWD],
output: ['o', 'Path you would like to save extracted strings to', 'dir', process.env.PWD],
format: ['f', 'Output format', ['json', 'namespaced-json', 'pot'], 'json'],
replace: ['r', 'Replace the contents of output file if it exists (Merges by default)', 'boolean', false],
sort: ['s', 'Sort translations in the output file in alphabetical order', 'boolean', false],
clean: ['c', 'Remove obsolete strings when merging', 'boolean', false],
experimental: ['e', 'Use experimental AST Service Parser', 'boolean', false]
});
const patterns: string[] = [
'/**/*.html',
'/**/*.ts'
];
const parsers: ParserInterface[] = [
new PipeParser(),
new DirectiveParser(),
options.experimental ? new AstServiceParser() : new ServiceParser()
];
let compiler: CompilerInterface;
let ext: string;
switch (options.format) {
case 'pot':
compiler = new PoCompiler();
ext = 'pot';
break;
case 'json':
compiler = new JsonCompiler();
ext = 'json';
break;
case 'namespaced-json':
compiler = new NamespacedJsonCompiler();
ext = 'json';
break;
}
const normalizedDir: string = path.resolve(options.dir);
const normalizedOutput: string = path.resolve(options.output);
let outputDir: string = normalizedOutput;
let outputFilename: string = `template.${ext}`;
if (!fs.existsSync(normalizedOutput) || !fs.statSync(normalizedOutput).isDirectory()) {
outputDir = path.dirname(normalizedOutput);
outputFilename = path.basename(normalizedOutput);
}
const outputPath: string = path.join(outputDir, outputFilename);
[normalizedDir, outputDir].forEach(dir => {
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
cli.fatal(`The path you supplied was not found: '${dir}'`);
}
});
try {
const extractor: Extractor = new Extractor(parsers, patterns);
cli.info(`Extracting strings from '${normalizedDir}'`);
const extracted: TranslationCollection = extractor.process(normalizedDir);
cli.ok(`* Extracted ${extracted.count()} strings`);
let collection: TranslationCollection = extracted;
if (!options.replace && fs.existsSync(outputPath)) {
const existing: TranslationCollection = compiler.parse(fs.readFileSync(outputPath, 'utf-8'));
if (existing.count() > 0) {
collection = extracted.union(existing);
cli.ok(`* Merged with ${existing.count()} existing strings`);
}
if (options.clean) {
const collectionCount = collection.count();
collection = collection.intersect(extracted);
const removeCount = collectionCount - collection.count();
if (removeCount > 0) {
cli.ok(`* Removed ${removeCount} obsolete strings`);
}
}
}
if (options.sort) {
collection = collection.sort();
}
fs.writeFileSync(outputPath, compiler.compile(collection));
cli.ok(`* Saved to '${outputPath}'`);
} catch (e) {
cli.fatal(e.toString());
}

View File

@@ -6,12 +6,12 @@ import * as flat from 'flat';
export class NamespacedJsonCompiler implements CompilerInterface {
public compile(collection: TranslationCollection): string {
const values = flat.unflatten(collection.values);
const values: {} = flat.unflatten(collection.values);
return JSON.stringify(values, null, '\t');
}
public parse(contents: string): TranslationCollection {
const values = flat.flatten(JSON.parse(contents));
const values: {} = flat.flatten(JSON.parse(contents));
return new TranslationCollection(values);
}

View File

@@ -1,3 +1 @@
declare module 'cli';
declare module 'flat';
declare module 'gettext-parser';

View File

@@ -1,163 +0,0 @@
import { ParserInterface } from './parser.interface';
import { TranslationCollection } from '../utils/translation.collection';
import { syntaxKindToName } from '../utils/ast-utils';
import * as ts from 'typescript';
export class AstServiceParser implements ParserInterface {
protected _sourceFile: ts.SourceFile;
protected _instancePropertyName: any;
protected _serviceClassName: string = 'TranslateService';
protected _serviceMethodNames: string[] = ['get', 'instant'];
public extract(contents: string, path?: string): TranslationCollection {
let collection: TranslationCollection = new TranslationCollection();
this._sourceFile = this._createSourceFile(path, contents);
this._instancePropertyName = this._getInstancePropertyName();
if (!this._instancePropertyName) {
return collection;
}
const callNodes = this._findCallNodes();
callNodes.forEach(callNode => {
const keys: string[] = this._getCallArgStrings(callNode);
if (keys && keys.length) {
collection = collection.addKeys(keys);
}
});
return collection;
}
protected _createSourceFile(path: string, contents: string): ts.SourceFile {
return ts.createSourceFile(path, contents, null, /*setParentNodes */ false);
}
/**
* Detect what the TranslateService instance property
* is called by inspecting constructor params
*/
protected _getInstancePropertyName(): string {
const constructorNode = this._findConstructorNode();
if (!constructorNode) {
return null;
}
const result = constructorNode.parameters.find(parameter => {
// Skip if visibility modifier is not present (we want it set as an instance property)
if (!parameter.modifiers) {
return false;
}
// Make sure className is of the correct type
const parameterType: ts.Identifier = (parameter.type as ts.TypeReferenceNode).typeName as ts.Identifier;
if (!parameterType) {
return false;
}
const className: string = parameterType.text;
if (className !== this._serviceClassName) {
return false;
}
return true;
});
if (result) {
return (result.name as ts.Identifier).text;
}
}
/**
* Find first constructor
*/
protected _findConstructorNode(): ts.ConstructorDeclaration {
const constructors = this._findNodes(this._sourceFile, ts.SyntaxKind.Constructor, true) as ts.ConstructorDeclaration[];
if (constructors.length) {
return constructors[0];
}
}
/**
* Find all calls to TranslateService methods
*/
protected _findCallNodes(node?: ts.Node): ts.CallExpression[] {
if (!node) {
node = this._sourceFile;
}
let callNodes = this._findNodes(node, ts.SyntaxKind.CallExpression) as ts.CallExpression[];
callNodes = callNodes
// Only call expressions with arguments
.filter(callNode => callNode.arguments.length > 0)
// More filters
.filter(callNode => {
const propAccess = callNode.getChildAt(0).getChildAt(0) as ts.PropertyAccessExpression;
if (!propAccess || propAccess.kind !== ts.SyntaxKind.PropertyAccessExpression) {
return false;
}
if (!propAccess.getFirstToken() || propAccess.getFirstToken().kind !== ts.SyntaxKind.ThisKeyword) {
return false;
}
if (propAccess.name.text !== this._instancePropertyName) {
return false;
}
const methodAccess = callNode.getChildAt(0) as ts.PropertyAccessExpression;
if (!methodAccess || methodAccess.kind !== ts.SyntaxKind.PropertyAccessExpression) {
return false;
}
if (!methodAccess.name || this._serviceMethodNames.indexOf(methodAccess.name.text) === -1) {
return false;
}
return true;
});
return callNodes;
}
/**
* Get strings from function call's first argument
*/
protected _getCallArgStrings(callNode: ts.CallExpression): string[] {
if (!callNode.arguments.length) {
return;
}
const firstArg = callNode.arguments[0];
switch (firstArg.kind) {
case ts.SyntaxKind.StringLiteral:
case ts.SyntaxKind.FirstTemplateToken:
return [(firstArg as ts.StringLiteral).text];
case ts.SyntaxKind.ArrayLiteralExpression:
return (firstArg as ts.ArrayLiteralExpression).elements
.map((element: ts.StringLiteral) => element.text);
case ts.SyntaxKind.Identifier:
console.log('WARNING: We cannot extract variable values passed to TranslateService (yet)');
break;
default:
console.log(`SKIP: Unknown argument type: '${syntaxKindToName(firstArg.kind)}'`, firstArg);
}
}
/**
* Find all child nodes of a kind
*/
protected _findNodes(node: ts.Node, kind: ts.SyntaxKind, onlyOne: boolean = false): ts.Node[] {
if (node.kind === kind && onlyOne) {
return [node];
}
const childrenNodes: ts.Node[] = node.getChildren(this._sourceFile);
const initialValue: ts.Node[] = node.kind === kind ? [node] : [];
return childrenNodes.reduce((result: ts.Node[], childNode: ts.Node) => {
return result.concat(this._findNodes(childNode, kind));
}, initialValue);
}
}

View File

@@ -1,59 +1,163 @@
import { ParserInterface } from './parser.interface';
import { TranslationCollection } from '../utils/translation.collection';
import { syntaxKindToName } from '../utils/ast-utils';
import * as ts from 'typescript';
export class ServiceParser implements ParserInterface {
protected _sourceFile: ts.SourceFile;
protected _instancePropertyName: any;
protected _serviceClassName: string = 'TranslateService';
protected _serviceMethodNames: string[] = ['get', 'instant'];
public extract(contents: string, path?: string): TranslationCollection {
let collection: TranslationCollection = new TranslationCollection();
const translateServiceVar = this._extractTranslateServiceVar(contents);
if (!translateServiceVar) {
this._sourceFile = this._createSourceFile(path, contents);
this._instancePropertyName = this._getInstancePropertyName();
if (!this._instancePropertyName) {
return collection;
}
const methodRegExp: RegExp = /(?:get|instant)\s*\(\s*(\[?\s*(['"`])([^\1\r\n]*)\2\s*\]?)/;
const regExp: RegExp = new RegExp(`\\.${translateServiceVar}\\.${methodRegExp.source}`, 'g');
let matches: RegExpExecArray;
while (matches = regExp.exec(contents)) {
if (this._stringContainsArray(matches[1])) {
collection = collection.addKeys(this._stringToArray(matches[1]));
} else {
collection = collection.add(matches[3]);
const callNodes = this._findCallNodes();
callNodes.forEach(callNode => {
const keys: string[] = this._getCallArgStrings(callNode);
if (keys && keys.length) {
collection = collection.addKeys(keys);
}
}
});
return collection;
}
/**
* Extracts name of TranslateService variable for use in patterns
*/
protected _extractTranslateServiceVar(contents: string): string {
const matches = contents.match(/([a-z0-9_]+)\s*:\s*TranslateService/i);
if (matches === null) {
return '';
}
return matches[1];
protected _createSourceFile(path: string, contents: string): ts.SourceFile {
return ts.createSourceFile(path, contents, null, /*setParentNodes */ false);
}
/**
* Checks if string contains an array
* Detect what the TranslateService instance property
* is called by inspecting constructor params
*/
protected _stringContainsArray(input: string): boolean {
return input.startsWith('[') && input.endsWith(']');
protected _getInstancePropertyName(): string {
const constructorNode = this._findConstructorNode();
if (!constructorNode) {
return null;
}
const result = constructorNode.parameters.find(parameter => {
// Skip if visibility modifier is not present (we want it set as an instance property)
if (!parameter.modifiers) {
return false;
}
// Make sure className is of the correct type
const parameterType: ts.Identifier = (parameter.type as ts.TypeReferenceNode).typeName as ts.Identifier;
if (!parameterType) {
return false;
}
const className: string = parameterType.text;
if (className !== this._serviceClassName) {
return false;
}
return true;
});
if (result) {
return (result.name as ts.Identifier).text;
}
}
/**
* Converts string to array
* Find first constructor
*/
protected _stringToArray(input: string): string[] {
if (this._stringContainsArray(input)) {
return eval(input);
protected _findConstructorNode(): ts.ConstructorDeclaration {
const constructors = this._findNodes(this._sourceFile, ts.SyntaxKind.Constructor, true) as ts.ConstructorDeclaration[];
if (constructors.length) {
return constructors[0];
}
}
/**
* Find all calls to TranslateService methods
*/
protected _findCallNodes(node?: ts.Node): ts.CallExpression[] {
if (!node) {
node = this._sourceFile;
}
return [];
let callNodes = this._findNodes(node, ts.SyntaxKind.CallExpression) as ts.CallExpression[];
callNodes = callNodes
// Only call expressions with arguments
.filter(callNode => callNode.arguments.length > 0)
// More filters
.filter(callNode => {
const propAccess = callNode.getChildAt(0).getChildAt(0) as ts.PropertyAccessExpression;
if (!propAccess || propAccess.kind !== ts.SyntaxKind.PropertyAccessExpression) {
return false;
}
if (!propAccess.getFirstToken() || propAccess.getFirstToken().kind !== ts.SyntaxKind.ThisKeyword) {
return false;
}
if (propAccess.name.text !== this._instancePropertyName) {
return false;
}
const methodAccess = callNode.getChildAt(0) as ts.PropertyAccessExpression;
if (!methodAccess || methodAccess.kind !== ts.SyntaxKind.PropertyAccessExpression) {
return false;
}
if (!methodAccess.name || this._serviceMethodNames.indexOf(methodAccess.name.text) === -1) {
return false;
}
return true;
});
return callNodes;
}
/**
* Get strings from function call's first argument
*/
protected _getCallArgStrings(callNode: ts.CallExpression): string[] {
if (!callNode.arguments.length) {
return;
}
const firstArg = callNode.arguments[0];
switch (firstArg.kind) {
case ts.SyntaxKind.StringLiteral:
case ts.SyntaxKind.FirstTemplateToken:
return [(firstArg as ts.StringLiteral).text];
case ts.SyntaxKind.ArrayLiteralExpression:
return (firstArg as ts.ArrayLiteralExpression).elements
.map((element: ts.StringLiteral) => element.text);
case ts.SyntaxKind.Identifier:
console.log('WARNING: We cannot extract variable values passed to TranslateService (yet)');
break;
default:
console.log(`SKIP: Unknown argument type: '${syntaxKindToName(firstArg.kind)}'`, firstArg);
}
}
/**
* Find all child nodes of a kind
*/
protected _findNodes(node: ts.Node, kind: ts.SyntaxKind, onlyOne: boolean = false): ts.Node[] {
if (node.kind === kind && onlyOne) {
return [node];
}
const childrenNodes: ts.Node[] = node.getChildren(this._sourceFile);
const initialValue: ts.Node[] = node.kind === kind ? [node] : [];
return childrenNodes.reduce((result: ts.Node[], childNode: ts.Node) => {
return result.concat(this._findNodes(childNode, kind));
}, initialValue);
}
}