Release v0.2.12

Cstring escapes

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "pofile",
-  "version": "0.2.9",
+  "version": "0.2.12",
   "authors": [
     "Ruben Vermeersch <ruben@rocketeer.be>"
   ],

dist/pofile.js

@@ -120,8 +120,32 @@ PO.parse = function (data) {
     function extract(string) {
         string = trim(string);
         string = string.replace(/^[^"]*"|"$/g, '');
-        string = string.replace(/\\"/g, '"');
-        string = string.replace(/\\\\/g, '\\');
+        string = string.replace(/\\([abtnvfr'"\\?]|([0-7]{3})|x([0-9a-fA-F]{2}))/g, function (match, esc, oct, hex) {
+            if (oct) {
+                return String.fromCharCode(parseInt(oct, 8));
+            }
+            if (hex) {
+                return String.fromCharCode(parseInt(hex, 16));
+            }
+            switch (esc) {
+                case 'a':
+                    return '\x07';
+                case 'b':
+                    return '\b';
+                case 't':
+                    return '\t';
+                case 'n':
+                    return '\n';
+                case 'v':
+                    return '\v';
+                case 'f':
+                    return '\f';
+                case 'r':
+                    return '\r';
+                default:
+                    return esc;
+            }
+        });
         return string;
     }
 
@@ -147,9 +171,9 @@ PO.parse = function (data) {
             for (var i = 0; i < flags.length; i++) {
                 item.flags[flags[i]] = true;
             }
-        } else if (line.match(/^#\s+/)) { // Translator comment
+        } else if (line.match(/^#($|\s+)/)) { // Translator comment
             finish();
-            item.comments.push(trim(line.replace(/^#\s+/, '')));
+            item.comments.push(trim(line.replace(/^#($|\s+)/, '')));
         } else if (line.match(/^#\./)) { // Extracted comment
             finish();
             item.extractedComments.push(trim(line.replace(/^#\./, '')));
@@ -213,8 +237,27 @@ PO.Item.prototype.toString = function () {
 
     // reverse what extract(string) method during PO.parse does
     var _escape = function (string) {
-        string = string.replace(/\\/g, '\\\\');
-        return string.replace(/"/g, '\\"');
+        // don't unescape \n, since string can never contain it
+        // since split('\n') is called on it
+        string = string.replace(/[\x07\b\t\v\f\r"\\]/g, function (match) {
+            switch (match) {
+                case '\x07':
+                    return '\\a';
+                case '\b':
+                    return '\\b';
+                case '\t':
+                    return '\\t';
+                case '\v':
+                    return '\\v';
+                case '\f':
+                    return '\\f';
+                case '\r':
+                    return '\\r';
+                default:
+                    return '\\' + match;
+            }
+        });
+        return string;
     };
 
     var _process = function (keyword, text, i) {
@@ -262,7 +305,12 @@ PO.Item.prototype.toString = function () {
                 });
             } else {
                 text = isArray(text) ? text.join() : text;
-                lines = lines.concat(mkObsolete + _process(keyword, text));
+                var processed = _process(keyword, text);
+                //handle \n in single-line texts (can not be handled in _escape)
+                for (var i = 1; i < processed.length - 1; i++) {
+                    processed[i] = processed[i].slice(0, -1) + '\\n"';
+                }
+                lines = lines.concat(mkObsolete + processed.join('\n' + mkObsolete));
             }
         }
     });

lib/po.js

@@ -119,8 +119,32 @@ PO.parse = function (data) {
     function extract(string) {
         string = trim(string);
         string = string.replace(/^[^"]*"|"$/g, '');
-        string = string.replace(/\\"/g, '"');
-        string = string.replace(/\\\\/g, '\\');
+        string = string.replace(/\\([abtnvfr'"\\?]|([0-7]{3})|x([0-9a-fA-F]{2}))/g, function (match, esc, oct, hex) {
+            if (oct) {
+                return String.fromCharCode(parseInt(oct, 8));
+            }
+            if (hex) {
+                return String.fromCharCode(parseInt(hex, 16));
+            }
+            switch (esc) {
+                case 'a':
+                    return '\x07';
+                case 'b':
+                    return '\b';
+                case 't':
+                    return '\t';
+                case 'n':
+                    return '\n';
+                case 'v':
+                    return '\v';
+                case 'f':
+                    return '\f';
+                case 'r':
+                    return '\r';
+                default:
+                    return esc;
+            }
+        });
         return string;
     }
 
@@ -146,9 +170,9 @@ PO.parse = function (data) {
             for (var i = 0; i < flags.length; i++) {
                 item.flags[flags[i]] = true;
             }
-        } else if (line.match(/^#\s+/)) { // Translator comment
+        } else if (line.match(/^#($|\s+)/)) { // Translator comment
             finish();
-            item.comments.push(trim(line.replace(/^#\s+/, '')));
+            item.comments.push(trim(line.replace(/^#($|\s+)/, '')));
         } else if (line.match(/^#\./)) { // Extracted comment
             finish();
             item.extractedComments.push(trim(line.replace(/^#\./, '')));
@@ -212,8 +236,27 @@ PO.Item.prototype.toString = function () {
 
     // reverse what extract(string) method during PO.parse does
     var _escape = function (string) {
-        string = string.replace(/\\/g, '\\\\');
-        return string.replace(/"/g, '\\"');
+        // don't unescape \n, since string can never contain it
+        // since split('\n') is called on it
+        string = string.replace(/[\x07\b\t\v\f\r"\\]/g, function (match) {
+            switch (match) {
+                case '\x07':
+                    return '\\a';
+                case '\b':
+                    return '\\b';
+                case '\t':
+                    return '\\t';
+                case '\v':
+                    return '\\v';
+                case '\f':
+                    return '\\f';
+                case '\r':
+                    return '\\r';
+                default:
+                    return '\\' + match;
+            }
+        });
+        return string;
     };
 
     var _process = function (keyword, text, i) {
@@ -261,7 +304,12 @@ PO.Item.prototype.toString = function () {
                 });
             } else {
                 text = isArray(text) ? text.join() : text;
-                lines = lines.concat(mkObsolete + _process(keyword, text));
+                var processed = _process(keyword, text);
+                //handle \n in single-line texts (can not be handled in _escape)
+                for (var i = 1; i < processed.length - 1; i++) {
+                    processed[i] = processed[i].slice(0, -1) + '\\n"';
+                }
+                lines = lines.concat(mkObsolete + processed.join('\n' + mkObsolete));
             }
         }
     });

package.json

@@ -1,7 +1,7 @@
 {
   "name": "pofile",
   "description": "Parse and serialize Gettext PO files.",
-  "version": "0.2.9",
+  "version": "0.2.12",
   "author": {
     "name": "Ruben Vermeersch",
     "email": "ruben@savanne.be",

test/fixtures/c-strings.po

@@ -17,3 +17,43 @@ msgstr ""
 
 msgid "The name field must not contain characters like \" or \\"
 msgstr ""
+
+# possibility to reorder items depending on locale
+#. Format of addresses
+#. %1$s is the street
+#. %2$s is the postal code
+#. %3$s is the city
+#. %4$s is the state
+#. %5$s is the country
+msgid ""
+"%1$s\n"
+"%2$s %3$s\n"
+"%4$s\n"
+"%5$s"
+msgstr ""
+
+# "i18"ned code
+#. used in <pre> environment, so don't remove any control sequences
+msgid ""
+"define('some/test/module', function () {\n"
+"\t'use strict';\n"
+"\treturn {};\n"
+"});\n"
+""
+msgstr ""
+"define('random/test/file', function () {\n"
+"\t'use strict';\n"
+"\treturn {};\n"
+"});\n"
+""
+
+# all one-letter escape characters
+# be aware, that \a, \b, \v, \f and \r should not be used
+# in i18ned messages (according to gettext tools)
+# however, they should be properly parsed, anyway
+msgid ""
+"\a\b\t\n"
+"\v\f\r"
+msgstr ""
+"\a\b\t\n"
+"\v\f\r"

test/fixtures/comment.po

@@ -19,3 +19,11 @@ msgstr ""
 #. Extracted comment
 msgid "Title, as plain text"
 msgstr "Attribut title, en tant que texte brut"
+
+#
+#.
+#:
+#, fuzzy
+msgid "Empty comment"
+msgstr "Empty"
+

test/parse.js

@@ -31,10 +31,24 @@ describe('Parse', function () {
         assert.equal(po.headers['Plural-Forms'], 'nplurals=3; plural=n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;');
     });
 
+    it('Handle empty comments', function (done) {
+        PO.load(__dirname + '/fixtures/comment.po', function (err, po) {
+            assert.equal(err, null);
+
+            var item = po.items[1];
+            assert.equal(item.msgid, 'Empty comment');
+            assert.equal(item.msgstr, 'Empty');
+            assert.deepEqual(item.comments, ['']);
+            assert.deepEqual(item.extractedComments, ['']);
+            assert.deepEqual(item.references, ['']);
+            done();
+        });
+    });
+
     it('Handles translator comments', function () {
         var po = PO.parse(fs.readFileSync(__dirname + '/fixtures/comment.po', 'utf8'));
         assert.notEqual(po, null);
-        assert.equal(po.items.length, 1);
+        assert.equal(po.items.length, 2);
 
         var item = po.items[0];
         assert.equal(item.msgid, 'Title, as plain text');
@@ -45,7 +59,7 @@ describe('Parse', function () {
     it('Handles extracted comments', function () {
         var po = PO.parse(fs.readFileSync(__dirname + '/fixtures/comment.po', 'utf8'));
         assert.notEqual(po, null);
-        assert.equal(po.items.length, 1);
+        assert.equal(po.items.length, 2);
 
         var item = po.items[0];
         assert.equal(item.msgid, 'Title, as plain text');
@@ -119,19 +133,29 @@ describe('Parse', function () {
     });
 
     describe('C-Strings', function () {
+        var po = PO.parse(fs.readFileSync(__dirname + '/fixtures/c-strings.po', 'utf8'));
         it('should parse the c-strings.po file', function () {
-            var po = PO.parse(fs.readFileSync(__dirname + '/fixtures/c-strings.po', 'utf8'));
-
             assert.notEqual(po, null);
         });
 
         it('should extract strings containing " and \\ characters', function () {
-            var po = PO.parse(fs.readFileSync(__dirname + '/fixtures/c-strings.po', 'utf8'));
-
             var items = po.items.filter(function (item) {
                 return (/^The name field must not contain/).test(item.msgid);
             });
             assert.equal(items[0].msgid, 'The name field must not contain characters like " or \\');
         });
+
+        it('should handle \n characters', function () {
+            var item = po.items[1];
+            assert.equal(item.msgid, '%1$s\n%2$s %3$s\n%4$s\n%5$s');
+        });
+
+        it('should handle \t characters', function () {
+            var item = po.items[2];
+            assert.equal(item.msgid, 'define(\'some/test/module\', function () {\n' +
+                '\t\'use strict\';\n' +
+                '\treturn {};\n' +
+                '});\n');
+        });
     });
 });

test/write.js

@@ -92,6 +92,15 @@ describe('Write', function () {
             assertHasLine(item.toString(), 'msgid "\\\\ should be written escaped"');
         });
 
+        it('should escape \\n', function () {
+            var item = new PO.Item();
+
+            item.msgid = '\n should be written escaped';
+            assertHasLine(item.toString(), 'msgid ""');
+            assertHasLine(item.toString(), '"\\n"');
+            assertHasLine(item.toString(), '" should be written escaped"');
+        });
+
         it('should write identical file after parsing a file', function () {
             var input = fs.readFileSync(__dirname + '/fixtures/c-strings.po', 'utf8');
             var po = PO.parse(input);