From 4c0207a77275d7c237733428dd600087eb74e16d Mon Sep 17 00:00:00 2001 From: ryan Date: Tue, 23 Jun 2009 16:32:52 +0000 Subject: [PATCH] Merge latest pomo. Works around mbstring.func_overload. Props nbachiyski. fixes #10236 for trunk git-svn-id: http://svn.automattic.com/wordpress/trunk@11626 1a063a9b-81f0-0310-95a4-ce76da25c4cd --- wp-includes/pomo/entry.php | 6 +- wp-includes/pomo/mo.php | 110 +++++++++++----- wp-includes/pomo/po.php | 211 ++++++++++++++++++++++++++++-- wp-includes/pomo/streams.php | 69 ++++++---- wp-includes/pomo/translations.php | 54 ++++++-- 5 files changed, 361 insertions(+), 89 deletions(-) diff --git a/wp-includes/pomo/entry.php b/wp-includes/pomo/entry.php index bb146a7d3e..feb5b34cf2 100644 --- a/wp-includes/pomo/entry.php +++ b/wp-includes/pomo/entry.php @@ -2,7 +2,7 @@ /** * Contains Translation_Entry class * - * @version $Id: entry.php 13 2008-04-21 12:03:37Z nbachiyski $ + * @version $Id: entry.php 115 2009-05-11 18:56:15Z nbachiyski $ * @package pomo * @subpackage entry */ @@ -48,9 +48,7 @@ class Translation_Entry { // get member variable values from args hash $object_varnames = array_keys(get_object_vars($this)); foreach ($args as $varname => $value) { - if (in_array($varname, $object_varnames)) { - $this->$varname = $value; - } + $this->$varname = $value; } if (isset($args['plural'])) $this->is_plural = true; if (!is_array($this->translations)) $this->translations = array(); diff --git a/wp-includes/pomo/mo.php b/wp-includes/pomo/mo.php index 6055838d16..0630ef285e 100644 --- a/wp-includes/pomo/mo.php +++ b/wp-includes/pomo/mo.php @@ -2,7 +2,7 @@ /** * Class for working with MO files * - * @version $Id: mo.php 33 2009-02-16 09:33:39Z nbachiyski $ + * @version $Id: mo.php 106 2009-04-23 19:48:22Z nbachiyski $ * @package pomo * @subpackage mo */ @@ -10,16 +10,10 @@ require_once dirname(__FILE__) . '/translations.php'; require_once dirname(__FILE__) . '/streams.php'; -class MO extends Translations { +class MO extends Gettext_Translations { var $_nplurals = 2; - function set_header($header, $value) { - parent::set_header($header, $value); - if ('Plural-Forms' == $header) - $this->_gettext_select_plural_form = $this->_make_gettext_select_plural_form($value); - } - /** * Fills up with the entries from MO file $filename * @@ -32,6 +26,73 @@ class MO extends Translations { } return $this->import_from_reader($reader); } + + function export_to_file($filename) { + $fh = fopen($filename, 'wb'); + if ( !$fh ) return false; + $entries = array_filter($this->entries, create_function('$e', 'return !empty($e->translations);')); + ksort($entries); + $magic = 0x950412de; + $revision = 0; + $total = count($entries) + 1; // all the headers are one entry + $originals_lenghts_addr = 28; + $translations_lenghts_addr = $originals_lenghts_addr + 8 * $total; + $size_of_hash = 0; + $hash_addr = $translations_lenghts_addr + 8 * $total; + $current_addr = $hash_addr; + fwrite($fh, pack('V*', $magic, $revision, $total, $originals_lenghts_addr, + $translations_lenghts_addr, $size_of_hash, $hash_addr)); + fseek($fh, $originals_lenghts_addr); + + // headers' msgid is an empty string + fwrite($fh, pack('VV', 0, $current_addr)); + $current_addr++; + $originals_table = chr(0); + + foreach($entries as $entry) { + $originals_table .= $this->export_original($entry) . chr(0); + $length = strlen($this->export_original($entry)); + fwrite($fh, pack('VV', $length, $current_addr)); + $current_addr += $length + 1; // account for the NULL byte after + } + + $exported_headers = $this->export_headers(); + fwrite($fh, pack('VV', strlen($exported_headers), $current_addr)); + $current_addr += strlen($exported_headers) + 1; + $translations_table = $exported_headers . chr(0); + + foreach($entries as $entry) { + $translations_table .= $this->export_translations($entry) . chr(0); + $length = strlen($this->export_translations($entry)); + fwrite($fh, pack('VV', $length, $current_addr)); + $current_addr += $length + 1; + } + + fwrite($fh, $originals_table); + fwrite($fh, $translations_table); + fclose($fh); + } + + function export_original($entry) { + //TODO: warnings for control characters + $exported = $entry->singular; + if ($entry->is_plural) $exported .= chr(0).$entry->plural; + if (!is_null($entry->context)) $exported = $entry->context . chr(4) . $exported; + return $exported; + } + + function export_translations($entry) { + //TODO: warnings for control characters + return implode(chr(0), $entry->translations); + } + + function export_headers() { + $exported = ''; + foreach($this->headers as $header => $value) { + $exported.= "$header: $value\n"; + } + return $exported; + } function get_byteorder($magic) { @@ -42,7 +103,7 @@ class MO extends Translations { $magic_little_64 = (int) 2500072158; // 0xde120495 $magic_big = ((int) - 569244523) && 0xFFFFFFFF; - + if ($magic_little == $magic || $magic_little_64 == $magic) { return 'little'; } else if ($magic_big == $magic) { @@ -63,22 +124,22 @@ class MO extends Translations { $revision = $reader->readint32(); $total = $reader->readint32(); // get addresses of array of lenghts and offsets for original string and translations - $originals_lo_addr = $reader->readint32(); - $translations_lo_addr = $reader->readint32(); + $originals_lenghts_addr = $reader->readint32(); + $translations_lenghts_addr = $reader->readint32(); - $reader->seekto($originals_lo_addr); - $originals_lo = $reader->readint32array($total * 2); // each of - $reader->seekto($translations_lo_addr); - $translations_lo = $reader->readint32array($total * 2); + $reader->seekto($originals_lenghts_addr); + $originals_lenghts = $reader->readint32array($total * 2); // each of + $reader->seekto($translations_lenghts_addr); + $translations_lenghts = $reader->readint32array($total * 2); $length = create_function('$i', 'return $i * 2 + 1;'); $offset = create_function('$i', 'return $i * 2 + 2;'); for ($i = 0; $i < $total; ++$i) { - $reader->seekto($originals_lo[$offset($i)]); - $original = $reader->read($originals_lo[$length($i)]); - $reader->seekto($translations_lo[$offset($i)]); - $translation = $reader->read($translations_lo[$length($i)]); + $reader->seekto($originals_lenghts[$offset($i)]); + $original = $reader->read($originals_lenghts[$length($i)]); + $reader->seekto($translations_lenghts[$offset($i)]); + $translation = $reader->read($translations_lenghts[$length($i)]); if ('' == $original) { $this->set_headers($this->make_headers($translation)); } else { @@ -88,17 +149,6 @@ class MO extends Translations { return true; } - function make_headers($translation) { - $headers = array(); - $lines = explode("\n", $translation); - foreach($lines as $line) { - $parts = explode(':', $line, 2); - if (!isset($parts[1])) continue; - $headers[trim($parts[0])] = trim($parts[1]); - } - return $headers; - } - /** * @static */ diff --git a/wp-includes/pomo/po.php b/wp-includes/pomo/po.php index 50520670e5..6c40c5a4ee 100644 --- a/wp-includes/pomo/po.php +++ b/wp-includes/pomo/po.php @@ -2,7 +2,7 @@ /** * Class for working with PO files * - * @version $Id: po.php 33 2009-02-16 09:33:39Z nbachiyski $ + * @version $Id: po.php 123 2009-05-13 19:35:43Z nbachiyski $ * @package pomo * @subpackage po */ @@ -16,8 +16,8 @@ ini_set('auto_detect_line_endings', 1); /** * Routines for working with PO files */ -class PO extends Translations { - +class PO extends Gettext_Translations { + /** * Exports headers to a PO entry @@ -75,7 +75,6 @@ class PO extends Translations { return fclose($fh); } - /** * Formats a string in PO-style * @@ -87,31 +86,59 @@ class PO extends Translations { $quote = '"'; $slash = '\\'; $newline = "\n"; - $tab = "\t"; $replaces = array( "$slash" => "$slash$slash", - "$tab" => '\t', "$quote" => "$slash$quote", + "\t" => '\t', ); + $string = str_replace(array_keys($replaces), array_values($replaces), $string); - $po = array(); - foreach (explode($newline, $string) as $line) { - $po[] = wordwrap($line, PO_MAX_LINE_LEN - 2, " $quote$newline$quote"); - } - $po = $quote.implode("${slash}n$quote$newline$quote", $po).$quote; + $po = $quote.implode("${slash}n$quote$newline$quote", explode($newline, $string)).$quote; // add empty string on first line for readbility - if (false !== strpos($po, $newline)) { + if (false !== strpos($string, $newline) && + (substr_count($string, $newline) > 1 || !($newline === substr($string, -strlen($newline))))) { $po = "$quote$quote$newline$po"; } // remove empty strings $po = str_replace("$newline$quote$quote", '', $po); return $po; } + + /** + * Gives back the original string from a PO-formatted string + * + * @static + * @param string $string PO-formatted string + * @return string enascaped string + */ + function unpoify($string) { + $escapes = array('t' => "\t", 'n' => "\n", '\\' => '\\'); + $lines = array_map('trim', explode("\n", $string)); + $lines = array_map(array('PO', 'trim_quotes'), $lines); + $unpoified = ''; + $previous_is_backslash = false; + foreach($lines as $line) { + preg_match_all('/./u', $line, $chars); + $chars = $chars[0]; + foreach($chars as $char) { + if (!$previous_is_backslash) { + if ('\\' == $char) + $previous_is_backslash = true; + else + $unpoified .= $char; + } else { + $previous_is_backslash = false; + $unpoified .= isset($escapes[$char])? $escapes[$char] : $char; + } + } + } + return $unpoified; + } /** - * Inserts $with in the beginning of every new line of $string and + * Inserts $with in the beginning of every new line of $string and * returns the modified string * * @static @@ -157,7 +184,7 @@ class PO extends Translations { if (!empty($entry->translator_comments)) $po[] = PO::comment_block($entry->translator_comments); if (!empty($entry->extracted_comments)) $po[] = PO::comment_block($entry->extracted_comments, '.'); if (!empty($entry->references)) $po[] = PO::comment_block(implode(' ', $entry->references), ':'); - if (!empty($entry->flags)) $po[] = PO::comment_block(implode("\n", $entry->flags), ','); + if (!empty($entry->flags)) $po[] = PO::comment_block(implode(", ", $entry->flags), ','); if (!is_null($entry->context)) $po[] = 'msgctxt '.PO::poify($entry->context); $po[] = 'msgid '.PO::poify($entry->singular); if (!$entry->is_plural) { @@ -173,5 +200,161 @@ class PO extends Translations { return implode("\n", $po); } + function import_from_file($filename) { + $f = fopen($filename, 'r'); + if (!$f) return false; + $lineno = 0; + while (true) { + $res = $this->read_entry($f, $lineno); + if (!$res) break; + if ($res['entry']->singular == '') { + $this->set_headers($this->make_headers($res['entry']->translations[0])); + } else { + $this->add_entry($res['entry']); + } + } + PO::read_line($f, 'clear'); + return $res !== false; + } + + function read_entry($f, $lineno = 0) { + $entry = new Translation_Entry(); + // where were we in the last step + // can be: comment, msgctxt, msgid, msgid_plural, msgstr, msgstr_plural + $context = ''; + $msgstr_index = 0; + $is_final = create_function('$context', 'return $context == "msgstr" || $context == "msgstr_plural";'); + while (true) { + $lineno++; + $line = PO::read_line($f); + if (!$line) { + if (feof($f)) { + if ($is_final($context)) + break; + elseif (!$context) // we haven't read a line and eof came + return null; + else + return false; + } else { + return false; + } + } + if ($line == "\n") continue; + $line = trim($line); + if (preg_match('/^#/', $line, $m)) { + // the comment is the start of a new entry + if ($is_final($context)) { + PO::read_line($f, 'put-back'); + $lineno--; + break; + } + // comments have to be at the beginning + if ($context && $context != 'comment') { + return false; + } + // add comment + $this->add_comment_to_entry($entry, $line);; + } elseif (preg_match('/^msgctxt\s+(".*")/', $line, $m)) { + if ($is_final($context)) { + PO::read_line($f, 'put-back'); + $lineno--; + break; + } + if ($context && $context != 'comment') { + return false; + } + $context = 'msgctxt'; + $entry->context .= PO::unpoify($m[1]); + } elseif (preg_match('/^msgid\s+(".*")/', $line, $m)) { + if ($is_final($context)) { + PO::read_line($f, 'put-back'); + $lineno--; + break; + } + if ($context && $context != 'msgctxt' && $context != 'comment') { + return false; + } + $context = 'msgid'; + $entry->singular .= PO::unpoify($m[1]); + } elseif (preg_match('/^msgid_plural\s+(".*")/', $line, $m)) { + if ($context != 'msgid') { + return false; + } + $context = 'msgid_plural'; + $entry->is_plural = true; + $entry->plural .= PO::unpoify($m[1]); + } elseif (preg_match('/^msgstr\s+(".*")/', $line, $m)) { + if ($context != 'msgid') { + return false; + } + $context = 'msgstr'; + $entry->translations = array(PO::unpoify($m[1])); + } elseif (preg_match('/^msgstr\[(\d+)\]\s+(".*")/', $line, $m)) { + if ($context != 'msgid_plural' && $context != 'msgstr_plural') { + return false; + } + $context = 'msgstr_plural'; + $msgstr_index = $m[1]; + $entry->translations[$m[1]] = PO::unpoify($m[2]); + } elseif (preg_match('/^".*"$/', $line)) { + $unpoified = PO::unpoify($line); + switch ($context) { + case 'msgid': + $entry->singular .= $unpoified; break; + case 'msgctxt': + $entry->context .= $unpoified; break; + case 'msgid_plural': + $entry->plural .= $unpoified; break; + case 'msgstr': + $entry->translations[0] .= $unpoified; break; + case 'msgstr_plural': + $entry->translations[$msgstr_index] .= $unpoified; break; + default: + return false; + } + } else { + return false; + } + } + if (array() == array_filter($entry->translations)) $entry->translations = array(); + return array('entry' => $entry, 'lineno' => $lineno); + } + + function read_line($f, $action = 'read') { + static $last_line = ''; + static $use_last_line = false; + if ('clear' == $action) { + $last_line = ''; + return true; + } + if ('put-back' == $action) { + $use_last_line = true; + return true; + } + $line = $use_last_line? $last_line : fgets($f); + $last_line = $line; + $use_last_line = false; + return $line; + } + + function add_comment_to_entry(&$entry, $po_comment_line) { + $first_two = substr($po_comment_line, 0, 2); + $comment = trim(substr($po_comment_line, 2)); + if ('#:' == $first_two) { + $entry->references = array_merge($entry->references, preg_split('/\s+/', $comment)); + } elseif ('#.' == $first_two) { + $entry->extracted_comments = trim($entry->extracted_comments . "\n" . $comment); + } elseif ('#,' == $first_two) { + $entry->flags = array_merge($entry->flags, preg_split('/,\s*/', $comment)); + } else { + $entry->translator_comments = trim($entry->translator_comments . "\n" . $comment); + } + } + + function trim_quotes($s) { + if ( substr($s, 0, 1) == '"') $s = substr($s, 1); + if ( substr($s, -1, 1) == '"') $s = substr($s, 0, -1); + return $s; + } } ?> diff --git a/wp-includes/pomo/streams.php b/wp-includes/pomo/streams.php index 3b0241af4c..6710746e8f 100644 --- a/wp-includes/pomo/streams.php +++ b/wp-includes/pomo/streams.php @@ -3,7 +3,7 @@ * Classes, which help reading streams of data from files. * Based on the classes from Danilo Segan * - * @version $Id: streams.php 33 2009-02-16 09:33:39Z nbachiyski $ + * @version $Id: streams.php 138 2009-06-23 13:22:09Z nbachiyski $ * @package pomo * @subpackage streams */ @@ -17,34 +17,48 @@ class POMO_StringReader { var $_pos; var $_str; - function POMO_StringReader($str = '') { - $this->_str = $str; - $this->_pos = 0; - } + function POMO_StringReader($str = '') { + $this->_str = $str; + $this->_pos = 0; + $this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr'); + } - function read($bytes) { - $data = substr($this->_str, $this->_pos, $bytes); - $this->_pos += $bytes; - if (strlen($this->_str)<$this->_pos) - $this->_pos = strlen($this->_str); + function _substr($string, $start, $length) { + if ($this->is_overloaded) { + return mb_substr($string,$start,$length,'ascii'); + } else { + return substr($string,$start,$length); + } + } + + function _strlen($string) { + if ($this->is_overloaded) { + return mb_strlen($string,'ascii'); + } else { + return strlen($string); + } + } - return $data; - } + function read($bytes) { + $data = $this->_substr($this->_str, $this->_pos, $bytes); + $this->_pos += $bytes; + if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str); + return $data; + } - function seekto($pos) { - $this->_pos = $pos; - if (strlen($this->_str)<$this->_pos) - $this->_pos = strlen($this->_str); - return $this->_pos; - } + function seekto($pos) { + $this->_pos = $pos; + if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str); + return $this->_pos; + } - function pos() { - return $this->_pos; - } + function pos() { + return $this->_pos; + } - function length() { - return strlen($this->_str); - } + function length() { + return $this->_strlen($this->_str); + } } @@ -53,10 +67,11 @@ class POMO_StringReader { */ class POMO_CachedFileReader extends POMO_StringReader { function POMO_CachedFileReader($filename) { + parent::POMO_StringReader(); $this->_str = file_get_contents($filename); if (false === $this->_str) return false; - $this->pos = 0; + $this->_pos = 0; } } @@ -96,7 +111,7 @@ class POMO_CachedIntFileReader extends POMO_CachedFileReader { */ function readint32() { $bytes = $this->read(4); - if (4 != strlen($bytes)) + if (4 != $this->_strlen($bytes)) return false; $endian_letter = ('big' == $this->endian)? 'N' : 'V'; $int = unpack($endian_letter, $bytes); @@ -112,7 +127,7 @@ class POMO_CachedIntFileReader extends POMO_CachedFileReader { */ function readint32array($count) { $bytes = $this->read(4 * $count); - if (4*$count != strlen($bytes)) + if (4*$count != $this->_strlen($bytes)) return false; $endian_letter = ('big' == $this->endian)? 'N' : 'V'; return unpack($endian_letter.$count, $bytes); diff --git a/wp-includes/pomo/translations.php b/wp-includes/pomo/translations.php index 0d73ee978c..a05ae6ceb2 100644 --- a/wp-includes/pomo/translations.php +++ b/wp-includes/pomo/translations.php @@ -2,7 +2,7 @@ /** * Class for a set of entries for translation and their associated headers * - * @version $Id: translations.php 35 2009-02-16 12:54:57Z nbachiyski $ + * @version $Id: translations.php 114 2009-05-11 17:30:38Z nbachiyski $ * @package pomo * @subpackage translations */ @@ -19,10 +19,13 @@ class Translations { * @param object &$entry * @return bool true on success, false if the entry doesn't have a key */ - function add_entry(&$entry) { + function add_entry($entry) { + if (is_array($entry)) { + $entry = new Translation_Entry($entry); + } $key = $entry->key(); if (false === $key) return false; - $this->entries[$key] = &$entry; + $this->entries[$key] = $entry; return true; } @@ -87,12 +90,24 @@ class Translations { $total_plural_forms = $this->get_plural_forms_count(); if ($translated && 0 <= $index && $index < $total_plural_forms && is_array($translated->translations) && - count($translated->translations) == $total_plural_forms) + isset($translated->translations[$index])) return $translated->translations[$index]; else return 1 == $count? $singular : $plural; } + /** + * Merge $other in the current object. + * + * @param Object &$other Another Translation object, whose translations will be merged in this one + * @return void + **/ + function merge_with(&$other) { + $this->entries = array_merge($this->entries, $other->entries); + } +} + +class Gettext_Translations extends Translations { /** * The gettext implmentation of select_plural_form. * @@ -130,7 +145,7 @@ class Translations { /** * Adds parantheses to the inner parts of ternary operators in * plural expressions, because PHP evaluates ternary oerators from left to right - * + * * @param string $expression the expression without parentheses * @return string the expression with parentheses added */ @@ -158,16 +173,27 @@ class Translations { } return rtrim($res, ';'); } - - /** - * Merge $other in the current object. - * - * @param Object &$other Another Translation object, whose translations will be merged in this one - * @return void - **/ - function merge_with(&$other) { - $this->entries = array_merge($this->entries, $other->entries); + + function make_headers($translation) { + $headers = array(); + // sometimes \ns are used instead of real new lines + $translation = str_replace('\n', "\n", $translation); + $lines = explode("\n", $translation); + foreach($lines as $line) { + $parts = explode(':', $line, 2); + if (!isset($parts[1])) continue; + $headers[trim($parts[0])] = trim($parts[1]); + } + return $headers; } + + function set_header($header, $value) { + parent::set_header($header, $value); + if ('Plural-Forms' == $header) + $this->_gettext_select_plural_form = $this->_make_gettext_select_plural_form($value); + } + + } ?>