Merge latest pomo. Works around mbstring.func_overload. Props nbachiyski. fixes #10236 for trunk

git-svn-id: http://svn.automattic.com/wordpress/trunk@11626 1a063a9b-81f0-0310-95a4-ce76da25c4cd

wp-includes/pomo/entry.php

@@ -2,7 +2,7 @@
 /**
  * Contains Translation_Entry class
  *
- * @version $Id: entry.php 13 2008-04-21 12:03:37Z nbachiyski $
+ * @version $Id: entry.php 115 2009-05-11 18:56:15Z nbachiyski $
  * @package pomo
  * @subpackage entry
  */
@@ -48,9 +48,7 @@ class Translation_Entry {
 		// get member variable values from args hash
 		$object_varnames = array_keys(get_object_vars($this));
 		foreach ($args as $varname => $value) {
-			if (in_array($varname, $object_varnames)) {
+			$this->$varname = $value;
-				$this->$varname = $value;
-			}
 		}
 		if (isset($args['plural'])) $this->is_plural = true;
 		if (!is_array($this->translations)) $this->translations = array();

wp-includes/pomo/mo.php

@@ -2,7 +2,7 @@
 /**
  * Class for working with MO files
  *
- * @version $Id: mo.php 33 2009-02-16 09:33:39Z nbachiyski $
+ * @version $Id: mo.php 106 2009-04-23 19:48:22Z nbachiyski $
  * @package pomo
  * @subpackage mo
  */
@@ -10,16 +10,10 @@
 require_once dirname(__FILE__) . '/translations.php';
 require_once dirname(__FILE__) . '/streams.php';
 
-class MO extends Translations {
+class MO extends Gettext_Translations {
 
 	var $_nplurals = 2;
 
-	function set_header($header, $value) {
-		parent::set_header($header, $value);
-		if ('Plural-Forms' == $header)
-			$this->_gettext_select_plural_form = $this->_make_gettext_select_plural_form($value);
-	}
-
 	/**
 	 * Fills up with the entries from MO file $filename
 	 *
@@ -33,6 +27,73 @@ class MO extends Translations {
 		return $this->import_from_reader($reader);
 	}
 	
+	function export_to_file($filename) {
+		$fh = fopen($filename, 'wb');
+		if ( !$fh ) return false;
+		$entries = array_filter($this->entries, create_function('$e', 'return !empty($e->translations);'));
+		ksort($entries);
+		$magic = 0x950412de;
+		$revision = 0;
+		$total = count($entries) + 1; // all the headers are one entry
+		$originals_lenghts_addr = 28;
+		$translations_lenghts_addr = $originals_lenghts_addr + 8 * $total;
+		$size_of_hash = 0;
+		$hash_addr = $translations_lenghts_addr + 8 * $total;
+		$current_addr = $hash_addr;
+		fwrite($fh, pack('V*', $magic, $revision, $total, $originals_lenghts_addr,
+			$translations_lenghts_addr, $size_of_hash, $hash_addr));
+		fseek($fh, $originals_lenghts_addr);
+		
+		// headers' msgid is an empty string
+		fwrite($fh, pack('VV', 0, $current_addr));
+		$current_addr++;
+		$originals_table = chr(0);
+
+		foreach($entries as $entry) {
+			$originals_table .= $this->export_original($entry) . chr(0);
+			$length = strlen($this->export_original($entry));
+			fwrite($fh, pack('VV', $length, $current_addr));
+			$current_addr += $length + 1; // account for the NULL byte after
+		}
+		
+		$exported_headers = $this->export_headers();
+		fwrite($fh, pack('VV', strlen($exported_headers), $current_addr));
+		$current_addr += strlen($exported_headers) + 1;
+		$translations_table = $exported_headers . chr(0);
+		
+		foreach($entries as $entry) {
+			$translations_table .= $this->export_translations($entry) . chr(0);
+			$length = strlen($this->export_translations($entry));
+			fwrite($fh, pack('VV', $length, $current_addr));
+			$current_addr += $length + 1;
+		}
+		
+		fwrite($fh, $originals_table);
+		fwrite($fh, $translations_table);
+		fclose($fh);
+	}
+	
+	function export_original($entry) {
+		//TODO: warnings for control characters
+		$exported = $entry->singular;
+		if ($entry->is_plural) $exported .= chr(0).$entry->plural;
+		if (!is_null($entry->context)) $exported = $entry->context . chr(4) . $exported;
+		return $exported;
+	}
+	
+	function export_translations($entry) {
+		//TODO: warnings for control characters
+		return implode(chr(0), $entry->translations);
+	}
+	
+	function export_headers() {
+		$exported = '';
+		foreach($this->headers as $header => $value) {
+			$exported.= "$header: $value\n";
+		}
+		return $exported;
+	}
+
 	function get_byteorder($magic) {
 
 		// The magic is 0x950412de
@@ -63,22 +124,22 @@ class MO extends Translations {
 		$revision = $reader->readint32();
 		$total = $reader->readint32();
 		// get addresses of array of lenghts and offsets for original string and translations
-		$originals_lo_addr = $reader->readint32();
+		$originals_lenghts_addr = $reader->readint32();
-		$translations_lo_addr = $reader->readint32();
+		$translations_lenghts_addr = $reader->readint32();
 
-		$reader->seekto($originals_lo_addr);
+		$reader->seekto($originals_lenghts_addr);
-		$originals_lo = $reader->readint32array($total * 2); // each of
+		$originals_lenghts = $reader->readint32array($total * 2); // each of 
-		$reader->seekto($translations_lo_addr);
+		$reader->seekto($translations_lenghts_addr);
-		$translations_lo = $reader->readint32array($total * 2);
+		$translations_lenghts = $reader->readint32array($total * 2);
 
 		$length = create_function('$i', 'return $i * 2 + 1;');
 		$offset = create_function('$i', 'return $i * 2 + 2;');
 
 		for ($i = 0; $i < $total; ++$i) {
-			$reader->seekto($originals_lo[$offset($i)]);
+			$reader->seekto($originals_lenghts[$offset($i)]);
-			$original = $reader->read($originals_lo[$length($i)]);
+			$original = $reader->read($originals_lenghts[$length($i)]);
-			$reader->seekto($translations_lo[$offset($i)]);
+			$reader->seekto($translations_lenghts[$offset($i)]);
-			$translation = $reader->read($translations_lo[$length($i)]);
+			$translation = $reader->read($translations_lenghts[$length($i)]);
 			if ('' == $original) {
 				$this->set_headers($this->make_headers($translation));
 			} else {
@@ -88,17 +149,6 @@ class MO extends Translations {
 		return true;
 	}
 
-	function make_headers($translation) {
-		$headers = array();
-		$lines = explode("\n", $translation);
-		foreach($lines as $line) {
-			$parts = explode(':', $line, 2);
-			if (!isset($parts[1])) continue;
-			$headers[trim($parts[0])] = trim($parts[1]);
-		}
-		return $headers;
-	}
-
 	/**
 	 * @static
 	 */

wp-includes/pomo/po.php

@@ -2,7 +2,7 @@
 /**
  * Class for working with PO files
  *
- * @version $Id: po.php 33 2009-02-16 09:33:39Z nbachiyski $
+ * @version $Id: po.php 123 2009-05-13 19:35:43Z nbachiyski $
  * @package pomo
  * @subpackage po
  */
@@ -16,7 +16,7 @@ ini_set('auto_detect_line_endings', 1);
 /**
  * Routines for working with PO files
  */
-class PO extends Translations {
+class PO extends Gettext_Translations {
 	
 
 	/**
@@ -75,7 +75,6 @@ class PO extends Translations {
 		return fclose($fh);
 	}
 
-
 	/**
 	 * Formats a string in PO-style
 	 *
@@ -87,22 +86,19 @@ class PO extends Translations {
 		$quote = '"';
 		$slash = '\\';
 		$newline = "\n";
-		$tab = "\t";
 
 		$replaces = array(
 			"$slash" 	=> "$slash$slash",
-			"$tab" 		=> '\t',
 			"$quote"	=> "$slash$quote",
+			"\t" 		=> '\t',
 		);
+
 		$string = str_replace(array_keys($replaces), array_values($replaces), $string);
 
-		$po = array();
+		$po = $quote.implode("${slash}n$quote$newline$quote", explode($newline, $string)).$quote;
-		foreach (explode($newline, $string) as $line) {
-			$po[] = wordwrap($line, PO_MAX_LINE_LEN - 2, " $quote$newline$quote");
-		}
-		$po = $quote.implode("${slash}n$quote$newline$quote", $po).$quote;
 		// add empty string on first line for readbility
-		if (false !== strpos($po, $newline)) {
+		if (false !== strpos($string, $newline) &&
+				(substr_count($string, $newline) > 1 || !($newline === substr($string, -strlen($newline))))) {
 			$po = "$quote$quote$newline$po";
 		}
 		// remove empty strings
@@ -110,6 +106,37 @@ class PO extends Translations {
 		return $po;
 	}
 	
+	/**
+	 * Gives back the original string from a PO-formatted string
+	 * 
+	 * @static
+	 * @param string $string PO-formatted string
+	 * @return string enascaped string
+	 */
+	function unpoify($string) {
+		$escapes = array('t' => "\t", 'n' => "\n", '\\' => '\\');
+		$lines = array_map('trim', explode("\n", $string));
+		$lines = array_map(array('PO', 'trim_quotes'), $lines);
+		$unpoified = '';
+		$previous_is_backslash = false;
+		foreach($lines as $line) {
+			preg_match_all('/./u', $line, $chars);
+			$chars = $chars[0];
+			foreach($chars as $char) {
+				if (!$previous_is_backslash) {
+					if ('\\' == $char)
+						$previous_is_backslash = true;
+					else
+						$unpoified .= $char;
+				} else {
+					$previous_is_backslash = false;
+					$unpoified .= isset($escapes[$char])? $escapes[$char] : $char;
+				}
+			}
+		}
+		return $unpoified;
+	}
+
 	/**
 	 * Inserts $with in the beginning of every new line of $string and 
 	 * returns the modified string
@@ -157,7 +184,7 @@ class PO extends Translations {
 		if (!empty($entry->translator_comments)) $po[] = PO::comment_block($entry->translator_comments);
 		if (!empty($entry->extracted_comments)) $po[] = PO::comment_block($entry->extracted_comments, '.');
 		if (!empty($entry->references)) $po[] = PO::comment_block(implode(' ', $entry->references), ':');
-		if (!empty($entry->flags)) $po[] = PO::comment_block(implode("\n", $entry->flags), ',');
+		if (!empty($entry->flags)) $po[] = PO::comment_block(implode(", ", $entry->flags), ',');
 		if (!is_null($entry->context)) $po[] = 'msgctxt '.PO::poify($entry->context);
 		$po[] = 'msgid '.PO::poify($entry->singular);
 		if (!$entry->is_plural) {
@@ -173,5 +200,161 @@ class PO extends Translations {
 		return implode("\n", $po);
 	}
 
+	function import_from_file($filename) {
+		$f = fopen($filename, 'r');
+		if (!$f) return false;
+		$lineno = 0;
+		while (true) {
+			$res = $this->read_entry($f, $lineno);
+			if (!$res) break;
+			if ($res['entry']->singular == '') {
+				$this->set_headers($this->make_headers($res['entry']->translations[0]));
+			} else {
+				$this->add_entry($res['entry']);
+			}
+		}
+		PO::read_line($f, 'clear');
+		return $res !== false;
+	}
+	
+	function read_entry($f, $lineno = 0) {
+		$entry = new Translation_Entry();
+		// where were we in the last step
+		// can be: comment, msgctxt, msgid, msgid_plural, msgstr, msgstr_plural
+		$context = '';
+		$msgstr_index = 0;
+		$is_final = create_function('$context', 'return $context == "msgstr" || $context == "msgstr_plural";');
+		while (true) {
+			$lineno++;
+			$line = PO::read_line($f);
+			if (!$line)  {
+				if (feof($f)) {
+					if ($is_final($context))
+						break;
+					elseif (!$context) // we haven't read a line and eof came
+						return null;
+					else
+						return false;
+				} else {
+					return false;
+				}
+			}
+			if ($line == "\n") continue;
+			$line = trim($line);
+			if (preg_match('/^#/', $line, $m)) {
+				// the comment is the start of a new entry
+				if ($is_final($context)) {
+					PO::read_line($f, 'put-back');
+					$lineno--;
+					break;
+				}
+				// comments have to be at the beginning
+				if ($context && $context != 'comment') {
+					return false;
+				}
+				// add comment
+				$this->add_comment_to_entry($entry, $line);;
+			} elseif (preg_match('/^msgctxt\s+(".*")/', $line, $m)) {
+				if ($is_final($context)) {
+					PO::read_line($f, 'put-back');
+					$lineno--;
+					break;
+				}
+				if ($context && $context != 'comment') {
+					return false;
+				}
+				$context = 'msgctxt';
+				$entry->context .= PO::unpoify($m[1]);
+			} elseif (preg_match('/^msgid\s+(".*")/', $line, $m)) {
+				if ($is_final($context)) {
+					PO::read_line($f, 'put-back');
+					$lineno--;
+					break;
+				}
+				if ($context && $context != 'msgctxt' && $context != 'comment') {
+					return false;
+				}
+				$context = 'msgid';
+				$entry->singular .= PO::unpoify($m[1]);
+			} elseif (preg_match('/^msgid_plural\s+(".*")/', $line, $m)) {
+				if ($context != 'msgid') {
+					return false;
+				}
+				$context = 'msgid_plural';
+				$entry->is_plural = true;
+				$entry->plural .= PO::unpoify($m[1]);
+			} elseif (preg_match('/^msgstr\s+(".*")/', $line, $m)) {
+				if ($context != 'msgid') {
+					return false;
+				}
+				$context = 'msgstr';
+				$entry->translations = array(PO::unpoify($m[1]));
+			} elseif (preg_match('/^msgstr\[(\d+)\]\s+(".*")/', $line, $m)) {
+				if ($context != 'msgid_plural' && $context != 'msgstr_plural') {
+					return false;
+				}
+				$context = 'msgstr_plural';
+				$msgstr_index = $m[1];
+				$entry->translations[$m[1]] = PO::unpoify($m[2]);
+			} elseif (preg_match('/^".*"$/', $line)) {
+				$unpoified = PO::unpoify($line);
+				switch ($context) {
+					case 'msgid':
+						$entry->singular .= $unpoified; break;
+					case 'msgctxt':
+						$entry->context .= $unpoified; break;
+					case 'msgid_plural':
+						$entry->plural .= $unpoified; break;
+					case 'msgstr':
+						$entry->translations[0] .= $unpoified; break;
+					case 'msgstr_plural':
+						$entry->translations[$msgstr_index] .= $unpoified; break;
+					default:
+						return false;
+				}
+			} else {
+				return false;
+			}
+		}
+		if (array() == array_filter($entry->translations)) $entry->translations = array();
+		return array('entry' => $entry, 'lineno' => $lineno);
+	}
+	
+	function read_line($f, $action = 'read') {
+		static $last_line = '';
+		static $use_last_line = false;
+		if ('clear' == $action) {
+			$last_line = '';
+			return true;
+		}
+		if ('put-back' == $action) {
+			$use_last_line = true;
+			return true;
+		}
+		$line = $use_last_line? $last_line : fgets($f);
+		$last_line = $line;
+		$use_last_line = false;
+		return $line;
+	}
+	
+	function add_comment_to_entry(&$entry, $po_comment_line) {
+		$first_two = substr($po_comment_line, 0, 2);
+		$comment = trim(substr($po_comment_line, 2));
+		if ('#:' == $first_two) {
+			$entry->references = array_merge($entry->references, preg_split('/\s+/', $comment));
+		} elseif ('#.' == $first_two) {
+			$entry->extracted_comments = trim($entry->extracted_comments . "\n" . $comment);
+		} elseif ('#,' == $first_two) {
+			$entry->flags = array_merge($entry->flags, preg_split('/,\s*/', $comment));
+		} else {
+			$entry->translator_comments = trim($entry->translator_comments . "\n" . $comment);
+		}
+	}
+	
+	function trim_quotes($s) {
+		if ( substr($s, 0, 1) == '"') $s = substr($s, 1);
+		if ( substr($s, -1, 1) == '"') $s = substr($s, 0, -1);
+		return $s;
+	}
 }
 ?>

wp-includes/pomo/streams.php

@@ -3,7 +3,7 @@
  * Classes, which help reading streams of data from files.
  * Based on the classes from Danilo Segan <danilo@kvota.net>
  *
- * @version $Id: streams.php 33 2009-02-16 09:33:39Z nbachiyski $
+ * @version $Id: streams.php 138 2009-06-23 13:22:09Z nbachiyski $
  * @package pomo
  * @subpackage streams
  */
@@ -17,34 +17,48 @@ class POMO_StringReader {
   var $_pos;
   var $_str;
 
-  function POMO_StringReader($str = '') {
+	function POMO_StringReader($str = '') {
-    $this->_str = $str;
+		$this->_str = $str;
-    $this->_pos = 0;
+		$this->_pos = 0;
-  }
+		$this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr');
+	}
 
-  function read($bytes) {
+	function _substr($string, $start, $length) {
-    $data = substr($this->_str, $this->_pos, $bytes);
+		if ($this->is_overloaded) {
-    $this->_pos += $bytes;
+			return mb_substr($string,$start,$length,'ascii');
-    if (strlen($this->_str)<$this->_pos)
+		} else {
-      $this->_pos = strlen($this->_str);
+			return substr($string,$start,$length);
+		}
+	}
 	
-    return $data;
+	function _strlen($string) {
-  }
+		if ($this->is_overloaded) {
+			return mb_strlen($string,'ascii');
+		} else {
+			return strlen($string);
+		}
+	}
 
-  function seekto($pos) {
+	function read($bytes) {
-    $this->_pos = $pos;
+		$data = $this->_substr($this->_str, $this->_pos, $bytes);
-    if (strlen($this->_str)<$this->_pos)
+		$this->_pos += $bytes;
-      $this->_pos = strlen($this->_str);
+		if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
-    return $this->_pos;
+		return $data;
-  }
+	}
 
-  function pos() {
+	function seekto($pos) {
-    return $this->_pos;
+		$this->_pos = $pos;
-  }
+		if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
+		return $this->_pos;
+	}
 
-  function length() {
+	function pos() {
-    return strlen($this->_str);
+		return $this->_pos;
-  }
+	}
+
+	function length() {
+		return $this->_strlen($this->_str);
+	}
 
 }
 
@@ -53,10 +67,11 @@ class POMO_StringReader {
  */
 class POMO_CachedFileReader extends POMO_StringReader {
 	function POMO_CachedFileReader($filename) {
+		parent::POMO_StringReader();
 		$this->_str = file_get_contents($filename);
 		if (false === $this->_str)
 			return false;
-		$this->pos = 0;
+		$this->_pos = 0;
 	}
 }
 
@@ -96,7 +111,7 @@ class POMO_CachedIntFileReader extends POMO_CachedFileReader {
 	 */
 	function readint32() {
 		$bytes = $this->read(4);
-		if (4 != strlen($bytes))
+		if (4 != $this->_strlen($bytes))
 			return false;
 		$endian_letter = ('big' == $this->endian)? 'N' : 'V';
 		$int = unpack($endian_letter, $bytes);
@@ -112,7 +127,7 @@ class POMO_CachedIntFileReader extends POMO_CachedFileReader {
 	 */
 	function readint32array($count) {
 		$bytes = $this->read(4 * $count);
-		if (4*$count != strlen($bytes))
+		if (4*$count != $this->_strlen($bytes))
 			return false;
 		$endian_letter = ('big' == $this->endian)? 'N' : 'V';
 		return unpack($endian_letter.$count, $bytes);

wp-includes/pomo/translations.php

@@ -2,7 +2,7 @@
 /**
  * Class for a set of entries for translation and their associated headers
  *
- * @version $Id: translations.php 35 2009-02-16 12:54:57Z nbachiyski $
+ * @version $Id: translations.php 114 2009-05-11 17:30:38Z nbachiyski $
  * @package pomo
  * @subpackage translations
  */
@@ -19,10 +19,13 @@ class Translations {
 	 * @param object &$entry
 	 * @return bool true on success, false if the entry doesn't have a key
 	 */
-	function add_entry(&$entry) {
+	function add_entry($entry) {
+		if (is_array($entry)) {
+			$entry = new Translation_Entry($entry);
+		}
 		$key = $entry->key();
 		if (false === $key) return false;
-		$this->entries[$key] = &$entry;
+		$this->entries[$key] = $entry;
 		return true;
 	}
 
@@ -87,12 +90,24 @@ class Translations {
 		$total_plural_forms = $this->get_plural_forms_count();
 		if ($translated && 0 <= $index && $index < $total_plural_forms &&
 				is_array($translated->translations) &&
-				count($translated->translations) == $total_plural_forms)
+				isset($translated->translations[$index]))
 			return $translated->translations[$index];
 		else
 			return 1 == $count? $singular : $plural;
 	}
 
+	/**
+	 * Merge $other in the current object.
+	 *
+	 * @param Object &$other Another Translation object, whose translations will be merged in this one
+	 * @return void
+	 **/
+	function merge_with(&$other) {
+		$this->entries = array_merge($this->entries, $other->entries);
+	}
+}
+
+class Gettext_Translations extends Translations {
 	/**
 	 * The gettext implmentation of select_plural_form.
 	 *
@@ -159,15 +174,26 @@ class Translations {
 		return rtrim($res, ';');
 	}
 	
-	/**
+	function make_headers($translation) {
-	 * Merge $other in the current object.
+		$headers = array();
-	 *
+		// sometimes \ns are used instead of real new lines
-	 * @param Object &$other Another Translation object, whose translations will be merged in this one
+		$translation = str_replace('\n', "\n", $translation);
-	 * @return void
+		$lines = explode("\n", $translation);
-	 **/
+		foreach($lines as $line) {
-	function merge_with(&$other) {
+			$parts = explode(':', $line, 2);
-		$this->entries = array_merge($this->entries, $other->entries);
+			if (!isset($parts[1])) continue;
+			$headers[trim($parts[0])] = trim($parts[1]);
+		}
+		return $headers;
 	}
+
+	function set_header($header, $value) {
+		parent::set_header($header, $value);
+		if ('Plural-Forms' == $header)
+			$this->_gettext_select_plural_form = $this->_make_gettext_select_plural_form($value);
+	}
+
+	
 }
 
 ?>