First run of introducing Stream-To-File for the WP_HTTP API. Reduces memory consumption during file downloads. Implemented in download_url() for upgraders. Props sivel. See #16236

git-svn-id: http://svn.automattic.com/wordpress/trunk@17555 1a063a9b-81f0-0310-95a4-ce76da25c4cd

wp-admin/includes/file.php

@@ -152,42 +152,6 @@ function list_files( $folder = '', $levels = 100 ) {
 	return $files;
 }
 
-/**
- * Determines a writable directory for temporary files.
- * Function's preference is to WP_CONTENT_DIR followed by the return value of <code>sys_get_temp_dir()</code>, before finally defaulting to /tmp/
- *
- * In the event that this function does not find a writable location, It may be overridden by the <code>WP_TEMP_DIR</code> constant in your <code>wp-config.php</code> file.
- *
- * @since 2.5.0
- *
- * @return string Writable temporary directory
- */
-function get_temp_dir() {
-	static $temp;
-	if ( defined('WP_TEMP_DIR') )
-		return trailingslashit(WP_TEMP_DIR);
-
-	if ( $temp )
-		return trailingslashit($temp);
-
-	$temp = WP_CONTENT_DIR . '/';
-	if ( is_dir($temp) && @is_writable($temp) )
-		return $temp;
-
-	if  ( function_exists('sys_get_temp_dir') ) {
-		$temp = sys_get_temp_dir();
-		if ( @is_writable($temp) )
-			return trailingslashit($temp);
-	}
-
-	$temp = ini_get('upload_tmp_dir');
-	if ( is_dir($temp) && @is_writable($temp) )
-		return trailingslashit($temp);
-
-	$temp = '/tmp/';
-	return $temp;
-}
-
 /**
  * Returns a filename of a Temporary unique file.
  * Please note that the calling function must unlink() this itself.
@@ -519,27 +483,18 @@ function download_url( $url, $timeout = 300 ) {
 	if ( ! $tmpfname )
 		return new WP_Error('http_no_file', __('Could not create Temporary file.'));
 
-	$handle = @fopen($tmpfname, 'wb');
+	$response = wp_remote_get( $url, array( 'timeout' => $timeout, 'stream' => true, 'filename' => $tmpfname ) );
-	if ( ! $handle )
-		return new WP_Error('http_no_file', __('Could not create Temporary file.'));
 
-	$response = wp_remote_get($url, array('timeout' => $timeout));
+	if ( is_wp_error( $response ) ) {
-
+		unlink( $tmpfname );
-	if ( is_wp_error($response) ) {
-		fclose($handle);
-		unlink($tmpfname);
 		return $response;
 	}
 
 	if ( $response['response']['code'] != '200' ){
-		fclose($handle);
+		unlink( $tmpfname );
-		unlink($tmpfname);
+		return new WP_Error( 'http_404', trim( $response['response']['message'] ) );
-		return new WP_Error('http_404', trim($response['response']['message']));
 	}
 
-	fwrite($handle, $response['body']);
-	fclose($handle);
-
 	return $tmpfname;
 }
 

wp-includes/class-http.php

@@ -97,7 +97,9 @@ class WP_Http {
 			'body' => null,
 			'compress' => false,
 			'decompress' => true,
-			'sslverify' => true
+			'sslverify' => true,
+			'stream' => false,
+			'filename' => null
 		);
 
 		
@@ -136,6 +138,18 @@ class WP_Http {
 		$r['local'] = $homeURL['host'] == $arrURL['host'] || 'localhost' == $arrURL['host'];
 		unset( $homeURL );
 
+		// If we are streaming to a file but no filename was given drop it in the WP temp dir
+		// and pick it's name using the basename of the $url
+		if ( $r['stream']  && empty( $r['filename'] ) )
+			$r['filename'] = get_temp_dir() . basename( $url );
+
+		// Force some settings if we are streaming to a file and check for existence and perms of destination directory
+		if ( $r['stream'] ) {
+			$r['blocking'] = true;
+			if ( ! is_writable( dirname( $r['filename'] ) ) )
+				return new WP_Error( 'http_request_failed', __( 'Destination directory for file streaming does not exist or is not writable.' ) );
+		}
+
 		if ( is_null( $r['headers'] ) )
 			$r['headers'] = array();
 
@@ -659,16 +673,49 @@ class WP_Http_Fsockopen {
 		}
 
 		$strResponse = '';
-		while ( ! feof($handle) )
+		$bodyStarted = false;
-			$strResponse .= fread($handle, 4096);
 
-		fclose($handle);
+		// If streaming to a file setup the file handle
+		if ( $r['stream'] ) {
+			if ( ! WP_DEBUG )
+				$stream_handle = @fopen( $r['filename'], 'w+' );
+			else
+				$stream_handle = fopen( $r['filename'], 'w+' );
+			if ( ! $stream_handle )
+				return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+			while ( ! feof($handle) ) {
+				$block = fread( $handle, 4096 );
+				if ( $bodyStarted ) {
+					fwrite( $stream_handle, $block );
+				} else {
+					$strResponse .= $block;
+					if ( strpos( $strResponse, "\r\n\r\n" ) ) {
+						$process = WP_Http::processResponse( $strResponse );
+						$bodyStarted = true;
+						fwrite( $stream_handle, $process['body'] );
+						unset( $strResponse );
+						$process['body'] = '';
+					}
+				}
+			}
+
+			fclose( $stream_handle );
+
+		} else { 
+			while ( ! feof($handle) )
+				$strResponse .= fread( $handle, 4096 );
+
+			$process = WP_Http::processResponse( $strResponse );
+			unset( $strResponse );
+		}
+
+		fclose( $handle );
 
 		if ( true === $secure_transport )
 			error_reporting($error_reporting);
 
-		$process = WP_Http::processResponse($strResponse);
+		$arrHeaders = WP_Http::processHeaders( $process['headers'] );
-		$arrHeaders = WP_Http::processHeaders($process['headers']);
 
 		// Is the response code within the 400 range?
 		if ( (int) $arrHeaders['response']['code'] >= 400 && (int) $arrHeaders['response']['code'] < 500 )
@@ -690,7 +737,7 @@ class WP_Http_Fsockopen {
 		if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($arrHeaders['headers']) )
 			$process['body'] = WP_Http_Encoding::decompress( $process['body'] );
 
-		return array('headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies']);
+		return array( 'headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies'], 'filename' => $r['filename'] );
 	}
 
 	/**
@@ -834,10 +881,26 @@ class WP_Http_Streams {
 			return array( 'headers' => array(), 'body' => '', 'response' => array('code' => false, 'message' => false), 'cookies' => array() );
 		}
 
-		$strResponse = stream_get_contents($handle);
+		if ( $r['stream'] ) {
-		$meta = stream_get_meta_data($handle);
+			if ( ! WP_DEBUG )
+				$stream_handle = @fopen( $r['filename'], 'w+' );
+			else
+				$stream_handle = fopen( $r['filename'], 'w+' );
 
-		fclose($handle);
+			if ( ! $stream_handle )
+				return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+			stream_copy_to_stream( $handle, $stream_handle );
+
+			fclose( $stream_handle );
+			$strResponse = '';
+		} else {
+			$strResponse = stream_get_contents( $handle );
+		}
+
+		$meta = stream_get_meta_data( $handle );
+
+		fclose( $handle );
 
 		$processedHeaders = array();
 		if ( isset( $meta['wrapper_data']['headers'] ) )
@@ -856,7 +919,7 @@ class WP_Http_Streams {
 		if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($processedHeaders['headers']) )
 			$strResponse = WP_Http_Encoding::decompress( $strResponse );
 
-		return array('headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies']);
+		return array( 'headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies'], 'filename' => $r['filename'] );
 	}
 
 	/**
@@ -1006,11 +1069,25 @@ class WP_Http_ExtHttp {
 		if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($theHeaders['headers']) )
 			$theBody = http_inflate( $theBody );
 
+		if ( $r['stream'] ) {
+			if ( !WP_DEBUG )
+				$stream_handle = @fopen( $r['filename'], 'w+' );
+			else
+				$stream_handle = fopen( $r['filename'], 'w+' );
+
+			if ( ! $stream_handle )
+				return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+			fwrite( $stream_handle, $theBody );
+			fclose( $stream_handle );
+			$theBody = '';
+		}
+
 		$theResponse = array();
 		$theResponse['code'] = $info['response_code'];
 		$theResponse['message'] = get_status_header_desc($info['response_code']);
 
-		return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies']);
+		return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] );
 	}
 
 	/**
@@ -1037,6 +1114,15 @@ class WP_Http_ExtHttp {
  */
 class WP_Http_Curl {
 
+	/**
+	 * Temporary header storage for use with streaming to a file.
+	 *
+	 * @since 3.2.0
+	 * @access private
+	 * @var string
+	 */
+	private $headers;
+
 	/**
 	 * Send a HTTP request to a URI using cURL extension.
 	 *
@@ -1121,9 +1207,20 @@ class WP_Http_Curl {
 		}
 
 		if ( true === $r['blocking'] )
-			curl_setopt( $handle, CURLOPT_HEADER, true );
+			curl_setopt( $handle, CURLOPT_HEADERFUNCTION, array( &$this, 'stream_headers' ) );
-		else
+
-			curl_setopt( $handle, CURLOPT_HEADER, false );
+		curl_setopt( $handle, CURLOPT_HEADER, false );
+
+		// If streaming to a file open a file handle, and setup our curl streaming handler
+		if ( $r['stream'] ) {
+			if ( ! WP_DEBUG )
+				$stream_handle = @fopen( $r['filename'], 'w+' );
+			else
+				$stream_handle = fopen( $r['filename'], 'w+' );
+			if ( ! $stream_handle )
+				return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+			curl_setopt( $handle, CURLOPT_FILE, $stream_handle );
+		}
 
 		// The option doesn't work with safe mode or when open_basedir is set.
 		if ( !ini_get('safe_mode') && !ini_get('open_basedir') && 0 !== $r['_redirection'] )
@@ -1155,48 +1252,58 @@ class WP_Http_Curl {
 		}
 
 		$theResponse = curl_exec( $handle );
+		$theBody = '';
+		$theHeaders = WP_Http::processHeaders( $this->headers );
 
-		if ( !empty($theResponse) ) {
+		if ( ! empty($theResponse) && ! is_bool( $theResponse ) ) // is_bool: when using $args['stream'], curl_exec will return (bool)true
-			$headerLength = curl_getinfo($handle, CURLINFO_HEADER_SIZE);
+			$theBody = $theResponse;
-			$theHeaders = trim( substr($theResponse, 0, $headerLength) );
+
-			if ( strlen($theResponse) > $headerLength )
+		// If no response, and It's not a HEAD request with valid headers returned
-				$theBody = substr( $theResponse, $headerLength );
+		if ( empty($theResponse) && 'HEAD' != $args['method'] && ! empty($this->headers) ) {
-			else
-				$theBody = '';
-			if ( false !== strpos($theHeaders, "\r\n\r\n") ) {
-				$headerParts = explode("\r\n\r\n", $theHeaders);
-				$theHeaders = $headerParts[ count($headerParts) -1 ];
-			}
-			$theHeaders = WP_Http::processHeaders($theHeaders);
-		} else {
 			if ( $curl_error = curl_error($handle) )
 				return new WP_Error('http_request_failed', $curl_error);
 			if ( in_array( curl_getinfo( $handle, CURLINFO_HTTP_CODE ), array(301, 302) ) )
 				return new WP_Error('http_request_failed', __('Too many redirects.'));
-
-			$theHeaders = array( 'headers' => array(), 'cookies' => array() );
-			$theBody = '';
 		}
 
+		unset( $this->headers );
+
 		$response = array();
 		$response['code'] = curl_getinfo( $handle, CURLINFO_HTTP_CODE );
 		$response['message'] = get_status_header_desc($response['code']);
 
 		curl_close( $handle );
 
+		if ( $r['stream'] )
+			fclose( $stream_handle );
+
 		// See #11305 - When running under safe mode, redirection is disabled above. Handle it manually.
-		if ( !empty($theHeaders['headers']['location']) && (ini_get('safe_mode') || ini_get('open_basedir')) && 0 !== $r['_redirection'] ) {
+		if ( ! empty( $theHeaders['headers']['location'] ) && ( ini_get( 'safe_mode' ) || ini_get( 'open_basedir' ) ) && 0 !== $r['_redirection'] ) {
 			if ( $r['redirection']-- > 0 ) {
-				return $this->request($theHeaders['headers']['location'], $r);
+				return $this->request( $theHeaders['headers']['location'], $r );
 			} else {
-				return new WP_Error('http_request_failed', __('Too many redirects.'));
+				return new WP_Error( 'http_request_failed', __( 'Too many redirects.' ) );
 			}
 		}
 
 		if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($theHeaders['headers']) )
 			$theBody = WP_Http_Encoding::decompress( $theBody );
 
-		return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies']);
+		return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] );
+	}
+
+	/**
+	 * Grab the headers of the cURL request
+	 *
+	 * Each header is sent individually to this callback, so we append to the $header property for temporary storage
+	 *
+	 * @since 3.2.0
+	 * @access private
+	 * @return int
+	 */
+	private function stream_headers( $handle, $headers ) {
+		$this->headers .= $headers;
+		return strlen( $headers );
 	}
 
 	/**

wp-includes/functions.php

@@ -2110,6 +2110,42 @@ function path_join( $base, $path ) {
 	return rtrim($base, '/') . '/' . ltrim($path, '/');
 }
 
+/**
+ * Determines a writable directory for temporary files.
+ * Function's preference is to WP_CONTENT_DIR followed by the return value of <code>sys_get_temp_dir()</code>, before finally defaulting to /tmp/
+ *
+ * In the event that this function does not find a writable location, It may be overridden by the <code>WP_TEMP_DIR</code> constant in your <code>wp-config.php</code> file.
+ *
+ * @since 2.5.0
+ *
+ * @return string Writable temporary directory
+ */
+function get_temp_dir() {
+	static $temp;
+	if ( defined('WP_TEMP_DIR') )
+		return trailingslashit(WP_TEMP_DIR);
+
+	if ( $temp )
+		return trailingslashit($temp);
+
+	$temp = WP_CONTENT_DIR . '/';
+	if ( is_dir($temp) && @is_writable($temp) )
+		return $temp;
+
+	if  ( function_exists('sys_get_temp_dir') ) {
+		$temp = sys_get_temp_dir();
+		if ( @is_writable($temp) )
+			return trailingslashit($temp);
+	}
+
+	$temp = ini_get('upload_tmp_dir');
+	if ( is_dir($temp) && @is_writable($temp) )
+		return trailingslashit($temp);
+
+	$temp = '/tmp/';
+	return $temp;
+}
+
 /**
  * Get an array containing the current upload directory's path and url.
  *