From 00dc7a57d634428fe5ff810724b84dd402d6339b Mon Sep 17 00:00:00 2001 From: dd32 Date: Fri, 25 Mar 2011 02:42:20 +0000 Subject: [PATCH] First run of introducing Stream-To-File for the WP_HTTP API. Reduces memory consumption during file downloads. Implemented in download_url() for upgraders. Props sivel. See #16236 git-svn-id: http://svn.automattic.com/wordpress/trunk@17555 1a063a9b-81f0-0310-95a4-ce76da25c4cd --- wp-admin/includes/file.php | 55 ++---------- wp-includes/class-http.php | 177 +++++++++++++++++++++++++++++-------- wp-includes/functions.php | 36 ++++++++ 3 files changed, 183 insertions(+), 85 deletions(-) diff --git a/wp-admin/includes/file.php b/wp-admin/includes/file.php index f330c3fa2f..b851e0824a 100644 --- a/wp-admin/includes/file.php +++ b/wp-admin/includes/file.php @@ -152,42 +152,6 @@ function list_files( $folder = '', $levels = 100 ) { return $files; } -/** - * Determines a writable directory for temporary files. - * Function's preference is to WP_CONTENT_DIR followed by the return value of sys_get_temp_dir(), before finally defaulting to /tmp/ - * - * In the event that this function does not find a writable location, It may be overridden by the WP_TEMP_DIR constant in your wp-config.php file. - * - * @since 2.5.0 - * - * @return string Writable temporary directory - */ -function get_temp_dir() { - static $temp; - if ( defined('WP_TEMP_DIR') ) - return trailingslashit(WP_TEMP_DIR); - - if ( $temp ) - return trailingslashit($temp); - - $temp = WP_CONTENT_DIR . '/'; - if ( is_dir($temp) && @is_writable($temp) ) - return $temp; - - if ( function_exists('sys_get_temp_dir') ) { - $temp = sys_get_temp_dir(); - if ( @is_writable($temp) ) - return trailingslashit($temp); - } - - $temp = ini_get('upload_tmp_dir'); - if ( is_dir($temp) && @is_writable($temp) ) - return trailingslashit($temp); - - $temp = '/tmp/'; - return $temp; -} - /** * Returns a filename of a Temporary unique file. * Please note that the calling function must unlink() this itself. @@ -519,27 +483,18 @@ function download_url( $url, $timeout = 300 ) { if ( ! $tmpfname ) return new WP_Error('http_no_file', __('Could not create Temporary file.')); - $handle = @fopen($tmpfname, 'wb'); - if ( ! $handle ) - return new WP_Error('http_no_file', __('Could not create Temporary file.')); + $response = wp_remote_get( $url, array( 'timeout' => $timeout, 'stream' => true, 'filename' => $tmpfname ) ); - $response = wp_remote_get($url, array('timeout' => $timeout)); - - if ( is_wp_error($response) ) { - fclose($handle); - unlink($tmpfname); + if ( is_wp_error( $response ) ) { + unlink( $tmpfname ); return $response; } if ( $response['response']['code'] != '200' ){ - fclose($handle); - unlink($tmpfname); - return new WP_Error('http_404', trim($response['response']['message'])); + unlink( $tmpfname ); + return new WP_Error( 'http_404', trim( $response['response']['message'] ) ); } - fwrite($handle, $response['body']); - fclose($handle); - return $tmpfname; } diff --git a/wp-includes/class-http.php b/wp-includes/class-http.php index 75f5bd3acd..25be416b6d 100644 --- a/wp-includes/class-http.php +++ b/wp-includes/class-http.php @@ -97,7 +97,9 @@ class WP_Http { 'body' => null, 'compress' => false, 'decompress' => true, - 'sslverify' => true + 'sslverify' => true, + 'stream' => false, + 'filename' => null ); @@ -136,6 +138,18 @@ class WP_Http { $r['local'] = $homeURL['host'] == $arrURL['host'] || 'localhost' == $arrURL['host']; unset( $homeURL ); + // If we are streaming to a file but no filename was given drop it in the WP temp dir + // and pick it's name using the basename of the $url + if ( $r['stream'] && empty( $r['filename'] ) ) + $r['filename'] = get_temp_dir() . basename( $url ); + + // Force some settings if we are streaming to a file and check for existence and perms of destination directory + if ( $r['stream'] ) { + $r['blocking'] = true; + if ( ! is_writable( dirname( $r['filename'] ) ) ) + return new WP_Error( 'http_request_failed', __( 'Destination directory for file streaming does not exist or is not writable.' ) ); + } + if ( is_null( $r['headers'] ) ) $r['headers'] = array(); @@ -659,16 +673,49 @@ class WP_Http_Fsockopen { } $strResponse = ''; - while ( ! feof($handle) ) - $strResponse .= fread($handle, 4096); + $bodyStarted = false; - fclose($handle); + // If streaming to a file setup the file handle + if ( $r['stream'] ) { + if ( ! WP_DEBUG ) + $stream_handle = @fopen( $r['filename'], 'w+' ); + else + $stream_handle = fopen( $r['filename'], 'w+' ); + if ( ! $stream_handle ) + return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) ); + + while ( ! feof($handle) ) { + $block = fread( $handle, 4096 ); + if ( $bodyStarted ) { + fwrite( $stream_handle, $block ); + } else { + $strResponse .= $block; + if ( strpos( $strResponse, "\r\n\r\n" ) ) { + $process = WP_Http::processResponse( $strResponse ); + $bodyStarted = true; + fwrite( $stream_handle, $process['body'] ); + unset( $strResponse ); + $process['body'] = ''; + } + } + } + + fclose( $stream_handle ); + + } else { + while ( ! feof($handle) ) + $strResponse .= fread( $handle, 4096 ); + + $process = WP_Http::processResponse( $strResponse ); + unset( $strResponse ); + } + + fclose( $handle ); if ( true === $secure_transport ) error_reporting($error_reporting); - $process = WP_Http::processResponse($strResponse); - $arrHeaders = WP_Http::processHeaders($process['headers']); + $arrHeaders = WP_Http::processHeaders( $process['headers'] ); // Is the response code within the 400 range? if ( (int) $arrHeaders['response']['code'] >= 400 && (int) $arrHeaders['response']['code'] < 500 ) @@ -690,7 +737,7 @@ class WP_Http_Fsockopen { if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($arrHeaders['headers']) ) $process['body'] = WP_Http_Encoding::decompress( $process['body'] ); - return array('headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies']); + return array( 'headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies'], 'filename' => $r['filename'] ); } /** @@ -834,10 +881,26 @@ class WP_Http_Streams { return array( 'headers' => array(), 'body' => '', 'response' => array('code' => false, 'message' => false), 'cookies' => array() ); } - $strResponse = stream_get_contents($handle); - $meta = stream_get_meta_data($handle); + if ( $r['stream'] ) { + if ( ! WP_DEBUG ) + $stream_handle = @fopen( $r['filename'], 'w+' ); + else + $stream_handle = fopen( $r['filename'], 'w+' ); - fclose($handle); + if ( ! $stream_handle ) + return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) ); + + stream_copy_to_stream( $handle, $stream_handle ); + + fclose( $stream_handle ); + $strResponse = ''; + } else { + $strResponse = stream_get_contents( $handle ); + } + + $meta = stream_get_meta_data( $handle ); + + fclose( $handle ); $processedHeaders = array(); if ( isset( $meta['wrapper_data']['headers'] ) ) @@ -856,7 +919,7 @@ class WP_Http_Streams { if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($processedHeaders['headers']) ) $strResponse = WP_Http_Encoding::decompress( $strResponse ); - return array('headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies']); + return array( 'headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies'], 'filename' => $r['filename'] ); } /** @@ -1006,11 +1069,25 @@ class WP_Http_ExtHttp { if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($theHeaders['headers']) ) $theBody = http_inflate( $theBody ); + if ( $r['stream'] ) { + if ( !WP_DEBUG ) + $stream_handle = @fopen( $r['filename'], 'w+' ); + else + $stream_handle = fopen( $r['filename'], 'w+' ); + + if ( ! $stream_handle ) + return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) ); + + fwrite( $stream_handle, $theBody ); + fclose( $stream_handle ); + $theBody = ''; + } + $theResponse = array(); $theResponse['code'] = $info['response_code']; $theResponse['message'] = get_status_header_desc($info['response_code']); - return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies']); + return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] ); } /** @@ -1037,6 +1114,15 @@ class WP_Http_ExtHttp { */ class WP_Http_Curl { + /** + * Temporary header storage for use with streaming to a file. + * + * @since 3.2.0 + * @access private + * @var string + */ + private $headers; + /** * Send a HTTP request to a URI using cURL extension. * @@ -1121,9 +1207,20 @@ class WP_Http_Curl { } if ( true === $r['blocking'] ) - curl_setopt( $handle, CURLOPT_HEADER, true ); - else - curl_setopt( $handle, CURLOPT_HEADER, false ); + curl_setopt( $handle, CURLOPT_HEADERFUNCTION, array( &$this, 'stream_headers' ) ); + + curl_setopt( $handle, CURLOPT_HEADER, false ); + + // If streaming to a file open a file handle, and setup our curl streaming handler + if ( $r['stream'] ) { + if ( ! WP_DEBUG ) + $stream_handle = @fopen( $r['filename'], 'w+' ); + else + $stream_handle = fopen( $r['filename'], 'w+' ); + if ( ! $stream_handle ) + return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) ); + curl_setopt( $handle, CURLOPT_FILE, $stream_handle ); + } // The option doesn't work with safe mode or when open_basedir is set. if ( !ini_get('safe_mode') && !ini_get('open_basedir') && 0 !== $r['_redirection'] ) @@ -1155,48 +1252,58 @@ class WP_Http_Curl { } $theResponse = curl_exec( $handle ); + $theBody = ''; + $theHeaders = WP_Http::processHeaders( $this->headers ); - if ( !empty($theResponse) ) { - $headerLength = curl_getinfo($handle, CURLINFO_HEADER_SIZE); - $theHeaders = trim( substr($theResponse, 0, $headerLength) ); - if ( strlen($theResponse) > $headerLength ) - $theBody = substr( $theResponse, $headerLength ); - else - $theBody = ''; - if ( false !== strpos($theHeaders, "\r\n\r\n") ) { - $headerParts = explode("\r\n\r\n", $theHeaders); - $theHeaders = $headerParts[ count($headerParts) -1 ]; - } - $theHeaders = WP_Http::processHeaders($theHeaders); - } else { + if ( ! empty($theResponse) && ! is_bool( $theResponse ) ) // is_bool: when using $args['stream'], curl_exec will return (bool)true + $theBody = $theResponse; + + // If no response, and It's not a HEAD request with valid headers returned + if ( empty($theResponse) && 'HEAD' != $args['method'] && ! empty($this->headers) ) { if ( $curl_error = curl_error($handle) ) return new WP_Error('http_request_failed', $curl_error); if ( in_array( curl_getinfo( $handle, CURLINFO_HTTP_CODE ), array(301, 302) ) ) return new WP_Error('http_request_failed', __('Too many redirects.')); - - $theHeaders = array( 'headers' => array(), 'cookies' => array() ); - $theBody = ''; } + unset( $this->headers ); + $response = array(); $response['code'] = curl_getinfo( $handle, CURLINFO_HTTP_CODE ); $response['message'] = get_status_header_desc($response['code']); curl_close( $handle ); + if ( $r['stream'] ) + fclose( $stream_handle ); + // See #11305 - When running under safe mode, redirection is disabled above. Handle it manually. - if ( !empty($theHeaders['headers']['location']) && (ini_get('safe_mode') || ini_get('open_basedir')) && 0 !== $r['_redirection'] ) { + if ( ! empty( $theHeaders['headers']['location'] ) && ( ini_get( 'safe_mode' ) || ini_get( 'open_basedir' ) ) && 0 !== $r['_redirection'] ) { if ( $r['redirection']-- > 0 ) { - return $this->request($theHeaders['headers']['location'], $r); + return $this->request( $theHeaders['headers']['location'], $r ); } else { - return new WP_Error('http_request_failed', __('Too many redirects.')); + return new WP_Error( 'http_request_failed', __( 'Too many redirects.' ) ); } } if ( true === $r['decompress'] && true === WP_Http_Encoding::should_decode($theHeaders['headers']) ) $theBody = WP_Http_Encoding::decompress( $theBody ); - return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies']); + return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] ); + } + + /** + * Grab the headers of the cURL request + * + * Each header is sent individually to this callback, so we append to the $header property for temporary storage + * + * @since 3.2.0 + * @access private + * @return int + */ + private function stream_headers( $handle, $headers ) { + $this->headers .= $headers; + return strlen( $headers ); } /** diff --git a/wp-includes/functions.php b/wp-includes/functions.php index 72843130a8..91cc775888 100644 --- a/wp-includes/functions.php +++ b/wp-includes/functions.php @@ -2110,6 +2110,42 @@ function path_join( $base, $path ) { return rtrim($base, '/') . '/' . ltrim($path, '/'); } +/** + * Determines a writable directory for temporary files. + * Function's preference is to WP_CONTENT_DIR followed by the return value of sys_get_temp_dir(), before finally defaulting to /tmp/ + * + * In the event that this function does not find a writable location, It may be overridden by the WP_TEMP_DIR constant in your wp-config.php file. + * + * @since 2.5.0 + * + * @return string Writable temporary directory + */ +function get_temp_dir() { + static $temp; + if ( defined('WP_TEMP_DIR') ) + return trailingslashit(WP_TEMP_DIR); + + if ( $temp ) + return trailingslashit($temp); + + $temp = WP_CONTENT_DIR . '/'; + if ( is_dir($temp) && @is_writable($temp) ) + return $temp; + + if ( function_exists('sys_get_temp_dir') ) { + $temp = sys_get_temp_dir(); + if ( @is_writable($temp) ) + return trailingslashit($temp); + } + + $temp = ini_get('upload_tmp_dir'); + if ( is_dir($temp) && @is_writable($temp) ) + return trailingslashit($temp); + + $temp = '/tmp/'; + return $temp; +} + /** * Get an array containing the current upload directory's path and url. *