Upgrade/Install: Automatically roll back to the previous version when an automatic plugin update results in a fatal error on the front end of the site.

This builds on the temporary backup system introduced in 6.3 to allow automatic updates to benefit from fatal error protection. A loopback request is performed to the home page of the site and the plugin is rolled back to its backed up version if a fatal error is observed.

For debugging and observability during beta, this change includes several calls to `error_log()` during the upgrade and rollback stages. These calls can be removed or placed behind a flag once we're ready for RC1.

Props costdev, johnbillion, mukesh27, afragen, audrasjb, justlevine, kirasong, peterwilsoncc

Fixes #58281

Built from https://develop.svn.wordpress.org/trunk@58128


git-svn-id: http://core.svn.wordpress.org/trunk@57593 1a063a9b-81f0-0310-95a4-ce76da25c4cd

wp-admin/includes/class-wp-automatic-updater.php

@@ -446,6 +446,29 @@ class WP_Automatic_Updater {
 			$allow_relaxed_file_ownership = true;
 		}
 
+		if ( 'plugin' === $type ) {
+			$was_active = is_plugin_active( $upgrader_item );
+			error_log( '    Upgrading plugin ' . var_export( $item->slug, true ) . '...' );
+		}
+
+		if ( 'theme' === $type ) {
+			error_log( '    Upgrading theme ' . var_export( $item->theme, true ) . '...' );
+		}
+
+		/*
+		 * Enable maintenance mode before upgrading the plugin or theme.
+		 *
+		 * This avoids potential non-fatal errors being detected
+		 * while scraping for a fatal error if some files are still
+		 * being moved.
+		 *
+		 * While these checks are intended only for plugins,
+		 * maintenance mode is enabled for all upgrade types as any
+		 * update could contain an error or warning, which could cause
+		 * the scrape to miss a fatal error in the plugin update.
+		 */
+		$upgrader->maintenance_mode( true );
+
 		// Boom, this site's about to get a whole new splash of paint!
 		$upgrade_result = $upgrader->upgrade(
 			$upgrader_item,
@@ -460,6 +483,17 @@ class WP_Automatic_Updater {
 			)
 		);
 
+		/*
+		 * After WP_Upgrader::upgrade() completes, maintenance mode is disabled.
+		 *
+		 * Re-enable maintenance mode while attempting to detect fatal errors
+		 * and potentially rolling back.
+		 *
+		 * This avoids errors if the site is visited while fatal errors exist
+		 * or while files are still being moved.
+		 */
+		$upgrader->maintenance_mode( true );
+
 		// If the filesystem is unavailable, false is returned.
 		if ( false === $upgrade_result ) {
 			$upgrade_result = new WP_Error( 'fs_unavailable', __( 'Could not access filesystem.' ) );
@@ -486,6 +520,92 @@ class WP_Automatic_Updater {
 			}
 		}
 
+		if ( 'theme' === $type ) {
+			error_log( '    Theme ' . var_export( $item->theme, true ) . ' has been upgraded.' );
+		}
+
+		if ( 'plugin' === $type ) {
+			error_log( '    Plugin ' . var_export( $item->slug, true ) . ' has been upgraded.' );
+			if ( is_plugin_inactive( $upgrader_item ) ) {
+				error_log( '    ' . var_export( $upgrader_item, true ) . ' is inactive and will not be checked for fatal errors.' );
+			}
+
+			if ( $was_active && ! is_wp_error( $upgrade_result ) ) {
+
+				/*
+				 * The usual time limit is five minutes. However, as a loopback request
+				 * is about to be performed, increase the time limit to account for this.
+				 */
+				if ( function_exists( 'set_time_limit' ) ) {
+					set_time_limit( 10 * MINUTE_IN_SECONDS );
+				}
+
+				/*
+				 * Avoids a race condition when there are 2 sequential plugins that have
+				 * fatal errors. It seems a slight delay is required for the loopback to
+				 * use the updated plugin code in the request. This can cause the second
+				 * plugin's fatal error checking to be inaccurate, and may also affect
+				 * subsequent plugin checks.
+				 */
+				sleep( 2 );
+
+				if ( $this->has_fatal_error() ) {
+					$upgrade_result = new WP_Error();
+					$temp_backup    = array(
+						array(
+							'dir'  => 'plugins',
+							'slug' => $item->slug,
+							'src'  => WP_PLUGIN_DIR,
+						),
+					);
+
+					$backup_restored = $upgrader->restore_temp_backup( $temp_backup );
+					if ( is_wp_error( $backup_restored ) ) {
+						$upgrade_result->add(
+							'plugin_update_fatal_error_rollback_failed',
+							sprintf(
+								/* translators: %s: The plugin's slug. */
+								__( "The update for '%s' contained a fatal error. The previously installed version could not be restored." ),
+								$item->slug
+							)
+						);
+
+						$upgrade_result->merge_from( $backup_restored );
+					} else {
+						$upgrade_result->add(
+							'plugin_update_fatal_error_rollback_successful',
+							sprintf(
+								/* translators: %s: The plugin's slug. */
+								__( "The update for '%s' contained a fatal error. The previously installed version has been restored." ),
+								$item->slug
+							)
+						);
+
+						$backup_deleted = $upgrader->delete_temp_backup( $temp_backup );
+						if ( is_wp_error( $backup_deleted ) ) {
+							$upgrade_result->merge_from( $backup_deleted );
+						}
+					}
+
+					/*
+					 * Should emails not be working, log the message(s) so that
+					 * the log file contains context for the fatal error,
+					 * and whether a rollback was performed.
+					 *
+					 * `trigger_error()` is not used as it outputs a stack trace
+					 * to this location rather than to the fatal error, which will
+					 * appear above this entry in the log file.
+					 */
+					error_log( '    ' . implode( "\n", $upgrade_result->get_error_messages() ) );
+				} else {
+					error_log( '    The update for ' . var_export( $item->slug, true ) . ' has no fatal errors.' );
+				}
+			}
+		}
+
+		// All processes are complete. Allow visitors to browse the site again.
+		$upgrader->maintenance_mode( false );
+
 		$this->update_results[ $type ][] = (object) array(
 			'item'     => $item,
 			'result'   => $upgrade_result,
@@ -514,6 +634,8 @@ class WP_Automatic_Updater {
 			return;
 		}
 
+		error_log( 'Automatic updates starting...' );
+
 		// Don't automatically run these things, as we'll handle it ourselves.
 		remove_action( 'upgrader_process_complete', array( 'Language_Pack_Upgrader', 'async_upgrade' ), 20 );
 		remove_action( 'upgrader_process_complete', 'wp_version_check' );
@@ -524,24 +646,35 @@ class WP_Automatic_Updater {
 		wp_update_plugins(); // Check for plugin updates.
 		$plugin_updates = get_site_transient( 'update_plugins' );
 		if ( $plugin_updates && ! empty( $plugin_updates->response ) ) {
+			error_log( '  Automatic plugin updates starting...' );
+
 			foreach ( $plugin_updates->response as $plugin ) {
 				$this->update( 'plugin', $plugin );
 			}
+
 			// Force refresh of plugin update information.
 			wp_clean_plugins_cache();
+
+			error_log( '  Automatic plugin updates complete.' );
 		}
 
 		// Next, those themes we all love.
 		wp_update_themes();  // Check for theme updates.
 		$theme_updates = get_site_transient( 'update_themes' );
 		if ( $theme_updates && ! empty( $theme_updates->response ) ) {
+			error_log( '  Automatic theme updates starting...' );
+
 			foreach ( $theme_updates->response as $theme ) {
 				$this->update( 'theme', (object) $theme );
 			}
 			// Force refresh of theme update information.
 			wp_clean_themes_cache();
+
+			error_log( '  Automatic theme updates complete.' );
 		}
 
+		error_log( 'Automatic updates complete.' );
+
 		// Next, process any core update.
 		wp_version_check(); // Check for core updates.
 		$core_update = find_core_auto_update();
@@ -1163,7 +1296,7 @@ class WP_Automatic_Updater {
 
 			// List failed plugin updates.
 			if ( ! empty( $failed_updates['plugin'] ) ) {
-				$body[] = __( 'These plugins failed to update:' );
+				$body[] = __( 'The following plugins failed to update. If there was a fatal error in the update, the previously installed version has been restored.' );
 
 				foreach ( $failed_updates['plugin'] as $item ) {
 					$body_message = '';
@@ -1551,4 +1684,84 @@ Thanks! -- The WordPress Team"
 
 		wp_mail( $email['to'], wp_specialchars_decode( $email['subject'] ), $email['body'], $email['headers'] );
 	}
+
+	/**
+	 * Performs a loopback request to check for potential fatal errors.
+	 *
+	 * Fatal errors cannot be detected unless maintenance mode is enabled.
+	 *
+	 * @since 6.6.0
+	 *
+	 * @global int $upgrading The Unix timestamp marking when upgrading WordPress began.
+	 *
+	 * @return bool Whether a fatal error was detected.
+	 */
+	protected function has_fatal_error() {
+		global $upgrading;
+
+		$maintenance_file = ABSPATH . '.maintenance';
+		if ( ! file_exists( $maintenance_file ) ) {
+			return false;
+		}
+
+		require $maintenance_file;
+		if ( ! is_int( $upgrading ) ) {
+			return false;
+		}
+
+		$scrape_key   = md5( $upgrading );
+		$scrape_nonce = (string) $upgrading;
+		$transient    = 'scrape_key_' . $scrape_key;
+		set_transient( $transient, $scrape_nonce, 30 );
+
+		$cookies       = wp_unslash( $_COOKIE );
+		$scrape_params = array(
+			'wp_scrape_key'   => $scrape_key,
+			'wp_scrape_nonce' => $scrape_nonce,
+		);
+		$headers       = array(
+			'Cache-Control' => 'no-cache',
+		);
+
+		/** This filter is documented in wp-includes/class-wp-http-streams.php */
+		$sslverify = apply_filters( 'https_local_ssl_verify', false );
+
+		// Include Basic auth in the loopback request.
+		if ( isset( $_SERVER['PHP_AUTH_USER'] ) && isset( $_SERVER['PHP_AUTH_PW'] ) ) {
+			$headers['Authorization'] = 'Basic ' . base64_encode( wp_unslash( $_SERVER['PHP_AUTH_USER'] ) . ':' . wp_unslash( $_SERVER['PHP_AUTH_PW'] ) );
+		}
+
+		// Time to wait for loopback request to finish.
+		$timeout = 50; // 50 seconds.
+
+		error_log( '    Scraping home page...' );
+
+		$needle_start = "###### wp_scraping_result_start:$scrape_key ######";
+		$needle_end   = "###### wp_scraping_result_end:$scrape_key ######";
+		$url          = add_query_arg( $scrape_params, home_url( '/' ) );
+		$response     = wp_remote_get( $url, compact( 'cookies', 'headers', 'timeout', 'sslverify' ) );
+
+		if ( is_wp_error( $response ) ) {
+			error_log( 'Loopback request failed: ' . $response->get_error_message() );
+			return true;
+		}
+
+		// If this outputs `true` in the log, it means there were no fatal errors detected.
+		error_log( var_export( substr( $response['body'], strpos( $response['body'], '###### wp_scraping_result_start:' ) ), true ) );
+
+		$body                   = wp_remote_retrieve_body( $response );
+		$scrape_result_position = strpos( $body, $needle_start );
+		$result                 = null;
+
+		if ( false !== $scrape_result_position ) {
+			$error_output = substr( $body, $scrape_result_position + strlen( $needle_start ) );
+			$error_output = substr( $error_output, 0, strpos( $error_output, $needle_end ) );
+			$result       = json_decode( trim( $error_output ), true );
+		}
+
+		delete_transient( $transient );
+
+		// Only fatal errors will result in a 'type' key.
+		return isset( $result['type'] );
+	}
 }

wp-admin/includes/class-wp-upgrader.php

@@ -901,6 +901,7 @@ class WP_Upgrader {
 		$this->skin->set_result( $result );
 
 		if ( is_wp_error( $result ) ) {
+			// An automatic plugin update will have already performed its rollback.
 			if ( ! empty( $options['hook_extra']['temp_backup'] ) ) {
 				$this->temp_restores[] = $options['hook_extra']['temp_backup'];
 
@@ -909,8 +910,12 @@ class WP_Upgrader {
 				 * Actions running on `shutdown` are immune to PHP timeouts,
 				 * so in case the failure was due to a PHP timeout,
 				 * it will still be able to properly restore the previous version.
+				 *
+				 * Zero arguments are accepted as a string can sometimes be passed
+				 * internally during actions, causing an error because
+				 * `WP_Upgrader::restore_temp_backup()` expects an array.
 				 */
-				add_action( 'shutdown', array( $this, 'restore_temp_backup' ) );
+				add_action( 'shutdown', array( $this, 'restore_temp_backup' ), 10, 0 );
 			}
 			$this->skin->error( $result );
 
@@ -983,15 +988,25 @@ class WP_Upgrader {
 	 */
 	public function maintenance_mode( $enable = false ) {
 		global $wp_filesystem;
+
+		if ( ! $wp_filesystem ) {
+			require_once ABSPATH . 'wp-admin/includes/file.php';
+			WP_Filesystem();
+		}
+
 		$file = $wp_filesystem->abspath() . '.maintenance';
 		if ( $enable ) {
+			if ( ! wp_doing_cron() ) {
 				$this->skin->feedback( 'maintenance_start' );
+			}
 			// Create maintenance file to signal that we are upgrading.
 			$maintenance_string = '<?php $upgrading = ' . time() . '; ?>';
 			$wp_filesystem->delete( $file );
 			$wp_filesystem->put_contents( $file, $maintenance_string, FS_CHMOD_FILE );
 		} elseif ( ! $enable && $wp_filesystem->exists( $file ) ) {
+			if ( ! wp_doing_cron() ) {
 				$this->skin->feedback( 'maintenance_end' );
+			}
 			$wp_filesystem->delete( $file );
 		}
 	}
@@ -1133,17 +1148,33 @@ class WP_Upgrader {
 	 * Restores the plugin or theme from temporary backup.
 	 *
 	 * @since 6.3.0
+	 * @since 6.6.0 Added the `$temp_backups` parameter.
 	 *
 	 * @global WP_Filesystem_Base $wp_filesystem WordPress filesystem subclass.
 	 *
+	 * @param array[] $temp_backups {
+	 *     Optional. An array of temporary backups.
+	 *
+	 *     @type array ...$0 {
+	 *         Information about the backup.
+	 *
+	 *         @type string $dir  The temporary backup location in the upgrade-temp-backup directory.
+	 *         @type string $slug The item's slug.
+	 *         @type string $src  The directory where the original is stored. For example, `WP_PLUGIN_DIR`.
+	 *     }
+	 * }
 	 * @return bool|WP_Error True on success, false on early exit, otherwise WP_Error.
 	 */
-	public function restore_temp_backup() {
+	public function restore_temp_backup( array $temp_backups = array() ) {
 		global $wp_filesystem;
 
 		$errors = new WP_Error();
 
-		foreach ( $this->temp_restores as $args ) {
+		if ( empty( $temp_backups ) ) {
+			$temp_backups = $this->temp_restores;
+		}
+
+		foreach ( $temp_backups as $args ) {
 			if ( empty( $args['slug'] ) || empty( $args['src'] ) || empty( $args['dir'] ) ) {
 				return false;
 			}
@@ -1186,17 +1217,33 @@ class WP_Upgrader {
 	 * Deletes a temporary backup.
 	 *
 	 * @since 6.3.0
+	 * @since 6.6.0 Added the `$temp_backups` parameter.
 	 *
 	 * @global WP_Filesystem_Base $wp_filesystem WordPress filesystem subclass.
 	 *
+	 * @param array[] $temp_backups {
+	 *     Optional. An array of temporary backups.
+	 *
+	 *     @type array ...$0 {
+	 *         Information about the backup.
+	 *
+	 *         @type string $dir  The temporary backup location in the upgrade-temp-backup directory.
+	 *         @type string $slug The item's slug.
+	 *         @type string $src  The directory where the original is stored. For example, `WP_PLUGIN_DIR`.
+	 *     }
+	 * }
 	 * @return bool|WP_Error True on success, false on early exit, otherwise WP_Error.
 	 */
-	public function delete_temp_backup() {
+	public function delete_temp_backup( array $temp_backups = array() ) {
 		global $wp_filesystem;
 
 		$errors = new WP_Error();
 
-		foreach ( $this->temp_backups as $args ) {
+		if ( empty( $temp_backups ) ) {
+			$temp_backups = $this->temp_backups;
+		}
+
+		foreach ( $temp_backups as $args ) {
 			if ( empty( $args['slug'] ) || empty( $args['dir'] ) ) {
 				return false;
 			}

wp-includes/load.php

@@ -420,6 +420,16 @@ function wp_is_maintenance_mode() {
 		return false;
 	}
 
+	// Don't enable maintenance mode while scraping for fatal errors.
+	if ( is_int( $upgrading ) && isset( $_REQUEST['wp_scrape_key'], $_REQUEST['wp_scrape_nonce'] ) ) {
+		$key   = stripslashes( $_REQUEST['wp_scrape_key'] );
+		$nonce = stripslashes( $_REQUEST['wp_scrape_nonce'] );
+
+		if ( md5( $upgrading ) === $key && (int) $nonce === $upgrading ) {
+			return false;
+		}
+	}
+
 	/**
 	 * Filters whether to enable maintenance mode.
 	 *

wp-includes/version.php

@@ -16,7 +16,7 @@
  *
  * @global string $wp_version
  */
-$wp_version = '6.6-alpha-58127';
+$wp_version = '6.6-alpha-58128';
 
 /**
  * Holds the WordPress DB revision, increments when changes are made to the WordPress DB schema.