WordPress-C
/
WordPress
mirror of https://github.com/WordPress/WordPress.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#43590: Use robots meta tag to better discourage search engines. 

This changes the "discourage search engines" option to output a `noindex, nofollow` robots meta tag. `Disallow: /` is removed from the `robots.txt` to allow search engines to discover they are requested not to index the site.

Disallowing search engines from accessing a site in the `robots.txt` file can result in search engines listing a site with a fragment (a listing without content).

Props donmhico, jonoaldersonwp.
Fixes #43590.


Built from https://develop.svn.wordpress.org/trunk@45928


git-svn-id: http://core.svn.wordpress.org/trunk@45739 1a063a9b-81f0-0310-95a4-ce76da25c4cd
This commit is contained in:
Peter Wilson 2019-09-02 02:27:55 +00:00
parent 8662d9a74d
commit b55ca66759
3 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
wp-includes/functions.php
View File

@ -1582,6 +1582,8 @@ function do_feed_atom( $for_comments ) {
* Displays the default robots.txt file content. * Displays the default robots.txt file content.
* *
* @since 2.1.0 * @since 2.1.0
* @since 5.3.0 Remove the "Disallow: /" output if search engine visiblity is
* discouraged in favor of robots meta HTML tag in wp_no_robots().
*/ */
function do_robots() { function do_robots() {
header( 'Content-Type: text/plain; charset=utf-8' ); header( 'Content-Type: text/plain; charset=utf-8' );
@ -1595,14 +1597,11 @@ function do_robots() {
$output = "User-agent: *\n"; $output = "User-agent: *\n";
$public = get_option( 'blog_public' ); $public = get_option( 'blog_public' );
if ( '0' == $public ) {
$output .= "Disallow: /\n"; $site_url = parse_url( site_url() );
} else { $path = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
$site_url = parse_url( site_url() ); $output .= "Disallow: $path/wp-admin/\n";
$path = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : ''; $output .= "Allow: $path/wp-admin/admin-ajax.php\n";
$output .= "Disallow: $path/wp-admin/\n";
$output .= "Allow: $path/wp-admin/admin-ajax.php\n";
}
/** /**
* Filters the robots.txt output. * Filters the robots.txt output.

10
wp-includes/general-template.php
View File

@ -2986,12 +2986,18 @@ function noindex() {
* Display a noindex meta tag. * Display a noindex meta tag.
* *
* Outputs a noindex meta tag that tells web robots not to index the page content. * Outputs a noindex meta tag that tells web robots not to index the page content.
* Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_no_robots' ); * Typical usage is as a {@see 'wp_head'} callback. add_action( 'wp_head', 'wp_no_robots' );
* *
* @since 3.3.0 * @since 3.3.0
* @since 5.3.0 Echo "noindex,nofollow" if search engine visibility is discouraged.
*/ */
function wp_no_robots() { function wp_no_robots() {
echo "<meta name='robots' content='noindex,follow' />\n"; if ( get_option( 'blog_public' ) ) {
echo "<meta name='robots' content='noindex,follow' />\n";
return;
}
echo "<meta name='robots' content='noindex,nofollow' />\n";
} }
/** /**

2
wp-includes/version.php
View File

@ -13,7 +13,7 @@
* *
* @global string $wp_version * @global string $wp_version
*/ */
$wp_version = '5.3-alpha-45927'; $wp_version = '5.3-alpha-45928';
/** /**
* Holds the WordPress DB revision, increments when changes are made to the WordPress DB schema. * Holds the WordPress DB revision, increments when changes are made to the WordPress DB schema.