diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
index a8fcdc7e6d9..15818283f72 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
@@ -436,7 +436,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
 
     // #10067: create-bwc-index.py deleted any doc with long_sort:[10-20]
     void assertDeleteByQueryWorked(String indexName, Version version) throws Exception {
-        if (version.onOrBefore(Version.V_1_0_0_Beta2)) {
+        if (version.onOrBefore(Version.V_1_0_0_Beta2) || version.onOrAfter(Version.V_2_0_0_beta1)) {
             // TODO: remove this once #10262 is fixed
             return;
         }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
index 6f5650374c6..61a9c3ac9bd 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
@@ -51,7 +51,7 @@ public class StringFieldMapperPositionIncrementGapTests extends ESSingleNodeTest
      * Asserts that the post-2.0 default is being applied.
      */
     public static void assertGapIsOneHundred(Client client, String indexName, String type) throws IOException {
-        testGap(client(), indexName, type, 100);
+        testGap(client, indexName, type, 100);
 
         // No match across gap using default slop with default positionIncrementGap
         assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two")).get(), 0);
diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py
index 1a2bfbdc489..22ba7f78ce5 100644
--- a/dev-tools/create_bwc_index.py
+++ b/dev-tools/create_bwc_index.py
@@ -237,15 +237,19 @@ def generate_index(client, version, index_name):
       }
     }
 
+  settings = {
+    'number_of_shards': 1,
+    'number_of_replicas': 0,
+  }
+  if version.startswith('0.') or version.startswith('1.'):
+    # Same as ES default (60 seconds), but missing the units to make sure they are inserted on upgrade:
+    settings['gc_deletes'] = '60000',
+    # Same as ES default (5 GB), but missing the units to make sure they are inserted on upgrade:
+    settings['merge.policy.max_merged_segment'] = '5368709120'
+    
+
   client.indices.create(index=index_name, body={
-      'settings': {
-          'number_of_shards': 1,
-          'number_of_replicas': 0,
-          # Same as ES default (60 seconds), but missing the units to make sure they are inserted on upgrade:
-          "gc_deletes": '60000',
-          # Same as ES default (5 GB), but missing the units to make sure they are inserted on upgrade:
-          "merge.policy.max_merged_segment": '5368709120'
-      },
+      'settings': settings,
       'mappings': mappings
   })
   health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0)
@@ -374,7 +378,8 @@ def create_bwc_index(cfg, version):
     # 10067: get a delete-by-query into the translog on upgrade.  We must do
     # this after the snapshot, because it calls flush.  Otherwise the index
     # will already have the deletions applied on upgrade.
-    delete_by_query(client, version, index_name, 'doc')
+    if version.startswith('0.') or version.startswith('1.'):
+      delete_by_query(client, version, index_name, 'doc')
 
     shutdown_node(node)
     node = None
diff --git a/dev-tools/get-bwc-version.py b/dev-tools/get-bwc-version.py
index a44c06af576..860fc2d3a63 100644
--- a/dev-tools/get-bwc-version.py
+++ b/dev-tools/get-bwc-version.py
@@ -65,7 +65,7 @@ def main():
     # 1.2.0 was pulled from download.elasticsearch.org because of routing bug:
     url = 'http://central.maven.org/maven2/org/elasticsearch/elasticsearch/1.2.0/%s' % filename
   else:
-    url = 'https://download.elasticsearch.org/elasticsearch/elasticsearch/%s' % filename
+    url = 'http://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/%s/%s' % (c.version, filename)
   print('Downloading %s' % url)
   urllib.request.urlretrieve(url, filename)