diff --git a/src/main/java/org/elasticsearch/plugins/PluginManager.java b/src/main/java/org/elasticsearch/plugins/PluginManager.java
index d9bde8df37b..c1b302f98db 100644
--- a/src/main/java/org/elasticsearch/plugins/PluginManager.java
+++ b/src/main/java/org/elasticsearch/plugins/PluginManager.java
@@ -186,7 +186,7 @@ public class PluginManager {
         if (FileSystemUtils.hasExtensions(extractLocation, ".java")) {
             debug("Plugin installation assumed to be site plugin, but contains source code, aborting installation...");
             FileSystemUtils.deleteRecursively(extractLocation);
-            return;
+            throw new IllegalArgumentException("Plugin installation assumed to be site plugin, but contains source code, aborting installation.");
         }
 
         File binFile = new File(extractLocation, "bin");
diff --git a/src/test/java/org/elasticsearch/plugin/PluginManagerTests.java b/src/test/java/org/elasticsearch/plugin/PluginManagerTests.java
index 7c75d354dd4..a788769cae9 100644
--- a/src/test/java/org/elasticsearch/plugin/PluginManagerTests.java
+++ b/src/test/java/org/elasticsearch/plugin/PluginManagerTests.java
@@ -104,6 +104,13 @@ public class PluginManagerTests extends AbstractIntegrationTest {
         assertPluginAvailable(nodeName, pluginName);
     }
 
+    @Test(expected = IllegalArgumentException.class)
+    public void testSitePluginWithSourceThrows() throws Exception {
+        String pluginName = "plugin-with-source";
+        URL url = PluginManagerTests.class.getResource("plugin_with_sourcefiles.zip");
+        downloadAndExtract(pluginName, "file://" + url.getFile());
+    }
+
     private static PluginManager pluginManager(String pluginUrl) {
         Tuple<Settings, Environment> initialSettings = InternalSettingsPreparer.prepareSettings(
                 ImmutableSettings.settingsBuilder().build(), false);
diff --git a/src/test/resources/org/elasticsearch/plugin/plugin_with_sourcefiles.zip b/src/test/resources/org/elasticsearch/plugin/plugin_with_sourcefiles.zip
new file mode 100644
index 00000000000..ccd194fc737
Binary files /dev/null and b/src/test/resources/org/elasticsearch/plugin/plugin_with_sourcefiles.zip differ