diff --git a/builder/azure/arm/config.go b/builder/azure/arm/config.go
index 6744287cc..eef7599a0 100644
--- a/builder/azure/arm/config.go
+++ b/builder/azure/arm/config.go
@@ -879,8 +879,12 @@ func assertRequiredParametersSet(c *Config, errs *packer.MultiError) {
 	}
 
 	if c.AllowedInboundIpAddresses != nil && len(c.AllowedInboundIpAddresses) >= 1 {
-		if ok, err := assertAllowedInboundIpAddresses(c.AllowedInboundIpAddresses, "allowed_inbound_ip_addresses"); !ok {
-			errs = packer.MultiErrorAppend(errs, err)
+		if c.VirtualNetworkName != "" {
+			errs = packer.MultiErrorAppend(errs, fmt.Errorf("If virtual_network_name is specified, allowed_inbound_ip_addresses cannot be specified"))
+		} else {
+			if ok, err := assertAllowedInboundIpAddresses(c.AllowedInboundIpAddresses, "allowed_inbound_ip_addresses"); !ok {
+				errs = packer.MultiErrorAppend(errs, err)
+			}
 		}
 	}
 
diff --git a/builder/azure/arm/config_test.go b/builder/azure/arm/config_test.go
index ce446b246..d90e31f44 100644
--- a/builder/azure/arm/config_test.go
+++ b/builder/azure/arm/config_test.go
@@ -308,7 +308,6 @@ func TestConfigShouldAcceptCorrectInboundIpAddresses(t *testing.T) {
 		"subscription_id":        "ignore",
 		"os_type":                constants.Target_Linux,
 		"communicator":           "none",
-		"virtual_network_name":   "MyVirtualNetwork",
 	}
 
 	config["allowed_inbound_ip_addresses"] = ipValue0
@@ -355,7 +354,6 @@ func TestConfigShouldRejectIncorrectInboundIpAddresses(t *testing.T) {
 		"subscription_id":        "ignore",
 		"os_type":                constants.Target_Linux,
 		"communicator":           "none",
-		"virtual_network_name":   "MyVirtualNetwork",
 	}
 
 	config["allowed_inbound_ip_addresses"] = []string{"127.0.0.1", "127.0.0.two"}
@@ -372,6 +370,32 @@ func TestConfigShouldRejectIncorrectInboundIpAddresses(t *testing.T) {
 	}
 }
 
+func TestConfigShouldRejectInboundIpAddressesWithVirtualNetwork(t *testing.T) {
+	config := map[string]interface{}{
+		"capture_name_prefix":          "ignore",
+		"capture_container_name":       "ignore",
+		"location":                     "ignore",
+		"image_url":                    "ignore",
+		"storage_account":              "ignore",
+		"resource_group_name":          "ignore",
+		"subscription_id":              "ignore",
+		"os_type":                      constants.Target_Linux,
+		"communicator":                 "none",
+		"allowed_inbound_ip_addresses": "127.0.0.1",
+	}
+
+	_, _, err := newConfig(config, getPackerConfiguration())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config["virtual_network_name"] = "some_vnet_name"
+	_, _, err = newConfig(config, getPackerConfiguration())
+	if err == nil {
+		t.Errorf("Expected configuration creation to fail, but it succeeded with allowed_inbound_ip_addresses and virtual_network_name both specified")
+	}
+}
+
 func TestConfigShouldDefaultToPublicCloud(t *testing.T) {
 	c, _, _ := newConfig(getArmBuilderConfiguration(), getPackerConfiguration())