iSharkFly-Docs
/
packer-cn
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

stop calculating spot bids; amazon changed the way spot instances are priced to be stable rather than bid-based, so when user sets auto, we can just default to the ondemand price and know that they'll get the same price as everyone else bidding at that time.

This commit is contained in:
Megan Marsh 2019-06-28 14:00:56 -07:00
parent 1678d66e31
commit c722afe707
8 changed files with 74 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
builder/amazon/common/run_config.go
View File

@ -153,20 +153,6 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
"block_duration_minutes must be multiple of 60"))
}
if c.SpotPrice == "auto" {
if c.SpotPriceAutoProduct == "" {
errs = append(errs, fmt.Errorf(
"spot_price_auto_product must be specified when spot_price is auto"))
}
}
if c.SpotPriceAutoProduct != "" {
if c.SpotPrice != "auto" {
errs = append(errs, fmt.Errorf(
"spot_price should be set to auto when spot_price_auto_product is specified"))
}
}
if c.SpotTags != nil {
if c.SpotPrice == "" || c.SpotPrice == "0" {
errs = append(errs, fmt.Errorf(

12
builder/amazon/common/run_config_test.go
View File

@ -117,7 +117,6 @@ func TestRunConfigPrepare_EnableT2UnlimitedBadWithSpotInstanceRequest(t *testing
c.InstanceType = "t2.micro"
c.EnableT2Unlimited = true
c.SpotPrice = "auto"
c.SpotPriceAutoProduct = "Linux/UNIX"
err := c.Prepare(nil)
if len(err) != 1 {
t.Fatalf("Should error if T2 Unlimited has been used in conjuntion with a Spot Price request")
@ -127,20 +126,9 @@ func TestRunConfigPrepare_EnableT2UnlimitedBadWithSpotInstanceRequest(t *testing
func TestRunConfigPrepare_SpotAuto(t *testing.T) {
c := testConfig()
c.SpotPrice = "auto"
if err := c.Prepare(nil); len(err) != 1 {
t.Fatalf("Should error if spot_price_auto_product is not set and spot_price is set to auto")
}
// Good - SpotPrice and SpotPriceAutoProduct are correctly set
c.SpotPriceAutoProduct = "foo"
if err := c.Prepare(nil); len(err) != 0 {
t.Fatalf("err: %s", err)
}
c.SpotPrice = ""
if err := c.Prepare(nil); len(err) != 1 {
t.Fatalf("Should error if spot_price is not set to auto and spot_price_auto_product is set")
}
}
func TestRunConfigPrepare_SSHPort(t *testing.T) {

68
builder/amazon/common/step_run_spot_instance.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"io/ioutil"
"log"
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
@ -35,7 +34,6 @@ type StepRunSpotInstance struct {
InstanceType string
SourceAMI string
SpotPrice string
SpotPriceProduct string
SpotTags TagMap
SpotInstanceTypes []string
Tags TagMap
@ -48,55 +46,6 @@ type StepRunSpotInstance struct {
spotRequest *ec2.SpotInstanceRequest
}
func (s *StepRunSpotInstance) CalculateSpotPrice(az string, ec2conn ec2iface.EC2API) (string, error) {
// Calculate the spot price for a given availability zone
spotPrice := s.SpotPrice
if spotPrice == "auto" {
// Detect the spot price
startTime := time.Now().Add(-1 * time.Hour)
resp, err := ec2conn.DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistoryInput{
InstanceTypes: []*string{&s.InstanceType},
ProductDescriptions: []*string{&s.SpotPriceProduct},
AvailabilityZone: &az,
StartTime: &startTime,
})
if err != nil {
return "", fmt.Errorf("Error finding spot price: %s", err)
}
var price float64
for _, history := range resp.SpotPriceHistory {
log.Printf("[INFO] Candidate spot price: %s", *history.SpotPrice)
current, err := strconv.ParseFloat(*history.SpotPrice, 64)
if err != nil {
log.Printf("[ERR] Error parsing spot price: %s", err)
continue
}
if price == 0 || current < price {
price = current
if az == "" {
az = *history.AvailabilityZone
}
}
}
if price == 0 {
return "", fmt.Errorf("No candidate spot prices found!")
} else {
// Add 0.5 cents to minimum spot bid to ensure capacity will be available
// Avoids price-too-low error in active markets which can fluctuate
price = price + 0.005
}
spotPrice = strconv.FormatFloat(price, 'f', -1, 64)
}
s.SpotPrice = spotPrice
return spotPrice, nil
}
func (s *StepRunSpotInstance) CreateTemplateData(userData *string, az string,
state multistep.StateBag, marketOptions *ec2.LaunchTemplateInstanceMarketOptionsRequest) *ec2.RequestLaunchTemplateData {
// Convert the BlockDeviceMapping into a
@ -213,17 +162,6 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag)
}
az := azConfig
ui.Message(fmt.Sprintf("Finding spot price for %s %s...",
s.SpotPriceProduct, s.InstanceType))
spotPrice, err := s.CalculateSpotPrice(az, ec2conn)
if err != nil {
state.Put("error", err)
ui.Error(err.Error())
return multistep.ActionHalt
}
ui.Message(fmt.Sprintf("Determined spot instance price of: %s.", spotPrice))
var instanceId string
ui.Say("Interpolating tags for spot instance...")
@ -247,8 +185,10 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag)
// instance yet
ec2Tags.Report(ui)
spotOptions := ec2.LaunchTemplateSpotMarketOptionsRequest{
MaxPrice: &s.SpotPrice,
spotOptions := ec2.LaunchTemplateSpotMarketOptionsRequest{}
// The default is to set the maximum price to the OnDemand price.
if s.SpotPrice != "auto" {
spotOptions.SetMaxPrice(s.SpotPrice)
}
if s.BlockDurationMinutes != 0 {
spotOptions.BlockDurationMinutes = &s.BlockDurationMinutes

39
builder/amazon/common/step_run_spot_instance_test.go
View File

@ -2,7 +2,6 @@ package common
import (
"bytes"
"strconv"
"testing"
"time"
@ -102,7 +101,6 @@ func getBasicStep() *StepRunSpotInstance {
InstanceType: "t2.micro",
SourceAMI: "",
SpotPrice: "auto",
SpotPriceProduct: "Linux/UNIX",
SpotTags: TagMap(nil),
Tags: TagMap{},
VolumeTags: TagMap(nil),
@ -112,20 +110,31 @@ func getBasicStep() *StepRunSpotInstance {
return &stepRunSpotInstance
}
func TestCalculateSpotPrice(t *testing.T) {
func TestCreateTemplateData(t *testing.T) {
state := tStateSpot()
stepRunSpotInstance := getBasicStep()
// Set spot price and spot price product
stepRunSpotInstance.SpotPrice = "auto"
stepRunSpotInstance.SpotPriceProduct = "Linux/UNIX"
ec2conn := getMockConnSpot()
// state := tStateSpot()
spotPrice, err := stepRunSpotInstance.CalculateSpotPrice("", ec2conn)
if err != nil {
t.Fatalf("Should not have had an error calculating spot price")
template := stepRunSpotInstance.CreateTemplateData(aws.String("userdata"), "az", state,
&ec2.LaunchTemplateInstanceMarketOptionsRequest{})
// expected := []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
// &ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{
// DeleteOnTermination: aws.Bool(true),
// DeviceIndex: aws.Int64(0),
// Groups: aws.StringSlice([]string{"sg-0b8984db72f213dc3"}),
// SubnetId: aws.String("subnet-077fde4e"),
// },
// }
// if expected != template.NetworkInterfaces {
if template.NetworkInterfaces == nil {
t.Fatalf("Template should have contained a networkInterface object: recieved %#v", template.NetworkInterfaces)
}
sp, _ := strconv.ParseFloat(spotPrice, 64)
expected := 0.008500
if sp != expected { // 0.003500 (from spot history) + .005
t.Fatalf("Expected spot price of \"0.008500\", not %s", spotPrice)
// Rerun, this time testing that we set security group IDs
state.Put("subnet_id", "")
template = stepRunSpotInstance.CreateTemplateData(aws.String("userdata"), "az", state,
&ec2.LaunchTemplateInstanceMarketOptionsRequest{})
if template.NetworkInterfaces != nil {
t.Fatalf("Template shouldn't contain network interfaces object if subnet_id is unset.")
}
}

15
builder/amazon/ebs/builder.go
View File

@ -64,6 +64,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
// Accumulate any errors
var errs *packer.MultiError
var warns []string
errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs,
b.config.AMIConfig.Prepare(&b.config.AccessConfig, &b.config.ctx)...)
@ -77,12 +79,20 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
"you use an AMI that already has either SR-IOV or ENA enabled."))
}
if b.config.RunConfig.SpotPriceAutoProduct != "" {
warns = append(warns, "spot_price_auto_product is deprecated and no "+
"longer necessary for Packer builds. In future versions of "+
"Packer, inclusion of spot_price_auto_product will error your "+
"builds. Please take a look at our current documentation to "+
"understand how Packer requests Spot instances.")
}
if errs != nil && len(errs.Errors) > 0 {
return nil, errs
return warns, errs
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, nil
return warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -121,7 +131,6 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
InstanceType: b.config.InstanceType,
SourceAMI: b.config.SourceAmi,
SpotPrice: b.config.SpotPrice,
SpotPriceProduct: b.config.SpotPriceAutoProduct,
SpotTags: b.config.SpotTags,
Tags: b.config.RunTags,
SpotInstanceTypes: b.config.SpotInstanceTypes,

15
builder/amazon/ebssurrogate/builder.go
View File

@ -64,6 +64,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
// Accumulate any errors
var errs *packer.MultiError
var warns []string
errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs,
@ -96,6 +97,14 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
"you use an AMI that already has either SR-IOV or ENA enabled."))
}
if b.config.RunConfig.SpotPriceAutoProduct != "" {
warns = append(warns, "spot_price_auto_product is deprecated and no "+
"longer necessary for Packer builds. In future versions of "+
"Packer, inclusion of spot_price_auto_product will error your "+
"builds. Please take a look at our current documentation to "+
"understand how Packer requests Spot instances.")
}
if b.config.Architecture == "" {
b.config.Architecture = "x86_64"
}
@ -110,11 +119,12 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
errs = packer.MultiErrorAppend(errs, errors.New(`The only valid ami_architecture values are "x86_64" and "arm64"`))
}
if errs != nil && len(errs.Errors) > 0 {
return nil, errs
return warns, errs
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, nil
return warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -151,7 +161,6 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
InstanceType: b.config.InstanceType,
SourceAMI: b.config.SourceAmi,
SpotPrice: b.config.SpotPrice,
SpotPriceProduct: b.config.SpotPriceAutoProduct,
SpotInstanceTypes: b.config.SpotInstanceTypes,
SpotTags: b.config.SpotTags,
Tags: b.config.RunTags,

14
builder/amazon/ebsvolume/builder.go
View File

@ -59,6 +59,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
// Accumulate any errors
var errs *packer.MultiError
var warns []string
errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs, b.config.launchBlockDevices.Prepare(&b.config.ctx)...)
@ -81,12 +82,20 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
"you use an AMI that already has either SR-IOV or ENA enabled."))
}
if b.config.RunConfig.SpotPriceAutoProduct != "" {
warns = append(warns, "spot_price_auto_product is deprecated and no "+
"longer necessary for Packer builds. In future versions of "+
"Packer, inclusion of spot_price_auto_product will error your "+
"builds. Please take a look at our current documentation to "+
"understand how Packer requests Spot instances.")
}
if errs != nil && len(errs.Errors) > 0 {
return nil, errs
return warns, errs
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, nil
return warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -121,7 +130,6 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
InstanceType: b.config.InstanceType,
SourceAMI: b.config.SourceAmi,
SpotPrice: b.config.SpotPrice,
SpotPriceProduct: b.config.SpotPriceAutoProduct,
SpotInstanceTypes: b.config.SpotInstanceTypes,
SpotTags: b.config.SpotTags,
Tags: b.config.RunTags,

14
builder/amazon/instance/builder.go
View File

@ -126,6 +126,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
// Accumulate any errors
var errs *packer.MultiError
var warns []string
errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs, b.config.BlockDevices.Prepare(&b.config.ctx)...)
errs = packer.MultiErrorAppend(errs,
@ -163,11 +164,19 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
"you use an AMI that already has either SR-IOV or ENA enabled."))
}
if b.config.RunConfig.SpotPriceAutoProduct != "" {
warns = append(warns, "spot_price_auto_product is deprecated and no "+
"longer necessary for Packer builds. In future versions of "+
"Packer, inclusion of spot_price_auto_product will error your "+
"builds. Please take a look at our current documentation to "+
"understand how Packer requests Spot instances.")
}
if errs != nil && len(errs.Errors) > 0 {
return nil, errs
return warns, errs
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, nil
return warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -203,7 +212,6 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
SourceAMI: b.config.SourceAmi,
SpotPrice: b.config.SpotPrice,
SpotInstanceTypes: b.config.SpotInstanceTypes,
SpotPriceProduct: b.config.SpotPriceAutoProduct,
Tags: b.config.RunTags,
SpotTags: b.config.SpotTags,
UserData: b.config.UserData,