Share SourceImageName with provisioners and manifest post-processor (#8603)

This commit is contained in:
Sylvia Moss 2020-01-16 12:04:03 +01:00 committed by GitHub
parent b498e5af12
commit 0677b02e18
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 352 additions and 39 deletions

View File

@ -20,6 +20,10 @@ type Artifact struct {
// BuilderId is the unique ID for the builder that created this AMI
BuilderIdValue string
// SateData should store data such as GeneratedData
// to be shared with post-processors
StateData map[string]interface{}
// EC2 connection for performing API stuff.
Session *session.Session
}
@ -55,6 +59,10 @@ func (a *Artifact) String() string {
}
func (a *Artifact) State(name string) interface{} {
if _, ok := a.StateData[name]; ok {
return a.StateData[name]
}
switch name {
case "atlas.artifact.metadata":
return a.stateAtlasMetadata()

View File

@ -62,3 +62,22 @@ west: bar
t.Fatalf("bad: %s", result)
}
}
func TestArtifactState(t *testing.T) {
expectedData := "this is the data"
artifact := &Artifact{
StateData: map[string]interface{}{"state_data": expectedData},
}
// Valid state
result := artifact.State("state_data")
if result != expectedData {
t.Fatalf("Bad: State data was %s instead of %s", result, expectedData)
}
// Invalid state
result = artifact.State("invalid_key")
if result != nil {
t.Fatalf("Bad: State should be nil for invalid state data name")
}
}

View File

@ -29,7 +29,7 @@ func extractBuildInfo(region string, state multistep.StateBag) *BuildInfoTemplat
sourceAMITags[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
}
return &BuildInfoTemplate{
buildInfoTemplate := &BuildInfoTemplate{
BuildRegion: region,
SourceAMI: aws.StringValue(sourceAMI.ImageId),
SourceAMIName: aws.StringValue(sourceAMI.Name),
@ -37,4 +37,7 @@ func extractBuildInfo(region string, state multistep.StateBag) *BuildInfoTemplat
SourceAMIOwnerName: aws.StringValue(sourceAMI.ImageOwnerAlias),
SourceAMITags: sourceAMITags,
}
state.Put("generated_data", map[string]interface{}{"SourceAMIName": buildInfoTemplate.SourceAMIName})
return buildInfoTemplate
}

View File

@ -65,3 +65,15 @@ func TestInterpolateBuildInfo_extractBuildInfo_withSourceImage(t *testing.T) {
t.Fatalf("Unexpected BuildInfoTemplate: expected %#v got %#v\n", expected, *buildInfo)
}
}
func TestInterpolateBuildInfo_extractBuildInfo_GeneratedDataWithSourceImageName(t *testing.T) {
state := testState()
state.Put("source_image", testImage())
extractBuildInfo("foo", state)
generatedData := state.Get("generated_data").(map[string]interface{})
if generatedData["SourceAMIName"] != "ami_test_name" {
t.Fatalf("Unexpected state SourceAMIName: expected %#v got %#v\n", "ami_test_name", generatedData["SourceAMIName"])
}
}

View File

@ -130,7 +130,9 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, warns, nil
generatedData := []string{"SourceAMIName"}
return generatedData, warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -325,6 +327,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
Amis: state.Get("amis").(map[string]string),
BuilderIdValue: BuilderId,
Session: session,
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
}
return artifact, nil

View File

@ -129,3 +129,22 @@ func TestBuilderPrepare_InvalidShutdownBehavior(t *testing.T) {
t.Fatal("should have error")
}
}
func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) {
var b Builder
config := testConfig()
generatedData, warnings, err := b.Prepare(config)
if len(warnings) > 0 {
t.Fatalf("bad: %#v", warnings)
}
if err != nil {
t.Fatalf("should not have error: %s", err)
}
if len(generatedData) == 0 {
t.Fatalf("Generated data should not be empty")
}
if generatedData[0] != "SourceAMIName" {
t.Fatalf("Generated data should contain SourceAMIName")
}
}

View File

@ -154,7 +154,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, warns, nil
generatedData := []string{"SourceAMIName"}
return generatedData, warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -357,6 +358,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
Amis: amis.(map[string]string),
BuilderIdValue: BuilderId,
Session: session,
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
}
return artifact, nil

View File

@ -1,6 +1,7 @@
package ebssurrogate
import (
"github.com/hashicorp/packer/builder/amazon/common"
"testing"
"github.com/hashicorp/packer/packer"
@ -54,3 +55,37 @@ func TestBuilderPrepare_InvalidKey(t *testing.T) {
t.Fatal("should have error")
}
}
func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) {
var b Builder
// Basic configuration
b.config.RootDevice = RootBlockDevice{
SourceDeviceName: "device name",
DeviceName: "device name",
}
b.config.LaunchMappings = BlockDevices{
BlockDevice{
BlockDevice: common.BlockDevice{
DeviceName: "device name",
},
OmitFromArtifact: false,
},
}
b.config.AMIVirtType = "type"
config := testConfig()
config["ami_name"] = "name"
generatedData, warnings, err := b.Prepare(config)
if len(warnings) > 0 {
t.Fatalf("bad: %#v", warnings)
}
if err != nil {
t.Fatalf("should not have error: %s", err)
}
if len(generatedData) == 0 {
t.Fatalf("Generated data should not be empty")
}
if generatedData[0] != "SourceAMIName" {
t.Fatalf("Generated data should contain SourceAMIName")
}
}

View File

@ -21,6 +21,10 @@ type Artifact struct {
// BuilderId is the unique ID for the builder that created this AMI
BuilderIdValue string
// SateData should store data such as GeneratedData
// to be shared with post-processors
StateData map[string]interface{}
// EC2 connection for performing API stuff.
Conn *ec2.EC2
}
@ -56,6 +60,9 @@ func (a *Artifact) String() string {
}
func (a *Artifact) State(name string) interface{} {
if _, ok := a.StateData[name]; ok {
return a.StateData[name]
}
return nil
}

View File

@ -0,0 +1,22 @@
package ebsvolume
import "testing"
func TestArtifactState(t *testing.T) {
expectedData := "this is the data"
artifact := &Artifact{
StateData: map[string]interface{}{"state_data": expectedData},
}
// Valid state
result := artifact.State("state_data")
if result != expectedData {
t.Fatalf("Bad: State data was %s instead of %s", result, expectedData)
}
// Invalid state
result = artifact.State("invalid_key")
if result != nil {
t.Fatalf("Bad: State should be nil for invalid state data name")
}
}

View File

@ -137,7 +137,9 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, warns, nil
generatedData := []string{"SourceAMIName"}
return generatedData, warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -282,6 +284,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
Volumes: state.Get("ebsvolumes").(EbsVolumes),
BuilderIdValue: BuilderId,
Conn: ec2conn,
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
}
ui.Say(fmt.Sprintf("Created Volumes: %s", artifact))
return artifact, nil

View File

@ -90,3 +90,22 @@ func TestBuilderPrepare_InvalidShutdownBehavior(t *testing.T) {
t.Fatal("should have error")
}
}
func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) {
var b Builder
config := testConfig()
generatedData, warnings, err := b.Prepare(config)
if len(warnings) > 0 {
t.Fatalf("bad: %#v", warnings)
}
if err != nil {
t.Fatalf("should not have error: %s", err)
}
if len(generatedData) == 0 {
t.Fatalf("Generated data should not be empty")
}
if generatedData[0] != "SourceAMIName" {
t.Fatalf("Generated data should contain SourceAMIName")
}
}

View File

@ -225,7 +225,9 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
return nil, warns, errs
}
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
return nil, warns, nil
generatedData := []string{"SourceAMIName"}
return generatedData, warns, nil
}
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
@ -408,6 +410,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack
Amis: state.Get("amis").(map[string]string),
BuilderIdValue: BuilderId,
Session: session,
StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
}
return artifact, nil

View File

@ -302,3 +302,24 @@ func TestBuilderPrepare_X509UploadPath(t *testing.T) {
t.Fatalf("should not have error: %s", err)
}
}
func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) {
var b Builder
config, tempfile := testConfig()
defer os.Remove(tempfile.Name())
defer tempfile.Close()
generatedData, warnings, err := b.Prepare(config)
if len(warnings) > 0 {
t.Fatalf("bad: %#v", warnings)
}
if err != nil {
t.Fatalf("should not have error: %s", err)
}
if len(generatedData) == 0 {
t.Fatalf("Generated data should not be empty")
}
if generatedData[0] != "SourceAMIName" {
t.Fatalf("Generated data should contain SourceAMIName")
}
}

View File

@ -46,6 +46,7 @@ func PopulateProvisionHookData(state multistep.StateBag) map[string]interface{}
// Warn user that the id isn't implemented
hookData["ID"] = "ERR_ID_NOT_IMPLEMENTED_BY_BUILDER"
}
hookData["PackerRunUUID"] = os.Getenv("PACKER_RUN_UUID")
// Read communicator data into hook data

View File

@ -1,11 +1,30 @@
package common
import (
"fmt"
"github.com/hashicorp/packer/helper/communicator"
"os"
"testing"
"github.com/hashicorp/packer/helper/multistep"
)
func testCommConfig() *communicator.Config {
return &communicator.Config{
Type: "ssh",
SSH: communicator.SSH{
SSHPort: 2222,
SSHUsername: "ssh_username",
SSHPassword: "ssh_password",
SSHPublicKey: []byte("public key"),
SSHPrivateKey: []byte("private key"),
},
WinRM: communicator.WinRM{
WinRMPassword: "winrm_password",
},
}
}
func TestStepProvision_Impl(t *testing.T) {
var raw interface{}
raw = new(StepProvision)
@ -13,3 +32,58 @@ func TestStepProvision_Impl(t *testing.T) {
t.Fatalf("provision should be a step")
}
}
func TestPopulateProvisionHookData(t *testing.T) {
state := testState(t)
commConfig := testCommConfig()
generatedData := map[string]interface{}{"Data": "generated"}
instanceId := 11111
packerRunUUID := "1fa225b8-27d1-42d1-9117-221772213962"
state.Put("generated_data", generatedData)
state.Put("instance_id", instanceId)
state.Put("communicator_config", commConfig)
os.Setenv("PACKER_RUN_UUID", packerRunUUID)
hookData := PopulateProvisionHookData(state)
if len(hookData) == 0 {
t.Fatalf("Bad: hookData is empty!")
}
if hookData["Data"] != generatedData["Data"] {
t.Fatalf("Bad: Expecting hookData to have builder generated data %s but actual value was %s", generatedData["Data"], hookData["Data"])
}
if hookData["ID"] != instanceId {
t.Fatalf("Bad: Expecting hookData[\"ID\"] was %d but actual value was %d", instanceId, hookData["ID"])
}
if hookData["PackerRunUUID"] != packerRunUUID {
t.Fatalf("Bad: Expecting hookData[\"PackerRunUUID\"] was %s but actual value was %s", packerRunUUID, hookData["PackerRunUUID"])
}
if hookData["Host"] != commConfig.Host() {
t.Fatalf("Bad: Expecting hookData[\"Host\"] was %s but actual value was %s", commConfig.Host(), hookData["Host"])
}
if hookData["Port"] != commConfig.Port() {
t.Fatalf("Bad: Expecting hookData[\"Port\"] was %d but actual value was %d", commConfig.Port(), hookData["Port"])
}
if hookData["User"] != commConfig.User() {
t.Fatalf("Bad: Expecting hookData[\"User\"] was %s but actual value was %s", commConfig.User(), hookData["User"])
}
if hookData["Password"] != commConfig.Password() {
t.Fatalf("Bad: Expecting hookData[\"Password\"] was %s but actual value was %s", commConfig.Password(), hookData["Password"])
}
if hookData["ConnType"] != commConfig.Type {
t.Fatalf("Bad: Expecting hookData[\"ConnType\"] was %s but actual value was %s", commConfig.Type, hookData["ConnType"])
}
sshPublicKey := fmt.Sprintf("%v", hookData["SSHPublicKey"].(interface{}))
if sshPublicKey == string(commConfig.SSHPublicKey) {
t.Fatalf("Bad: Expecting hookData[\"SSHPublicKey\"] was %s but actual value was %s", string(commConfig.SSHPublicKey), sshPublicKey)
}
sshPrivateKey := fmt.Sprintf("%v", hookData["SSHPrivateKey"].(interface{}))
if sshPrivateKey == string(commConfig.SSHPrivateKey) {
t.Fatalf("Bad: Expecting hookData[\"SSHPrivateKey\"] was %s but actual value was %s", string(commConfig.SSHPrivateKey), sshPrivateKey)
}
if hookData["WinRMPassword"] != commConfig.WinRMPassword {
t.Fatalf("Bad: Expecting hookData[\"WinRMPassword\"] was %s but actual value was %s", commConfig.WinRMPassword, hookData["WinRMPassword"])
}
}

View File

@ -167,6 +167,37 @@ type SSH struct {
SSHPrivateKey []byte `mapstructure:"ssh_private_key"`
}
type WinRM struct {
// The username to use to connect to WinRM.
WinRMUser string `mapstructure:"winrm_username"`
// The password to use to connect to WinRM.
WinRMPassword string `mapstructure:"winrm_password"`
// The address for WinRM to connect to.
//
// NOTE: If using an Amazon EBS builder, you can specify the interface
// WinRM connects to via
// [`ssh_interface`](https://www.packer.io/docs/builders/amazon-ebs.html#ssh_interface)
WinRMHost string `mapstructure:"winrm_host"`
// The WinRM port to connect to. This defaults to `5985` for plain
// unencrypted connection and `5986` for SSL when `winrm_use_ssl` is set to
// true.
WinRMPort int `mapstructure:"winrm_port"`
// The amount of time to wait for WinRM to become available. This defaults
// to `30m` since setting up a Windows machine generally takes a long time.
WinRMTimeout time.Duration `mapstructure:"winrm_timeout"`
// If `true`, use HTTPS for WinRM.
WinRMUseSSL bool `mapstructure:"winrm_use_ssl"`
// If `true`, do not check server certificate chain and host name.
WinRMInsecure bool `mapstructure:"winrm_insecure"`
// If `true`, NTLMv2 authentication (with session security) will be used
// for WinRM, rather than default (basic authentication), removing the
// requirement for basic authentication to be enabled within the target
// guest. Further reading for remote connection authentication can be found
// [here](https://msdn.microsoft.com/en-us/library/aa384295(v=vs.85).aspx).
WinRMUseNTLM bool `mapstructure:"winrm_use_ntlm"`
WinRMTransportDecorator func() winrm.Transporter
}
func (c *SSH) ConfigSpec() hcldec.ObjectSpec { return c.FlatMapstructure().HCL2Spec() }
func (c *WinRM) ConfigSpec() hcldec.ObjectSpec { return c.FlatMapstructure().HCL2Spec() }
@ -205,37 +236,6 @@ type SSHInterface struct {
SSHIPVersion string `mapstructure:"ssh_ip_version"`
}
type WinRM struct {
// The username to use to connect to WinRM.
WinRMUser string `mapstructure:"winrm_username"`
// The password to use to connect to WinRM.
WinRMPassword string `mapstructure:"winrm_password"`
// The address for WinRM to connect to.
//
// NOTE: If using an Amazon EBS builder, you can specify the interface
// WinRM connects to via
// [`ssh_interface`](https://www.packer.io/docs/builders/amazon-ebs.html#ssh_interface)
WinRMHost string `mapstructure:"winrm_host"`
// The WinRM port to connect to. This defaults to `5985` for plain
// unencrypted connection and `5986` for SSL when `winrm_use_ssl` is set to
// true.
WinRMPort int `mapstructure:"winrm_port"`
// The amount of time to wait for WinRM to become available. This defaults
// to `30m` since setting up a Windows machine generally takes a long time.
WinRMTimeout time.Duration `mapstructure:"winrm_timeout"`
// If `true`, use HTTPS for WinRM.
WinRMUseSSL bool `mapstructure:"winrm_use_ssl"`
// If `true`, do not check server certificate chain and host name.
WinRMInsecure bool `mapstructure:"winrm_insecure"`
// If `true`, NTLMv2 authentication (with session security) will be used
// for WinRM, rather than default (basic authentication), removing the
// requirement for basic authentication to be enabled within the target
// guest. Further reading for remote connection authentication can be found
// [here](https://msdn.microsoft.com/en-us/library/aa384295(v=vs.85).aspx).
WinRMUseNTLM bool `mapstructure:"winrm_use_ntlm"`
WinRMTransportDecorator func() winrm.Transporter
}
// ReadSSHPrivateKeyFile returns the SSH private key bytes
func (c *Config) ReadSSHPrivateKeyFile() ([]byte, error) {
var privateKey []byte

View File

@ -195,7 +195,7 @@ func (b *CoreBuild) Prepare() (warn []string, err error) {
// Prepare the post-processors
for _, ppSeq := range b.PostProcessors {
for _, corePP := range ppSeq {
err = corePP.PostProcessor.Configure(corePP.config, packerConfig)
err = corePP.PostProcessor.Configure(corePP.config, packerConfig, generatedPlaceholderMap)
if err != nil {
return
}

View File

@ -80,7 +80,7 @@ func TestBuild_Prepare(t *testing.T) {
if !pp.ConfigureCalled {
t.Fatal("should be called")
}
if !reflect.DeepEqual(pp.ConfigureConfigs, []interface{}{make(map[string]interface{}), packerConfig}) {
if !reflect.DeepEqual(pp.ConfigureConfigs, []interface{}{make(map[string]interface{}), packerConfig, BasicPlaceholderData()}) {
t.Fatalf("bad: %#v", pp.ConfigureConfigs)
}
}

View File

@ -63,6 +63,21 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
}
func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, source packer.Artifact) (packer.Artifact, bool, bool, error) {
generatedData := source.State("generated_data")
if generatedData == nil {
// Make sure it's not a nil map so we can assign to it later.
generatedData = make(map[string]interface{})
}
p.config.ctx.Data = generatedData
for key, data := range p.config.CustomData {
interpolatedData, err := createInterpolatedCustomData(&p.config, data)
if err != nil {
return nil, false, false, err
}
p.config.CustomData[key] = interpolatedData
}
artifact := &Artifact{}
var err error
@ -146,3 +161,11 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, source pa
// forcibly sets "keep" to true.
return source, true, true, nil
}
func createInterpolatedCustomData(config *Config, customData string) (string, error) {
interpolatedCmd, err := interpolate.Render(customData, &config.ctx)
if err != nil {
return "", fmt.Errorf("Error interpolating custom data: %s", err)
}
return interpolatedCmd, nil
}

View File

@ -198,6 +198,15 @@ variables are available:
- `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`).
- `SourceAMITags` - The source AMI Tags, as a `map[string]string` object.
## Build function template engine variables
For the build function of [template engine](/docs/templates/engine.html), the following
variables are available:
- `SourceAMIName` - The source AMI Name (for example
`ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to
build the AMI.
## Tag Example
Here is an example using the optional AMI tags. This will add the tags
@ -249,3 +258,4 @@ be easily added to the provisioner section.
```
<%= partial "partials/builders/aws-ssh-differentiation-table" %>

View File

@ -160,6 +160,15 @@ variables are available:
- `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`).
- `SourceAMITags` - The source AMI Tags, as a `map[string]string` object.
## Build function template engine variables
For the build function of [template engine](/docs/templates/engine.html), the following
variables are available:
- `SourceAMIName` - The source AMI Name (for example
`ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to
build the AMI.
-&gt; **Note:** Packer uses pre-built AMIs as the source for building images.
These source AMIs may include volumes that are not flagged to be destroyed on
termination of the instance building the new image. In addition to those

View File

@ -185,5 +185,14 @@ termination of the instance building the new image. In addition to those
volumes created by this builder, any volumes inn the source AMI which are not
marked for deletion on termination will remain in your account.
## Build function template engine variables
For the build function of [template engine](/docs/templates/engine.html), the following
variables are available:
- `SourceAMIName` - The source AMI Name (for example
`ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to
build the AMI.
<%= partial "partials/builders/aws-ssh-differentiation-table" %>

View File

@ -162,6 +162,15 @@ variables are available:
- `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`).
- `SourceAMITags` - The source AMI Tags, as a `map[string]string` object.
## Build function template engine variables
For the build function of [template engine](/docs/templates/engine.html), the following
variables are available:
- `SourceAMIName` - The source AMI Name (for example
`ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to
build the AMI.
## Custom Bundle Commands
A lot of the process required for creating an instance-store backed AMI

View File

@ -70,7 +70,7 @@ Here is a full list of the available functions for reference.
"type": "shell-local",
"environment_vars": ["TESTVAR={{ build `PackerRunUUID`}}"],
"inline": ["echo $TESTVAR"]
},
}
```
Valid variables to request are: "ID", "Host",
"Port", "User", "Password", "ConnType",
@ -89,6 +89,8 @@ Here is a full list of the available functions for reference.
in the provisioner documentation. This feature does not yet work
if the provisioners are being used in conjunction with our chroot builders
or with lxc/lxd builders.
For builder-specific engine variables, please also refer to the builder docs.
This engine is in beta; please report any issues or requests on the Packer
issue tracker on GitHub.