Use the hashicorp/go-getter to download files
* removed packer.Cache and references since packer.Cache is never used except in the download step. The download step now uses the new func packer.CachePath(targetPath) for this, the behavior is the same.
* removed download code from packer that was reimplemented into the go-getter library: progress bar, http download restart, checksuming from file, skip already downloaded files, symlinking, make a download cancellable by context.
* on windows if packer is running without symlinking rights and we are getting a local file, the file will be copied instead to avoid errors.
* added unit tests for step_download that are now CI tested on windows, mac & linux.
* files are now downloaded under cache dir `sha1(filename + "?checksum=" + checksum) + file_extension`
* since the output dir is based on the source url and the checksum, when the checksum fails, the file is auto deleted.
* a download file is protected and locked by a file lock,
* updated docs
* updated go modules and vendors
2019-03-13 07:11:58 -04:00
|
|
|
package rpc
|
|
|
|
|
|
|
|
import (
|
|
|
|
"io"
|
|
|
|
"log"
|
|
|
|
"net/rpc"
|
|
|
|
|
2020-11-12 17:44:02 -05:00
|
|
|
"github.com/hashicorp/packer/packer-plugin-sdk/random"
|
Use the hashicorp/go-getter to download files
* removed packer.Cache and references since packer.Cache is never used except in the download step. The download step now uses the new func packer.CachePath(targetPath) for this, the behavior is the same.
* removed download code from packer that was reimplemented into the go-getter library: progress bar, http download restart, checksuming from file, skip already downloaded files, symlinking, make a download cancellable by context.
* on windows if packer is running without symlinking rights and we are getting a local file, the file will be copied instead to avoid errors.
* added unit tests for step_download that are now CI tested on windows, mac & linux.
* files are now downloaded under cache dir `sha1(filename + "?checksum=" + checksum) + file_extension`
* since the output dir is based on the source url and the checksum, when the checksum fails, the file is auto deleted.
* a download file is protected and locked by a file lock,
* updated docs
* updated go modules and vendors
2019-03-13 07:11:58 -04:00
|
|
|
)
|
|
|
|
|
|
|
|
// TrackProgress starts a pair of ProgressTrackingClient and ProgressProgressTrackingServer
|
|
|
|
// that will send the size of each read bytes of stream.
|
|
|
|
// In order to track an operation on the terminal side.
|
|
|
|
func (u *Ui) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
|
|
|
|
pl := &TrackProgressParameters{
|
|
|
|
Src: src,
|
|
|
|
CurrentSize: currentSize,
|
|
|
|
TotalSize: totalSize,
|
|
|
|
}
|
|
|
|
var trackingID string
|
|
|
|
if err := u.client.Call("Ui.NewTrackProgress", pl, &trackingID); err != nil {
|
|
|
|
log.Printf("Error in Ui.NewTrackProgress RPC call: %s", err)
|
|
|
|
return stream
|
|
|
|
}
|
|
|
|
cli := &ProgressTrackingClient{
|
|
|
|
id: trackingID,
|
|
|
|
client: u.client,
|
|
|
|
stream: stream,
|
|
|
|
}
|
|
|
|
return cli
|
|
|
|
}
|
|
|
|
|
|
|
|
type ProgressTrackingClient struct {
|
|
|
|
id string
|
|
|
|
client *rpc.Client
|
|
|
|
stream io.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read will send len(b) over the wire instead of it's content
|
|
|
|
func (u *ProgressTrackingClient) Read(b []byte) (read int, err error) {
|
|
|
|
defer func() {
|
|
|
|
if err := u.client.Call("Ui"+u.id+".Add", read, new(interface{})); err != nil {
|
|
|
|
log.Printf("Error in ProgressTrackingClient.Read RPC call: %s", err)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return u.stream.Read(b)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *ProgressTrackingClient) Close() error {
|
|
|
|
log.Printf("closing")
|
|
|
|
if err := u.client.Call("Ui"+u.id+".Close", nil, new(interface{})); err != nil {
|
|
|
|
log.Printf("Error in ProgressTrackingClient.Close RPC call: %s", err)
|
|
|
|
}
|
|
|
|
return u.stream.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
type TrackProgressParameters struct {
|
|
|
|
Src string
|
|
|
|
TotalSize int64
|
|
|
|
CurrentSize int64
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ui *UiServer) NewTrackProgress(pl *TrackProgressParameters, reply *string) error {
|
|
|
|
// keep identifier as is for now
|
|
|
|
srvr := &ProgressTrackingServer{
|
|
|
|
id: *reply,
|
|
|
|
}
|
|
|
|
|
|
|
|
*reply = pl.Src + random.AlphaNum(6)
|
|
|
|
srvr.stream = ui.ui.TrackProgress(pl.Src, pl.CurrentSize, pl.TotalSize, nopReadCloser{})
|
|
|
|
err := ui.register("Ui"+*reply, srvr)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("failed to register ProgressTrackingServer at %s: %s", *reply, err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type ProgressTrackingServer struct {
|
|
|
|
id string
|
|
|
|
stream io.ReadCloser
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *ProgressTrackingServer) Add(size int, _ *interface{}) error {
|
|
|
|
stubBytes := make([]byte, size, size)
|
|
|
|
t.stream.Read(stubBytes)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *ProgressTrackingServer) Close(_, _ *interface{}) error {
|
|
|
|
t.stream.Close()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type nopReadCloser struct {
|
|
|
|
}
|
|
|
|
|
|
|
|
func (nopReadCloser) Close() error { return nil }
|
|
|
|
func (nopReadCloser) Read(b []byte) (int, error) { return len(b), nil }
|