From f17007d54608b1a2c6ed91666f537881586c416f Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 27 May 2020 14:40:06 -0500 Subject: [PATCH 1/9] Removed the sentinelSignaller chan from the vmware builder parsers and its unit-tests, and also commented all of the goroutines that are used. --- builder/vmware/common/driver_parser.go | 984 ++++++++++++-------- builder/vmware/common/driver_parser_test.go | 42 +- 2 files changed, 588 insertions(+), 438 deletions(-) diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 37f45e54e..c78c673e5 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -12,111 +12,118 @@ import ( "strings" ) -type sentinelSignaller chan struct{} - /** low-level parsing */ // strip the comments and extraneous newlines from a byte channel -func uncomment(eof sentinelSignaller, in <-chan byte) (chan byte, sentinelSignaller) { +func uncomment(in <-chan byte) chan byte { out := make(chan byte) - eoc := make(sentinelSignaller) go func(in <-chan byte, out chan byte) { var endofline bool - for stillReading := true; stillReading; { - select { - case <-eof: - stillReading = false - case ch, ok := <-in: - if !ok { - break - } - switch ch { - case '#': - endofline = true - case '\n': - if endofline { - endofline = false - } - } - if !endofline { - out <- ch - } + + for { + by, ok := <-in + if !ok { + break + } + + // If we find a comment, then everything until the end of line + // needs to be culled. We keep track of that using the `endofline` + // flag. + if by == '#' { + endofline = true + + } else if by == '\n' && endofline { + endofline = false + } + + // If we're not in the processing of culling bytes, then write what + // we've read into our output chan. + if !endofline { + out <- by } } - close(eoc) + close(out) }(in, out) - return out, eoc + return out } // convert a byte channel into a channel of pseudo-tokens -func tokenizeDhcpConfig(eof sentinelSignaller, in chan byte) (chan string, sentinelSignaller) { +func tokenizeDhcpConfig(in chan byte) chan string { var state string var quote bool - eot := make(sentinelSignaller) - out := make(chan string) go func(out chan string) { - for stillReading := true; stillReading; { - select { - case <-eof: - stillReading = false + for { + by, ok := <-in + if !ok { + break + } - case ch, ok := <-in: - if !ok { - break - } - if quote { - if ch == '"' { - out <- state + string(ch) - state, quote = "", false - continue - } - state += string(ch) + // If we're in a quote, then we continue until we're not in a quote + // before we start looing for tokens + if quote { + if by == '"' { + out <- state + string(by) + state, quote = "", false continue } + state += string(by) + continue + } - switch ch { - case '"': - quote = true - state += string(ch) + switch by { + case '"': + // Otherwise we're outside any quotes and can process bytes normaly + quote = true + state += string(by) + continue + + case '\r': + fallthrough + case '\n': + fallthrough + case '\t': + fallthrough + case ' ': + // Whitespace is a separator, so we check to see if there's any state. + // If so, then write our state prior to resetting. + + if len(state) == 0 { continue + } + out <- state + state = "" - case '\r': - fallthrough - case '\n': - fallthrough - case '\t': - fallthrough - case ' ': - if len(state) == 0 { - continue - } + case '{': + fallthrough + case '}': + fallthrough + case ';': + // If we encounter a brace or a semicolon, then we need to emit our + // state and then the byte because it can be part of the token. + + if len(state) > 0 { out <- state - state = "" - - case '{': - fallthrough - case '}': - fallthrough - case ';': - if len(state) > 0 { - out <- state - } - out <- string(ch) - state = "" - - default: - state += string(ch) } + out <- string(by) + state = "" + + default: + // Just a byte which needs to be aggregated into our state + state += string(by) } } + + // If we still have any data left, then make sure to emit that if len(state) > 0 { out <- state } - close(eot) + + // Close our channel since we're responsible for it. + close(out) }(out) - return out, eot + return out } /** mid-level parsing */ @@ -162,33 +169,34 @@ func parseTokenParameter(in chan string) tkParameter { for { token, ok := <-in if !ok { - goto leave + break } + // If there's no name for this parameter yet, then the first token + // is our name. Snag it into our struct, and grab the next one. if result.name == "" { result.name = token continue } - switch token { - case "{": - fallthrough - case "}": - fallthrough - case ";": - goto leave - default: + + // If encounter any braces or line-terminators, then we're done parsing. + // Anything else we find are just operands we need to keep track of. + if strings.ContainsAny("{};", token) { + break + } else { result.operand = append(result.operand, token) } } -leave: return result } // convert a channel of pseudo-tokens into an tkGroup tree */ -func parseDhcpConfig(eof sentinelSignaller, in chan string) (tkGroup, error) { +func parseDhcpConfig(in chan string) (tkGroup, error) { var tokens []string var result tkGroup + // This utility function takes a list of tokens and line-terminates them + // before sending them to parseTokenParameter(). toParameter := func(tokens []string) tkParameter { out := make(chan string) go func(out chan string) { @@ -201,121 +209,173 @@ func parseDhcpConfig(eof sentinelSignaller, in chan string) (tkGroup, error) { return parseTokenParameter(out) } - for stillReading, currentgroup := true, &result; stillReading; { - select { - case <-eof: - stillReading = false + // Start building our tree using result as our root node + node := &result + for { + tk, ok := <-in + if !ok { + break + } + + switch tk { + case "{": + // If our next token is an opening brace, then we need to collect our + // current aggregated tokens to parse, push our current node onto the + // tree, and then pivot into it. Then we can reset our tokens for the child. + + grp := &tkGroup{parent: node} + grp.id = toParameter(tokens) + + node.groups = append(node.groups, grp) + node = grp - case tk := <-in: - switch tk { - case "{": - grp := &tkGroup{parent: currentgroup} - grp.id = toParameter(tokens) - currentgroup.groups = append(currentgroup.groups, grp) - currentgroup = grp - case "}": - if currentgroup.parent == nil { - return tkGroup{}, fmt.Errorf("Unable to close the global declaration") - } - if len(tokens) > 0 { - return tkGroup{}, fmt.Errorf("List of tokens was left unterminated : %v", tokens) - } - currentgroup = currentgroup.parent - case ";": - arg := toParameter(tokens) - currentgroup.params = append(currentgroup.params, arg) - default: - tokens = append(tokens, tk) - continue - } tokens = []string{} + + case "}": + // Otherwise if it's a closing brace, then we need to pop back up to + // the parent node and resume parsing. If we have any tokens, then + // that was because they were unterminated. Raise an error in that case. + + if node.parent == nil { + return tkGroup{}, fmt.Errorf("Refusing to close the global declaration") + } + if len(tokens) > 0 { + return tkGroup{}, fmt.Errorf("List of tokens was left unterminated : %v", tokens) + } + node = node.parent + + tokens = []string{} + + case ";": + // If we encounter a line-terminator, then the list of tokens we've been + // aggregating are ready to be parsed. Afterwards, we can write them + // to our current tree node. + + arg := toParameter(tokens) + node.params = append(node.params, arg) + tokens = []string{} + + default: + // Anything else requires us to aggregate our token into our list, and + // try grabbing the next one. + + tokens = append(tokens, tk) } } return result, nil } -func tokenizeNetworkMapConfig(eof sentinelSignaller, in chan byte) (chan string, sentinelSignaller) { - var ch byte +func tokenizeNetworkMapConfig(in chan byte) chan string { var state string var quote bool var lastnewline bool - eot := make(sentinelSignaller) + // This logic is very similar to tokenizeDhcpConfig except she needs to handle + // braces, and we don't. This is the only major difference from us. out := make(chan string) go func(out chan string) { - for stillReading := true; stillReading; { - select { - case <-eof: - stillReading = false - - case ch = <-in: - if quote { - if ch == '"' { - out <- state + string(ch) - state, quote = "", false - continue - } - state += string(ch) - continue - } - - switch ch { - case '"': - quote = true - state += string(ch) - continue - - case '\r': - fallthrough - case '\t': - fallthrough - case ' ': - if len(state) == 0 { - continue - } - out <- state - state = "" - - case '\n': - if lastnewline { - continue - } - if len(state) > 0 { - out <- state - } - out <- string(ch) - state = "" - lastnewline = true - continue - - case '.': - fallthrough - case '=': - if len(state) > 0 { - out <- state - } - out <- string(ch) - state = "" - - default: - state += string(ch) - } - lastnewline = false + for { + by, ok := <-in + if !ok { + break } + + // If we're currently inide a quote, then we need to continue until + // we encounter the closing quote. We'll keep collecting our state + // in the meantime. + if quote { + if by == '"' { + out <- state + string(by) + state, quote = "", false + continue + } + state += string(by) + continue + } + + switch by { + case '"': + // If we encounter a quote, then we need to transition into our + // quote-parsing state that keeps collecting data until the closing + // quote is encountered. + + quote = true + state += string(by) + continue + + case '\r': + fallthrough + case '\t': + fallthrough + case ' ': + // Whitespace is considered a separator, so if we encounter this + // then we can write our current state, and then reset. + + if len(state) == 0 { + continue + } + out <- state + state = "" + + case '\n': + // Newlines are a somewhat special case because they separate each + // attribute/line-item, and they can repeat. We need to preserve + // this token, so we write our current state, then the newline. + // We also maintain a flag so that we can consolidate multiple + // newlines together. + + if lastnewline { + continue + } + if len(state) > 0 { + out <- state + } + out <- string(by) + state = "" + lastnewline = true + continue + + case '.': + fallthrough + case '=': + // These characters separate attributes or tokens from one another, + // so they result in writing the state, the character, and then reset. + + if len(state) > 0 { + out <- state + } + out <- string(by) + state = "" + + default: + // Any byte we couldn't parse just gets aggregated for the next pass. + state += string(by) + } + + // If we made it here, then we can guarantee that the we didn't just + // process a newline. Clear this flag for the next one we find. + lastnewline = false } + + // If there's anything left in our state, then the last line was just not + // newline-terminated. This is a common occurrence, so write our current + // state before we finish. if len(state) > 0 { out <- state } - close(eot) + close(out) }(out) - return out, eot + return out } -func parseNetworkMapConfig(eof sentinelSignaller, in chan string) (NetworkMap, error) { - var unsorted map[string]map[string]string +func parseNetworkMapConfig(in chan string) (NetworkMap, error) { var state []string + unsorted := make(map[string]map[string]string) + // A network map has the following syntax "network.attribute = value". This + // closure is responsible for using the "network" as a key into the `unsorted` + // mapping, and then assigning the "value" into it keyed by the "attribute". addResult := func(network string, attribute string, value string) error { _, ok := unsorted[network] if !ok { @@ -332,53 +392,67 @@ func parseNetworkMapConfig(eof sentinelSignaller, in chan string) (NetworkMap, e return nil } - stillReading := true - for unsorted = make(map[string]map[string]string); stillReading; { - select { - case <-eof: + // Loop through all of our tokens making sure to update our unsorted map. + for { + tk, ok := <-in + if !ok { + // If our token channel is closed, then check to see if we've + // collected 3 items in our state. If so, then we can add this + // final attribute/value before we leave. if len(state) == 3 { err := addResult(state[0], state[1], state[2]) if err != nil { return nil, err } } - stillReading = false - case tk := <-in: - switch tk { - case ".": - if len(state) != 1 { - return nil, fmt.Errorf("Missing network index") - } - case "=": - if len(state) != 2 { - return nil, fmt.Errorf("Assignment to empty attribute") - } - case "\n": - if len(state) == 0 { - continue - } - if len(state) != 3 { - return nil, fmt.Errorf("Invalid attribute assignment : %v", state) - } - err := addResult(state[0], state[1], state[2]) - if err != nil { - return nil, err - } - state = make([]string, 0) - default: - state = append(state, tk) + break + } + + // This switch makes sure we encounter these tokens in the correct order. + switch tk { + case ".": + if len(state) != 1 { + return nil, fmt.Errorf("Missing network index") } + + case "=": + if len(state) != 2 { + return nil, fmt.Errorf("Assignment to empty attribute") + } + + case "\n": + if len(state) == 0 { + continue + } + if len(state) != 3 { + return nil, fmt.Errorf("Invalid attribute assignment : %v", state) + } + err := addResult(state[0], state[1], state[2]) + if err != nil { + return nil, err + } + state = make([]string, 0) + + default: + state = append(state, tk) } } + + // Go through our unsorted map, and collect all of the keys for "network". result := make([]map[string]string, 0) var keys []string for k := range unsorted { keys = append(keys, k) } + + // This way we can sort them. sort.Strings(keys) + + // And then collect all of them into a list to return to the caller. for _, k := range keys { result = append(result, unsorted[k]) } + return result, nil } @@ -429,9 +503,11 @@ type pParameterHardware struct { func (e pParameterHardware) repr() string { res := make([]string, 0) + for _, v := range e.address { res = append(res, fmt.Sprintf("%02x", v)) } + return fmt.Sprintf("hardware-address:%s[%s]", e.class, strings.Join(res, ":")) } @@ -528,6 +604,7 @@ func (e *pDeclaration) repr() string { if e.parent != nil { res = fmt.Sprintf("%s parent:%s", res, e.parent.short()) } + return fmt.Sprintf("%s\n%s\n%s\n", res, strings.Join(parameters, "\n"), strings.Join(groups, "\n")) } @@ -570,6 +647,7 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) != 2 { return nil, fmt.Errorf("Invalid number of parameters for pParameterInclude : %v", val.operand) } + name := val.operand[0] return pParameterInclude{filename: name}, nil @@ -577,6 +655,7 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) != 2 { return nil, fmt.Errorf("Invalid number of parameters for pParameterOption : %v", val.operand) } + name, value := val.operand[0], val.operand[1] return pParameterOption{name: name, value: value}, nil @@ -588,6 +667,7 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) < 1 { return nil, fmt.Errorf("Invalid number of parameters for pParameterGrant : %v", val.operand) } + attribute := strings.Join(val.operand, " ") return pParameterGrant{verb: strings.ToLower(val.name), attribute: attribute}, nil @@ -595,14 +675,17 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) < 1 { return nil, fmt.Errorf("Invalid number of parameters for pParameterRange4 : %v", val.operand) } + idxAddress := map[bool]int{true: 1, false: 0}[strings.ToLower(val.operand[0]) == "bootp"] if len(val.operand) > 2+idxAddress { return nil, fmt.Errorf("Invalid number of parameters for pParameterRange : %v", val.operand) } + if idxAddress+1 > len(val.operand) { res := net.ParseIP(val.operand[idxAddress]) return pParameterRange4{min: res, max: res}, nil } + addr1 := net.ParseIP(val.operand[idxAddress]) addr2 := net.ParseIP(val.operand[idxAddress+1]) return pParameterRange4{min: addr1, max: addr2}, nil @@ -615,6 +698,7 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(cidr) != 2 { return nil, fmt.Errorf("Unknown ipv6 format : %v", address) } + address := net.ParseIP(cidr[0]) bits, err := strconv.Atoi(cidr[1]) if err != nil { @@ -632,6 +716,7 @@ func parseParameter(val tkParameter) (pParameter, error) { for i := networkSize / 8; i < totalSize/8; i++ { broadcast[i] = byte(0xff) } + octetIndex := network[networkSize/8] bitsLeft := (uint32)(hostSize % 8) broadcast[octetIndex] = network[octetIndex] | ((1 << bitsLeft) - 1) @@ -642,11 +727,13 @@ func parseParameter(val tkParameter) (pParameter, error) { res := net.ParseIP(address) return pParameterRange6{min: res, max: res}, nil } + if len(val.operand) == 2 { addr := net.ParseIP(val.operand[0]) if strings.ToLower(val.operand[1]) == "temporary" { return pParameterRange6{min: addr, max: addr}, nil } + other := net.ParseIP(val.operand[1]) return pParameterRange6{min: addr, max: other}, nil } @@ -656,10 +743,12 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) != 3 { return nil, fmt.Errorf("Invalid number of parameters for pParameterRange6 : %v", val.operand) } + bits, err := strconv.Atoi(val.operand[2]) if err != nil { return nil, fmt.Errorf("Invalid bits for pParameterPrefix6 : %v", val.operand[2]) } + minaddr := net.ParseIP(val.operand[0]) maxaddr := net.ParseIP(val.operand[1]) return pParameterPrefix6{min: minaddr, max: maxaddr, bits: bits}, nil @@ -668,6 +757,7 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) != 2 { return nil, fmt.Errorf("Invalid number of parameters for pParameterHardware : %v", val.operand) } + class := val.operand[0] octets := strings.Split(val.operand[1], ":") address := make([]byte, 0) @@ -678,6 +768,7 @@ func parseParameter(val tkParameter) (pParameter, error) { } address = append(address, byte(b)) } + return pParameterHardware{class: class, address: address}, nil case "fixed-address": @@ -694,34 +785,42 @@ func parseParameter(val tkParameter) (pParameter, error) { if len(val.operand) != 3 { return nil, fmt.Errorf("Invalid number of parameters for pParameterClientMatch : %v", val.operand) } + if val.operand[0] != "option" { return nil, fmt.Errorf("Invalid match parameter : %v", val.operand[0]) } + optionName := val.operand[1] optionData := val.operand[2] return pParameterClientMatch{name: optionName, data: optionData}, nil default: length := len(val.operand) + if length < 1 { return pParameterBoolean{parameter: val.name, truancy: true}, nil + } else if length > 1 { if val.operand[0] == "=" { return pParameterExpression{parameter: val.name, expression: strings.Join(val.operand[1:], "")}, nil } } + if length != 1 { return nil, fmt.Errorf("Invalid number of parameters for pParameterOther : %v", val.operand) } + if strings.ToLower(val.name) == "not" { return pParameterBoolean{parameter: val.operand[0], truancy: false}, nil } + return pParameterOther{parameter: val.name, value: val.operand[0]}, nil } } func parseTokenGroup(val tkGroup) (*pDeclaration, error) { params := val.id.operand + switch val.id.name { case "group": return &pDeclaration{id: pDeclarationGroup{}}, nil @@ -749,6 +848,7 @@ func parseTokenGroup(val tkGroup) (*pDeclaration, error) { return &pDeclaration{id: pDeclarationSubnet4{net.IPNet{IP: subnet, Mask: mask}}}, nil } } + case "subnet6": if len(params) == 1 { ip6 := strings.SplitN(params[0], "/", 2) @@ -761,10 +861,12 @@ func parseTokenGroup(val tkGroup) (*pDeclaration, error) { return &pDeclaration{id: pDeclarationSubnet6{net.IPNet{IP: address, Mask: net.CIDRMask(prefix, net.IPv6len*8)}}}, nil } } + case "shared-network": if len(params) == 1 { return &pDeclaration{id: pDeclarationShared{name: params[0]}}, nil } + case "": return &pDeclaration{id: pDeclarationGlobal{}}, nil } @@ -772,8 +874,8 @@ func parseTokenGroup(val tkGroup) (*pDeclaration, error) { } func flattenDhcpConfig(root tkGroup) (*pDeclaration, error) { - var result *pDeclaration result, err := parseTokenGroup(root) + if err != nil { return nil, err } @@ -785,6 +887,7 @@ func flattenDhcpConfig(root tkGroup) (*pDeclaration, error) { } result.parameters = append(result.parameters, param) } + for _, p := range root.groups { group, err := flattenDhcpConfig(*p) if err != nil { @@ -793,6 +896,7 @@ func flattenDhcpConfig(root tkGroup) (*pDeclaration, error) { group.parent = result result.declarations = append(result.declarations, *group) } + return result, nil } @@ -870,16 +974,14 @@ func createDeclaration(node pDeclaration) configDeclaration { func (e *configDeclaration) repr() string { var result []string - var res []string - - res = make([]string, 0) + res := make([]string, 0) for _, v := range e.id { res = append(res, v.repr()) } result = append(result, strings.Join(res, ",")) if len(e.address) > 0 { - res = make([]string, 0) + res := make([]string, 0) for _, v := range e.address { res = append(res, v.repr()) } @@ -903,17 +1005,19 @@ func (e *configDeclaration) repr() string { } if len(e.hostid) > 0 { - res = make([]string, 0) + res := make([]string, 0) for _, v := range e.hostid { res = append(res, v.repr()) } result = append(result, fmt.Sprintf("hostid : %v", strings.Join(res, " "))) } + return strings.Join(result, "\n") + "\n" } func (e *configDeclaration) IP4() (net.IP, error) { var result []string + for _, entry := range e.address { switch entry.(type) { case pParameterAddress4: @@ -922,8 +1026,10 @@ func (e *configDeclaration) IP4() (net.IP, error) { } } } + if len(result) > 1 { return nil, fmt.Errorf("More than one address4 returned : %v", result) + } else if len(result) == 0 { return nil, fmt.Errorf("No IP4 addresses found") } @@ -931,14 +1037,18 @@ func (e *configDeclaration) IP4() (net.IP, error) { if res := net.ParseIP(result[0]); res != nil { return res, nil } + res, err := net.ResolveIPAddr("ip4", result[0]) if err != nil { return nil, err } + return res.IP, nil } + func (e *configDeclaration) IP6() (net.IP, error) { var result []string + for _, entry := range e.address { switch entry.(type) { case pParameterAddress6: @@ -947,8 +1057,10 @@ func (e *configDeclaration) IP6() (net.IP, error) { } } } + if len(result) > 1 { return nil, fmt.Errorf("More than one address6 returned : %v", result) + } else if len(result) == 0 { return nil, fmt.Errorf("No IP6 addresses found") } @@ -956,23 +1068,28 @@ func (e *configDeclaration) IP6() (net.IP, error) { if res := net.ParseIP(result[0]); res != nil { return res, nil } + res, err := net.ResolveIPAddr("ip6", result[0]) if err != nil { return nil, err } return res.IP, nil } + func (e *configDeclaration) Hardware() (net.HardwareAddr, error) { var result []pParameterHardware + for _, addr := range e.address { switch addr.(type) { case pParameterHardware: result = append(result, addr.(pParameterHardware)) } } + if len(result) > 0 { return nil, fmt.Errorf("More than one hardware address returned : %v", result) } + res := make(net.HardwareAddr, 0) for _, by := range result[0].address { res = append(res, by) @@ -984,20 +1101,29 @@ func (e *configDeclaration) Hardware() (net.HardwareAddr, error) { type DhcpConfiguration []configDeclaration func ReadDhcpConfiguration(fd *os.File) (DhcpConfiguration, error) { - fromfile, eof := consumeFile(fd) - uncommented, eoc := uncomment(eof, fromfile) - tokenized, eot := tokenizeDhcpConfig(eoc, uncommented) - parsetree, err := parseDhcpConfig(eot, tokenized) + fromfile := consumeFile(fd) + uncommented := uncomment(fromfile) + tokenized := tokenizeDhcpConfig(uncommented) + + // Parse the tokenized DHCP configuration into a tree. We need it as a tree + // because DHCP declarations can inherit options from their parent + parsetree, err := parseDhcpConfig(tokenized) if err != nil { return nil, err } + // Flatten the tree into a list of pDeclaration objects. This is responsible + // for actually propagating options from the parent pDeclaration into all of + // its children. global, err := flattenDhcpConfig(parsetree) if err != nil { return nil, err } + // This closure is just to the goro that follows it in recursively walking + // through all of the declarations and writing them individually to a chan. var walkDeclarations func(root pDeclaration, out chan *configDeclaration) + walkDeclarations = func(root pDeclaration, out chan *configDeclaration) { res := createDeclaration(root) out <- &res @@ -1006,12 +1132,15 @@ func ReadDhcpConfiguration(fd *os.File) (DhcpConfiguration, error) { } } + // That way this goro can take each individual declaration and write it to + // a channel. each := make(chan *configDeclaration) go func(out chan *configDeclaration) { walkDeclarations(*global, out) out <- nil }(each) + // For this loop to convert it into a itemized list. var result DhcpConfiguration for decl := <-each; decl != nil; decl = <-each { result = append(result, *decl) @@ -1081,28 +1210,33 @@ type NetworkNameMapper interface { } func ReadNetworkMap(fd *os.File) (NetworkMap, error) { + fromfile := consumeFile(fd) + uncommented := uncomment(fromfile) + tokenized := tokenizeNetworkMapConfig(uncommented) - fromfile, eof := consumeFile(fd) - uncommented, eoc := uncomment(eof, fromfile) - tokenized, eot := tokenizeNetworkMapConfig(eoc, uncommented) - - result, err := parseNetworkMapConfig(eot, tokenized) + // Now that we've tokenized the network map, we just need to parse it into + // a list of maps. + result, err := parseNetworkMapConfig(tokenized) if err != nil { return nil, err } + return result, nil } func (e NetworkMap) NameIntoDevices(name string) ([]string, error) { var devices []string + for _, val := range e { if strings.ToLower(val["name"]) == strings.ToLower(name) { devices = append(devices, val["device"]) } } + if len(devices) > 0 { return devices, nil } + return make([]string, 0), fmt.Errorf("Network name not found : %v", name) } func (e NetworkMap) DeviceIntoName(device string) (string, error) { @@ -1111,100 +1245,114 @@ func (e NetworkMap) DeviceIntoName(device string) (string, error) { return val["name"], nil } } + return "", fmt.Errorf("Device name not found : %v", device) } func (e *NetworkMap) repr() string { var result []string + for idx, val := range *e { result = append(result, fmt.Sprintf("network%d.name = \"%s\"", idx, val["name"])) result = append(result, fmt.Sprintf("network%d.device = \"%s\"", idx, val["device"])) } + return strings.Join(result, "\n") } /*** parser for VMware Fusion's networking file */ -func tokenizeNetworkingConfig(eof sentinelSignaller, in chan byte) (chan string, sentinelSignaller) { - var ch byte +func tokenizeNetworkingConfig(in chan byte) chan string { var state string var repeat_newline bool - eot := make(sentinelSignaller) - out := make(chan string) go func(out chan string) { - for reading := true; reading; { - select { - case <-eof: - reading = false - - case ch = <-in: - switch ch { - case '\r': - fallthrough - case '\t': - fallthrough - case ' ': - if len(state) == 0 { - continue - } - out <- state - state = "" - case '\n': - if repeat_newline { - continue - } - if len(state) > 0 { - out <- state - } - out <- string(ch) - state = "" - repeat_newline = true - continue - default: - state += string(ch) - } - repeat_newline = false + for { + by, ok := <-in + if !ok { + break } + + switch by { + case '\r': + fallthrough + case '\t': + fallthrough + case ' ': + // If we receive whitespace, then this is just a write to our + // state and then we reset. + if len(state) == 0 { + continue + } + out <- state + state = "" + + case '\n': + // Newlines can repeat, so this case is responsible for writing + // to the chan, and consolidating multiple newlines into singles. + if repeat_newline { + continue + } + if len(state) > 0 { + out <- state + } + out <- string(by) + state = "" + repeat_newline = true + continue + + default: + // Anything other bytes just need to be aggregated into a string. + state += string(by) + } + repeat_newline = false } + + // If there's anything left in our state after the chan has been closed, + // then the input just wasn't terminated properly. It's still valid, so + // write we have to the channel. if len(state) > 0 { out <- state } - close(eot) }(out) - return out, eot + return out } -func splitNetworkingConfig(eof sentinelSignaller, in chan string) (chan []string, sentinelSignaller) { - var out chan []string +func splitNetworkingConfig(in chan string) chan []string { + out := make(chan []string) - eos := make(sentinelSignaller) + // This goroutine is simple in that it takes a chan of tokens, and splits + // them across the newlines. - out = make(chan []string) go func(out chan []string) { row := make([]string, 0) - for reading := true; reading; { - select { - case <-eof: - reading = false + for { + tk, ok := <-in + if !ok { + break + } - case tk := <-in: - switch tk { - case "\n": - if len(row) > 0 { - out <- row - } - row = make([]string, 0) - default: - row = append(row, tk) + if tk == "\n" { + // If we received a newline token, then we need to write our + // aggregated list of tokens and reset our "splitting" state. + + if len(row) > 0 { + out <- row } + + row = make([]string, 0) + + } else { + // Anything else just requires us to aggregate the token into + // our list. + row = append(row, tk) } } + if len(row) > 0 { out <- row } - close(eos) }(out) - return out, eos + return out } /// All token types in networking file. @@ -1437,8 +1585,7 @@ func (e networkingCommandEntry) Entry() reflect.Value { } func (e networkingCommandEntry) Repr() string { - var result map[string]interface{} - result = make(map[string]interface{}) + result := make(map[string]interface{}) entryN, entry := e.Name(), e.Entry() entryT := entry.Type() @@ -1454,13 +1601,13 @@ func parseNetworkingCommand_answer(row []string) (*networkingCommandEntry, error if len(row) != 2 { return nil, fmt.Errorf("Expected %d arguments. Received only %d.", 2, len(row)) } + vnet := networkingVNET{value: row[0]} if !vnet.Valid() { return nil, fmt.Errorf("Invalid format for VNET.") } - value := row[1] - result := networkingCommandEntry_answer{vnet: vnet, value: value} + result := networkingCommandEntry_answer{vnet: vnet, value: row[1]} return &networkingCommandEntry{entry: result, answer: &result}, nil } func parseNetworkingCommand_remove_answer(row []string) (*networkingCommandEntry, error) { @@ -1635,6 +1782,7 @@ func parseNetworkingCommand_remove_nat_prefix(row []string) (*networkingCommandE if !strings.HasPrefix(row[1], "/") { return nil, fmt.Errorf("Expected second argument to begin with \"/\". : %v", row[1]) } + prefix, err := strconv.Atoi(row[1][1:]) if err != nil { return nil, fmt.Errorf("Unable to parse prefix out of second argument. : %v", row[1]) @@ -1671,37 +1819,35 @@ func NetworkingParserByCommand(command string) *func([]string) (*networkingComma return nil } -func parseNetworkingConfig(eof sentinelSignaller, rows chan []string) (chan networkingCommandEntry, sentinelSignaller) { - var out chan networkingCommandEntry +func parseNetworkingConfig(rows chan []string) chan networkingCommandEntry { + out := make(chan networkingCommandEntry) - eop := make(sentinelSignaller) - - out = make(chan networkingCommandEntry) go func(in chan []string, out chan networkingCommandEntry) { - for reading := true; reading; { - select { - case <-eof: - reading = false - case row := <-in: - if len(row) >= 1 { - parser := NetworkingParserByCommand(row[0]) - if parser == nil { - log.Printf("Invalid command : %v", row) - continue - } - callback := *parser - entry, err := callback(row[1:]) - if err != nil { - log.Printf("Unable to parse command : %v %v", err, row) - continue - } - out <- *entry + for { + row, ok := <-in + if !ok { + break + } + + if len(row) >= 1 { + parser := NetworkingParserByCommand(row[0]) + if parser == nil { + log.Printf("Invalid command : %v", row) + continue } + + callback := *parser + + entry, err := callback(row[1:]) + if err != nil { + log.Printf("Unable to parse command : %v %v", err, row) + continue + } + out <- *entry } } - close(eop) }(rows, out) - return out, eop + return out } type NetworkingConfig struct { @@ -1717,7 +1863,7 @@ func (c NetworkingConfig) repr() string { return fmt.Sprintf("answer -> %v\nnat_portfwd -> %v\ndhcp_mac_to_ip -> %v\nbridge_mapping -> %v\nnat_prefix -> %v", c.answer, c.nat_portfwd, c.dhcp_mac_to_ip, c.bridge_mapping, c.nat_prefix) } -func flattenNetworkingConfig(eof sentinelSignaller, in chan networkingCommandEntry) NetworkingConfig { +func flattenNetworkingConfig(in chan networkingCommandEntry) NetworkingConfig { var result NetworkingConfig var vmnet int @@ -1727,96 +1873,106 @@ func flattenNetworkingConfig(eof sentinelSignaller, in chan networkingCommandEnt result.bridge_mapping = make(map[string]int) result.nat_prefix = make(map[int][]int) - for reading := true; reading; { - select { - case <-eof: - reading = false - case e := <-in: - switch e.entry.(type) { - case networkingCommandEntry_answer: - vnet := e.answer.vnet - answers, exists := result.answer[vnet.Number()] - if !exists { - answers = make(map[string]string) - result.answer[vnet.Number()] = answers - } - answers[vnet.Option()] = e.answer.value - case networkingCommandEntry_remove_answer: - vnet := e.remove_answer.vnet - answers, exists := result.answer[vnet.Number()] - if exists { - delete(answers, vnet.Option()) - } else { - log.Printf("Unable to remove answer %s as specified by `remove_answer`.\n", vnet.Repr()) - } - case networkingCommandEntry_add_nat_portfwd: - vmnet = e.add_nat_portfwd.vnet - protoport := fmt.Sprintf("%s/%d", e.add_nat_portfwd.protocol, e.add_nat_portfwd.port) - target := fmt.Sprintf("%s:%d", e.add_nat_portfwd.target_host, e.add_nat_portfwd.target_port) - portfwds, exists := result.nat_portfwd[vmnet] - if !exists { - portfwds = make(map[string]string) - result.nat_portfwd[vmnet] = portfwds - } - portfwds[protoport] = target - case networkingCommandEntry_remove_nat_portfwd: - vmnet = e.remove_nat_portfwd.vnet - protoport := fmt.Sprintf("%s/%d", e.remove_nat_portfwd.protocol, e.remove_nat_portfwd.port) - portfwds, exists := result.nat_portfwd[vmnet] - if exists { - delete(portfwds, protoport) - } else { - log.Printf("Unable to remove nat port-forward %s from interface %s%d as requested by `remove_nat_portfwd`.\n", protoport, NetworkingInterfacePrefix, vmnet) - } - case networkingCommandEntry_add_dhcp_mac_to_ip: - vmnet = e.add_dhcp_mac_to_ip.vnet - dhcpmacs, exists := result.dhcp_mac_to_ip[vmnet] - if !exists { - dhcpmacs = make(map[string]net.IP) - result.dhcp_mac_to_ip[vmnet] = dhcpmacs - } - dhcpmacs[e.add_dhcp_mac_to_ip.mac.String()] = e.add_dhcp_mac_to_ip.ip - case networkingCommandEntry_remove_dhcp_mac_to_ip: - vmnet = e.remove_dhcp_mac_to_ip.vnet - dhcpmacs, exists := result.dhcp_mac_to_ip[vmnet] - if exists { - delete(dhcpmacs, e.remove_dhcp_mac_to_ip.mac.String()) - } else { - log.Printf("Unable to remove dhcp_mac_to_ip entry %v from interface %s%d as specified by `remove_dhcp_mac_to_ip`.\n", e.remove_dhcp_mac_to_ip, NetworkingInterfacePrefix, vmnet) - } - case networkingCommandEntry_add_bridge_mapping: - intf := e.add_bridge_mapping.intf - if _, err := intf.Interface(); err != nil { - log.Printf("Interface \"%s\" as specified by `add_bridge_mapping` was not found on the current platform. This is a non-critical error. Ignoring.", intf.name) - } - result.bridge_mapping[intf.name] = e.add_bridge_mapping.vnet - case networkingCommandEntry_remove_bridge_mapping: - intf := e.remove_bridge_mapping.intf - if _, err := intf.Interface(); err != nil { - log.Printf("Interface \"%s\" as specified by `remove_bridge_mapping` was not found on the current platform. This is a non-critical error. Ignoring.", intf.name) - } - delete(result.bridge_mapping, intf.name) - case networkingCommandEntry_add_nat_prefix: - vmnet = e.add_nat_prefix.vnet - _, exists := result.nat_prefix[vmnet] - if exists { - result.nat_prefix[vmnet] = append(result.nat_prefix[vmnet], e.add_nat_prefix.prefix) - } else { - result.nat_prefix[vmnet] = []int{e.add_nat_prefix.prefix} - } - case networkingCommandEntry_remove_nat_prefix: - vmnet = e.remove_nat_prefix.vnet - prefixes, exists := result.nat_prefix[vmnet] - if exists { - for index := 0; index < len(prefixes); index++ { - if prefixes[index] == e.remove_nat_prefix.prefix { - result.nat_prefix[vmnet] = append(prefixes[:index], prefixes[index+1:]...) - break - } + for { + e, ok := <-in + if !ok { + break + } + + switch e.entry.(type) { + case networkingCommandEntry_answer: + vnet := e.answer.vnet + answers, exists := result.answer[vnet.Number()] + if !exists { + answers = make(map[string]string) + result.answer[vnet.Number()] = answers + } + answers[vnet.Option()] = e.answer.value + + case networkingCommandEntry_remove_answer: + vnet := e.remove_answer.vnet + answers, exists := result.answer[vnet.Number()] + if exists { + delete(answers, vnet.Option()) + } else { + log.Printf("Unable to remove answer %s as specified by `remove_answer`.\n", vnet.Repr()) + } + + case networkingCommandEntry_add_nat_portfwd: + vmnet = e.add_nat_portfwd.vnet + protoport := fmt.Sprintf("%s/%d", e.add_nat_portfwd.protocol, e.add_nat_portfwd.port) + target := fmt.Sprintf("%s:%d", e.add_nat_portfwd.target_host, e.add_nat_portfwd.target_port) + portfwds, exists := result.nat_portfwd[vmnet] + if !exists { + portfwds = make(map[string]string) + result.nat_portfwd[vmnet] = portfwds + } + portfwds[protoport] = target + + case networkingCommandEntry_remove_nat_portfwd: + vmnet = e.remove_nat_portfwd.vnet + protoport := fmt.Sprintf("%s/%d", e.remove_nat_portfwd.protocol, e.remove_nat_portfwd.port) + portfwds, exists := result.nat_portfwd[vmnet] + if exists { + delete(portfwds, protoport) + } else { + log.Printf("Unable to remove nat port-forward %s from interface %s%d as requested by `remove_nat_portfwd`.\n", protoport, NetworkingInterfacePrefix, vmnet) + } + + case networkingCommandEntry_add_dhcp_mac_to_ip: + vmnet = e.add_dhcp_mac_to_ip.vnet + dhcpmacs, exists := result.dhcp_mac_to_ip[vmnet] + if !exists { + dhcpmacs = make(map[string]net.IP) + result.dhcp_mac_to_ip[vmnet] = dhcpmacs + } + dhcpmacs[e.add_dhcp_mac_to_ip.mac.String()] = e.add_dhcp_mac_to_ip.ip + + case networkingCommandEntry_remove_dhcp_mac_to_ip: + vmnet = e.remove_dhcp_mac_to_ip.vnet + dhcpmacs, exists := result.dhcp_mac_to_ip[vmnet] + if exists { + delete(dhcpmacs, e.remove_dhcp_mac_to_ip.mac.String()) + } else { + log.Printf("Unable to remove dhcp_mac_to_ip entry %v from interface %s%d as specified by `remove_dhcp_mac_to_ip`.\n", e.remove_dhcp_mac_to_ip, NetworkingInterfacePrefix, vmnet) + } + + case networkingCommandEntry_add_bridge_mapping: + intf := e.add_bridge_mapping.intf + if _, err := intf.Interface(); err != nil { + log.Printf("Interface \"%s\" as specified by `add_bridge_mapping` was not found on the current platform. This is a non-critical error. Ignoring.", intf.name) + } + result.bridge_mapping[intf.name] = e.add_bridge_mapping.vnet + + case networkingCommandEntry_remove_bridge_mapping: + intf := e.remove_bridge_mapping.intf + if _, err := intf.Interface(); err != nil { + log.Printf("Interface \"%s\" as specified by `remove_bridge_mapping` was not found on the current platform. This is a non-critical error. Ignoring.", intf.name) + } + delete(result.bridge_mapping, intf.name) + + case networkingCommandEntry_add_nat_prefix: + vmnet = e.add_nat_prefix.vnet + _, exists := result.nat_prefix[vmnet] + if exists { + result.nat_prefix[vmnet] = append(result.nat_prefix[vmnet], e.add_nat_prefix.prefix) + } else { + result.nat_prefix[vmnet] = []int{e.add_nat_prefix.prefix} + } + + case networkingCommandEntry_remove_nat_prefix: + vmnet = e.remove_nat_prefix.vnet + prefixes, exists := result.nat_prefix[vmnet] + if exists { + for index := 0; index < len(prefixes); index++ { + if prefixes[index] == e.remove_nat_prefix.prefix { + result.nat_prefix[vmnet] = append(prefixes[:index], prefixes[index+1:]...) + break } - } else { - log.Printf("Unable to remove nat prefix /%d from interface %s%d as specified by `remove_nat_prefix`.\n", e.remove_nat_prefix.prefix, NetworkingInterfacePrefix, vmnet) } + + } else { + log.Printf("Unable to remove nat prefix /%d from interface %s%d as specified by `remove_nat_prefix`.\n", e.remove_nat_prefix.prefix, NetworkingInterfacePrefix, vmnet) } } } @@ -1825,11 +1981,12 @@ func flattenNetworkingConfig(eof sentinelSignaller, in chan networkingCommandEnt // Constructor for networking file func ReadNetworkingConfig(fd *os.File) (NetworkingConfig, error) { + // start piecing together different parts of the file - fromfile, eof := consumeFile(fd) - tokenized, eot := tokenizeNetworkingConfig(eof, fromfile) - rows, eos := splitNetworkingConfig(eot, tokenized) - entries, eop := parseNetworkingConfig(eos, rows) + fromfile := consumeFile(fd) + tokenized := tokenizeNetworkingConfig(fromfile) + rows := splitNetworkingConfig(tokenized) + entries := parseNetworkingConfig(rows) // parse the version parsed_version, err := networkingReadVersion(<-rows) @@ -1844,7 +2001,7 @@ func ReadNetworkingConfig(fd *os.File) (NetworkingConfig, error) { } // convert to a configuration - result := flattenNetworkingConfig(eop, entries) + result := flattenNetworkingConfig(entries) return result, nil } @@ -1858,8 +2015,7 @@ const ( ) func networkingConfig_InterfaceTypes(config NetworkingConfig) map[int]NetworkingType { - var result map[int]NetworkingType - result = make(map[int]NetworkingType) + result := make(map[int]NetworkingType) // defaults result[0] = NetworkingType_BRIDGED @@ -1873,6 +2029,7 @@ func networkingConfig_InterfaceTypes(config NetworkingConfig) map[int]Networking // walk through answers finding out which ones are nat versus hostonly for vmnet, table := range config.answer { + // everything should be defined as a virtual adapter... if table["VIRTUAL_ADAPTER"] == "yes" { @@ -1886,12 +2043,13 @@ func networkingConfig_InterfaceTypes(config NetworkingConfig) map[int]Networking // distinguish between nat or hostonly if table["NAT"] == "yes" { result[vmnet] = NetworkingType_NAT + } else { result[vmnet] = NetworkingType_HOSTONLY } - // if it's not a virtual_adapter, then it must be an alias (really a bridge). } else { + // if it's not a virtual_adapter, then it must be an alias (really a bridge). result[vmnet] = NetworkingType_BRIDGED } } @@ -1909,8 +2067,8 @@ func networkingConfig_NamesToVmnet(config NetworkingConfig) map[NetworkingType][ sort.Ints(keys) // build result dictionary - var result map[NetworkingType][]int - result = make(map[NetworkingType][]int) + result := make(map[NetworkingType][]int) + for i := 0; i < len(keys); i++ { t := types[keys[i]] result[t] = append(result[t], keys[i]) @@ -1928,10 +2086,13 @@ func (e NetworkingConfig) NameIntoDevices(name string) ([]string, error) { var networkingType NetworkingType if name == "hostonly" && len(netmapper[NetworkingType_HOSTONLY]) > 0 { networkingType = NetworkingType_HOSTONLY + } else if name == "nat" && len(netmapper[NetworkingType_NAT]) > 0 { networkingType = NetworkingType_NAT + } else if name == "bridged" && len(netmapper[NetworkingType_BRIDGED]) > 0 { networkingType = NetworkingType_BRIDGED + } else { return make([]string, 0), fmt.Errorf("Network name not found: %v", name) } @@ -1957,8 +2118,10 @@ func (e NetworkingConfig) DeviceIntoName(device string) (string, error) { switch network { case NetworkingType_HOSTONLY: return "hostonly", nil + case NetworkingType_NAT: return "nat", nil + case NetworkingType_BRIDGED: return "bridged", nil } @@ -1966,9 +2129,8 @@ func (e NetworkingConfig) DeviceIntoName(device string) (string, error) { } /** generic async file reader */ -func consumeFile(fd *os.File) (chan byte, sentinelSignaller) { - fromfile := make(chan byte) - eof := make(sentinelSignaller) +func consumeFile(fd *os.File) chan byte { + fromFile := make(chan byte) go func() { b := make([]byte, 1) for { @@ -1978,9 +2140,9 @@ func consumeFile(fd *os.File) (chan byte, sentinelSignaller) { // ErrClosed may appear since file is closed and this goroutine still left running break } - fromfile <- b[0] + fromFile <- b[0] } - close(eof) + close(fromFile) }() - return fromfile, eof + return fromFile } diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index 58d739a63..a7083cb3c 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -7,35 +7,29 @@ import ( "path/filepath" ) -func consumeString(s string) (out chan byte, eos sentinelSignaller) { - eos = make(sentinelSignaller) +func consumeString(s string) (out chan byte) { out = make(chan byte) go func() { for _, ch := range s { out <- byte(ch) } - close(eos) close(out) }() return } func uncommentFromString(s string) string { - inCh, eos := consumeString(s) - out, eoc := uncomment(eos, inCh) + inCh := consumeString(s) + out := uncomment(inCh) result := "" for reading := true; reading; { - select { - case <-eoc: - reading = false - case item, ok := <-out: - if ok { - result += string(item) - } + if item, ok := <-out; !ok { + break + } else { + result += string(item) } } - close(out) return result } @@ -107,21 +101,17 @@ func TestParserUncomment(t *testing.T) { } func tokenizeDhcpConfigFromString(s string) []string { - inCh, eos := consumeString(s) - out, eoc := tokenizeDhcpConfig(eos, inCh) + inCh := consumeString(s) + out := tokenizeDhcpConfig(inCh) result := make([]string, 0) - for reading := true; reading; { - select { - case <-eoc: - reading = false - case item, ok := <-out: - if ok { - result = append(result, item) - } + for { + if item, ok := <-out; !ok { + break + } else { + result = append(result, item) } } - close(out) return result } @@ -224,16 +214,14 @@ func consumeDhcpConfig(items []string) (tkGroup, error) { out := make(chan string) tch := consumeTokens(items) - end := make(sentinelSignaller) go func() { for item := range tch { out <- item } - close(end) close(out) }() - return parseDhcpConfig(end, out) + return parseDhcpConfig(out) } func compareSlice(a, b []string) bool { From efb775accb7ee73ab2da7cc73a0dc1a0d0a5d8f1 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 27 May 2020 18:27:36 -0500 Subject: [PATCH 2/9] Added a number of parsing combinators to the vmware builder and some minor tests for them to help with parsing dhcpd leases. --- builder/vmware/common/driver_parser.go | 87 +++++++++++ builder/vmware/common/driver_parser_test.go | 154 ++++++++++++++++++++ 2 files changed, 241 insertions(+) diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index c78c673e5..cd63c1345 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -2146,3 +2146,90 @@ func consumeFile(fd *os.File) chan byte { }() return fromFile } + +/** Consume a byte channel until a terminal byte is reached, and write each list of bytes to a channel */ +func consumeUntilSentinel(sentinel byte, in chan byte) (result []byte, ok bool) { + + // This is a simple utility that will consume from a channel until a sentinel + // byte has been reached. Consumed data is returned in `result, and if + // there's no more data to read, then `ok` will be false. + for ok = true; ; { + if by, success := <-in; !success { + ok = false + break + + } else if by == sentinel { + break + + } else { + result = append(result, by) + + } + } + return +} + +/** Simple utility to ignore chars when consuming a channel */ +func filterOutCharacters(ignore []byte, in chan byte) chan byte { + out := make(chan byte) + + go func(ignore_s string) { + for { + if by, ok := <-in; !ok { + break + + } else if !strings.ContainsAny(ignore_s, string(by)) { + out <- by + } + } + close(out) + }(string(ignore)) + + return out +} + +/** +This consumes bytes within a pair of some bytes, like parentheses, brackets, braces... + +We start by reading bytes until we encounter openByte. These will be returned as +the first parameter. Then we can enter a goro and consume bytes until we get to +closeByte. At that point we're done, and suicide. +**/ +func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan byte) { + result := make([]byte, 0) + + // Consume until we get to openByte. We'll return what we consumed because + // it isn't actually relevant to what we're trying to accomplish. + for by := range in { + if by == openByte { + break + } + result = append(result, by) + } + + // Now we can feed input to our goro and a consumer can see what's contained + // between their requested pairs + out := make(chan byte) + go func(out chan byte) { + by := openByte + + // We only made it here because we received an openByte, so let's make + // sure we send it down the channel. + out <- by + + // Now just spin in a loop shipping bytes down the channel until we hit + // closeByte, or we're at the very end...whichever comes first. + for ok := true; by != closeByte; { + by, ok = <-in + if !ok { + by = closeByte + } + out <- by + } + close(out) + }(out) + + // Return what we consumed, and a channel that yields everything in between + // the openByte and closeByte pair. + return result, out +} diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index a7083cb3c..b52e6739e 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -3,6 +3,7 @@ package common import ( "testing" + "bytes" "os" "path/filepath" ) @@ -480,3 +481,156 @@ func TestParserReadNetworkMap(t *testing.T) { } } } + +func collectIntoString(in chan byte) string { + result := "" + for item := range in { + result += string(item) + } + return result +} + +func TestParserConsumeUntilSentinel(t *testing.T) { + + test_1 := "consume until a semicolon; yeh?" + expected_1 := "consume until a semicolon" + + ch := consumeString(test_1) + resultch, _ := consumeUntilSentinel(';', ch) + result := string(resultch) + if expected_1 != result { + t.Errorf("expected %#v, got %#v", expected_1, result) + } + + test_2 := "; this is only a semi" + expected_2 := "" + + ch = consumeString(test_2) + resultch, _ = consumeUntilSentinel(';', ch) + result = string(resultch) + if expected_2 != result { + t.Errorf("expected %#v, got %#v", expected_2, result) + } +} + +func TestParserFilterCharacters(t *testing.T) { + + test_1 := []string{" ", "ignore all spaces"} + expected_1 := "ignoreallspaces" + + ch := consumeString(test_1[1]) + outch := filterOutCharacters(bytes.NewBufferString(test_1[0]).Bytes(), ch) + result := collectIntoString(outch) + if result != expected_1 { + t.Errorf("expected %#v, got %#v", expected_1, result) + } + + test_2 := []string{"\n\v\t\r ", "ignore\nall\rwhite\v\v space "} + expected_2 := "ignoreallwhitespace" + + ch = consumeString(test_2[1]) + outch = filterOutCharacters(bytes.NewBufferString(test_2[0]).Bytes(), ch) + result = collectIntoString(outch) + if result != expected_2 { + t.Errorf("expected %#v, got %#v", expected_2, result) + } +} + +func TestParserConsumeOpenClosePair(t *testing.T) { + test_1 := "(everything)" + expected_1 := []string{"", test_1} + + testch := consumeString(test_1) + prefix, ch := consumeOpenClosePair('(', ')', testch) + if string(prefix) != expected_1[0] { + t.Errorf("expected prefix %#v, got %#v", expected_1[0], prefix) + } + result := collectIntoString(ch) + if result != expected_1[1] { + t.Errorf("expected %#v, got %#v", expected_1[1], test_1) + } + + test_2 := "prefixed (everything)" + expected_2 := []string{"prefixed ", "(everything)"} + + testch = consumeString(test_2) + prefix, ch = consumeOpenClosePair('(', ')', testch) + if string(prefix) != expected_2[0] { + t.Errorf("expected prefix %#v, got %#v", expected_2[0], prefix) + } + result = collectIntoString(ch) + if result != expected_2[1] { + t.Errorf("expected %#v, got %#v", expected_2[1], test_2) + } + + test_3 := "this(is()suffixed" + expected_3 := []string{"this", "(is()"} + + testch = consumeString(test_3) + prefix, ch = consumeOpenClosePair('(', ')', testch) + if string(prefix) != expected_3[0] { + t.Errorf("expected prefix %#v, got %#v", expected_3[0], prefix) + } + result = collectIntoString(ch) + if result != expected_3[1] { + t.Errorf("expected %#v, got %#v", expected_3[1], test_2) + } +} + +func TestParserCombinators(t *testing.T) { + + test_1 := "across # ignore\nmultiple lines;" + expected_1 := "across multiple lines" + + ch := consumeString(test_1) + inch := uncomment(ch) + whch := filterOutCharacters([]byte{'\n'}, inch) + resultch, _ := consumeUntilSentinel(';', whch) + result := string(resultch) + if expected_1 != result { + t.Errorf("expected %#v, got %#v", expected_1, result) + } + + test_2 := "lease blah {\n blah\r\n# skipping this line\nblahblah # ignore semicolon;\n last item;\n\n };;;;;;" + expected_2 := []string{"lease blah ", "{ blahblahblah last item; }"} + + ch = consumeString(test_2) + inch = uncomment(ch) + whch = filterOutCharacters([]byte{'\n', '\v', '\r'}, inch) + prefix, pairch := consumeOpenClosePair('{', '}', whch) + + result = collectIntoString(pairch) + if string(prefix) != expected_2[0] { + t.Errorf("expected prefix %#v, got %#v", expected_2[0], prefix) + } + if result != expected_2[1] { + t.Errorf("expected %#v, got %#v", expected_2[1], result) + } + + test_3 := "lease blah { # comment\n item 1;\n item 2;\n } not imortant" + expected_3_prefix := "lease blah " + expected_3 := []string{"{ item 1", " item 2", " }"} + + sch := consumeString(test_3) + inch = uncomment(sch) + wch := filterOutCharacters([]byte{'\n', '\v', '\r'}, inch) + lease, itemch := consumeOpenClosePair('{', '}', wch) + if string(lease) != expected_3_prefix { + t.Errorf("expected %#v, got %#v", expected_3_prefix, string(lease)) + } + + result_3 := []string{} + for reading := true; reading; { + item, ok := consumeUntilSentinel(';', itemch) + result_3 = append(result_3, string(item)) + if !ok { + reading = false + } + } + + for index := range expected_3 { + if expected_3[index] != result_3[index] { + t.Errorf("expected index %d as %#v, got %#v", index, expected_3[index], result_3[index]) + } + } +} From e2a7c317eb0fa0f96fd750c9a9573e5d773b23d3 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Wed, 27 May 2020 21:35:24 -0500 Subject: [PATCH 3/9] Implemented the dhcpd lease entry decoder and the address decoder for the vmware builder. --- builder/vmware/common/driver_parser.go | 120 ++++++++++++++++++++ builder/vmware/common/driver_parser_test.go | 48 ++++++++ 2 files changed, 168 insertions(+) diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index cd63c1345..745de40c4 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -1,15 +1,19 @@ package common import ( + "bytes" + "encoding/hex" "fmt" "log" "math" "net" "os" "reflect" + "regexp" "sort" "strconv" "strings" + "time" ) /** low-level parsing */ @@ -2233,3 +2237,119 @@ func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan // the openByte and closeByte pair. return result, out } + +// Basic decoding of a dhcpd lease address +func decodeDhcpdLeaseBytes(input string) ([]byte, error) { + processed := &bytes.Buffer{} + + // Split the string into pieces as we'll need to validate it. + for _, item := range strings.Split(input, ":") { + if len(item) != 2 { + return []byte{}, fmt.Errorf("bytes are not well-formed (%v)", input) + } + processed.WriteString(item) + } + + length := hex.DecodedLen(processed.Len()) + + // Decode the processed data into the result... + result := make([]byte, length) + if n, err := hex.Decode(result, processed.Bytes()); err != nil { + return []byte{}, err + + // Check that our decode length corresponds to what was intended + } else if n != length { + return []byte{}, fmt.Errorf("expected to decode %d bytes, got %d instead", length, n) + } + + // ...and then return it. + return result, nil +} + +/*** Dhcp Leases */ +type dhcpLeaseEntry struct { + address string + starts, ends time.Time + ether, uid []byte + extra []string +} + +func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { + + // Build the regexes we'll use to legitimately parse each item + ipLineRe := regexp.MustCompile(`^lease (.+?) {$`) + startTimeLineRe := regexp.MustCompile(`^\s*starts \d (.+?);$`) + endTimeLineRe := regexp.MustCompile(`^\s*ends \d (.+?);$`) + macLineRe := regexp.MustCompile(`^\s*hardware ethernet (.+?);$`) + uidLineRe := regexp.MustCompile(`^\s*uid (.+?);$`) + + /// Read up to the lease item and validate that it actually matches + lease, ch := consumeOpenClosePair('{', '}', in) + + matches := ipLineRe.FindStringSubmatch(string(lease)) + if matches == nil { + return nil + } + + // If we found a lease match and we're definitely beginning a lease + // entry, then create our storage. + if by, ok := <-ch; ok && by == '{' { + entry = &dhcpLeaseEntry{address: matches[1]} + + // Otherwise we bail. + } else { + return nil + } + + /// Now we can parse the inside of the block. + for insideBraces := true; insideBraces; { + item, ok := consumeUntilSentinel(';', in) + item_s := string(item) + + if !ok { + insideBraces = false + } + + // Parse out the start time + matches = startTimeLineRe.FindStringSubmatch(item_s) + if matches != nil { + entry.starts, _ = time.Parse("2006/01/02 15:04:05", matches[1]) + continue + } + + // Parse out the end time + matches = endTimeLineRe.FindStringSubmatch(item_s) + if matches != nil { + entry.ends, _ = time.Parse("2006/01/02 15:04:05", matches[1]) + continue + } + + // Parse out the hardware ethernet + matches = macLineRe.FindStringSubmatch(item_s) + if matches != nil { + entry.ether, _ = decodeDhcpdLeaseBytes(item_s) + continue + } + + // Parse out the uid + matches = uidLineRe.FindStringSubmatch(item_s) + if matches != nil { + entry.uid, _ = decodeDhcpdLeaseBytes(item_s) + continue + } + + // Just stash it for now because we have no idea what it is. + entry.extra = append(entry.extra, strings.TrimSpace(item_s)) + } + + return entry +} + +func ReadDhcpdLeases(fd *os.File) ([]dhcpLeaseEntry, error) { + fch := consumeFile(fd) + uncommentedch := uncomment(fch) + wch := filterOutCharacters([]byte{'\n', '\r', '\v'}, uncommentedch) + close(wch) + + return []dhcpLeaseEntry{}, fmt.Errorf("Not implemented yet!") +} diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index b52e6739e..1d2e489ca 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -634,3 +634,51 @@ func TestParserCombinators(t *testing.T) { } } } + +func TestParserDhcpdLeaseBytesDecoder(t *testing.T) { + test_1 := "00:0d:0e:0a:0d:00" + expected_1 := []byte{0, 13, 14, 10, 13, 0} + + result, err := decodeDhcpdLeaseBytes(test_1) + if err != nil { + t.Errorf("unable to decode address: %s", err) + } + if bytes.Compare(result, expected_1) != 0 { + t.Errorf("expected %v, got %v", expected_1, result) + } + + test_2 := "11" + expected_2 := []byte{17} + + result, err = decodeDhcpdLeaseBytes(test_2) + if err != nil { + t.Errorf("unable to decode address: %s", err) + } + if bytes.Compare(result, expected_2) != 0 { + t.Errorf("expected %v, got %v", expected_2, result) + } + + failtest_1 := "" + result, err = decodeDhcpdLeaseBytes(failtest_1) + if err == nil { + t.Errorf("expected decoding error: %s", err) + } + + failtest_2 := "000000" + result, err = decodeDhcpdLeaseBytes(failtest_2) + if err == nil { + t.Errorf("expected decoding error: %s", err) + } + + failtest_3 := "000:00" + result, err = decodeDhcpdLeaseBytes(failtest_3) + if err == nil { + t.Errorf("expected decoding error: %s", err) + } + + failtest_4 := "00:00:" + result, err = decodeDhcpdLeaseBytes(failtest_4) + if err == nil { + t.Errorf("expected decoding error: %s", err) + } +} From afa46969857c0730fe6626918f52ff525b2c4114 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 28 May 2020 01:16:42 -0500 Subject: [PATCH 4/9] Tightened the regular expressions used by the vmware builder to parse the dhcpd leases, and added support for parsing the weekdays. --- builder/vmware/common/driver_parser.go | 48 ++++++++++------- builder/vmware/common/driver_parser_test.go | 58 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 18 deletions(-) diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 745de40c4..406a444d8 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -2207,8 +2207,10 @@ func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan for by := range in { if by == openByte { break + + } else { + result = append(result, by) } - result = append(result, by) } // Now we can feed input to our goro and a consumer can see what's contained @@ -2228,6 +2230,7 @@ func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan if !ok { by = closeByte } + out <- by } close(out) @@ -2268,20 +2271,21 @@ func decodeDhcpdLeaseBytes(input string) ([]byte, error) { /*** Dhcp Leases */ type dhcpLeaseEntry struct { - address string - starts, ends time.Time - ether, uid []byte - extra []string + address string + starts, ends time.Time + starts_weekday, ends_weekday int + ether, uid []byte + extra []string } func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { // Build the regexes we'll use to legitimately parse each item - ipLineRe := regexp.MustCompile(`^lease (.+?) {$`) - startTimeLineRe := regexp.MustCompile(`^\s*starts \d (.+?);$`) - endTimeLineRe := regexp.MustCompile(`^\s*ends \d (.+?);$`) - macLineRe := regexp.MustCompile(`^\s*hardware ethernet (.+?);$`) - uidLineRe := regexp.MustCompile(`^\s*uid (.+?);$`) + ipLineRe := regexp.MustCompile(`lease\s+(.+?)\s*$`) + startTimeLineRe := regexp.MustCompile(`starts\s+(\d+)\s+(.+?)\s*$`) + endTimeLineRe := regexp.MustCompile(`ends\s+(\d+)\s+(.+?)\s*$`) + macLineRe := regexp.MustCompile(`hardware\s+ethernet\s+(.+?)\s*$`) + uidLineRe := regexp.MustCompile(`uid\s+(.+?)\s*$`) /// Read up to the lease item and validate that it actually matches lease, ch := consumeOpenClosePair('{', '}', in) @@ -2291,19 +2295,19 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { return nil } - // If we found a lease match and we're definitely beginning a lease - // entry, then create our storage. if by, ok := <-ch; ok && by == '{' { + // If we found a lease match and we're definitely beginning a lease + // entry, then create our storage. entry = &dhcpLeaseEntry{address: matches[1]} - // Otherwise we bail. } else { + // Otherwise we bail. return nil } /// Now we can parse the inside of the block. for insideBraces := true; insideBraces; { - item, ok := consumeUntilSentinel(';', in) + item, ok := consumeUntilSentinel(';', ch) item_s := string(item) if !ok { @@ -2313,28 +2317,36 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { // Parse out the start time matches = startTimeLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.starts, _ = time.Parse("2006/01/02 15:04:05", matches[1]) + entry.starts, _ = time.Parse("2006/01/02 15:04:05", matches[2]) + entry.starts_weekday, _ = strconv.Atoi(matches[1]) continue } // Parse out the end time matches = endTimeLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.ends, _ = time.Parse("2006/01/02 15:04:05", matches[1]) + entry.ends, _ = time.Parse("2006/01/02 15:04:05", matches[2]) + entry.ends_weekday, _ = strconv.Atoi(matches[1]) continue } // Parse out the hardware ethernet matches = macLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.ether, _ = decodeDhcpdLeaseBytes(item_s) + entry.ether, _ = decodeDhcpdLeaseBytes(matches[1]) continue } // Parse out the uid matches = uidLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.uid, _ = decodeDhcpdLeaseBytes(item_s) + entry.uid, _ = decodeDhcpdLeaseBytes(matches[1]) + continue + } + + // Check to see if we're terminating the brace, so we can skip + // to the next iteration. + if strings.HasSuffix(item_s, "}") { continue } diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index 1d2e489ca..70ac59e7b 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -4,6 +4,7 @@ import ( "testing" "bytes" + "encoding/hex" "os" "path/filepath" ) @@ -682,3 +683,60 @@ func TestParserDhcpdLeaseBytesDecoder(t *testing.T) { t.Errorf("expected decoding error: %s", err) } } + +func consumeLeaseString(s string) chan byte { + sch := consumeString(s) + uncommentedch := uncomment(sch) + return filterOutCharacters([]byte{'\n', '\r', '\v'}, uncommentedch) +} + +func TestParserReadDhcpdLeaseEntry(t *testing.T) { + test_1 := "lease 127.0.0.1 {\nhardware ethernet 00:11:22:33 ;\nuid 00:11 ;\n }" + expected_1 := map[string]string{ + "address": "127.0.0.1", + "ether": "00112233", + "uid": "0011", + } + + result := readDhcpdLeaseEntry(consumeLeaseString(test_1)) + if result.address != expected_1["address"] { + t.Errorf("expected address %v, got %v", expected_1["address"], result.address) + } + if hex.EncodeToString(result.ether) != expected_1["ether"] { + t.Errorf("expected ether %v, got %v", expected_1["ether"], hex.EncodeToString(result.ether)) + } + if hex.EncodeToString(result.uid) != expected_1["uid"] { + t.Errorf("expected uid %v, got %v", expected_1["uid"], hex.EncodeToString(result.uid)) + } + + test_2 := " \n\t lease 192.168.21.254{ hardware\n ethernet 44:55:66:77:88:99;uid 00:1\n1:22:3\r3:44;\n starts 57005 2006/01/02 15:04:05;ends 57005 2006/01/03 15:04:05;\tunknown item1; unknown item2; } " + expected_2 := map[string]string{ + "address": "192.168.21.254", + "ether": "445566778899", + "uid": "0011223344", + "starts": "2006-01-02 15:04:05 +0000 UTC", + "ends": "2006-01-03 15:04:05 +0000 UTC", + } + result = readDhcpdLeaseEntry(consumeLeaseString(test_2)) + if result.address != expected_2["address"] { + t.Errorf("expected address %v, got %v", expected_2["address"], result.address) + } + if hex.EncodeToString(result.ether) != expected_2["ether"] { + t.Errorf("expected ether %v, got %v", expected_2["ether"], hex.EncodeToString(result.ether)) + } + if hex.EncodeToString(result.uid) != expected_2["uid"] { + t.Errorf("expected uid %v, got %v", expected_2["uid"], hex.EncodeToString(result.uid)) + } + if result.starts.String() != expected_2["starts"] { + t.Errorf("expected starts %v, got %v", expected_2["starts"], result.starts) + } + if result.ends.String() != expected_2["ends"] { + t.Errorf("expected ends %v, got %v", expected_2["ends"], result.ends) + } + if result.starts_weekday != 57005 { + t.Errorf("expected starts weekday %v, got %v", 57005, result.starts_weekday) + } + if result.ends_weekday != 57005 { + t.Errorf("expected ends weekday %v, got %v", 57005, result.ends_weekday) + } +} From cf5332fa20d3476056f22dbe8f15e33769691045 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 28 May 2020 01:51:22 -0500 Subject: [PATCH 5/9] Updated the dhcpd lease entry parser in the vmware builder to return errors if any are encountered. --- builder/vmware/common/driver_parser.go | 42 +++++++++++++++------ builder/vmware/common/driver_parser_test.go | 10 ++++- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 406a444d8..56afa6bc6 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -2278,7 +2278,7 @@ type dhcpLeaseEntry struct { extra []string } -func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { +func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry, err error) { // Build the regexes we'll use to legitimately parse each item ipLineRe := regexp.MustCompile(`lease\s+(.+?)\s*$`) @@ -2292,7 +2292,8 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { matches := ipLineRe.FindStringSubmatch(string(lease)) if matches == nil { - return nil + res := strings.TrimSpace(string(lease)) + return &dhcpLeaseEntry{extra: []string{res}}, fmt.Errorf("Unable to parse lease entry (%#v)", string(lease)) } if by, ok := <-ch; ok && by == '{' { @@ -2300,9 +2301,14 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { // entry, then create our storage. entry = &dhcpLeaseEntry{address: matches[1]} - } else { - // Otherwise we bail. - return nil + } else if ok { + // If we didn't see a begin brace, then this entry is mangled which + // means that we should probably ail + return &dhcpLeaseEntry{address: matches[1]}, fmt.Errorf("Missing parameters for lease entry %v", matches[1]) + + } else if !ok { + // If our channel is closed, so we bail "cleanly". + return nil, nil } /// Now we can parse the inside of the block. @@ -2317,30 +2323,42 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { // Parse out the start time matches = startTimeLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.starts, _ = time.Parse("2006/01/02 15:04:05", matches[2]) - entry.starts_weekday, _ = strconv.Atoi(matches[1]) + if entry.starts, err = time.Parse("2006/01/02 15:04:05", matches[2]); err != nil { + log.Printf("Error trying to parse start time (%v) for entry %v", matches[2], entry.address) + } + if entry.starts_weekday, err = strconv.Atoi(matches[1]); err != nil { + log.Printf("Error trying to parse start weekday (%v) for entry %v", matches[1], entry.address) + } continue } // Parse out the end time matches = endTimeLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.ends, _ = time.Parse("2006/01/02 15:04:05", matches[2]) - entry.ends_weekday, _ = strconv.Atoi(matches[1]) + if entry.ends, err = time.Parse("2006/01/02 15:04:05", matches[2]); err != nil { + log.Printf("Error trying to parse end time (%v) for entry %v", matches[2], entry.address) + } + if entry.ends_weekday, err = strconv.Atoi(matches[1]); err != nil { + log.Printf("Error trying to parse end weekday (%v) for entry %v", matches[1], entry.address) + } continue } // Parse out the hardware ethernet matches = macLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.ether, _ = decodeDhcpdLeaseBytes(matches[1]) + if entry.ether, err = decodeDhcpdLeaseBytes(matches[1]); err != nil { + log.Printf("Error trying to parse hardware ethernet address (%v) for entry %v", matches[1], entry.address) + } continue } // Parse out the uid matches = uidLineRe.FindStringSubmatch(item_s) if matches != nil { - entry.uid, _ = decodeDhcpdLeaseBytes(matches[1]) + if entry.uid, err = decodeDhcpdLeaseBytes(matches[1]); err != nil { + log.Printf("Error trying to parse uid (%v) for entry %v", matches[1], entry.address) + } continue } @@ -2354,7 +2372,7 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry) { entry.extra = append(entry.extra, strings.TrimSpace(item_s)) } - return entry + return entry, nil } func ReadDhcpdLeases(fd *os.File) ([]dhcpLeaseEntry, error) { diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index 70ac59e7b..d43011463 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -698,7 +698,10 @@ func TestParserReadDhcpdLeaseEntry(t *testing.T) { "uid": "0011", } - result := readDhcpdLeaseEntry(consumeLeaseString(test_1)) + result, err := readDhcpdLeaseEntry(consumeLeaseString(test_1)) + if err != nil { + t.Errorf("error parsing entry: %v", err) + } if result.address != expected_1["address"] { t.Errorf("expected address %v, got %v", expected_1["address"], result.address) } @@ -717,7 +720,10 @@ func TestParserReadDhcpdLeaseEntry(t *testing.T) { "starts": "2006-01-02 15:04:05 +0000 UTC", "ends": "2006-01-03 15:04:05 +0000 UTC", } - result = readDhcpdLeaseEntry(consumeLeaseString(test_2)) + result, err = readDhcpdLeaseEntry(consumeLeaseString(test_2)) + if err != nil { + t.Errorf("error parsing entry: %v", err) + } if result.address != expected_2["address"] { t.Errorf("expected address %v, got %v", expected_2["address"], result.address) } From 020b00552267586ddfab5ba802c446570b075052 Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 28 May 2020 03:08:50 -0500 Subject: [PATCH 6/9] Finished implementing the final dhcpd lease entries parser for the vmware builders and also included its own unit tests. --- builder/vmware/common/driver_parser.go | 40 ++++++- builder/vmware/common/driver_parser_test.go | 110 ++++++++++++++++++ .../common/testdata/dhcpd-example.leases | 32 +++++ 3 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 builder/vmware/common/testdata/dhcpd-example.leases diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 56afa6bc6..2dc828d24 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -2290,6 +2290,12 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry, err error) { /// Read up to the lease item and validate that it actually matches lease, ch := consumeOpenClosePair('{', '}', in) + // If we couldn't read the lease, then this item is busted and we're prolly + // done reading the channel. + if len(lease) == 0 { + return nil, nil + } + matches := ipLineRe.FindStringSubmatch(string(lease)) if matches == nil { res := strings.TrimSpace(string(lease)) @@ -2303,7 +2309,7 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry, err error) { } else if ok { // If we didn't see a begin brace, then this entry is mangled which - // means that we should probably ail + // means that we should probably bail return &dhcpLeaseEntry{address: matches[1]}, fmt.Errorf("Missing parameters for lease entry %v", matches[1]) } else if !ok { @@ -2375,11 +2381,37 @@ func readDhcpdLeaseEntry(in chan byte) (entry *dhcpLeaseEntry, err error) { return entry, nil } -func ReadDhcpdLeases(fd *os.File) ([]dhcpLeaseEntry, error) { +func ReadDhcpdLeaseEntries(fd *os.File) ([]dhcpLeaseEntry, error) { fch := consumeFile(fd) uncommentedch := uncomment(fch) wch := filterOutCharacters([]byte{'\n', '\r', '\v'}, uncommentedch) - close(wch) - return []dhcpLeaseEntry{}, fmt.Errorf("Not implemented yet!") + result := make([]dhcpLeaseEntry, 0) + errors := make([]error, 0) + + // Consume dhcpd lease entries from the channel until we just plain run out. + for i := 0; ; i += 1 { + if entry, err := readDhcpdLeaseEntry(wch); entry == nil { + // If our entry is nil, then we've run out of input and finished + // parsing the file to completion. + break + + } else if err != nil { + // If we received an error, then log it and keep track of it. This + // way we can warn the user later which entries we had issues with. + log.Printf("Error parsing dhcpd lease entry #%d: %s", 1+i, err) + errors = append(errors, err) + + } else { + // If we've parsed an entry successfully, then aggregate it to + // our slice of results. + result = append(result, *entry) + } + } + + // If we received any errors then include alongside our results. + if len(errors) > 0 { + return result, fmt.Errorf("Errors found while parsing dhcpd lease entries: %v", errors) + } + return result, nil } diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index d43011463..5381a3e13 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -746,3 +746,113 @@ func TestParserReadDhcpdLeaseEntry(t *testing.T) { t.Errorf("expected ends weekday %v, got %v", 57005, result.ends_weekday) } } + +func TestParserReadDhcpdLeases(t *testing.T) { + f, err := os.Open(filepath.Join("testdata", "dhcpd-example.leases")) + if err != nil { + t.Fatalf("Unable to open dhcpd.leases sample: %s", err) + } + defer f.Close() + + results, err := ReadDhcpdLeaseEntries(f) + if err != nil { + t.Fatalf("Error reading lease: %s", err) + } + + // some simple utilities + filter_address := func(address string, items []dhcpLeaseEntry) (result []dhcpLeaseEntry) { + for _, item := range items { + if item.address == address { + result = append(result, item) + } + } + return + } + + find_uid := func(uid string, items []dhcpLeaseEntry) *dhcpLeaseEntry { + for _, item := range items { + if uid == hex.EncodeToString(item.uid) { + return &item + } + } + return nil + } + + find_ether := func(ether string, items []dhcpLeaseEntry) *dhcpLeaseEntry { + for _, item := range items { + if ether == hex.EncodeToString(item.ether) { + return &item + } + } + return nil + } + + // actual unit tests + test_1 := map[string]string{ + "address": "127.0.0.19", + "uid": "010dead099aabb", + "ether": "0dead099aabb", + } + test_1_findings := filter_address(test_1["address"], results) + if len(test_1_findings) != 2 { + t.Errorf("expected %d matching entries, got %d", 2, len(test_1_findings)) + } else { + res := find_ether(test_1["ether"], test_1_findings) + if res == nil { + t.Errorf("unable to find item with ether %v", test_1["ether"]) + } else if hex.EncodeToString(res.uid) != test_1["uid"] { + t.Errorf("expected uid %s, got %s", test_1["uid"], hex.EncodeToString(res.uid)) + } + } + + test_2 := map[string]string{ + "address": "127.0.0.19", + "uid": "010dead0667788", + "ether": "0dead0667788", + } + test_2_findings := filter_address(test_2["address"], results) + if len(test_2_findings) != 2 { + t.Errorf("expected %d matching entries, got %d", 2, len(test_2_findings)) + } else { + res := find_ether(test_2["ether"], test_2_findings) + if res == nil { + t.Errorf("unable to find item with ether %v", test_2["ether"]) + } else if hex.EncodeToString(res.uid) != test_2["uid"] { + t.Errorf("expected uid %s, got %s", test_2["uid"], hex.EncodeToString(res.uid)) + } + } + + test_3 := map[string]string{ + "address": "127.0.0.17", + "uid": "010dead0334455", + "ether": "0dead0667788", + } + test_3_findings := filter_address(test_3["address"], results) + if len(test_3_findings) != 2 { + t.Errorf("expected %d matching entries, got %d", 2, len(test_3_findings)) + } else { + res := find_uid(test_3["uid"], test_3_findings) + if res == nil { + t.Errorf("unable to find item with uid %v", test_3["uid"]) + } else if hex.EncodeToString(res.ether) != test_3["ether"] { + t.Errorf("expected ethernet hardware %s, got %s", test_3["ether"], hex.EncodeToString(res.ether)) + } + } + + test_4 := map[string]string{ + "address": "127.0.0.17", + "uid": "010dead0001122", + "ether": "0dead0667788", + } + test_4_findings := filter_address(test_4["address"], results) + if len(test_4_findings) != 2 { + t.Errorf("expected %d matching entries, got %d", 2, len(test_4_findings)) + } else { + res := find_uid(test_4["uid"], test_4_findings) + if res == nil { + t.Errorf("unable to find item with uid %v", test_4["uid"]) + } else if hex.EncodeToString(res.ether) != test_4["ether"] { + t.Errorf("expected ethernet hardware %s, got %s", test_4["ether"], hex.EncodeToString(res.ether)) + } + } +} diff --git a/builder/vmware/common/testdata/dhcpd-example.leases b/builder/vmware/common/testdata/dhcpd-example.leases new file mode 100644 index 000000000..da95d2955 --- /dev/null +++ b/builder/vmware/common/testdata/dhcpd-example.leases @@ -0,0 +1,32 @@ +# This entry is normal +lease 127.0.0.17 { + starts 3 2020/05/13 12:00:37; + ends 3 2020/05/13 12:30:37; + hardware ethernet 0d:ea:d0:66:77:88; + uid 01:0d:ea:d0:00:11:22; +} + +# This entry has tabs +lease 127.0.0.17 { + starts 6 2020/06/12 22:28:54; + ends 6 2020/06/12 22:58:54; + hardware ethernet 0d:ea:d0:66:77:88; + uid 01:0d:ea:d0:33:44:55; +} + +# These next two entries have the same address, but different uids +lease 127.0.0.19 { + starts 4 2020/05/28 11:35:06; + ends 4 2020/05/28 12:05:06; + hardware ethernet 0d:ea:d0:66:77:88; + uid 01:0d:ea:d0:66:77:88; + client-hostname "VAGRANT-ADA-LUZ"; +} + +lease 127.0.0.19 { + starts 5 2020/08/20 20:32:03; + ends 5 2020/08/20 21:02:03; + hardware ethernet 0d:ea:d0:99:aa:bb; + uid 01:0d:ea:d0:99:aa:bb; + client-hostname "WINDOWS-SQABBAS"; +} From ad7194c92046b1cb12404f69fa213def29ebc53e Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 28 May 2020 12:48:19 -0500 Subject: [PATCH 7/9] Rewrote the GuestIP implementation in the vmware builder to use the new dhcpd lease parsers instead of the old regexp-based logic. --- builder/vmware/common/driver.go | 34 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go index 84e8fb9c7..13db32a68 100644 --- a/builder/vmware/common/driver.go +++ b/builder/vmware/common/driver.go @@ -376,7 +376,7 @@ func (d *VmwareDriver) GuestIP(state multistep.StateBag) (string, error) { return "", fmt.Errorf("no DHCP leases path found for device %s", device) } - // open up the lease and read its contents + // open up the path to the dhcpd leases fh, err := os.Open(dhcpLeasesPath) if err != nil { log.Printf("Error while reading DHCP lease path file %s: %s", dhcpLeasesPath, err.Error()) @@ -384,11 +384,16 @@ func (d *VmwareDriver) GuestIP(state multistep.StateBag) (string, error) { } defer fh.Close() - dhcpBytes, err := ioutil.ReadAll(fh) + // and then read its contents + leaseEntries, err := ReadDhcpdLeaseEntries(fh) if err != nil { return "", err } + // Parse our MAC address again. There's no need to check for an + // error because we've already parsed this successfully. + hwaddr, _ := net.ParseMAC(MACAddress) + // start grepping through the file looking for fields that we care about var lastIp string var lastLeaseEnd time.Time @@ -396,34 +401,19 @@ func (d *VmwareDriver) GuestIP(state multistep.StateBag) (string, error) { var curIp string var curLeaseEnd time.Time - ipLineRe := regexp.MustCompile(`^lease (.+?) {$`) - endTimeLineRe := regexp.MustCompile(`^\s*ends \d (.+?);$`) - macLineRe := regexp.MustCompile(`^\s*hardware ethernet (.+?);$`) + for _, entry := range leaseEntries { - for _, line := range strings.Split(string(dhcpBytes), "\n") { - // Need to trim off CR character when running in windows - line = strings.TrimRight(line, "\r") - - matches := ipLineRe.FindStringSubmatch(line) - if matches != nil { - lastIp = matches[1] - continue - } - - matches = endTimeLineRe.FindStringSubmatch(line) - if matches != nil { - lastLeaseEnd, _ = time.Parse("2006/01/02 15:04:05", matches[1]) - continue - } + lastIp = entry.address + lastLeaseEnd = entry.ends // If the mac address matches and this lease ends farther in the // future than the last match we might have, then choose it. - matches = macLineRe.FindStringSubmatch(line) - if matches != nil && strings.EqualFold(matches[1], MACAddress) && curLeaseEnd.Before(lastLeaseEnd) { + if bytes.Compare(hwaddr, entry.ether) == 0 && curLeaseEnd.Before(lastLeaseEnd) { curIp = lastIp curLeaseEnd = lastLeaseEnd } } + if curIp != "" { return curIp, nil } From 0609909f1a6b59d603f4eb4c816f2fc255a6c3ae Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 28 May 2020 13:53:49 -0500 Subject: [PATCH 8/9] Fixed some lint issues. --- builder/vmware/common/driver.go | 2 +- builder/vmware/common/driver_parser.go | 3 ++- builder/vmware/common/driver_parser_test.go | 12 ++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go index 13db32a68..51c866de5 100644 --- a/builder/vmware/common/driver.go +++ b/builder/vmware/common/driver.go @@ -408,7 +408,7 @@ func (d *VmwareDriver) GuestIP(state multistep.StateBag) (string, error) { // If the mac address matches and this lease ends farther in the // future than the last match we might have, then choose it. - if bytes.Compare(hwaddr, entry.ether) == 0 && curLeaseEnd.Before(lastLeaseEnd) { + if bytes.Equal(hwaddr, entry.ether) && curLeaseEnd.Before(lastLeaseEnd) { curIp = lastIp curLeaseEnd = lastLeaseEnd } diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 2dc828d24..58292f736 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -2225,7 +2225,8 @@ func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan // Now just spin in a loop shipping bytes down the channel until we hit // closeByte, or we're at the very end...whichever comes first. - for ok := true; by != closeByte; { + var ok bool + for by != closeByte { by, ok = <-in if !ok { by = closeByte diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index 5381a3e13..1b9a066a8 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -644,7 +644,7 @@ func TestParserDhcpdLeaseBytesDecoder(t *testing.T) { if err != nil { t.Errorf("unable to decode address: %s", err) } - if bytes.Compare(result, expected_1) != 0 { + if !bytes.Equal(result, expected_1) { t.Errorf("expected %v, got %v", expected_1, result) } @@ -655,30 +655,30 @@ func TestParserDhcpdLeaseBytesDecoder(t *testing.T) { if err != nil { t.Errorf("unable to decode address: %s", err) } - if bytes.Compare(result, expected_2) != 0 { + if !bytes.Equal(result, expected_2) { t.Errorf("expected %v, got %v", expected_2, result) } failtest_1 := "" - result, err = decodeDhcpdLeaseBytes(failtest_1) + _, err = decodeDhcpdLeaseBytes(failtest_1) if err == nil { t.Errorf("expected decoding error: %s", err) } failtest_2 := "000000" - result, err = decodeDhcpdLeaseBytes(failtest_2) + _, err = decodeDhcpdLeaseBytes(failtest_2) if err == nil { t.Errorf("expected decoding error: %s", err) } failtest_3 := "000:00" - result, err = decodeDhcpdLeaseBytes(failtest_3) + _, err = decodeDhcpdLeaseBytes(failtest_3) if err == nil { t.Errorf("expected decoding error: %s", err) } failtest_4 := "00:00:" - result, err = decodeDhcpdLeaseBytes(failtest_4) + _, err = decodeDhcpdLeaseBytes(failtest_4) if err == nil { t.Errorf("expected decoding error: %s", err) } From 898321bcfeeba20cc3fff5a1b4f1280433daa2da Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Thu, 4 Jun 2020 05:07:07 -0500 Subject: [PATCH 9/9] Added suggestions from @SwampDragons to the parsers from the vmware builders, included unit-tests and sample file for fusion's networking file, and fixed a few situations where the channel wasn't being closed. --- builder/vmware/common/driver_parser.go | 22 +- builder/vmware/common/driver_parser_test.go | 228 ++++++++++++++++++ .../vmware/common/testdata/networking-example | 19 ++ 3 files changed, 263 insertions(+), 6 deletions(-) create mode 100644 builder/vmware/common/testdata/networking-example diff --git a/builder/vmware/common/driver_parser.go b/builder/vmware/common/driver_parser.go index 58292f736..d080b307a 100644 --- a/builder/vmware/common/driver_parser.go +++ b/builder/vmware/common/driver_parser.go @@ -704,6 +704,7 @@ func parseParameter(val tkParameter) (pParameter, error) { } address := net.ParseIP(cidr[0]) + bits, err := strconv.Atoi(cidr[1]) if err != nil { return nil, err @@ -1038,10 +1039,12 @@ func (e *configDeclaration) IP4() (net.IP, error) { return nil, fmt.Errorf("No IP4 addresses found") } + // Try and parse it as an IP4. If so, then it's good to return it as-is. if res := net.ParseIP(result[0]); res != nil { return res, nil } + // Otherwise make an attempt to resolve it to an address. res, err := net.ResolveIPAddr("ip4", result[0]) if err != nil { return nil, err @@ -1069,10 +1072,12 @@ func (e *configDeclaration) IP6() (net.IP, error) { return nil, fmt.Errorf("No IP6 addresses found") } + // If we were able to parse it into an IP, then we can just return it. if res := net.ParseIP(result[0]); res != nil { return res, nil } + // Otherwise, try to resolve it into an address. res, err := net.ResolveIPAddr("ip6", result[0]) if err != nil { return nil, err @@ -1277,8 +1282,6 @@ func tokenizeNetworkingConfig(in chan byte) chan string { } switch by { - case '\r': - fallthrough case '\t': fallthrough case ' ': @@ -1290,16 +1293,20 @@ func tokenizeNetworkingConfig(in chan byte) chan string { out <- state state = "" + case '\r': + // If windows has tampered with our newlines, then we normalize + // things by converting its value from 0x0d to 0x0a. + fallthrough case '\n': // Newlines can repeat, so this case is responsible for writing - // to the chan, and consolidating multiple newlines into singles. + // to the chan, and consolidating multiple newlines into a single. if repeat_newline { continue } if len(state) > 0 { out <- state } - out <- string(by) + out <- "\n" state = "" repeat_newline = true continue @@ -1317,6 +1324,7 @@ func tokenizeNetworkingConfig(in chan byte) chan string { if len(state) > 0 { out <- state } + close(out) }(out) return out } @@ -1355,6 +1363,7 @@ func splitNetworkingConfig(in chan string) chan []string { if len(row) > 0 { out <- row } + close(out) }(out) return out } @@ -1698,7 +1707,7 @@ func parseNetworkingCommand_add_dhcp_mac_to_ip(row []string) (*networkingCommand } ip := net.ParseIP(row[2]) - if ip != nil { + if ip == nil { return nil, fmt.Errorf("Unable to parse third argument as ipv4. : %v", row[2]) } @@ -1850,6 +1859,7 @@ func parseNetworkingConfig(rows chan []string) chan networkingCommandEntry { out <- *entry } } + close(out) }(rows, out) return out } @@ -2197,7 +2207,7 @@ This consumes bytes within a pair of some bytes, like parentheses, brackets, bra We start by reading bytes until we encounter openByte. These will be returned as the first parameter. Then we can enter a goro and consume bytes until we get to -closeByte. At that point we're done, and suicide. +closeByte. At that point we're done, and we can exit. **/ func consumeOpenClosePair(openByte, closeByte byte, in chan byte) ([]byte, chan byte) { result := make([]byte, 0) diff --git a/builder/vmware/common/driver_parser_test.go b/builder/vmware/common/driver_parser_test.go index 1b9a066a8..01abffdbb 100644 --- a/builder/vmware/common/driver_parser_test.go +++ b/builder/vmware/common/driver_parser_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "os" "path/filepath" + "strings" ) func consumeString(s string) (out chan byte) { @@ -20,6 +21,13 @@ func consumeString(s string) (out chan byte) { return } +func collectIntoStringList(in chan string) (result []string) { + for item := range in { + result = append(result, item) + } + return +} + func uncommentFromString(s string) string { inCh := consumeString(s) out := uncomment(inCh) @@ -856,3 +864,223 @@ func TestParserReadDhcpdLeases(t *testing.T) { } } } + +func TestParserTokenizeNetworkingConfig(t *testing.T) { + tests := []string{ + "words words words", + "newlines\n\n\n\n\n\n\n\nnewlines\r\r\r\r\r\r\r\rnewlines\n\n\n\n", + " newline-less", + } + expects := [][]string{ + []string{"words", "words", "words"}, + []string{"newlines", "\n", "newlines", "\n", "newlines", "\n"}, + []string{"newline-less"}, + } + + for testnum := 0; testnum < len(tests); testnum += 1 { + inCh := consumeString(tests[testnum]) + outCh := tokenizeNetworkingConfig(inCh) + result := collectIntoStringList(outCh) + + expected := expects[testnum] + if len(result) != len(expected) { + t.Errorf("test %d expected %d items, got %d instead", 1+testnum, len(expected), len(result)) + continue + } + + ok := true + for index := 0; index < len(expected); index += 1 { + if result[index] != expected[index] { + ok = false + } + } + if !ok { + t.Errorf("test %d expected %#v, got %#v", 1+testnum, expects[testnum], result) + } + } +} + +func TestParserSplitNetworkingConfig(t *testing.T) { + tests := []string{ + "this is a story\n\n\nabout some newlines", + "\n\n\nthat can begin and end with newlines\n\n\n", + " in\n\n\nsome\ncases\nit\ncan\nend\nwith\nan\nempty\nstring\n\n\n\n", + "\n\n\nand\nbegin\nwith\nan\nempty\nstring ", + } + expects := [][]string{ + []string{"this is a story", "about some newlines"}, + []string{"that can begin and end with newlines"}, + []string{"in", "some", "cases", "it", "can", "end", "with", "an", "empty", "string"}, + []string{"and", "begin", "with", "an", "empty", "string"}, + } + + for testnum := 0; testnum < len(tests); testnum += 1 { + inCh := consumeString(tests[testnum]) + stringCh := tokenizeNetworkingConfig(inCh) + outCh := splitNetworkingConfig(stringCh) + + result := make([]string, 0) + for item := range outCh { + result = append(result, strings.Join(item, " ")) + } + + expected := expects[testnum] + if len(result) != len(expected) { + t.Errorf("test %d expected %d items, got %d instead", 1+testnum, len(expected), len(result)) + continue + } + + ok := true + for index := 0; index < len(expected); index += 1 { + if result[index] != expected[index] { + ok = false + } + } + if !ok { + t.Errorf("test %d expected %#v, got %#v", 1+testnum, expects[testnum], result) + } + } +} + +func TestParserParseNetworkingConfigVersion(t *testing.T) { + success_tests := []string{"VERSION=4,2"} + failure_tests := []string{ + "VERSION=1=2", + "VERSION=3,4,5", + "VERSION=a,b", + } + + for testnum := 0; testnum < len(success_tests); testnum += 1 { + test := []string{success_tests[testnum]} + if _, err := networkingReadVersion(test); err != nil { + t.Errorf("success-test %d parsing failed: %v", 1+testnum, err) + } + } + + for testnum := 0; testnum < len(success_tests); testnum += 1 { + test := []string{failure_tests[testnum]} + if _, err := networkingReadVersion(test); err == nil { + t.Errorf("failure-test %d should have failed", 1+testnum) + } + } +} + +func TestParserParseNetworkingConfigEntries(t *testing.T) { + tests := []string{ + "answer VNET_999_ANYTHING option", + "remove_answer VNET_123_ALSOANYTHING", + "add_nat_portfwd 24 udp 42 127.0.0.1 24", + "remove_nat_portfwd 42 tcp 2502", + "add_dhcp_mac_to_ip 57005 00:0d:0e:0a:0d:00 127.0.0.2", + "remove_dhcp_mac_to_ip 57005 00:0d:0e:0a:0d:00", + "add_bridge_mapping string 51", + "remove_bridge_mapping string", + "add_nat_prefix 57005 /24", + "remove_nat_prefix 57005 /31", + } + + for testnum := 0; testnum < len(tests); testnum += 1 { + test := strings.Split(tests[testnum], " ") + parser := NetworkingParserByCommand(test[0]) + if parser == nil { + t.Errorf("test %d unable to parse command: %#v", 1+testnum, test) + continue + } + operand_parser := *parser + + _, err := operand_parser(test[1:]) + if err != nil { + t.Errorf("test %d unable to parse command parameters %#v: %v", 1+testnum, test, err) + } + } +} + +func TestParserReadNetworingConfig(t *testing.T) { + expected_answer_vnet_1 := map[string]string{ + "DHCP": "yes", + "DHCP_CFG_HASH": "01F4CE0D79A1599698B6E5814CCB68058BB0ED5E", + "HOSTONLY_NETMASK": "255.255.255.0", + "HOSTONLY_SUBNET": "192.168.70.0", + "NAT": "no", + "VIRTUAL_ADAPTER": "yes", + } + + f, err := os.Open(filepath.Join("testdata", "networking-example")) + if err != nil { + t.Fatalf("Unable to open networking-example sample: %v", err) + } + defer f.Close() + + config, err := ReadNetworkingConfig(f) + if err != nil { + t.Fatalf("error parsing networking-example: %v", err) + } + + if vnet, ok := config.answer[1]; ok { + for ans_key := range expected_answer_vnet_1 { + result, ok := vnet[ans_key] + if !ok { + t.Errorf("unable to find key %s in VNET_%d answer", ans_key, 1) + continue + } + + if result != expected_answer_vnet_1[ans_key] { + t.Errorf("expected key %s for VNET_%d to be %v, got %v", ans_key, 1, expected_answer_vnet_1[ans_key], result) + } + } + + } else { + t.Errorf("unable to find VNET_%d answer", 1) + } + + expected_answer_vnet_8 := map[string]string{ + "DHCP": "yes", + "DHCP_CFG_HASH": "C30F14F65A0FE4B5DCC6C67497D7A8A33E5E538C", + "HOSTONLY_NETMASK": "255.255.255.0", + "HOSTONLY_SUBNET": "172.16.41.0", + "NAT": "yes", + "VIRTUAL_ADAPTER": "yes", + } + + if vnet, ok := config.answer[8]; ok { + for ans_key := range expected_answer_vnet_8 { + result, ok := vnet[ans_key] + if !ok { + t.Errorf("unable to find key %s in VNET_%d answer", ans_key, 8) + continue + } + + if result != expected_answer_vnet_8[ans_key] { + t.Errorf("expected key %s for VNET_%d to be %v, got %v", ans_key, 8, expected_answer_vnet_8[ans_key], result) + } + } + + } else { + t.Errorf("unable to find VNET_%d answer", 8) + } + + expected_nat_portfwd_8 := map[string]string{ + "tcp/2200": "172.16.41.129:3389", + "tcp/2201": "172.16.41.129:3389", + "tcp/2222": "172.16.41.129:22", + "tcp/3389": "172.16.41.131:3389", + "tcp/55985": "172.16.41.129:5985", + "tcp/55986": "172.16.41.129:5986", + } + + if vnet, ok := config.nat_portfwd[8-1]; ok { + for nat_key := range expected_nat_portfwd_8 { + result, ok := vnet[nat_key] + if !ok { + t.Errorf("unable to find key %s in VNET_%d nat_portfwd", nat_key, 8) + continue + } + + if result != expected_nat_portfwd_8[nat_key] { + t.Errorf("expected key %s for VNET_%d to be %v, got %v", nat_key, 8, expected_nat_portfwd_8[nat_key], result) + } + } + } else { + t.Errorf("unable to find VNET_%d answer", 8-1) + } +} diff --git a/builder/vmware/common/testdata/networking-example b/builder/vmware/common/testdata/networking-example new file mode 100644 index 000000000..7117ac8f6 --- /dev/null +++ b/builder/vmware/common/testdata/networking-example @@ -0,0 +1,19 @@ +VERSION=1,0 +answer VNET_1_DHCP yes +answer VNET_1_DHCP_CFG_HASH 01F4CE0D79A1599698B6E5814CCB68058BB0ED5E +answer VNET_1_HOSTONLY_NETMASK 255.255.255.0 +answer VNET_1_HOSTONLY_SUBNET 192.168.70.0 +answer VNET_1_NAT no +answer VNET_1_VIRTUAL_ADAPTER yes +answer VNET_8_DHCP yes +answer VNET_8_DHCP_CFG_HASH C30F14F65A0FE4B5DCC6C67497D7A8A33E5E538C +answer VNET_8_HOSTONLY_NETMASK 255.255.255.0 +answer VNET_8_HOSTONLY_SUBNET 172.16.41.0 +answer VNET_8_NAT yes +answer VNET_8_VIRTUAL_ADAPTER yes +add_nat_portfwd 8 tcp 2200 172.16.41.129 3389 +add_nat_portfwd 8 tcp 2201 172.16.41.129 3389 +add_nat_portfwd 8 tcp 2222 172.16.41.129 22 +add_nat_portfwd 8 tcp 3389 172.16.41.131 3389 +add_nat_portfwd 8 tcp 55985 172.16.41.129 5985 +add_nat_portfwd 8 tcp 55986 172.16.41.129 5986