Skip to content

Commit

Permalink
Add support for HCN v2 endpoint and add unit tests (#2343)
Browse files Browse the repository at this point in the history 
* Add support for HCN v2 endpoint and add unit tests
* switch to HCN v2 endpoint API instead of HNS v1 endpoint API
* Support parsing routes in GCS when we setup the network interfaces
* [breaking] update gcs bridge LCOW network adapter type with new fields that better
align with v2 endpoint
* Add unit tests for new GCS side changes
* Add legacy policy based routing for lcow and an annotation to toggle use

Signed-off-by: Kathryn Baldauf <[email protected]>

---------

Signed-off-by: Kathryn Baldauf <[email protected]>
  • Loading branch information
@katiewasnothere
katiewasnothere authored Jan 15, 2025
1 parent bac751f commit 8d81359
Show file tree
Hide file tree
Showing 11 changed files with 1,119 additions and 197 deletions.
8 changes: 8 additions & 0 deletions internal/annotations/annotations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,12 @@ const (
// UVMConsolePipe is the name of the named pipe that the UVM console is connected to. This works only for non-SNP
// scenario, for SNP use a debugger.
UVMConsolePipe = "io.microsoft.virtualmachine.console.pipe"

// NetworkingPolicyBasedRouting toggles on the ability to set policy based routing in the
// guest for LCOW.
//
// TODO(katiewasnothere): The goal of this annotation was to be used as a fallback if the
// work to support multiple custom network routes per adapter in LCOW breaks existing
// LCOW scenarios. Ideally, this annotation should be removed if no issues are found.
NetworkingPolicyBasedRouting = "io.microsoft.virtualmachine.lcow.network.policybasedrouting"
)
251 changes: 139 additions & 112 deletions internal/guest/network/netns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,33 @@ import (
"os/exec"
"runtime"
"strconv"
"strings"
"time"

"github.com/Microsoft/hcsshim/internal/guest/prot"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)

var (
// function definitions for mocking configureLink
netlinkAddrAdd = netlink.AddrAdd
netlinkRouteAdd = netlink.RouteAdd
netlinkRuleAdd = netlink.RuleAdd
)

const (
ipv4GwDestination = "0.0.0.0/0"
ipv4EmptyGw = "0.0.0.0"
ipv6GwDestination = "::/0"
ipv6EmptyGw = "::"

unreachableErrStr = "network is unreachable"
)

// MoveInterfaceToNS moves the adapter with interface name `ifStr` to the network namespace
// of `pid`.
func MoveInterfaceToNS(ifStr string, pid int) error {
Expand Down Expand Up @@ -67,7 +84,7 @@ func DoInNetNS(ns netns.NsHandle, run func() error) error {
//
// This function MUST be used in tandem with `DoInNetNS` or some other means that ensures that the goroutine
// executing this code stays on the same thread.
func NetNSConfig(ctx context.Context, ifStr string, nsPid int, adapter *prot.NetworkAdapter) error {
func NetNSConfig(ctx context.Context, ifStr string, nsPid int, adapter *guestresource.LCOWNetworkAdapter) error {
ctx, entry := log.S(ctx, logrus.Fields{
"ifname": ifStr,
"pid": nsPid,
Expand Down Expand Up @@ -101,29 +118,14 @@ func NetNSConfig(ctx context.Context, ifStr string, nsPid int, adapter *prot.Net
}

// Configure the interface
if adapter.NatEnabled {
entry.Tracef("Configuring interface with NAT: %s/%d gw=%s",
adapter.AllocatedIPAddress,
adapter.HostIPPrefixLength, adapter.HostIPAddress)
metric := 1
if adapter.EnableLowMetric {
metric = 500
}
if len(adapter.IPConfigs) != 0 {
entry.Debugf("Configuring interface with NAT: %v", adapter)

// Bring the interface up
if err := netlink.LinkSetUp(link); err != nil {
return errors.Wrapf(err, "netlink.LinkSetUp(%#v) failed", link)
}
if err := assignIPToLink(ctx, ifStr, nsPid, link,
adapter.AllocatedIPAddress, adapter.HostIPAddress, adapter.HostIPPrefixLength,
adapter.EnableLowMetric, metric,
); err != nil {
return err
return fmt.Errorf("netlink.LinkSetUp(%#v) failed: %w", link, err)
}
if err := assignIPToLink(ctx, ifStr, nsPid, link,
adapter.AllocatedIPv6Address, adapter.HostIPv6Address, adapter.HostIPv6PrefixLength,
adapter.EnableLowMetric, metric,
); err != nil {
if err := configureLink(ctx, link, adapter); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -186,107 +188,132 @@ func NetNSConfig(ctx context.Context, ifStr string, nsPid int, adapter *prot.Net
return nil
}

func assignIPToLink(ctx context.Context,
ifStr string,
nsPid int,
func configureLink(ctx context.Context,
link netlink.Link,
allocatedIP string,
gatewayIP string,
prefixLen uint8,
enableLowMetric bool,
metric int,
adapter *guestresource.LCOWNetworkAdapter,
) error {
entry := log.G(ctx)
entry.WithFields(logrus.Fields{
"link": link.Attrs().Name,
"IP": allocatedIP,
"prefixLen": prefixLen,
"gateway": gatewayIP,
"metric": metric,
}).Trace("assigning IP address")
if allocatedIP == "" {
return nil
}
// Set IP address
ip, addr, err := net.ParseCIDR(allocatedIP + "/" + strconv.FormatUint(uint64(prefixLen), 10))
if err != nil {
return errors.Wrapf(err, "parsing address %s/%d failed", allocatedIP, prefixLen)
}
// the IP address field in addr is masked, so replace it with the original ip address
addr.IP = ip
entry.WithFields(logrus.Fields{
"allocatedIP": ip,
"IP": addr,
}).Debugf("parsed ip address %s/%d", allocatedIP, prefixLen)
ipAddr := &netlink.Addr{IPNet: addr, Label: ""}
if err := netlink.AddrAdd(link, ipAddr); err != nil {
return errors.Wrapf(err, "netlink.AddrAdd(%#v, %#v) failed", link, ipAddr)
}
if gatewayIP == "" {
return nil
}
// Set gateway
gw := net.ParseIP(gatewayIP)
if gw == nil {
return errors.Wrapf(err, "parsing gateway address %s failed", gatewayIP)
}
var table int
for _, ipConfig := range adapter.IPConfigs {
log.G(ctx).WithFields(logrus.Fields{
"link": link.Attrs().Name,
"IP": ipConfig.IPAddress,
"prefixLen": ipConfig.PrefixLength,
}).Debug("assigning IP address")

if !addr.Contains(gw) {
// In the case that a gw is not part of the subnet we are setting gw for,
// a new addr containing this gw address need to be added into the link to avoid getting
// unreachable error when adding this out-of-subnet gw route
entry.Debugf("gw is outside of the subnet: Configure %s in %d with: %s/%d gw=%s\n",
ifStr, nsPid, allocatedIP, prefixLen, gatewayIP)

// net library's ParseIP call always returns an array of length 16, so we
// need to first check if the address is IPv4 or IPv6 before calculating
// the mask length. See https://pkg.go.dev/net#ParseIP.
ml := 8
if gw.To4() != nil {
ml *= net.IPv4len
} else if gw.To16() != nil {
ml *= net.IPv6len
} else {
return fmt.Errorf("gw IP is neither IPv4 nor IPv6: %v", gw)
// Set IP address
ip, addr, err := net.ParseCIDR(ipConfig.IPAddress + "/" + strconv.FormatUint(uint64(ipConfig.PrefixLength), 10))
if err != nil {
return fmt.Errorf("parsing address %s/%d failed: %w", ipConfig.IPAddress, ipConfig.PrefixLength, err)
}
addr2 := &net.IPNet{
IP: gw,
Mask: net.CIDRMask(ml, ml)}
ipAddr2 := &netlink.Addr{IPNet: addr2, Label: ""}
if err := netlink.AddrAdd(link, ipAddr2); err != nil {
return errors.Wrapf(err, "netlink.AddrAdd(%#v, %#v) failed", link, ipAddr2)
// the IP address field in addr is masked, so replace it with the original ip address
addr.IP = ip
log.G(ctx).WithFields(logrus.Fields{
"allocatedIP": ip,
"IP": addr,
}).Debugf("parsed ip address %s/%d", ipConfig.IPAddress, ipConfig.PrefixLength)
ipAddr := &netlink.Addr{IPNet: addr, Label: ""}
if err := netlinkAddrAdd(link, ipAddr); err != nil {
return fmt.Errorf("netlink.AddrAdd(%#v, %#v) failed: %w", link, ipAddr, err)
}

if adapter.EnableLowMetric {
// add a route rule for the new interface so packets coming on this interface
// always go out the same interface
_, ml := addr.Mask.Size()
srcNet := &net.IPNet{
IP: net.ParseIP(ipConfig.IPAddress),
Mask: net.CIDRMask(ml, ml),
}
rule := netlink.NewRule()
rule.Table = 101
rule.Src = srcNet
rule.Priority = 5

if err := netlinkRuleAdd(rule); err != nil {
return errors.Wrapf(err, "netlink.RuleAdd(%#v) failed", rule)
}
table = rule.Table
}
}

var table int
if enableLowMetric {
// add a route rule for the new interface so packets coming on this interface
// always go out the same interface
_, ml := addr.Mask.Size()
srcNet := &net.IPNet{
IP: net.ParseIP(allocatedIP),
Mask: net.CIDRMask(ml, ml),
for _, r := range adapter.Routes {
log.G(ctx).WithField("route", r).Debugf("adding a route to interface %s", link.Attrs().Name)

if (r.DestinationPrefix == ipv4GwDestination || r.DestinationPrefix == ipv6GwDestination) &&
(r.NextHop == ipv4EmptyGw || r.NextHop == ipv6EmptyGw) {
// this indicates no default gateway was added for this interface
continue
}
rule := netlink.NewRule()
rule.Table = 101
rule.Src = srcNet
rule.Priority = 5

if err := netlink.RuleAdd(rule); err != nil {
return errors.Wrapf(err, "netlink.RuleAdd(%#v) failed", rule)
// dst will be nil when setting default gateways
var dst *net.IPNet
if !(r.DestinationPrefix == ipv4GwDestination || r.DestinationPrefix == ipv6GwDestination) {
dstIP, dstAddr, err := net.ParseCIDR(r.DestinationPrefix)
if err != nil {
return fmt.Errorf("parsing route dst address %s failed: %w", r.DestinationPrefix, err)
}
dstAddr.IP = dstIP
dst = dstAddr
}

// gw can be nil when setting something like
// ip route add 10.0.0.0/16 dev eth0
gw := net.ParseIP(r.NextHop)
if gw == nil && dst == nil {
return fmt.Errorf("gw and destination cannot both be nil")
}

metric := int(r.Metric)
if adapter.EnableLowMetric && r.Metric == 0 {
// set a low metric only if the endpoint didn't already have a metric
// configured
metric = 500
}
route := netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: link.Attrs().Index,
Gw: gw,
Dst: dst,
Priority: metric,
// table will be set to 101 for the legacy policy based routing support
Table: table,
}
if err := netlinkRouteAdd(&route); err != nil {
// unfortunately, netlink library doesn't have great error handling,
// so we have to rely on the string error here
if strings.Contains(err.Error(), unreachableErrStr) && gw != nil {
// In the case that a gw is not part of the subnet we are setting gw for,
// a new addr containing this gw address needs to be added into the link to avoid getting
// unreachable error when adding this out-of-subnet gw route
log.G(ctx).Infof("gw is outside of the subnet: %v", gw)

// net library's ParseIP call always returns an array of length 16, so we
// need to first check if the address is IPv4 or IPv6 before calculating
// the mask length. See https://pkg.go.dev/net#ParseIP.
ml := 8
if gw.To4() != nil {
ml *= net.IPv4len
} else if gw.To16() != nil {
ml *= net.IPv6len
} else {
return fmt.Errorf("gw IP is neither IPv4 nor IPv6: %v", gw)
}
addr2 := &net.IPNet{
IP: gw,
Mask: net.CIDRMask(ml, ml)}
ipAddr2 := &netlink.Addr{IPNet: addr2, Label: ""}
if err := netlinkAddrAdd(link, ipAddr2); err != nil {
return fmt.Errorf("netlink.AddrAdd(%#v, %#v) failed: %w", link, ipAddr2, err)
}

// try adding the route again
if err := netlinkRouteAdd(&route); err != nil {
return fmt.Errorf("netlink.RouteAdd(%#v) failed: %w", route, err)
}
} else {
return fmt.Errorf("netlink.RouteAdd(%#v) failed: %w", route, err)
}
}
table = rule.Table
}
// add the default route in that interface specific table
route := netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: link.Attrs().Index,
Gw: gw,
Table: table,
Priority: metric,
}
if err := netlink.RouteAdd(&route); err != nil {
return errors.Wrapf(err, "netlink.RouteAdd(%#v) failed", route)
}
return nil
}
Loading

0 comments on commit 8d81359

Please sign in to comment.