diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index 0a41156e1..fa7265bfc 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -31,6 +31,7 @@ import ( cmn "github.com/loxilb-io/loxilb/common" opt "github.com/loxilb-io/loxilb/options" + "github.com/loxilb-io/loxilb/pkg/utils" tk "github.com/loxilb-io/loxilib" nlp "github.com/vishvananda/netlink" "golang.org/x/sys/unix" @@ -565,6 +566,11 @@ func AddVxLANBridgeNoHook(vxlanid int, epIntfName string) int { _, err := nlp.LinkByName(VxlanBridgeName) if err != nil { + hwAddr, err := utils.GenerateRandomMAC() + if err != nil { + tk.LogIt(tk.LogError, "nlp: Error generating hwAddr\n") + return 403 + } EndpointInterface, err := nlp.LinkByName(epIntfName) if err != nil { tk.LogIt(tk.LogWarning, "nlp: Endpoint interface finding Fail\n") @@ -577,8 +583,9 @@ func AddVxLANBridgeNoHook(vxlanid int, epIntfName string) int { } VxlanDev := &nlp.Vxlan{ LinkAttrs: nlp.LinkAttrs{ - Name: VxlanBridgeName, - MTU: 9000, // Static Value for Vxlan in loxiLB + Name: VxlanBridgeName, + MTU: 9000, // Static Value for Vxlan in loxiLB + HardwareAddr: hwAddr, }, SrcAddr: LocalIPs[0].IP, VtepDevIndex: EndpointInterface.Attrs().Index, @@ -1271,24 +1278,28 @@ func AddRouteNoHook(DestinationIPNet, gateway, proto string) int { return ret } -func GetRouteNoHook(destination string) ([]string, error) { +func GetRouteNoHook(destination string) ([]string, string, error) { var gws []string + var src string dst := net.ParseIP(destination) if dst == nil { - return []string{}, errors.New("invalid destination") + return []string{}, "", errors.New("invalid destination") } rts, err := nlp.RouteGet(dst) if err != nil { - return []string{}, errors.New("invalid rt destination") + return []string{}, "", errors.New("invalid rt destination") } for _, rt := range rts { + if src == "" { + src = rt.Src.String() + } gws = append(gws, rt.Gw.String()) } - return gws, nil + return gws, src, nil } func DelRouteNoHook(DestinationIPNet string) int { diff --git a/loxilb-ebpf b/loxilb-ebpf index 73f23493a..56e9de522 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 73f23493ab81f3ffe1c71b9650c186d5115a5e48 +Subproject commit 56e9de522ce8990bdcc160192bac49fc71f9de91 diff --git a/pkg/loxinet/cluster.go b/pkg/loxinet/cluster.go index f62b96970..d833b8c3a 100644 --- a/pkg/loxinet/cluster.go +++ b/pkg/loxinet/cluster.go @@ -87,6 +87,10 @@ type CIStateH struct { ClusterIf string OGw []string OGw6 []string + OSrc string + OSrc6 string + initRules bool + initRules6 bool } func (ci *CIStateH) BFDSessionNotify(instance string, remote string, ciState string) { @@ -285,11 +289,11 @@ func CIInit(args CIKAArgs) *CIStateH { nCIh.ClusterGw = gw.String() nCIh.ClusterGw6 = gw6.String() - nCIh.OGw, _ = nlp.GetRouteNoHook("8.8.8.8") - nCIh.OGw6, _ = nlp.GetRouteNoHook("2001:4860:4860::8888") + nCIh.OGw, nCIh.OSrc, _ = nlp.GetRouteNoHook("8.8.8.8") + nCIh.OGw6, nCIh.OSrc6, _ = nlp.GetRouteNoHook("2001:4860:4860::8888") - tk.LogIt(tk.LogInfo, "Cluster IP address %s GW %s oGW %v\n", ip.String(), nCIh.ClusterGw, nCIh.OGw) - tk.LogIt(tk.LogInfo, "Cluster IP6 address %s GW6 %s oGw6 %v\n", ip6.String(), nCIh.ClusterGw, nCIh.OGw6) + tk.LogIt(tk.LogInfo, "Cluster IP address %s GW %s oGW %v oSrc %v \n", ip.String(), nCIh.ClusterGw, nCIh.OGw, nCIh.OSrc) + tk.LogIt(tk.LogInfo, "Cluster IP6 address %s GW6 %s oGw6 %v oSrc6 %v\n", ip6.String(), nCIh.ClusterGw, nCIh.OGw6, nCIh.OSrc6) } @@ -384,7 +388,7 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) { if add { found := false if tk.IsNetIPv4(dest) { - gws, _ := nlp.GetRouteNoHook("8.8.8.8") + gws, _, _ := nlp.GetRouteNoHook("8.8.8.8") for _, gw := range gws { if gw == dest { found = true @@ -395,10 +399,16 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) { if !found { nlp.DelRouteNoHook("0.0.0.0/0") nlp.AddRouteNoHook("0.0.0.0/0", dest, "static") + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"} + _, err := mh.zr.Rules.DeleteFwRule(fwarg) + if err != nil { + tk.LogIt(tk.LogError, "Failed to delete egress snat for cluster %s\n", mh.has.ClusterNet) + } + } } else { found = false - gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888") + gws, _, _ := nlp.GetRouteNoHook("2001:4860:4860::8888") for _, gw := range gws { if gw == dest { found = true @@ -408,12 +418,17 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) { if !found { nlp.DelRouteNoHook("::/0") nlp.AddRouteNoHook("::/0", dest, "static") + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "::/0"} + _, err := mh.zr.Rules.DeleteFwRule(fwarg) + if err != nil { + tk.LogIt(tk.LogError, "Failed to delete egress snat for cluster %s\n", mh.has.ClusterNet6) + } } } } else { found := false if tk.IsNetIPv4(dest) { - gws, _ := nlp.GetRouteNoHook("8.8.8.8") + gws, _, _ := nlp.GetRouteNoHook("8.8.8.8") for _, gw := range gws { if gw == dest { found = true @@ -422,13 +437,33 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) { } if found { nlp.DelRouteNoHook("0.0.0.0/0") - for _, gw := range mh.has.OGw { + for i, gw := range mh.has.OGw { nlp.AddRouteNoHook("0.0.0.0/0", gw, "static") + if i == 0 { + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"} + fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc} + _, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts) + if err != nil { + tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet, err) + } + } } + } else if !h.initRules { + for i := range mh.has.OGw { + if i == 0 { + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet, DstIP: "0.0.0.0/0"} + fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc} + _, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts) + if err != nil { + tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet, err) + } + } + } + h.initRules = true } } else { found = false - gws, _ := nlp.GetRouteNoHook("2001:4860:4860::8888") + gws, _, _ := nlp.GetRouteNoHook("2001:4860:4860::8888") for _, gw := range gws { if gw == dest { found = true @@ -437,9 +472,29 @@ func (h *CIStateH) CIAddClusterRoute(dest string, add bool) { } if found { nlp.DelRouteNoHook("::/0") - for _, gw := range mh.has.OGw { + for i, gw := range mh.has.OGw6 { nlp.AddRouteNoHook("::", gw, "static") + if i == 0 { + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "0.0.0.0/0"} + fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc6} + _, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts) + if err != nil { + tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet6, err) + } + } + } + } else if !h.initRules6 { + for i := range mh.has.OGw6 { + if i == 0 { + fwarg := cmn.FwRuleArg{SrcIP: mh.has.ClusterNet6, DstIP: "0.0.0.0/0"} + fwOpts := cmn.FwOptArg{DoSnat: true, OnDefault: true, ToIP: mh.has.OSrc6} + _, err := mh.zr.Rules.AddFwRule(fwarg, fwOpts) + if err != nil { + tk.LogIt(tk.LogError, "Failed to create egress snat for cluster %s:%s\n", mh.has.ClusterNet6, err) + } + } } + h.initRules6 = true } } } diff --git a/pkg/loxinet/rules.go b/pkg/loxinet/rules.go index c9f4304d2..54c1414c0 100644 --- a/pkg/loxinet/rules.go +++ b/pkg/loxinet/rules.go @@ -2081,10 +2081,12 @@ func (R *RuleH) AddFwRule(fwRule cmn.FwRuleArg, fwOptArgs cmn.FwOptArg) (int, er eFw := R.tables[RtFw].eMap[rt.ruleKey()] if eFw != nil { - if eFw.act.action.(*ruleFwOpts).opt.fwMark != fwOptArgs.Mark { - eFw.Fw2DP(DpRemove) - eFw.act.action.(*ruleFwOpts).opt.fwMark = fwOptArgs.Mark - eFw.Fw2DP(DpCreate) + if !fwOptArgs.DoSnat { + if eFw.act.action.(*ruleFwOpts).opt.fwMark != fwOptArgs.Mark { + eFw.Fw2DP(DpRemove) + eFw.act.action.(*ruleFwOpts).opt.fwMark = fwOptArgs.Mark + eFw.Fw2DP(DpCreate) + } } // If a FW rule already exists return RuleExistsErr, errors.New("fwrule-exists error") @@ -3107,6 +3109,9 @@ func (r *ruleEnt) Fw2DP(work DpWorkT) int { nWork.FwVal2 = at.opt.fwMark nWork.FwRecord = at.opt.record nWork.OnDflt = at.opt.onDflt + if nWork.OnDflt && work == DpRemove { + r.sync = 0 + } default: return -1 } diff --git a/pkg/utils/net.go b/pkg/utils/net.go index 7de69a8bc..ca99712ba 100644 --- a/pkg/utils/net.go +++ b/pkg/utils/net.go @@ -19,6 +19,7 @@ package utils import ( "bytes" "context" + "crypto/rand" "crypto/tls" "crypto/x509" "encoding/binary" @@ -543,3 +544,16 @@ func IPHostCIDRString(ip net.IP) string { return ip.String() + "/128" } } + +func GenerateRandomMAC() (net.HardwareAddr, error) { + mac := make([]byte, 6) + _, err := rand.Read(mac) + if err != nil { + return nil, err + } + + mac[0] |= 0x02 + mac[0] &= 0xfe + + return net.HardwareAddr(mac), nil +}