gobgpd: support graceful-restart restarting-speaker behavior

use `-r` option to start gobgpd in restarting-speaker mode

Signed-off-by: ISHIDA Wataru <ishida.wataru@lab.ntt.co.jp>

7 files changed, 176 insertions, 23 deletions

diff --git a/config/bgp_configs.go b/config/bgp_configs.go
index 17baf726..d6c68782 100644
--- a/config/bgp_configs.go
+++ b/config/bgp_configs.go
@@ -1709,6 +1709,12 @@ type MpGracefulRestartState struct {
 	// original -> bgp-op:advertised
 	//bgp-op:advertised's original type is boolean
 	Advertised bool `mapstructure:"advertised"`
+	// original -> gobgp:end-of-rib-received
+	//gobgp:end-of-rib-received's original type is boolean
+	EndOfRibReceived bool `mapstructure:"end-of-rib-received"`
+	// original -> gobgp:end-of-rib-sent
+	//gobgp:end-of-rib-sent's original type is boolean
+	EndOfRibSent bool `mapstructure:"end-of-rib-sent"`
 }
 
 //struct for container bgp-mp:config
@@ -1787,6 +1793,8 @@ type GracefulRestartState struct {
 	LocalRestarting bool `mapstructure:"local-restarting"`
 	// original -> bgp-op:mode
 	Mode Mode `mapstructure:"mode"`
+	// original -> gobgp:deferral-time
+	DeferralTime uint16 `mapstructure:"deferral-time"`
 }
 
 //struct for container bgp:config
@@ -1802,6 +1810,8 @@ type GracefulRestartConfig struct {
 	// original -> bgp:helper-only
 	//bgp:helper-only's original type is boolean
 	HelperOnly bool `mapstructure:"helper-only"`
+	// original -> gobgp:deferral-time
+	DeferralTime uint16 `mapstructure:"deferral-time"`
 }
 
 //struct for container bgp:graceful-restart
diff --git a/config/default.go b/config/default.go
index 1d29ec4d..fe7d547b 100644
--- a/config/default.go
+++ b/config/default.go
@@ -146,6 +146,13 @@ func SetDefaultConfigValues(v *viper.Viper, b *Bgp) error {
 				// equal to the HOLDTIME carried in the OPEN.
 				n.GracefulRestart.Config.RestartTime = uint16(n.Timers.Config.HoldTime)
 			}
+			if !vv.IsSet("neighbor.graceful-restart.config.deferral-time") {
+				// RFC 4724 4.1. Procedures for the Restarting Speaker
+				// The value of this timer should be large
+				// enough, so as to provide all the peers of the Restarting Speaker with
+				// enough time to send all the routes to the Restarting Speaker
+				n.GracefulRestart.Config.DeferralTime = uint16(360)
+			}
 		}
 		b.Neighbors[idx] = n
 	}
diff --git a/gobgpd/main.go b/gobgpd/main.go
index ca950d29..dd12fec6 100644
--- a/gobgpd/main.go
+++ b/gobgpd/main.go
@@ -38,16 +38,17 @@ func main() {
 	signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM)
 
 	var opts struct {
-		ConfigFile    string `short:"f" long:"config-file" description:"specifying a config file"`
-		ConfigType    string `short:"t" long:"config-type" description:"specifying config type (toml, yaml, json)" default:"toml"`
-		LogLevel      string `short:"l" long:"log-level" description:"specifying log level"`
-		LogPlain      bool   `short:"p" long:"log-plain" description:"use plain format for logging (json by default)"`
-		UseSyslog     string `short:"s" long:"syslog" description:"use syslogd"`
-		Facility      string `long:"syslog-facility" description:"specify syslog facility"`
-		DisableStdlog bool   `long:"disable-stdlog" description:"disable standard logging"`
-		CPUs          int    `long:"cpus" description:"specify the number of CPUs to be used"`
-		Ops           bool   `long:"openswitch" description:"openswitch mode"`
-		GrpcPort      int    `long:"grpc-port" description:"grpc port" default:"50051"`
+		ConfigFile      string `short:"f" long:"config-file" description:"specifying a config file"`
+		ConfigType      string `short:"t" long:"config-type" description:"specifying config type (toml, yaml, json)" default:"toml"`
+		LogLevel        string `short:"l" long:"log-level" description:"specifying log level"`
+		LogPlain        bool   `short:"p" long:"log-plain" description:"use plain format for logging (json by default)"`
+		UseSyslog       string `short:"s" long:"syslog" description:"use syslogd"`
+		Facility        string `long:"syslog-facility" description:"specify syslog facility"`
+		DisableStdlog   bool   `long:"disable-stdlog" description:"disable standard logging"`
+		CPUs            int    `long:"cpus" description:"specify the number of CPUs to be used"`
+		Ops             bool   `long:"openswitch" description:"openswitch mode"`
+		GrpcPort        int    `short:"g" long:"grpc-port" description:"grpc port" default:"50051"`
+		GracefulRestart bool   `short:"r" long:"graceful-restart" description:"flag restart-state in graceful-restart capability"`
 	}
 	_, err := flags.Parse(&opts)
 	if err != nil {
@@ -187,6 +188,13 @@ func main() {
 				bgpConfig = &newConfig.Bgp
 				bgpServer.SetRpkiConfig(newConfig.Bgp.RpkiServers)
 				added = newConfig.Bgp.Neighbors
+				if opts.GracefulRestart {
+					for i, n := range added {
+						if n.GracefulRestart.Config.Enabled {
+							added[i].GracefulRestart.State.LocalRestarting = true
+						}
+					}
+				}
 				deleted = []config.Neighbor{}
 				updated = []config.Neighbor{}
 			} else {
diff --git a/server/grpc_server.go b/server/grpc_server.go
index a15143ee..1cce90bb 100644
--- a/server/grpc_server.go
+++ b/server/grpc_server.go
@@ -71,6 +71,7 @@ const (
 	REQ_BMP_NEIGHBORS
 	REQ_BMP_GLOBAL
 	REQ_BMP_ADJ_IN
+	REQ_DEFERRAL_TIMER_EXPIRED
 )
 
 type Server struct {
diff --git a/server/peer.go b/server/peer.go
index 5b4fd701..da76bf93 100644
--- a/server/peer.go
+++ b/server/peer.go
@@ -138,13 +138,14 @@ func (peer *Peer) getBestFromLocal(rfList []bgp.RouteFamily) ([]*table.Path, []*
 	return pathList, filtered
 }
 
-func (peer *Peer) handleBGPmessage(e *FsmMsg) ([]*table.Path, []*bgp.BGPMessage) {
+func (peer *Peer) handleBGPmessage(e *FsmMsg) ([]*table.Path, []*bgp.BGPMessage, []bgp.RouteFamily) {
 	m := e.MsgData.(*bgp.BGPMessage)
 	log.WithFields(log.Fields{
 		"Topic": "Peer",
 		"Key":   peer.conf.Config.NeighborAddress,
 		"data":  m,
 	}).Debug("received")
+	eor := []bgp.RouteFamily{}
 
 	switch m.Header.Type {
 	case bgp.BGP_MSG_ROUTE_REFRESH:
@@ -167,7 +168,7 @@ func (peer *Peer) handleBGPmessage(e *FsmMsg) ([]*table.Path, []*bgp.BGPMessage)
 				path.IsWithdraw = true
 				accepted = append(accepted, path)
 			}
-			return nil, table.CreateUpdateMsgFromPaths(accepted)
+			return nil, table.CreateUpdateMsgFromPaths(accepted), eor
 		} else {
 			log.WithFields(log.Fields{
 				"Topic": "Peer",
@@ -181,14 +182,24 @@ func (peer *Peer) handleBGPmessage(e *FsmMsg) ([]*table.Path, []*bgp.BGPMessage)
 			peer.adjRibIn.Update(e.PathList)
 			paths := make([]*table.Path, 0, len(e.PathList))
 			for _, path := range e.PathList {
+				if path.IsEOR() {
+					family := path.GetRouteFamily()
+					log.WithFields(log.Fields{
+						"Topic":         "Peer",
+						"Key":           peer.conf.Config.NeighborAddress,
+						"AddressFamily": family,
+					}).Debug("EOR received")
+					eor = append(eor, family)
+					continue
+				}
 				if path.Filtered(peer.ID()) != table.POLICY_DIRECTION_IN {
 					paths = append(paths, path)
 				}
 			}
-			return paths, nil
+			return paths, nil, eor
 		}
 	}
-	return nil, nil
+	return nil, nil, eor
 }
 
 func (peer *Peer) startFSMHandler(incoming, stateCh chan *FsmMsg) {
diff --git a/server/server.go b/server/server.go
index f98d4f4c..a833fa7f 100644
--- a/server/server.go
+++ b/server/server.go
@@ -874,7 +874,7 @@ func (server *BgpServer) propagateUpdate(peer *Peer, pathList []*table.Path) ([]
 		dsts := rib.ProcessPaths(append(pathList, moded...))
 		server.validatePaths(dsts, false)
 		for _, targetPeer := range server.neighborMap {
-			if !targetPeer.isRouteServerClient() || targetPeer.fsm.state != bgp.BGP_FSM_ESTABLISHED {
+			if !targetPeer.isRouteServerClient() || targetPeer.fsm.state != bgp.BGP_FSM_ESTABLISHED || targetPeer.fsm.pConf.GracefulRestart.State.LocalRestarting {
 				continue
 			}
 			sendPathList := make([]*table.Path, 0, len(dsts))
@@ -913,7 +913,13 @@ func (server *BgpServer) propagateUpdate(peer *Peer, pathList []*table.Path) ([]
 		}
 
 		for _, targetPeer := range server.neighborMap {
-			if targetPeer.isRouteServerClient() || targetPeer.fsm.state != bgp.BGP_FSM_ESTABLISHED {
+			if targetPeer.isRouteServerClient() || targetPeer.fsm.state != bgp.BGP_FSM_ESTABLISHED || targetPeer.fsm.pConf.GracefulRestart.State.LocalRestarting {
+				if targetPeer.fsm.pConf.GracefulRestart.State.LocalRestarting {
+					log.WithFields(log.Fields{
+						"Topic": "Peer",
+						"Key":   targetPeer.conf.Config.NeighborAddress,
+					}).Debug("now syncing, suppress sending updates")
+				}
 				continue
 			}
 			pathList := make([]*table.Path, len(sendPathList))
@@ -963,10 +969,29 @@ func (server *BgpServer) handleFSMMessage(peer *Peer, e *FsmMsg) []*SenderMsg {
 			// update for export policy
 			laddr, _ := peer.fsm.LocalHostPort()
 			peer.conf.Transport.Config.LocalAddress = laddr
-			pathList, _ := peer.getBestFromLocal(peer.configuredRFlist())
-			if len(pathList) > 0 {
-				peer.adjRibOut.Update(pathList)
-				msgs = append(msgs, newSenderMsg(peer, table.CreateUpdateMsgFromPaths(pathList)))
+			if !peer.fsm.pConf.GracefulRestart.State.LocalRestarting {
+				pathList, _ := peer.getBestFromLocal(peer.configuredRFlist())
+				if len(pathList) > 0 {
+					peer.adjRibOut.Update(pathList)
+					msgs = append(msgs, newSenderMsg(peer, table.CreateUpdateMsgFromPaths(pathList)))
+				}
+			} else {
+				// RFC 4724 4.1
+				// Once the session between the Restarting Speaker and the Receiving
+				// Speaker is re-established, the Restarting Speaker will receive and
+				// process BGP messages from its peers.  However, it MUST defer route
+				// selection for an address family until it either (a) ...snip...
+				// or (b) the Selection_Deferral_Timer referred to below has expired.
+				deferral := peer.fsm.pConf.GracefulRestart.Config.DeferralTime
+				log.WithFields(log.Fields{
+					"Topic": "Peer",
+					"Key":   peer.conf.Config.NeighborAddress,
+				}).Debugf("now syncing, suppress sending updates. start deferral timer(%d)", deferral)
+				time.AfterFunc(time.Second*time.Duration(deferral), func() {
+					req := NewGrpcRequest(REQ_DEFERRAL_TIMER_EXPIRED, peer.conf.Config.NeighborAddress, bgp.RouteFamily(0), nil)
+					server.GrpcReqCh <- req
+					<-req.ResponseCh
+				})
 			}
 		} else {
 			if server.shutdown && nextState == bgp.BGP_FSM_IDLE {
@@ -996,7 +1021,7 @@ func (server *BgpServer) handleFSMMessage(peer *Peer, e *FsmMsg) []*SenderMsg {
 		case *bgp.MessageError:
 			msgs = append(msgs, newSenderMsg(peer, []*bgp.BGPMessage{bgp.NewBGPNotificationMessage(m.TypeCode, m.SubTypeCode, m.Data)}))
 		case *bgp.BGPMessage:
-			pathList, msgList := peer.handleBGPmessage(e)
+			pathList, msgList, eor := peer.handleBGPmessage(e)
 
 			if m.Header.Type == bgp.BGP_MSG_UPDATE && server.watchers.watching(WATCHER_EVENT_UPDATE_MSG) {
 				_, y := peer.fsm.capMap[bgp.BGP_CAP_FOUR_OCTET_AS_NUMBER]
@@ -1045,6 +1070,58 @@ func (server *BgpServer) handleFSMMessage(peer *Peer, e *FsmMsg) []*SenderMsg {
 					}
 				}
 			}
+
+			if len(eor) > 0 {
+				// RFC 4724 4.1
+				// Once the session between the Restarting Speaker and the Receiving
+				// Speaker is re-established, ...snip... it MUST defer route
+				// selection for an address family until it either (a) receives the
+				// End-of-RIB marker from all its peers (excluding the ones with the
+				// "Restart State" bit set in the received capability and excluding the
+				// ones that do not advertise the graceful restart capability) or ...snip...
+				if peer.fsm.pConf.GracefulRestart.State.LocalRestarting {
+					var end bool
+					for _, f := range eor {
+						end = true
+						for i, a := range peer.fsm.pConf.AfiSafis {
+							if g, _ := bgp.GetRouteFamily(string(a.AfiSafiName)); f == g {
+								peer.fsm.pConf.AfiSafis[i].MpGracefulRestart.State.EndOfRibReceived = true
+							}
+							if s := a.MpGracefulRestart.State; s.Enabled && !s.EndOfRibReceived {
+								end = false
+							}
+						}
+					}
+					if end {
+						log.WithFields(log.Fields{
+							"Topic": "Peer",
+							"Key":   peer.conf.Config.NeighborAddress,
+						}).Debug("all family's EOR received")
+						peer.fsm.pConf.GracefulRestart.State.LocalRestarting = false
+					}
+					allEnd := true
+					for _, p := range server.neighborMap {
+						if p.fsm.pConf.GracefulRestart.State.LocalRestarting {
+							allEnd = false
+						}
+					}
+					if allEnd {
+						for _, p := range server.neighborMap {
+							if !p.isGracefulRestartEnabled() {
+								continue
+							}
+							pathList, _ := p.getBestFromLocal(p.configuredRFlist())
+							if len(pathList) > 0 {
+								p.adjRibOut.Update(pathList)
+								msgs = append(msgs, newSenderMsg(p, table.CreateUpdateMsgFromPaths(pathList)))
+							}
+						}
+						log.WithFields(log.Fields{
+							"Topic": "Server",
+						}).Info("sync finished")
+					}
+				}
+			}
 		default:
 			log.WithFields(log.Fields{
 				"Topic": "Peer",
@@ -1942,7 +2019,7 @@ func (server *BgpServer) handleGrpc(grpcReq *GrpcRequest) []*SenderMsg {
 			break
 		}
 		fallthrough
-	case REQ_NEIGHBOR_SOFT_RESET_OUT:
+	case REQ_NEIGHBOR_SOFT_RESET_OUT, REQ_DEFERRAL_TIMER_EXPIRED:
 		peers, err := reqToPeers(grpcReq)
 		if err != nil {
 			break
@@ -1954,6 +2031,19 @@ func (server *BgpServer) handleGrpc(grpcReq *GrpcRequest) []*SenderMsg {
 			if peer.fsm.state != bgp.BGP_FSM_ESTABLISHED {
 				continue
 			}
+
+			if grpcReq.RequestType == REQ_DEFERRAL_TIMER_EXPIRED {
+				if peer.fsm.pConf.GracefulRestart.State.LocalRestarting {
+					peer.fsm.pConf.GracefulRestart.State.LocalRestarting = false
+					log.WithFields(log.Fields{
+						"Topic": "Peer",
+						"Key":   peer.conf.Config.NeighborAddress,
+					}).Debug("deferral timer expired")
+				} else {
+					continue
+				}
+			}
+
 			families := []bgp.RouteFamily{grpcReq.RouteFamily}
 			if families[0] == bgp.RouteFamily(0) {
 				families = peer.configuredRFlist()
@@ -1965,7 +2055,7 @@ func (server *BgpServer) handleGrpc(grpcReq *GrpcRequest) []*SenderMsg {
 				peer.adjRibOut.Update(pathList)
 				msgs = append(msgs, newSenderMsg(peer, table.CreateUpdateMsgFromPaths(pathList)))
 			}
-			if len(filtered) > 0 {
+			if grpcReq.RequestType != REQ_DEFERRAL_TIMER_EXPIRED && len(filtered) > 0 {
 				withdrawnList := make([]*table.Path, 0, len(filtered))
 				for _, p := range filtered {
 					found := false
diff --git a/tools/pyang_plugins/gobgp.yang b/tools/pyang_plugins/gobgp.yang
index 1760ad28..092a2908 100644
--- a/tools/pyang_plugins/gobgp.yang
+++ b/tools/pyang_plugins/gobgp.yang
@@ -10,6 +10,7 @@ module gobgp {
   // import some basic types
   import openconfig-bgp { prefix bgp; }
   import openconfig-bgp-types { prefix bgp-types; }
+  import openconfig-bgp-multiprotocol { prefix bgp-mp; }
   import openconfig-routing-policy {prefix rpol; }
   import openconfig-policy-types {prefix ptypes; }
   import openconfig-bgp-policy {prefix bgp-pol; }
@@ -572,6 +573,30 @@ module gobgp {
     uses gobgp-neighbor-timer;
    }
 
+  augment "/bgp:bgp/bgp:neighbors/bgp:neighbor/bgp:afi-safis/bgp:afi-safi/bgp:graceful-restart/bgp:state" {
+    description "additional graceful-restart status";
+    leaf end-of-rib-received {
+      type boolean;
+    }
+    leaf end-of-rib-sent {
+      type boolean;
+    }
+  }
+
+  augment "/bgp:bgp/bgp:neighbors/bgp:neighbor/bgp:graceful-restart/bgp:config" {
+    description "additional graceful-restart status";
+    leaf deferral-time {
+      type uint16;
+    }
+  }
+
+  augment "/bgp:bgp/bgp:neighbors/bgp:neighbor/bgp:graceful-restart/bgp:state" {
+    description "additional graceful-restart status";
+    leaf deferral-time {
+      type uint16;
+    }
+  }
+
   augment "/bgp:bgp/bgp:peer-groups/bgp:peer-group" {
     description "route server configuration for peer-group";
     uses gobgp-route-server-config-set;
@@ -742,4 +767,5 @@ module gobgp {
         }
     }
   }
+
 }