test/lib/{bird,gobgp,quagga,yabgp}: Wait for daemon boot

Currently, scenario test library does not wait for the daemons boot in
containers at starting up or after sending SIGHUP for reloading new
configurations.
So some test cases (e.g., graceful_restart_test.py) fails occasionally
to insert static routes into GoBGP after reloading new configurations.

This patch fixes to wait for the daemons boot and improves the stability
of scenario tests.

Note: This patch does not introduce these improvements to
BagpipeContainer and ExaBGPContainer, because;
- Docker image for BaGPipe, which used in scenario test library, is too
  old and BagpipeContainer does not seem to be used in the current test
  cases.
- The version of ExaBGP in "osrg/exabgp" Docker image has no way to ask
  the daemon status other than "ps" command ("ps" is already done, but
  not enough). If "exabgpcli" is available on ExaBGP container, which is
  required the next release of 4.0.2 or 3.4.21, we can use it to check
  whether ExaBGP is started up or not.

Signed-off-by: IWASE Yusuke <iwase.yusuke0@gmail.com>

4 files changed, 77 insertions, 18 deletions

diff --git a/test/lib/bird.py b/test/lib/bird.py
index 8077760f..1207e335 100644
--- a/test/lib/bird.py
+++ b/test/lib/bird.py
@@ -25,6 +25,7 @@ from lib.base import (
     BGPContainer,
     CmdBuffer,
     try_several_times,
+    wait_for_completion,
 )
 
 
@@ -48,6 +49,13 @@ class BirdContainer(BGPContainer):
         local(cmd)
         self.local('{0}/start.sh'.format(self.SHARED_VOLUME))
 
+    def _wait_for_boot(self):
+        def _f():
+            ret = self.local('birdc show status > /dev/null 2>&1; echo $?', capture=True)
+            return ret == '0'
+
+        return wait_for_completion(_f)
+
     def run(self):
         super(BirdContainer, self).run()
         self.reload_config()
@@ -81,13 +89,16 @@ class BirdContainer(BGPContainer):
                     if 'bird' in line:
                         running = True
                 return running
+
             if _is_running():
                 self.local('birdc configure')
             else:
                 self._start_bird()
-            time.sleep(1)
+
+            self._wait_for_boot()
             if not _is_running():
                 raise RuntimeError()
+
         try_several_times(_reload)
 
 
diff --git a/test/lib/gobgp.py b/test/lib/gobgp.py
index f0bff6d7..ef2d2e96 100644
--- a/test/lib/gobgp.py
+++ b/test/lib/gobgp.py
@@ -30,6 +30,7 @@ import yaml
 import collections
 
 from lib.base import (
+    wait_for_completion,
     BGPContainer,
     CmdBuffer,
     BGP_ATTR_TYPE_AS_PATH,
@@ -131,6 +132,28 @@ class GoBGPContainer(BGPContainer):
         cmd = '{0} -f {1}/ospfd.conf'.format(daemon_bin, self.QUAGGA_VOLUME)
         self.local(cmd, detach=True)
 
+    def _get_enabled_quagga_daemons(self):
+        daemons = []
+        if self.zebra:
+            daemons.append('zebra')
+            if self.ospfd_config:
+                daemons.append('ospfd')
+        return daemons
+
+    def _wait_for_boot(self):
+        def _f_gobgp():
+            ret = self.local('gobgp global > /dev/null 2>&1; echo $?', capture=True)
+            return ret == '0'
+
+        for daemon in self._get_enabled_quagga_daemons():
+            def _f_quagga():
+                ret = self.local("vtysh -d {0} -c 'show run' > /dev/null 2>&1; echo $?".format(daemon), capture=True)
+                return ret == '0'
+
+            wait_for_completion(_f_quagga)
+
+        wait_for_completion(_f_gobgp)
+
     def run(self):
         super(GoBGPContainer, self).run()
         if self.zebra:
@@ -138,6 +161,7 @@ class GoBGPContainer(BGPContainer):
             if self.ospfd_config:
                 self._start_ospfd()
         self._start_gobgp()
+        self._wait_for_boot()
         return self.WAIT_FOR_BOOT
 
     @staticmethod
@@ -529,14 +553,10 @@ class GoBGPContainer(BGPContainer):
             f.writelines(str(c))
 
     def reload_config(self):
-        daemon = ['gobgpd']
-        if self.zebra:
-            daemon.append('zebra')
-            if self.ospfd_config:
-                daemon.append('ospfd')
-        for d in daemon:
-            cmd = '/usr/bin/pkill {0} -SIGHUP'.format(d)
-            self.local(cmd)
+        for daemon in self._get_enabled_quagga_daemons():
+            self.local('pkill {0} -SIGHUP'.format(daemon), capture=True)
+        self.local('pkill gobgpd -SIGHUP', capture=True)
+        self._wait_for_boot()
         for v in chain.from_iterable(self.routes.itervalues()):
             if v['rf'] == 'ipv4' or v['rf'] == 'ipv6':
                 r = CmdBuffer(' ')
diff --git a/test/lib/quagga.py b/test/lib/quagga.py
index 6d65f061..e657b439 100644
--- a/test/lib/quagga.py
+++ b/test/lib/quagga.py
@@ -22,6 +22,7 @@ from itertools import chain
 import netaddr
 
 from lib.base import (
+    wait_for_completion,
     BGPContainer,
     OSPFContainer,
     CmdBuffer,
@@ -56,8 +57,23 @@ class QuaggaBGPContainer(BGPContainer):
         # }
         self.bgpd_config = bgpd_config or {}
 
+    def _get_enabled_daemons(self):
+        daemons = ['bgpd']
+        if self.zebra:
+            daemons.append('zebra')
+        return daemons
+
+    def _wait_for_boot(self):
+        for daemon in self._get_enabled_daemons():
+            def _f():
+                ret = self.local("vtysh -d {0} -c 'show run' > /dev/null 2>&1; echo $?".format(daemon), capture=True)
+                return ret == '0'
+
+            wait_for_completion(_f)
+
     def run(self):
         super(QuaggaBGPContainer, self).run()
+        self._wait_for_boot()
         return self.WAIT_FOR_BOOT
 
     def get_global_rib(self, prefix='', rf='ipv4'):
@@ -261,12 +277,9 @@ class QuaggaBGPContainer(BGPContainer):
             return self.local("vtysh -d bgpd {0}".format(cmd), capture=True)
 
     def reload_config(self):
-        daemon = ['bgpd']
-        if self.zebra:
-            daemon.append('zebra')
-        for d in daemon:
-            cmd = '/usr/bin/pkill {0} -SIGHUP'.format(d)
-            self.local(cmd, capture=True)
+        for daemon in self._get_enabled_daemons():
+            self.local('pkill {0} -SIGHUP'.format(daemon), capture=True)
+        self._wait_for_boot()
 
 
 class RawQuaggaBGPContainer(QuaggaBGPContainer):
diff --git a/test/lib/yabgp.py b/test/lib/yabgp.py
index c3e5e0ac..fb13620b 100644
--- a/test/lib/yabgp.py
+++ b/test/lib/yabgp.py
@@ -18,7 +18,6 @@ from __future__ import print_function
 
 import json
 import os
-import time
 
 from fabric import colors
 from fabric.api import local
@@ -29,6 +28,7 @@ from lib.base import (
     BGPContainer,
     CmdBuffer,
     try_several_times,
+    wait_for_completion,
 )
 
 
@@ -55,11 +55,17 @@ class YABGPContainer(BGPContainer):
             ' --config-file {0}/yabgp.ini'.format(self.SHARED_VOLUME),
             detach=True)
 
+    def _wait_for_boot(self):
+        return wait_for_completion(self._curl_is_running)
+
     def run(self):
         super(YABGPContainer, self).run()
         # self.create_config() is called in super class
         self._copy_helper_app()
-        self._start_yabgp()
+        # To start YABGP, it is required to configure neighbor settings, so
+        # here does not start YABGP yet.
+        # self._start_yabgp()
+        # self._wait_for_boot()
         return self.WAIT_FOR_BOOT
 
     def create_config(self):
@@ -108,12 +114,21 @@ class YABGPContainer(BGPContainer):
                 self.local('/usr/bin/pkill -9 python')
 
             self._start_yabgp()
-            time.sleep(self.WAIT_FOR_BOOT)
+            self._wait_for_boot()
             if not _is_running():
                 raise RuntimeError()
 
         try_several_times(_reload)
 
+    def _curl_is_running(self):
+        c = CmdBuffer(' ')
+        c << "curl -X GET"
+        c << "-u admin:admin"
+        c << "-H 'Content-Type: application/json'"
+        c << "http://localhost:8801/v1/"
+        c << "> /dev/null 2>&1; echo $?"
+        return self.local(str(c), capture=True) == '0'
+
     def _curl_send_update(self, path, peer):
         c = CmdBuffer(' ')
         c << "curl -X POST"