From be7e6206bc3706214a0aab762ca1af4a782ad27c Mon Sep 17 00:00:00 2001 From: IWASE Yusuke Date: Tue, 16 Jan 2018 12:57:55 +0900 Subject: test/lib/{bird,gobgp,quagga,yabgp}: Wait for daemon boot Currently, scenario test library does not wait for the daemons boot in containers at starting up or after sending SIGHUP for reloading new configurations. So some test cases (e.g., graceful_restart_test.py) fails occasionally to insert static routes into GoBGP after reloading new configurations. This patch fixes to wait for the daemons boot and improves the stability of scenario tests. Note: This patch does not introduce these improvements to BagpipeContainer and ExaBGPContainer, because; - Docker image for BaGPipe, which used in scenario test library, is too old and BagpipeContainer does not seem to be used in the current test cases. - The version of ExaBGP in "osrg/exabgp" Docker image has no way to ask the daemon status other than "ps" command ("ps" is already done, but not enough). If "exabgpcli" is available on ExaBGP container, which is required the next release of 4.0.2 or 3.4.21, we can use it to check whether ExaBGP is started up or not. Signed-off-by: IWASE Yusuke --- test/lib/bird.py | 13 ++++++++++++- test/lib/gobgp.py | 36 ++++++++++++++++++++++++++++-------- test/lib/quagga.py | 25 +++++++++++++++++++------ test/lib/yabgp.py | 21 ++++++++++++++++++--- 4 files changed, 77 insertions(+), 18 deletions(-) diff --git a/test/lib/bird.py b/test/lib/bird.py index 8077760f..1207e335 100644 --- a/test/lib/bird.py +++ b/test/lib/bird.py @@ -25,6 +25,7 @@ from lib.base import ( BGPContainer, CmdBuffer, try_several_times, + wait_for_completion, ) @@ -48,6 +49,13 @@ class BirdContainer(BGPContainer): local(cmd) self.local('{0}/start.sh'.format(self.SHARED_VOLUME)) + def _wait_for_boot(self): + def _f(): + ret = self.local('birdc show status > /dev/null 2>&1; echo $?', capture=True) + return ret == '0' + + return wait_for_completion(_f) + def run(self): super(BirdContainer, self).run() self.reload_config() @@ -81,13 +89,16 @@ class BirdContainer(BGPContainer): if 'bird' in line: running = True return running + if _is_running(): self.local('birdc configure') else: self._start_bird() - time.sleep(1) + + self._wait_for_boot() if not _is_running(): raise RuntimeError() + try_several_times(_reload) diff --git a/test/lib/gobgp.py b/test/lib/gobgp.py index f0bff6d7..ef2d2e96 100644 --- a/test/lib/gobgp.py +++ b/test/lib/gobgp.py @@ -30,6 +30,7 @@ import yaml import collections from lib.base import ( + wait_for_completion, BGPContainer, CmdBuffer, BGP_ATTR_TYPE_AS_PATH, @@ -131,6 +132,28 @@ class GoBGPContainer(BGPContainer): cmd = '{0} -f {1}/ospfd.conf'.format(daemon_bin, self.QUAGGA_VOLUME) self.local(cmd, detach=True) + def _get_enabled_quagga_daemons(self): + daemons = [] + if self.zebra: + daemons.append('zebra') + if self.ospfd_config: + daemons.append('ospfd') + return daemons + + def _wait_for_boot(self): + def _f_gobgp(): + ret = self.local('gobgp global > /dev/null 2>&1; echo $?', capture=True) + return ret == '0' + + for daemon in self._get_enabled_quagga_daemons(): + def _f_quagga(): + ret = self.local("vtysh -d {0} -c 'show run' > /dev/null 2>&1; echo $?".format(daemon), capture=True) + return ret == '0' + + wait_for_completion(_f_quagga) + + wait_for_completion(_f_gobgp) + def run(self): super(GoBGPContainer, self).run() if self.zebra: @@ -138,6 +161,7 @@ class GoBGPContainer(BGPContainer): if self.ospfd_config: self._start_ospfd() self._start_gobgp() + self._wait_for_boot() return self.WAIT_FOR_BOOT @staticmethod @@ -529,14 +553,10 @@ class GoBGPContainer(BGPContainer): f.writelines(str(c)) def reload_config(self): - daemon = ['gobgpd'] - if self.zebra: - daemon.append('zebra') - if self.ospfd_config: - daemon.append('ospfd') - for d in daemon: - cmd = '/usr/bin/pkill {0} -SIGHUP'.format(d) - self.local(cmd) + for daemon in self._get_enabled_quagga_daemons(): + self.local('pkill {0} -SIGHUP'.format(daemon), capture=True) + self.local('pkill gobgpd -SIGHUP', capture=True) + self._wait_for_boot() for v in chain.from_iterable(self.routes.itervalues()): if v['rf'] == 'ipv4' or v['rf'] == 'ipv6': r = CmdBuffer(' ') diff --git a/test/lib/quagga.py b/test/lib/quagga.py index 6d65f061..e657b439 100644 --- a/test/lib/quagga.py +++ b/test/lib/quagga.py @@ -22,6 +22,7 @@ from itertools import chain import netaddr from lib.base import ( + wait_for_completion, BGPContainer, OSPFContainer, CmdBuffer, @@ -56,8 +57,23 @@ class QuaggaBGPContainer(BGPContainer): # } self.bgpd_config = bgpd_config or {} + def _get_enabled_daemons(self): + daemons = ['bgpd'] + if self.zebra: + daemons.append('zebra') + return daemons + + def _wait_for_boot(self): + for daemon in self._get_enabled_daemons(): + def _f(): + ret = self.local("vtysh -d {0} -c 'show run' > /dev/null 2>&1; echo $?".format(daemon), capture=True) + return ret == '0' + + wait_for_completion(_f) + def run(self): super(QuaggaBGPContainer, self).run() + self._wait_for_boot() return self.WAIT_FOR_BOOT def get_global_rib(self, prefix='', rf='ipv4'): @@ -261,12 +277,9 @@ class QuaggaBGPContainer(BGPContainer): return self.local("vtysh -d bgpd {0}".format(cmd), capture=True) def reload_config(self): - daemon = ['bgpd'] - if self.zebra: - daemon.append('zebra') - for d in daemon: - cmd = '/usr/bin/pkill {0} -SIGHUP'.format(d) - self.local(cmd, capture=True) + for daemon in self._get_enabled_daemons(): + self.local('pkill {0} -SIGHUP'.format(daemon), capture=True) + self._wait_for_boot() class RawQuaggaBGPContainer(QuaggaBGPContainer): diff --git a/test/lib/yabgp.py b/test/lib/yabgp.py index c3e5e0ac..fb13620b 100644 --- a/test/lib/yabgp.py +++ b/test/lib/yabgp.py @@ -18,7 +18,6 @@ from __future__ import print_function import json import os -import time from fabric import colors from fabric.api import local @@ -29,6 +28,7 @@ from lib.base import ( BGPContainer, CmdBuffer, try_several_times, + wait_for_completion, ) @@ -55,11 +55,17 @@ class YABGPContainer(BGPContainer): ' --config-file {0}/yabgp.ini'.format(self.SHARED_VOLUME), detach=True) + def _wait_for_boot(self): + return wait_for_completion(self._curl_is_running) + def run(self): super(YABGPContainer, self).run() # self.create_config() is called in super class self._copy_helper_app() - self._start_yabgp() + # To start YABGP, it is required to configure neighbor settings, so + # here does not start YABGP yet. + # self._start_yabgp() + # self._wait_for_boot() return self.WAIT_FOR_BOOT def create_config(self): @@ -108,12 +114,21 @@ class YABGPContainer(BGPContainer): self.local('/usr/bin/pkill -9 python') self._start_yabgp() - time.sleep(self.WAIT_FOR_BOOT) + self._wait_for_boot() if not _is_running(): raise RuntimeError() try_several_times(_reload) + def _curl_is_running(self): + c = CmdBuffer(' ') + c << "curl -X GET" + c << "-u admin:admin" + c << "-H 'Content-Type: application/json'" + c << "http://localhost:8801/v1/" + c << "> /dev/null 2>&1; echo $?" + return self.local(str(c), capture=True) == '0' + def _curl_send_update(self, path, peer): c = CmdBuffer(' ') c << "curl -X POST" -- cgit v1.2.3