scenario_test: add retry routine in each test

4 files changed, 156 insertions, 66 deletions

diff --git a/test/scenario_test/docker_control.py b/test/scenario_test/docker_control.py
index 2d68748f..1afab078 100644
--- a/test/scenario_test/docker_control.py
+++ b/test/scenario_test/docker_control.py
@@ -16,6 +16,8 @@
 from fabric.api import local
 import re
 import os
+import time
+
 
 GOBGP_CONTAINER_NAME = "gobgp"
 GOBGP_ADDRESS_0 = {"IPv4": "10.0.255.1",
@@ -148,7 +150,7 @@ def docker_containers_get():
 
 
 def docker_container_set_ipaddress(bridge, name, address):
-    cmd = "pipework " + bridge["BRIDGE_NAME"] + " -i eth-" + bridge["BRIDGE_NAME"]\
+    cmd = "pipework " + bridge["BRIDGE_NAME"] + " -i e" + bridge["BRIDGE_NAME"]\
           + " " + name + " " + address
     local(cmd, capture=True)
 
@@ -160,7 +162,7 @@ def docker_container_run_quagga(quagga_num, bridge):
     local(cmd, capture=True)
     quagga_address = BASE_NET[bridge["BRIDGE_NAME"]][IP_VERSION] + str(quagga_num) + BASE_MASK[IP_VERSION]
     docker_container_set_ipaddress(bridge, quagga_name, quagga_address)
-    # restart quagga supervisord deamon in docker container for reset the retry of opensent message quagga sends
+    # restart the quagga after the docker container has become IP reachable
     cmd = 'docker kill --signal="HUP" ' + quagga_name
     local(cmd, capture=True)
 
@@ -334,6 +336,8 @@ def make_config_append(quagga_num, go_path, bridge):
     local(cmd, capture=True)
 
 
+
+
 def reload_config():
     cmd = "docker exec gobgp /usr/bin/pkill gobgp -SIGHUP"
     local(cmd, capture=True)
@@ -471,6 +475,7 @@ def init_malformed_test_env_executor(conf_file, use_local):
 
     change_owner_to_root(CONFIG_DIR)
     start_gobgp()
+    # time.sleep(5)
     # run quagga docker container
     docker_container_run_quagga(1, BRIDGE_0)
     start_exabgp(conf_file)
diff --git a/test/scenario_test/route_server_ipv4_v6_test.py b/test/scenario_test/route_server_ipv4_v6_test.py
index 286bb915..bff5bad5 100644
--- a/test/scenario_test/route_server_ipv4_v6_test.py
+++ b/test/scenario_test/route_server_ipv4_v6_test.py
@@ -42,7 +42,9 @@ class GoBGPIPv6Test(unittest.TestCase):
     append_quagga = 10
     remove_quagga = 10
     append_quagga_best = 20
-    sleep_time = 20
+    initial_wait_time = 10
+    wait_per_retry = 5
+    retry_limit = (60 - initial_wait_time) / wait_per_retry
 
     def __init__(self, *args, **kwargs):
         super(GoBGPIPv6Test, self).__init__(*args, **kwargs)
@@ -57,15 +59,16 @@ class GoBGPIPv6Test(unittest.TestCase):
         use_local = parser_option.use_local
         go_path = parser_option.go_path
         fab.init_ipv6_test_env_executor(self.quagga_num, use_local, go_path)
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait (" + str(self.initial_wait_time) + " second)"
+        time.sleep(self.initial_wait_time)
         fab.docker_container_ipv6_quagga_append_executor([3, 4], go_path)
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait (" + str(self.initial_wait_time) + " second)"
+        time.sleep(self.initial_wait_time)
         if self.check_load_config() is False:
             return
 
         addresses = self.get_neighbor_address(self.gobgp_config)
+        self.retry_routine_for_stete(addresses)
 
         for address in addresses:
             # get neighbor state and remote ip from gobgp connections
@@ -77,6 +80,7 @@ class GoBGPIPv6Test(unittest.TestCase):
             remote_ip = neighbor['conf']['remote_ip']
             self.assertEqual(address[0], remote_ip)
             self.assertEqual(state, "BGP_FSM_ESTABLISHED")
+            print "state" + state
 
     def test_02_ipv4_ipv6_received_route(self):
         print "test_ipv4_ipv6_received_route"
@@ -152,6 +156,36 @@ class GoBGPIPv6Test(unittest.TestCase):
                                     exist_n += 1
                             self.assertEqual(exist_n, 1)
 
+    def retry_routine_for_stete(self, addresses):
+        inprepar_quagga = True
+        retry_count = 0
+        while inprepar_quagga:
+            if retry_count != 0:
+                print "please wait more (" + str(self.wait_per_retry) + " second)"
+                time.sleep(self.wait_per_retry)
+            if retry_count >= self.retry_limit:
+                print "retry limit"
+                break
+            retry_count += 1
+            success_count = 0
+            for address in addresses:
+                # get neighbor state and remote ip from gobgp connections
+                url = "http://" + self.gobgp_ip + ":" + self.gobgp_port + "/v1/bgp/neighbor/" + address[0]
+                try:
+                    r = requests.get(url)
+                    neighbor = json.loads(r.text)
+                except Exception:
+                    continue
+                if neighbor is None:
+                    continue
+                state = neighbor['info']['bgp_state']
+                remote_ip = neighbor['conf']['remote_ip']
+                if address[0] == remote_ip and state == "BGP_FSM_ESTABLISHED":
+                    success_count += 1
+            if success_count == len(addresses):
+                inprepar_quagga = False
+        time.sleep(self.wait_per_retry)
+
     def load_gobgp_config(self):
         try:
             self.gobgp_config = toml.loads(open(self.gobgp_config_file).read())
diff --git a/test/scenario_test/route_server_malformed_test.py b/test/scenario_test/route_server_malformed_test.py
index 077ad847..dba94c8d 100644
--- a/test/scenario_test/route_server_malformed_test.py
+++ b/test/scenario_test/route_server_malformed_test.py
@@ -27,7 +27,10 @@ from noseplugin import OptionParser
 from noseplugin import parser_option
 
 
-sleep_time = 20
+initial_wait_time = 10
+wait_per_retry = 5
+retry_limit = (60 - initial_wait_time) / wait_per_retry
+
 gobgp_ip = "10.0.255.1"
 gobgp_port = "8080"
 gobgp_config_file = "/tmp/gobgp/gobgpd.conf"
@@ -73,8 +76,8 @@ def test_malformed_packet():
         conf_file = pwd + "/exabgp_test_conf/" + pkey
         if os.path.isfile(conf_file) is True:
             fab.init_malformed_test_env_executor(pkey, use_local)
-            print "please wait " + str(sleep_time) + " second"
-            time.sleep(sleep_time)
+            print "please wait (" + str(initial_wait_time) + " second)"
+            time.sleep(initial_wait_time)
             yield check_func, pkey, pattern[pkey]
 
         else:
@@ -84,40 +87,62 @@ def test_malformed_packet():
 
 
 def check_func(exabgp_conf, result):
+    inprepar_quagga = True
+    inprepar_exabgp = True
+    retry_count = 0
     # get neighbor addresses from gobgpd.conf
     addresses = get_neighbor_address()
-    # check whether the service of gobgp is normally
     url = "http://" + gobgp_ip + ":" + gobgp_port + "/v1/bgp/neighbors"
-    r = requests.get(url)
-    neighbors = json.loads(r.text)
-
-    assert len(neighbors) == len(addresses)
-
-    for neighbor in neighbors:
-        state = neighbor['info']['bgp_state']
-        remote_ip = neighbor['conf']['remote_ip']
-        e_transitions = neighbor['info']['fsm_established_transitions']
-        if remote_ip == "10.0.0.1":
-            print "check of [ " + remote_ip + " ]"
-            assert state == "BGP_FSM_ESTABLISHED"
-            assert e_transitions == 1
-        else:
-            print "check of [ " + remote_ip + " ]"
-            assert remote_ip == "10.0.0.100"
-
-    # get notification message from exabgp log
-    err_msg = fab.get_notification_from_exabgp_log()
-    notification = None
-    parse_msg = re.search(r'error.*', err_msg)
-    if parse_msg is not None:
-        notification_src = parse_msg.group(0)[5:]
-        notification = notification_src[1:-1]
-
+    neighbors = None
+    q_address = ""
+    e_address = ""
+    q_transitions = 0
+    q_state = ""
+    notification = ""
+    while inprepar_quagga or inprepar_exabgp:
+        if retry_count != 0:
+            print "please wait more (" + str(wait_per_retry) + " second)"
+            time.sleep(wait_per_retry)
+        if retry_count >= retry_limit:
+            print "retry limit"
+            break
+        retry_count += 1
+        # check whether the service of gobgp is normally
+        try:
+            r = requests.get(url)
+            neighbors = json.loads(r.text)
+        except Exception:
+            continue
+        if neighbors is None:
+            continue
+        for neighbor in neighbors:
+            remote_ip = neighbor['conf']['remote_ip']
+            if remote_ip == "10.0.0.1":
+                q_state = neighbor['info']['bgp_state']
+                q_transitions = neighbor['info']['fsm_established_transitions']
+                q_address = remote_ip
+                if q_state == "BGP_FSM_ESTABLISHED":
+                    inprepar_quagga = False
+            else:
+                e_address = remote_ip
+        # get notification message from exabgp log
+        err_msg = fab.get_notification_from_exabgp_log()
+        parse_msg = re.search(r'error.*', err_msg)
+        if parse_msg is not None:
+            notification_src = parse_msg.group(0)[5:]
+            notification = notification_src[1:-1]
+            inprepar_exabgp = False
+
+    assert neighbors is not None, "neighbors is None"
+    assert len(neighbors) == len(addresses), "neighbors = " + len(neighbors) + ", addresses = " + len(addresses)
+    print "check of [ " + q_address + " ]"
+    assert q_state == "BGP_FSM_ESTABLISHED", "q_state = " + q_state
+    assert q_transitions == 1, "q_transitions = " + q_transitions
+    print "check of [ " + e_address + " ]"
     print "notification message : "
     print " >>> " + str(notification)
-
     # check notification messege
-    assert notification == result
+    assert notification == result, "notification = " + notification
 
 
 # get address of each neighbor from gobpg configration
diff --git a/test/scenario_test/route_server_test.py b/test/scenario_test/route_server_test.py
index de4b60fc..b093063e 100644
--- a/test/scenario_test/route_server_test.py
+++ b/test/scenario_test/route_server_test.py
@@ -42,7 +42,9 @@ class GoBGPTest(unittest.TestCase):
     append_quagga = 10
     remove_quagga = 10
     append_quagga_best = 20
-    sleep_time = 20
+    initial_wait_time = 10
+    wait_per_retry = 5
+    retry_limit = (60 - initial_wait_time) / wait_per_retry
 
     def __init__(self, *args, **kwargs):
         super(GoBGPTest, self).__init__(*args, **kwargs)
@@ -58,12 +60,13 @@ class GoBGPTest(unittest.TestCase):
         go_path = parser_option.go_path
         fab.init_test_env_executor(self.quagga_num, use_local, go_path)
 
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait " + str(self.initial_wait_time) + " second"
+        time.sleep(self.initial_wait_time)
         if self.check_load_config() is False:
             return
 
         addresses = self.get_neighbor_address(self.gobgp_config)
+        self.retry_routine_for_state(addresses, "BGP_FSM_ESTABLISHED")
 
         for address in addresses:
             # get neighbor state and remote ip from gobgp connections
@@ -92,21 +95,17 @@ class GoBGPTest(unittest.TestCase):
             for quagga_config in self.quagga_configs:
                 if quagga_config.peer_ip == address:
                     for c_dest in quagga_config.destinations.itervalues():
-                        # print "config : ", c_dest.prefix, "my ip !!!"
                         g_dests = local_rib['Destinations']
                         exist_n = 0
                         for g_dest in g_dests:
-                            # print "gobgp : ", g_dest['Prefix']
                             if c_dest.prefix == g_dest['Prefix']:
                                 exist_n += 1
                         self.assertEqual(exist_n, 0)
                 else:
                     for c_dest in quagga_config.destinations.itervalues():
-                        # print "config : ", c_dest.prefix"
                         g_dests = local_rib['Destinations']
                         exist_n = 0
                         for g_dest in g_dests:
-                            # print "gobgp : ", g_dest['Prefix']
                             if c_dest.prefix == g_dest['Prefix']:
                                 exist_n += 1
                         self.assertEqual(exist_n, 1)
@@ -126,9 +125,7 @@ class GoBGPTest(unittest.TestCase):
                     for c_dest in quagga_config.destinations.itervalues():
                         exist_n = 0
                         for c_path in c_dest.paths:
-                            # print "conf : ", c_path.network, c_path.nexthop, "my ip !!!"
                             for q_path in q_rib:
-                                # print "quag : ", q_path['Network'], q_path['Next Hop']
                                 if c_path.network.split("/")[0] == q_path['Network'] and "0.0.0.0" == q_path['Next Hop']:
                                     exist_n += 1
                             self.assertEqual(exist_n, 1)
@@ -136,9 +133,7 @@ class GoBGPTest(unittest.TestCase):
                     for c_dest in quagga_config.destinations.itervalues():
                         exist_n = 0
                         for c_path in c_dest.paths:
-                            # print "conf : ", c_path.network, c_path.nexthop
                             for q_path in q_rib:
-                                # print "quag : ", q_path['Network'], q_path['Next Hop']
                                 if c_path.network.split("/")[0] == q_path['Network'] and c_path.nexthop == q_path['Next Hop']:
                                     exist_n += 1
                             self.assertEqual(exist_n, 1)
@@ -152,9 +147,10 @@ class GoBGPTest(unittest.TestCase):
         go_path = parser_option.go_path
         # append new quagga container
         fab.docker_container_quagga_append_executor(self.append_quagga, go_path)
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait " + str(self.initial_wait_time) + " second"
+        time.sleep(self.initial_wait_time)
         append_quagga_address = "10.0.0." + str(self.append_quagga)
+        self.retry_routine_for_state([append_quagga_address], "BGP_FSM_ESTABLISHED")
 
         # get neighbor state and remote ip of new quagga
         print "check of [" + append_quagga_address + " ]"
@@ -240,9 +236,10 @@ class GoBGPTest(unittest.TestCase):
 
         # remove quagga container
         fab.docker_container_quagga_removed_executor(self.remove_quagga)
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait " + str(self.initial_wait_time) + " second"
+        time.sleep(self.initial_wait_time)
         removed_quagga_address = "10.0.0." + str(self.remove_quagga)
+        self.retry_routine_for_state([removed_quagga_address], "BGP_FSM_ACTIVE")
 
         # get neighbor state and remote ip of removed quagga
         print "check of [" + removed_quagga_address + " ]"
@@ -334,8 +331,8 @@ class GoBGPTest(unittest.TestCase):
 
         go_path = parser_option.go_path
         fab.docker_container_make_bestpath_env_executor(self.append_quagga_best, go_path)
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait " + str(self.initial_wait_time) + " second"
+        time.sleep(self.initial_wait_time)
 
         print "add neighbor setting"
         tn = qaccess.login("11.0.0.20")
@@ -352,18 +349,48 @@ class GoBGPTest(unittest.TestCase):
         qaccess.add_neighbor(tn, "65003", "12.0.0.20", "65020")
         qaccess.add_neighbor_metric(tn, "65003", "10.0.255.1", "100")
 
-        print "please wait " + str(self.sleep_time) + " second"
-        time.sleep(self.sleep_time)
+        print "please wait " + str(self.initial_wait_time) + " second"
+        time.sleep(self.initial_wait_time)
 
         check_address = "10.0.0.1"
         target_network = "192.168.20.0"
         ans_nexthop = "10.0.0.3"
 
         print "check of [ " + check_address + " ]"
-        self.check_bestpath(check_address, target_network, ans_nexthop)
+        self.retry_routine_for_bestpath(check_address, target_network, ans_nexthop)
+
+    def retry_routine_for_state(self, addresses, allow_state):
+        inprepar_quagga = True
+        retry_count = 0
+        while inprepar_quagga:
+            if retry_count != 0:
+                print "please wait more (" + str(self.wait_per_retry) + " second)"
+                time.sleep(self.wait_per_retry)
+            if retry_count >= self.retry_limit:
+                print "retry limit"
+                break
+            retry_count += 1
+            success_count = 0
+            for address in addresses:
+                # get neighbor state and remote ip from gobgp connections
+                url = "http://" + self.gobgp_ip + ":" + self.gobgp_port + "/v1/bgp/neighbor/" + address
+                try:
+                    r = requests.get(url)
+                    neighbor = json.loads(r.text)
+                except Exception:
+                    continue
+                if neighbor is None:
+                    continue
+                state = neighbor['info']['bgp_state']
+                remote_ip = neighbor['conf']['remote_ip']
+                if address == remote_ip and state == allow_state:
+                    success_count += 1
+            if success_count == len(addresses):
+                inprepar_quagga = False
+        time.sleep(self.wait_per_retry)
 
     # load configration from gobgp(gobgpd.conf)
-    def check_bestpath(self, check_address, target_network, ans_nexthop):
+    def retry_routine_for_bestpath(self, check_address, target_network, ans_nexthop):
         # get local-rib
         rep_nexthop = ""
         target_exist = False
@@ -372,7 +399,6 @@ class GoBGPTest(unittest.TestCase):
         local_rib = json.loads(r.text)
         g_dests = local_rib['Destinations']
         for g_dest in g_dests:
-            # print "prefix : ", g_dest['Prefix']
             best_path_idx = g_dest['BestPathIdx']
             if target_network == g_dest['Prefix']:
                 target_exist = True
@@ -380,9 +406,9 @@ class GoBGPTest(unittest.TestCase):
                 idx = 0
                 if len(g_paths) < 2:
                     print "target path has not been bestpath selected yet."
-                    print "please wait " + str(self.sleep_time/2) + " second more."
-                    time.sleep(self.sleep_time/2)
-                    self.check_bestpath(check_address, target_network, ans_nexthop)
+                    print "please wait more (" + str(self.wait_per_retry) + " second)"
+                    time.sleep(self.wait_per_retry)
+                    self.retry_routine_for_bestpath(check_address, target_network, ans_nexthop)
                     return
                 for g_path in g_paths:
                     print "best_path_Idx: " + str(best_path_idx) + "idx: " + str(idx)
@@ -392,9 +418,9 @@ class GoBGPTest(unittest.TestCase):
                     idx += 1
         if target_exist is False:
             print "target path has not been receive yet."
-            print "please wait " + str(self.sleep_time/2) + " second more."
-            time.sleep(self.sleep_time/2)
-            self.check_bestpath(check_address, target_network, ans_nexthop)
+            print "please wait more (" + str(self.wait_per_retry) + " second)"
+            time.sleep(self.wait_per_retry)
+            self.retry_routine_for_bestpath(check_address, target_network, ans_nexthop)
             return
         self.assertEqual(ans_nexthop, rep_nexthop)