test: increse the number of concurrent test execution

Signed-off-by: ISHIDA Wataru <ishida.wataru@lab.ntt.co.jp>

4 files changed, 97 insertions, 94 deletions

diff --git a/test/scenario_test/ci-scripts/jenkins-build-script.sh b/test/scenario_test/ci-scripts/jenkins-build-script.sh
index 953c3d09..50e694b0 100644
--- a/test/scenario_test/ci-scripts/jenkins-build-script.sh
+++ b/test/scenario_test/ci-scripts/jenkins-build-script.sh
@@ -18,6 +18,12 @@ git log | head -20
 sudo docker rmi $(sudo docker images | grep "^<none>" | awk '{print $3}')
 sudo docker rm -f $(sudo docker ps -a -q)
 
+for link in $(ip li | awk '/(_br|veth)/{sub(":","", $2); print $2}')
+do
+    sudo ip li set down $link
+    sudo ip li del $link
+done
+
 sudo fab -f $GOBGP/test/scenario_test/lib/base.py make_gobgp_ctn --set tag=$GOBGP_IMAGE
 
 cd $GOBGP/gobgpd
diff --git a/test/scenario_test/lib/base.py b/test/scenario_test/lib/base.py
index 7922bbdf..1bba6f5b 100644
--- a/test/scenario_test/lib/base.py
+++ b/test/scenario_test/lib/base.py
@@ -16,6 +16,7 @@
 from fabric.api import local, lcd
 from fabric import colors
 from fabric.utils import indent
+from fabric.state import env
 
 import netaddr
 import os
@@ -40,15 +41,26 @@ BGP_ATTR_TYPE_COMMUNITIES = 8
 BGP_ATTR_TYPE_MP_REACH_NLRI = 14
 BGP_ATTR_TYPE_EXTENDED_COMMUNITIES = 16
 
+env.abort_exception = RuntimeError
+
+def try_several_times(f, t=3, s=1):
+    e = None
+    for i in range(t):
+        try:
+            r = f()
+        except RuntimeError as e:
+            time.sleep(s)
+        else:
+            return r
+    raise e
+
 
 def get_bridges():
-    return local("brctl show | awk 'NR > 1{print $1}'",
-                 capture=True).split('\n')
+    return try_several_times(lambda : local("brctl show | awk 'NR > 1{print $1}'", capture=True)).split('\n')
 
 
 def get_containers():
-    return local("docker ps -a | awk 'NR > 1 {print $NF}'",
-                 capture=True).split('\n')
+    return try_several_times(lambda : local("docker ps -a | awk 'NR > 1 {print $NF}'", capture=True)).split('\n')
 
 
 class CmdBuffer(list):
@@ -100,18 +112,17 @@ class Bridge(object):
             # throw away first network address
             self.next_ip_address()
 
-        if self.name in get_bridges():
-            self.delete()
-
-        local("ip link add {0} type bridge".format(self.name), capture=True)
-        local("ip link set up dev {0}".format(self.name), capture=True)
+        def f():
+            if self.name in get_bridges():
+                self.delete()
+            local("ip link add {0} type bridge".format(self.name))
+        try_several_times(f)
+        try_several_times(lambda : local("ip link set up dev {0}".format(self.name)))
 
         self.self_ip = self_ip
         if self_ip:
             self.ip_addr = self.next_ip_address()
-            local("ip addr add {0} dev {1}".format(self.ip_addr, self.name),
-                  capture=True)
-
+            try_several_times(lambda :local("ip addr add {0} dev {1}".format(self.ip_addr, self.name)))
         self.ctns = []
 
     def next_ip_address(self):
@@ -127,8 +138,8 @@ class Bridge(object):
             ctn.pipework(self, '0/0', name)
 
     def delete(self):
-        local("ip link set down dev {0}".format(self.name), capture=True)
-        local("ip link delete {0} type bridge".format(self.name), capture=True)
+        try_several_times(lambda : local("ip link set down dev {0}".format(self.name)))
+        try_several_times(lambda : local("ip link delete {0} type bridge".format(self.name)))
 
 
 class Container(object):
@@ -159,13 +170,7 @@ class Container(object):
         for sv in self.shared_volumes:
             c << "-v {0}:{1}".format(sv[0], sv[1])
         c << "--name {0} -id {1}".format(self.docker_name(), self.image)
-        for i in range(3):
-            try:
-                self.id = local(str(c), capture=True)
-            except:
-                time.sleep(1)
-            else:
-                break
+        self.id = try_several_times(lambda : local(str(c), capture=True))
         self.is_running = True
         self.local("ip li set up dev lo")
         return 0
@@ -188,7 +193,7 @@ class Container(object):
             intf_name = "eth1"
         c << "{0} {1}".format(self.docker_name(), ip_addr)
         self.ip_addrs.append((intf_name, ip_addr, bridge))
-        return local(str(c), capture=True)
+        try_several_times(lambda :local(str(c)))
 
     def local(self, cmd, capture=False, flag=''):
         return local("docker exec {0} {1} {2}".format(flag,
diff --git a/test/scenario_test/lib/exabgp.py b/test/scenario_test/lib/exabgp.py
index cf5fbc1a..a3be012b 100644
--- a/test/scenario_test/lib/exabgp.py
+++ b/test/scenario_test/lib/exabgp.py
@@ -34,18 +34,19 @@ class ExaBGPContainer(BGPContainer):
         self.local(str(cmd), flag='-d')
 
     def _update_exabgp(self):
+        if self.exabgp_path == '':
+            return
         c = CmdBuffer()
         c << '#!/bin/bash'
 
         remotepath = '/root/exabgp'
         localpath = self.exabgp_path
-        if localpath != '':
-            local('cp -r {0} {1}'.format(localpath, self.config_dir))
-            c << 'cp {0}/etc/exabgp/exabgp.env {1}'.format(remotepath, self.SHARED_VOLUME)
-            c << 'sed -i -e \'s/all = false/all = true/g\' {0}/exabgp.env'.format(self.SHARED_VOLUME)
-            c << 'cp -r {0}/exabgp {1}'.format(self.SHARED_VOLUME,
-                                               remotepath[:-1*len('exabgp')])
-            c << 'cp {0}/exabgp.env {1}/etc/exabgp/'.format(self.SHARED_VOLUME, remotepath)
+        local('cp -r {0} {1}'.format(localpath, self.config_dir))
+        c << 'cp {0}/etc/exabgp/exabgp.env {1}'.format(remotepath, self.SHARED_VOLUME)
+        c << 'sed -i -e \'s/all = false/all = true/g\' {0}/exabgp.env'.format(self.SHARED_VOLUME)
+        c << 'cp -r {0}/exabgp {1}'.format(self.SHARED_VOLUME,
+                                           remotepath[:-1*len('exabgp')])
+        c << 'cp {0}/exabgp.env {1}/etc/exabgp/'.format(self.SHARED_VOLUME, remotepath)
         cmd = 'echo "{0:s}" > {1}/update.sh'.format(c, self.config_dir)
         local(cmd, capture=True)
         cmd = 'chmod 755 {0}/update.sh'.format(self.config_dir)
@@ -134,16 +135,27 @@ class ExaBGPContainer(BGPContainer):
             cmd << '}'
 
         with open('{0}/exabgpd.conf'.format(self.config_dir), 'w') as f:
+            print colors.yellow('[{0}\'s new config]'.format(self.name))
             print colors.yellow(str(cmd))
             f.write(str(cmd))
 
     def reload_config(self):
-        ps = self.local('ps', capture=True)
-        running = False
-        for line in ps.split('\n')[1:]:
-            if 'python' in line:
-                running = True
-        if running:
-            self.local('/usr/bin/pkill python -SIGUSR1')
-        else:
-             self._start_exabgp()
+        if len(self.peers) == 0:
+            return
+
+        def _reload():
+            def _is_running():
+                ps = self.local('ps', capture=True)
+                running = False
+                for line in ps.split('\n')[1:]:
+                    if 'python' in line:
+                        running = True
+                return running
+            if _is_running():
+                self.local('/usr/bin/pkill python -SIGUSR1')
+            else:
+                self._start_exabgp()
+            time.sleep(1)
+            if not _is_running():
+                raise RuntimeError()
+        try_several_times(_reload)
diff --git a/test/scenario_test/run_all_tests.sh b/test/scenario_test/run_all_tests.sh
index 2ff55e6f..87999c91 100755
--- a/test/scenario_test/run_all_tests.sh
+++ b/test/scenario_test/run_all_tests.sh
@@ -29,62 +29,6 @@ fi
 
 cd $GOBGP/test/scenario_test
 
-# route server malformed message test
-NUM=$(sudo -E python route_server_malformed_test.py -s 2> /dev/null | awk '/invalid/{print $NF}')
-PARALLEL_NUM=4
-for (( i = 0; i < $(( $NUM / $PARALLEL_NUM + 1)); ++i ))
-do
-    sudo docker rm -f $(sudo docker ps -a -q)
-
-    PIDS=()
-    for (( j = $((PARALLEL_NUM * $i + 1)); j < $((PARALLEL_NUM * ($i+1) + 1)); ++j))
-    do
-        sudo -E python route_server_malformed_test.py --gobgp-image $GOBGP_IMAGE --test-prefix mal$j --test-index $j -s -x --gobgp-log-level debug --with-xunit --xunit-file=${WS}/nosetest_malform${j}.xml &
-        PIDS=("${PIDS[@]}" $!)
-        if [ $j -eq $NUM ]; then
-            break
-        fi
-        sleep 4
-    done
-
-    for (( j = 0; j < ${#PIDS[@]}; ++j ))
-    do
-        wait ${PIDS[$j]}
-        if [ $? != 0 ]; then
-            exit 1
-        fi
-    done
-
-done
-
-# route server policy test
-NUM=$(sudo -E python route_server_policy_test.py -s 2> /dev/null | awk '/invalid/{print $NF}')
-PARALLEL_NUM=4
-for (( i = 0; i < $(( NUM / PARALLEL_NUM + 1)); ++i ))
-do
-    sudo docker rm -f $(sudo docker ps -a -q)
-
-    PIDS=()
-    for (( j = $((PARALLEL_NUM * $i + 1)); j < $((PARALLEL_NUM * ($i+1) + 1)); ++j))
-    do
-        sudo -E python route_server_policy_test.py --gobgp-image $GOBGP_IMAGE --test-prefix p$j --test-index $j -s -x --gobgp-log-level debug --with-xunit --xunit-file=${WS}/nosetest_policy${j}.xml &
-        PIDS=("${PIDS[@]}" $!)
-        if [ $j -eq $NUM ]; then
-            break
-        fi
-        sleep 4
-    done
-
-    for (( j = 0; j < ${#PIDS[@]}; ++j ))
-    do
-        wait ${PIDS[$j]}
-        if [ $? != 0 ]; then
-            exit 1
-        fi
-    done
-
-done
-
 PIDS=()
 
 # route server test
@@ -111,6 +55,16 @@ PIDS=("${PIDS[@]}" $!)
 sudo -E python flow_spec_test.py --gobgp-image $GOBGP_IMAGE --test-prefix flow -s -x --with-xunit --xunit-file=${WS}/nosetest_flow.xml &
 PIDS=("${PIDS[@]}" $!)
 
+# route server malformed message test
+NUM=$(sudo -E python route_server_malformed_test.py -s 2> /dev/null | awk '/invalid/{print $NF}')
+PARALLEL_NUM=10
+for (( i = 1; i < $(( $NUM + 1)); ++i ))
+do
+    sudo -E python route_server_malformed_test.py --gobgp-image $GOBGP_IMAGE --test-prefix mal$i --test-index $i -s -x --gobgp-log-level debug --with-xunit --xunit-file=${WS}/nosetest_malform${i}.xml &
+    PIDS=("${PIDS[@]}" $!)
+    sleep 3
+done
+
 for (( i = 0; i < ${#PIDS[@]}; ++i ))
 do
     wait ${PIDS[$i]}
@@ -119,5 +73,31 @@ do
     fi
 done
 
+# route server policy test
+NUM=$(sudo -E python route_server_policy_test.py -s 2> /dev/null | awk '/invalid/{print $NF}')
+PARALLEL_NUM=25
+for (( i = 0; i < $(( NUM / PARALLEL_NUM + 1)); ++i ))
+do
+    PIDS=()
+    for (( j = $((PARALLEL_NUM * $i + 1)); j < $((PARALLEL_NUM * ($i+1) + 1)); ++j))
+    do
+        sudo -E python route_server_policy_test.py --gobgp-image $GOBGP_IMAGE --test-prefix p$j --test-index $j -s -x --gobgp-log-level debug --with-xunit --xunit-file=${WS}/nosetest_policy${j}.xml &
+        PIDS=("${PIDS[@]}" $!)
+        if [ $j -eq $NUM ]; then
+            break
+        fi
+        sleep 3
+    done
+
+    for (( j = 0; j < ${#PIDS[@]}; ++j ))
+    do
+        wait ${PIDS[$j]}
+        if [ $? != 0 ]; then
+            exit 1
+        fi
+    done
+
+done
+
 echo 'all tests passed successfully'
 exit 0