[dts] [‘dts-v1’ 3/9] Add some params and functions related to the virtual test
Liu, Yong
yong.liu at intel.com
Mon May 18 09:26:26 CEST 2015
Jiajia, please see my comments below.
> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of sjiajiax
> Sent: Monday, May 18, 2015 1:07 PM
> To: dts at dpdk.org
> Subject: [dts] [‘dts-v1’ 3/9] Add some params and functions related to the
> virtual test
>
> Signed-off-by: sjiajiax <sunx.jiajia at intel.com>
> ---
> framework/dts.py | 92 +++++++++++-------------------------------
> -----
> framework/exception.py | 27 ++++++++++++++
> framework/logger.py | 69 +++++++++++++++++++++++++++++------
> framework/main.py | 7 +++-
> framework/project_dpdk.py | 62 ++++++++++++++++++++++----------
> 5 files changed, 157 insertions(+), 100 deletions(-)
>
> diff --git a/framework/dts.py b/framework/dts.py
> index c9ecccb..c0df4e9 100644
> --- a/framework/dts.py
> +++ b/framework/dts.py
> @@ -49,6 +49,7 @@ from test_case import TestCase
> from test_result import Result
> from stats_reporter import StatsReporter
> from excel_reporter import ExcelReporter
> +from utils import *
> from exception import TimeoutException
> from logger import getLogger
> import logger
> @@ -57,6 +58,7 @@ import sys
> reload(sys)
> sys.setdefaultencoding('UTF8')
>
> +PROJECT_MODULE_PREFIX = 'project_'
>
> debug_mode = False
> config = None
> @@ -73,44 +75,12 @@ result = None
> excel_report = None
> stats = None
> log_handler = None
> +Package = ''
> +Patches = []
> drivername = ""
> interrupttypr = ""
>
>
> -def RED(text):
> - return "\x1B[" + "31;1m" + text + "\x1B[" + "0m"
> -
> -
> -def BLUE(text):
> - return "\x1B[" + "36;1m" + text + "\x1B[" + "0m"
> -
> -
> -def GREEN(text):
> - return "\x1B[" + "32;1m" + text + "\x1B[" + "0m"
> -
> -
> -def regexp(s, to_match, allString=False):
> - """
> - Ensure that the re `to_match' only has one group in it.
> - """
> -
> - scanner = re.compile(to_match, re.DOTALL)
> - if allString:
> - return scanner.findall(s)
> - m = scanner.search(s)
> - if m is None:
> - log_handler.warning("Failed to match " + to_match + " in the
> string " + s)
> - return None
> - return m.group(1)
> -
> -
> -def pprint(some_dict):
> - """
> - Print JSON format dictionary object.
> - """
> - return json.dumps(some_dict, sort_keys=True, indent=4)
> -
> -
> def report(text, frame=False, annex=False):
> """
> Save report text into rst file.
> @@ -132,36 +102,6 @@ def close_crb_sessions():
> log_handler.info("DTS ended")
>
>
> -def get_nic_driver(pci_id):
> - """
> - Return linux driver for specified pci device
> - """
> - driverlist = dict(zip(NICS.values(), DRIVERS.keys()))
> - try:
> - driver = DRIVERS[driverlist[pci_id]]
> - except Exception as e:
> - driver = None
> - return driver
> -
> -
> -def accepted_nic(pci_id):
> - """
> - Return True if the pci_id is a known NIC card in the settings file
> and if
> - it is selected in the execution file, otherwise it returns False.
> - """
> - if pci_id not in NICS.values():
> - return False
> -
> - if nic is 'any':
> - return True
> -
> - else:
> - if pci_id == NICS[nic]:
> - return True
> -
> - return False
> -
> -
> def get_crb_os(crb):
> if 'OS' in crb:
> return crb['OS']
> @@ -220,9 +160,10 @@ def get_project_obj(project_name, super_class,
> crbInst, serializer):
> """
> Load project module and return crb instance.
> """
> + global PROJECT_MODULE_PREFIX
> project_obj = None
> try:
> - project_module = __import__("project_" + project_name)
> + project_module = __import__(PROJECT_MODULE_PREFIX + project_name)
>
> for project_subclassname, project_subclass in
> get_subclasses(project_module, super_class):
> project_obj = project_subclass(crbInst, serializer)
> @@ -267,19 +208,20 @@ def dts_log_execution(log_handler):
> pass
>
>
> -def dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir,
> nic):
> +def dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir, nic,
> virttype):
> """
> Create dts dut/tester instance and initialize them.
> """
> global dut
> global tester
> - serializer.set_serialized_filename('../.%s.cache' % crbInst['IP'])
> + serializer.set_serialized_filename('.%s.cache' % crbInst['IP'])
> serializer.load_from_file()
>
> dut = get_project_obj(project, Dut, crbInst, serializer)
> tester = get_project_obj(project, Tester, crbInst, serializer)
> dut.tester = tester
> tester.dut = dut
> + dut.set_virttype(virttype)
> dut.set_speedup_options(read_cache, skip_setup)
> dut.set_directory(base_dir)
> dut.set_nic_type(nic)
> @@ -337,7 +279,6 @@ def dts_run_target(crbInst, targets, test_suites, nic):
> if 'nic_type' not in paramDict:
> paramDict['nic_type'] = 'any'
> nic = 'any'
> - result.nic = nic
>
> dts_run_suite(crbInst, test_suites, target, nic)
>
> @@ -359,7 +300,9 @@ def dts_run_suite(crbInst, test_suites, target, nic):
> test_module = __import__('TestSuite_' + test_suite)
> for test_classname, test_class in get_subclasses(test_module,
> TestCase):
>
> - test_suite = test_class(dut, tester, target)
> + test_suite = test_class(dut, tester, target, test_suite)
> + result.nic = test_suite.nic
> +
> dts_log_testsuite(test_suite, log_handler, test_classname)
>
> log_handler.info("\nTEST SUITE : " + test_classname)
> @@ -386,7 +329,7 @@ def dts_run_suite(crbInst, test_suites, target, nic):
>
> def run_all(config_file, pkgName, git, patch, skip_setup,
> read_cache, project, suite_dir, test_cases,
> - base_dir, output_dir, verbose, debug):
> + base_dir, output_dir, verbose, virttype, debug):
> """
> Main process of DTS, it will run all test suites in the config file.
> """
> @@ -400,6 +343,12 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
> global stats
> global log_handler
> global debug_mode
> + global Package
> + global Patches
> +
> + # save global variable
> + Package = pkgName
> + Patches = patch
>
> # prepare the output folder
> if not os.path.exists(output_dir):
> @@ -466,7 +415,8 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
> result.dut = dutIP
>
> # init dut, tester crb
> - dts_crbs_init(crbInst, skip_setup, read_cache, project, base_dir,
> nics)
> + dts_crbs_init(
> + crbInst, skip_setup, read_cache, project, base_dir, nics,
> virttype)
>
> # Run DUT prerequisites
> if dts_run_prerequisties(pkgName, patch) is False:
> diff --git a/framework/exception.py b/framework/exception.py
> index be38c16..98dedf4 100644
> --- a/framework/exception.py
> +++ b/framework/exception.py
> @@ -46,3 +46,30 @@ class SSHConnectionException(Exception):
>
> def __str__(self):
> return 'Error trying to connect with %s' % self.host
> +
> +
> +class SSHSessionDeadException(Exception):
> +
> + """
> + SSH session is not alive.
> + It can no longer be used.
> + """
> +
> + def __init__(self, host):
> + self.host = host
> +
> + def __str__(self):
> + return 'SSH session with %s has been dead' % self.host
> +
> +
> +class StartVMFailedException(Exception):
> +
> + """
> + Start VM failed.
> + """
> +
> + def __init__(self, error):
> + self.error = error
> +
> + def __str__(self):
> + return repr(self.error)
> diff --git a/framework/logger.py b/framework/logger.py
> index 1829e18..5597e33 100644
> --- a/framework/logger.py
> +++ b/framework/logger.py
> @@ -35,6 +35,9 @@ import sys
> import inspect
> import re
>
> +from settings import LOG_NAME_SEP
> +from utils import RED
> +
> """
> DTS logger module with several log level. DTS framwork and TestSuite log
> will saved into different log files.
> @@ -58,6 +61,9 @@ logging.SUITE_TESTER_OUTPUT = logging.DEBUG + 4
> logging.DTS_IXIA_CMD = logging.INFO + 5
> logging.DTS_IXIA_OUTPUT = logging.DEBUG + 5
>
> +logging.DTS_VIRTDUT_CMD = logging.INFO + 6
> +logging.DTS_VIRTDUT_OUTPUT = logging.DEBUG + 6
> +
> logging.addLevelName(logging.DTS_DUT_CMD, 'DTS_DUT_CMD')
> logging.addLevelName(logging.DTS_DUT_OUTPUT, 'DTS_DUT_OUTPUT')
> logging.addLevelName(logging.DTS_DUT_RESULT, 'DTS_DUT_RESUTL')
> @@ -66,6 +72,12 @@ logging.addLevelName(logging.DTS_TESTER_CMD,
> 'DTS_TESTER_CMD')
> logging.addLevelName(logging.DTS_TESTER_OUTPUT, 'DTS_TESTER_OUTPUT')
> logging.addLevelName(logging.DTS_TESTER_RESULT, 'DTS_TESTER_RESULT')
>
> +logging.addLevelName(logging.DTS_IXIA_CMD, 'DTS_IXIA_CMD')
> +logging.addLevelName(logging.DTS_IXIA_OUTPUT, 'DTS_IXIA_OUTPUT')
> +
> +logging.addLevelName(logging.DTS_VIRTDUT_CMD, 'VIRTDUT_CMD')
> +logging.addLevelName(logging.DTS_VIRTDUT_OUTPUT, 'VIRTDUT_OUTPUT')
> +
> logging.addLevelName(logging.SUITE_DUT_CMD, 'SUITE_DUT_CMD')
> logging.addLevelName(logging.SUITE_DUT_OUTPUT, 'SUITE_DUT_OUTPUT')
>
> @@ -82,15 +94,18 @@ stream_fmt = '%(color)s%(levelname)20s: %(message)s' +
> RESET_COLOR
> log_dir = None
>
>
> -def RED(text):
> - return "\x1B[" + "31;1m" + text + "\x1B[" + "0m"
> -
> -
> def set_verbose():
> global verbose
> verbose = True
>
>
> +def add_salt(salt, msg):
> + if not salt:
> + return msg
> + else:
> + return '[%s] ' % salt + str(msg)
> +
> +
> class BaseLoggerAdapter(logging.LoggerAdapter):
> """
> Upper layer of original logging module.
> @@ -132,6 +147,12 @@ class BaseLoggerAdapter(logging.LoggerAdapter):
> def dts_ixia_output(self, msg, *args, **kwargs):
> self.log(logging.DTS_IXIA_OUTPUT, msg, *args, **kwargs)
>
> + def dts_virtdut_cmd(self, msg, *args, **kwargs):
> + self.log(logging.DTS_VIRTDUT_CMD, msg, *args, **kwargs)
> +
> + def dts_virtdut_output(self, msg, *args, **kwargs):
> + self.log(logging.DTS_VIRTDUT_OUTPUT, msg, *args, **kwargs)
> +
>
> class ColorHandler(logging.StreamHandler):
> """
> @@ -150,6 +171,8 @@ class ColorHandler(logging.StreamHandler):
> logging.SUITE_TESTER_CMD: '', # SYSTEM
> logging.DTS_IXIA_CMD: '', # SYSTEM
> logging.DTS_IXIA_OUTPUT: '', # SYSTEM
> + logging.DTS_VIRTDUT_CMD: '', # SYSTEM
> + logging.DTS_VIRTDUT_OUTPUT: '', # SYSTEM
> logging.WARN: '\033[01;33m', # BOLD YELLOW
> logging.DTS_DUT_RESULT: '\033[01;34m', # BOLD BLUE
> logging.DTS_TESTER_RESULT: '\033[01;34m', # BOLD BLUE
> @@ -189,6 +212,8 @@ class DTSLOG(BaseLoggerAdapter):
> self.crb = crb
> super(DTSLOG, self).__init__(self.logger, dict(crb=self.crb))
>
> + self.salt = ''
> +
> self.fh = None
> self.ch = None
>
> @@ -221,24 +246,28 @@ class DTSLOG(BaseLoggerAdapter):
> """
> DTS warnning level log function.
> """
> + message = add_salt(self.salt, message)
> self.logger.log(self.warn_lvl, message)
>
> def info(self, message):
> """
> DTS information level log function.
> """
> + message = add_salt(self.salt, message)
> self.logger.log(self.info_lvl, message)
>
> def error(self, message):
> """
> DTS error level log function.
> """
> + message = add_salt(self.salt, message)
> self.logger.log(self.error_lvl, message)
>
> def debug(self, message):
> """
> DTS debug level log function.
> """
> + message = add_salt(self.salt, message)
> self.logger.log(self.debug_lvl, message)
>
> def set_logfile_path(self, path):
> @@ -270,17 +299,34 @@ class DTSLOG(BaseLoggerAdapter):
> ch = ColorHandler()
> self.__log_hander(fh, ch)
>
> - if crb == "dut":
> + def set_salt(crb, start_flag):
> + if LOG_NAME_SEP in crb:
> + old = '%s%s' % (start_flag, LOG_NAME_SEP)
> + if not self.salt:
> + self.salt = crb.replace(old, '', 1)
> +
> + if crb.startswith('dut'):
> self.info_lvl = logging.DTS_DUT_CMD
> self.debug_lvl = logging.DTS_DUT_OUTPUT
> self.warn_lvl = logging.DTS_DUT_RESULT
> - elif crb == "tester":
> +
> + set_salt(crb, 'dut')
> + elif crb.startswith('tester'):
> self.info_lvl = logging.DTS_TESTER_CMD
> self.debug_lvl = logging.DTS_TESTER_OUTPUT
> self.warn_lvl = logging.DTS_TESTER_RESULT
> - elif crb == "ixia":
> +
> + set_salt(crb, 'tester')
> + elif crb.startswith('ixia'):
> self.info_lvl = logging.DTS_IXIA_CMD
> self.debug_lvl = logging.DTS_IXIA_OUTPUT
> +
> + set_salt(crb, 'ixia')
> + elif crb.startswith('virtdut'):
> + self.info_lvl = logging.DTS_VIRTDUT_CMD
> + self.debug_lvl = logging.DTS_VIRTDUT_OUTPUT
> +
> + set_salt(crb, 'virtdut')
> else:
> self.error_lvl = logging.ERROR
> self.warn_lvl = logging.WARNING
> @@ -296,15 +342,18 @@ class DTSLOG(BaseLoggerAdapter):
> ch = ColorHandler()
> self.__log_hander(fh, ch)
>
> - if crb == "dut":
> + if crb == 'dut':
> self.info_lvl = logging.SUITE_DUT_CMD
> self.debug_lvl = logging.SUITE_DUT_OUTPUT
> - elif crb == "tester":
> + elif crb == 'tester':
> self.info_lvl = logging.SUITE_TESTER_CMD
> self.debug_lvl = logging.SUITE_TESTER_OUTPUT
> - elif crb == "ixia":
> + elif crb == 'ixia':
> self.info_lvl = logging.DTS_IXIA_CMD
> self.debug_lvl = logging.DTS_IXIA_OUTPUT
> + elif crb == 'virtdut':
> + self.info_lvl = logging.DTS_VIRTDUT_CMD
> + self.debug_lvl = logging.DTS_VIRTDUT_OUTPUT
>
> def logger_exit(self):
> """
> diff --git a/framework/main.py b/framework/main.py
> index 3e467d0..0496b20 100755
> --- a/framework/main.py
> +++ b/framework/main.py
> @@ -117,6 +117,10 @@ parser.add_argument('-v', '--verbose',
> action='store_true',
> help='enable verbose output, all message output on
> screen')
>
> +parser.add_argument('--virttype',
> + default='kvm',
> + help='set virt type,support libvirt,xen,kvm')
> +
> parser.add_argument('--debug',
> action='store_true',
> help='enable debug mode, user can enter debug mode in
> process')
> @@ -136,4 +140,5 @@ if args.git is not None:
> dts.run_all(args.config_file, args.snapshot, args.git,
> args.patch, args.skip_setup, args.read_cache,
> args.project, args.suite_dir, args.test_cases,
> - args.dir, args.output, args.verbose, args.debug)
> + args.dir, args.output, args.verbose,args.virttype,
> + args.debug)
> diff --git a/framework/project_dpdk.py b/framework/project_dpdk.py
> index 8963924..67bd492 100644
> --- a/framework/project_dpdk.py
> +++ b/framework/project_dpdk.py
> @@ -39,7 +39,7 @@ from crb import Crb
> from dut import Dut
> from tester import Tester
> from logger import getLogger
> -from settings import IXIA
> +from settings import IXIA, accepted_nic
>
>
> class DPDKdut(Dut):
> @@ -50,8 +50,9 @@ class DPDKdut(Dut):
> """
>
> def __init__(self, crb, serializer):
> - self.NAME = 'dut'
> +
> super(DPDKdut, self).__init__(crb, serializer)
> + self.testpmd = None
>
> def set_target(self, target):
> """
> @@ -60,6 +61,7 @@ class DPDKdut(Dut):
> Set hugepage on DUT and install modules required by DPDK.
> Configure default ixgbe PMD function.
> """
> + self.target = target
> self.set_toolchain(target)
>
> # set env variable
> @@ -99,7 +101,16 @@ class DPDKdut(Dut):
> out = self.send_expect("lsmod | grep igb_uio", "#")
> if "igb_uio" in out:
> self.send_expect("rmmod -f igb_uio", "#", 70)
> - self.send_expect("insmod ./" + target + "/kmod/igb_uio.ko",
> "#", 60)
> + if "rte_dom0_mm" in self.send_expect("lsmod |grep
> rte_dom0_mm", "#"):
> + self.send_expect("rmmod -f rte_dmo0_mm", "#", 70)
> + self.send_expect(
> + "insmod ./" + target + "/kmod/igb_uio.ko", "#", 60)
"rte_dom0_mm" module only can be built out when DOM0 enabled in DPDK configuration file.
We should not expected it existed in host. Make sure DOM0 configuration enabled and build target with modified configuration.
> + if self.virttype == 'xen':
> + self.send_expect(
> + "insmod ./" + target + "/kmod/rte_dom0_mm.ko", "#",
> 60)
> + self.send_expect(
> + r'echo 1024 > /sys/kernel/mm/dom0-mm/memsize-
> mB/memsize', "#", 70)
> +
Better to use macro replace of 1024, these settings should be defined in settings.py.
> out = self.send_expect("lsmod | grep igb_uio", "#")
> assert ("igb_uio" in out), "Failed to insmod igb_uio"
>
> @@ -110,7 +121,7 @@ class DPDKdut(Dut):
> binding_list = ''
>
> for (pci_bus, pci_id) in self.pci_devices_info:
> - if dts.accepted_nic(pci_id):
> + if accepted_nic(pci_id):
> binding_list += '%s,' % (pci_bus)
>
> self.send_expect("kldunload if_ixgbe.ko", "#")
> @@ -158,6 +169,9 @@ class DPDKdut(Dut):
> """
> # clean all
> self.send_expect("rm -rf " + target, "#")
> + if self.virttype == 'xen':
> + self.send_expect("sed -i -e
> 's/CONFIG_RTE_LIBRTE_XEN_DOM0=.*$/"
> + + "CONFIG_RTE_LIBRTE_XEN_DOM0=y/'
> config/common_linuxapp", "# ", 30)
>
> # compile
> out = self.send_expect("make -j install T=%s %s" % (target,
> extra_options), "# ", 120)
> @@ -178,9 +192,9 @@ class DPDKdut(Dut):
> self.send_expect("rm -rf " + target, "#")
>
> # compile
> - out = self.send_expect("make -j %d install T=%s CC=gcc48" %
> (self.number_of_cores,
> -
> target),
> - "#", 120)
> + out = self.send_expect(
> + "make -j %d install T=%s CC=gcc48" %
> + (self.number_of_cores, target), "#", 120)
>
> if("Error" in out or "No rule to make" in out):
> self.logger.error("ERROR - try without '-j'")
> @@ -191,10 +205,7 @@ class DPDKdut(Dut):
> assert ("Error" not in out), "Compilation error..."
> assert ("No rule to make" not in out), "No rule to make error..."
>
> - def prerequisites(self, pkgName, patch):
> - """
> - Copy DPDK package to DUT and apply patch files.
> - """
> + def prepare_package(self, pkgName, patch):
> if not self.skip_setup:
> assert (os.path.isfile(pkgName) is True), "Invalid package"
>
> @@ -202,7 +213,7 @@ class DPDKdut(Dut):
> # ToDo: make this configurable
> dst_dir = "/tmp/"
>
> - out = self.send_expect("ls %s && cd %s" % (dst_dir, p_dir),
> + out = self.send_expect("ll %s && cd %s" % (dst_dir, p_dir),
> "#", verify=True)
Command "ll" may not existed on every OS, please just use basic "ls" command.
> if out == -1:
> raise ValueError("Directiry %s or %s does not exist,"
> @@ -249,6 +260,17 @@ class DPDKdut(Dut):
> (self.base_dir, dst_dir + p),
> "# ")
> assert "****" not in out
>
> + self.session.copy_file_to("dep/aclrule.tgz", dst_dir)
> + # unpack acl rule
> + out = self.send_expect("tar zxf %saclrule.tgz -C %s" %
> (dst_dir, p_dir), "# ", 20, verify=True)
> + if out == -1:
> + raise ValueError("acl rule extract failure!!!")
> +
These files related with l3fwd acl suite, we should merged these files in following with acl suite.
Please remove these codes from your patch set.
> + def prerequisites(self, pkgName, patch):
> + """
> + Copy DPDK package to DUT and apply patch files.
> + """
> + self.prepare_package(pkgName, patch)
> self.dut_prerequisites()
>
> def bind_interfaces_linux(self, driver='igb_uio', nics_to_bind=None):
> @@ -354,16 +376,20 @@ class DPDKtester(Tester):
> total_huge_pages = self.get_total_huge_pages()
> if total_huge_pages == 0:
> self.mount_huge_pages()
> - self.set_huge_pages(4096)
> + self.set_huge_pages(1024)
>
> self.session.copy_file_to("dep/tgen.tgz")
> self.session.copy_file_to("dep/tclclient.tgz")
> + self.session.copy_file_to("dep/aclpcap.tgz")
> # unpack tgen
> out = self.send_expect("tar zxf tgen.tgz", "# ")
> assert "Error" not in out
> # unpack tclclient
> out = self.send_expect("tar zxf tclclient.tgz", "# ")
> assert "Error" not in out
These files related with l3fwd acl suite, we should merged these files in following with acl suite.
Please remove these codes from your patch set.
> + # unpacl ACL pcap files
> + out = self.send_expect("tar zxf aclpcap.tgz", "# ")
> + assert "Error" not in out
>
> self.send_expect("modprobe uio", "# ")
>
> @@ -386,10 +412,10 @@ class DPDKtester(Tester):
> """
> hugepages_size = self.send_expect("awk '/Hugepagesize/ {print
> $2}' /proc/meminfo", "# ")
>
> - if int(hugepages_size) < (1024 * 1024):
> - arch_huge_pages = hugepages if hugepages > 0 else 4096
> + if int(hugepages_size) < (2048 * 2048):
> + arch_huge_pages = hugepages if hugepages > 0 else 2048
Better to use macro replace of 2048, these settings should be defined in settings.py.
> total_huge_pages = self.get_total_huge_pages()
>
> - self.mount_huge_pages()
> - if total_huge_pages != arch_huge_pages:
> - self.set_huge_pages(arch_huge_pages)
> + self.mount_huge_pages()
> + if total_huge_pages != arch_huge_pages:
> + self.set_huge_pages(arch_huge_pages)
> --
> 1.9.0
More information about the dts
mailing list