[dts] [next] [PATCH 1/3] framework: support argument: --update-expected.
Lijuan Tu
lijuan.tu at intel.com
Fri Jul 20 17:22:02 CEST 2018
A DPDK Performance Test Lab are established, and Specification requires
DTS to support an --update-expected argument which will cause the
script to update values based on the results of the current test run.
Signed-off-by: Lijuan Tu <lijuan.tu at intel.com>
---
framework/config.py | 11 +++++++++++
framework/dts.py | 6 +++++-
framework/main.py | 6 +++++-
framework/settings.py | 1 +
framework/test_case.py | 22 ++++++++++++++++++----
5 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/framework/config.py b/framework/config.py
index 628fc6d..0b112d8 100644
--- a/framework/config.py
+++ b/framework/config.py
@@ -147,6 +147,17 @@ class SuiteConf(UserConf):
return case_cfg
+ def update_case_config(self, case_name=""):
+ """
+ update section (case_name) of the configure file
+ """
+ update_suite_cfg_obj = UserConf(self.config_file)
+ update_suite_cfg = update_suite_cfg_obj.load_section(case_name)
+ for key in update_suite_cfg_obj.conf.options(case_name):
+ update_suite_cfg_obj.conf.set(
+ case_name, key, str(self.suite_cfg[key]))
+ update_suite_cfg_obj.conf.write(open(self.config_file, 'w'))
+
class VirtConf(UserConf):
diff --git a/framework/dts.py b/framework/dts.py
index 0b2240c..4435418 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target):
def run_all(config_file, pkgName, git, patch, skip_setup,
read_cache, project, suite_dir, test_cases,
base_dir, output_dir, verbose, virttype, debug,
- debugcase, re_run, commands):
+ debugcase, re_run, commands, update_expected):
"""
Main process of DTS, it will run all test suites in the config file.
"""
@@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
if debugcase is True:
settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes')
+ # enable update-expected
+ if update_expected is True:
+ settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes')
+
# init log_handler handler
if verbose is True:
logger.set_verbose()
diff --git a/framework/main.py b/framework/main.py
index 0aa54fd..9d7ef31 100755
--- a/framework/main.py
+++ b/framework/main.py
@@ -143,6 +143,10 @@ parser.add_argument('--commands',
help='run command on tester or dut. The command format is ' +
'[commands]:dut|tester:pre-init|post-init:check|ignore')
+parser.add_argument('--update-expected',
+ action='store_true',
+ help='update expected values based on test results')
+
args = parser.parse_args()
@@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
args.patch, args.skip_setup, args.read_cache,
args.project, args.suite_dir, args.test_cases,
args.dir, args.output, args.verbose,args.virttype,
- args.debug, args.debugcase, args.re_run, args.commands)
+ args.debug, args.debugcase, args.re_run, args.commands, args.update_expected)
diff --git a/framework/settings.py b/framework/settings.py
index 07c3ac6..2561ddb 100644
--- a/framework/settings.py
+++ b/framework/settings.py
@@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
DTS_PARALLEL_SETTING = "DTS_PARALLEL_ENABLE"
+UPDATE_EXPECTED = "DTS_UPDATE_EXPECTED_ENABLE"
"""
diff --git a/framework/test_case.py b/framework/test_case.py
index a84e2bb..ffa03db 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -40,7 +40,9 @@ import time
from exception import VerifyFailure, TimeoutException
from settings import DRIVERS, NICS, get_nic_name, load_global_setting
-from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING, DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
+from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
+from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
+from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME
from rst import RstReport
from test_result import ResultTable, Result
from logger import getLogger
@@ -254,10 +256,7 @@ class TestCase(object):
self._rst_obj.write_title("Test Case: " + case_name)
# load suite configuration file here for rerun command
- self._suite_conf = SuiteConf(self.suite_name)
- self._suite_cfg = self._suite_conf.suite_cfg
self._case_cfg = self._suite_conf.load_case_config(case_name)
- del(self._suite_conf)
case_result = True
if self._check_inst is not None:
@@ -315,6 +314,9 @@ class TestCase(object):
self._suite_result.test_case_failed(trace)
self.logger.error('Test Case %s Result ERROR: ' % (case_name) + trace)
finally:
+ # update expected
+ if load_global_setting(UPDATE_EXPECTED) == "yes":
+ self._suite_conf.update_case_config(SUITE_SECTION_NAME)
self.tear_down()
return case_result
@@ -375,6 +377,18 @@ class TestCase(object):
"""
return self._suite_cfg
+ def update_suite_cfg(self, suite_cfg):
+ """
+ Update suite based configuration
+ """
+ self._suite_cfg = suite_cfg
+
+ def update_suite_cfg_ele(self, key, value):
+ """
+ update one element of suite configuration
+ """
+ self._suite_cfg[key]=value
+
def execute_tear_downall(self):
"""
execute suite tear_down_all function
--
1.8.3.1
More information about the dts
mailing list