[dts] [next] [PATCH V2 1/3] framework: support argument: --update-expected.

[Liu, Yong]

Lijuan,
Some comments are inline.

Thanks,
Marvin

> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of Lijuan Tu
> Sent: Monday, July 23, 2018 7:19 PM
> To: dts at dpdk.org
> Cc: Tu, Lijuan <lijuan.tu at intel.com>
> Subject: [dts] [next] [PATCH V2 1/3] framework: support argument: --
> update-expected.
> 
> A DPDK Performance Test Lab has been established, and Specification
> requires
> DTS to support an --update-expected argument which will cause the
> script to update values based on the results of the current test run.
> 
> When "--update-expected" added in bash command, and there is
> "update-expected = Ture" in suite configuration file. All objects in
> configuration file will be updated.
> 
> Take single core performance test for example:
> If "./dts --update-expected" and "update-expected = Ture" in
> conf/nic_single_core_perf.cfg, all objects will be updated in
> conf/nic_single_core_perf.cfg
> 
> Signed-off-by: Lijuan Tu <lijuan.tu at intel.com>
> ---
>  framework/config.py    | 11 +++++++++++
>  framework/dts.py       |  6 +++++-
>  framework/main.py      |  6 +++++-
>  framework/settings.py  |  1 +
>  framework/test_case.py | 24 ++++++++++++++++++++----
>  5 files changed, 42 insertions(+), 6 deletions(-)
> 
> diff --git a/framework/config.py b/framework/config.py
> index 628fc6d..0b112d8 100644
> --- a/framework/config.py
> +++ b/framework/config.py
> @@ -147,6 +147,17 @@ class SuiteConf(UserConf):
> 
>          return case_cfg
> 
> +    def update_case_config(self, case_name=""):
> +        """
> +        update section (case_name) of the configure file
> +        """
> +        update_suite_cfg_obj = UserConf(self.config_file)
> +        update_suite_cfg = update_suite_cfg_obj.load_section(case_name)
> +        for key in update_suite_cfg_obj.conf.options(case_name):
> +            update_suite_cfg_obj.conf.set(
> +                case_name, key, str(self.suite_cfg[key]))
> +        update_suite_cfg_obj.conf.write(open(self.config_file, 'w'))
> +
> 
>  class VirtConf(UserConf):
> 
> diff --git a/framework/dts.py b/framework/dts.py
> index 0b2240c..4435418 100644
> --- a/framework/dts.py
> +++ b/framework/dts.py
> @@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target):
>  def run_all(config_file, pkgName, git, patch, skip_setup,
>              read_cache, project, suite_dir, test_cases,
>              base_dir, output_dir, verbose, virttype, debug,
> -            debugcase, re_run, commands):
> +            debugcase, re_run, commands, update_expected):
>      """
>      Main process of DTS, it will run all test suites in the config file.
>      """
> @@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
>      if debugcase is True:
>          settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes')
> 
> +    # enable update-expected
> +    if update_expected is True:
> +        settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes')
> +
>      # init log_handler handler
>      if verbose is True:
>          logger.set_verbose()
> diff --git a/framework/main.py b/framework/main.py
> index 0aa54fd..9d7ef31 100755
> --- a/framework/main.py
> +++ b/framework/main.py
> @@ -143,6 +143,10 @@ parser.add_argument('--commands',
>                      help='run command on tester or dut. The command
> format is ' +
>                      '[commands]:dut|tester:pre-init|post-
> init:check|ignore')
> 
> +parser.add_argument('--update-expected',
> +                    action='store_true',
> +                    help='update expected values based on test results')
> +
>  args = parser.parse_args()
> 
> 
> @@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
>              args.patch, args.skip_setup, args.read_cache,
>              args.project, args.suite_dir, args.test_cases,
>              args.dir, args.output, args.verbose,args.virttype,
> -            args.debug, args.debugcase, args.re_run, args.commands)
> +            args.debug, args.debugcase, args.re_run, args.commands,
> args.update_expected)
> diff --git a/framework/settings.py b/framework/settings.py
> index 07c3ac6..2561ddb 100644
> --- a/framework/settings.py
> +++ b/framework/settings.py
> @@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
>  DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
>  DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
>  DTS_PARALLEL_SETTING = "DTS_PARALLEL_ENABLE"
> +UPDATE_EXPECTED = "DTS_UPDATE_EXPECTED_ENABLE"
> 
> 
>  """
> diff --git a/framework/test_case.py b/framework/test_case.py
> index a84e2bb..228544c 100644
> --- a/framework/test_case.py
> +++ b/framework/test_case.py
> @@ -40,7 +40,9 @@ import time
> 
>  from exception import VerifyFailure, TimeoutException
>  from settings import DRIVERS, NICS, get_nic_name, load_global_setting
> -from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING,
> DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
> +from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
> +from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
> +from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME
>  from rst import RstReport
>  from test_result import ResultTable, Result
>  from logger import getLogger
> @@ -254,10 +256,7 @@ class TestCase(object):
>          self._rst_obj.write_title("Test Case: " + case_name)
> 
>          # load suite configuration file here for rerun command
> -        self._suite_conf = SuiteConf(self.suite_name)
> -        self._suite_cfg = self._suite_conf.suite_cfg
>          self._case_cfg = self._suite_conf.load_case_config(case_name)
> -        del(self._suite_conf)
> 

This is useful for reload case configuration. You can skip this if update expected is enabled.

>          case_result = True
>          if self._check_inst is not None:
> @@ -315,6 +314,11 @@ class TestCase(object):
>              self._suite_result.test_case_failed(trace)
>              self.logger.error('Test Case %s Result ERROR: ' % (case_name)
> + trace)
>          finally:
> +            # update expected
> +            if load_global_setting(UPDATE_EXPECTED) == "yes" and \
> +                self.get_suite_cfg().has_key('update_expected') and \
> +                self.get_suite_cfg()['update_expected'] == True:
> +                self._suite_conf.update_case_config(SUITE_SECTION_NAME)
>              self.tear_down()
>              return case_result
> 
> @@ -375,6 +379,18 @@ class TestCase(object):
>          """
>          return self._suite_cfg
> 
> +    def update_suite_cfg(self, suite_cfg):
> +

Lijuan,
I can't check where this function was called, is it useful?

        """
> +        Update suite based configuration
> +        """
> +        self._suite_cfg = suite_cfg
> +
> +    def update_suite_cfg_ele(self, key, value):
> +        """
> +        update one element of suite configuration
> +        """
> +        self._suite_cfg[key]=value
> +
Same comment as previous one, please add space between equal mark.

>      def execute_tear_downall(self):
>          """
>          execute suite tear_down_all function
> --
> 1.8.3.1