[dts] [PATCH] tests: add TestSuite_runtime_queue_number.py

[Peng, Yuan]

Hi Marvin,

I have updated the patch according to your comments.
Please help to review it again.

Thank you very much!
Yuan.

-----Original Message-----
From: Liu, Yong 
Sent: Monday, March 12, 2018 2:13 PM
To: Peng, Yuan <yuan.peng at intel.com>; dts at dpdk.org
Cc: Peng, Yuan <yuan.peng at intel.com>
Subject: RE: [dts] [PATCH] tests: add TestSuite_runtime_queue_number.py

Yuan,
Some comments are inline.

Thanks,
Marvin

> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of Peng Yuan
> Sent: Monday, March 05, 2018 3:17 PM
> To: dts at dpdk.org
> Cc: Peng, Yuan <yuan.peng at intel.com>
> Subject: [dts] [PATCH] tests: add TestSuite_runtime_queue_number.py
> 
> Set the VF max queue number when running the DPDK APP instead of compiling.
> 
> Signed-off-by: Peng Yuan <yuan.peng at intel.com>
> 
> diff --git a/tests/TestSuite_runtime_queue_number.py
> b/tests/TestSuite_runtime_queue_number.py
> new file mode 100644
> index 0000000..11931d1
> --- /dev/null
> +++ b/tests/TestSuite_runtime_queue_number.py
> @@ -0,0 +1,257 @@
> +# BSD LICENSE
> +#
> +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +# All rights reserved.

Should be "2010-2018". 
> +#
> +# Redistribution and use in source and binary forms, with or without 
> +# modification, are permitted provided that the following conditions 
> +# are met:
> +#
> +#   * Redistributions of source code must retain the above copyright
> +#     notice, this list of conditions and the following disclaimer.
> +#   * Redistributions in binary form must reproduce the above copyright
> +#     notice, this list of conditions and the following disclaimer in
> +#     the documentation and/or other materials provided with the
> +#     distribution.
> +#   * Neither the name of Intel Corporation nor the names of its
> +#     contributors may be used to endorse or promote products derived
> +#     from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
> +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # 
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 
> +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
> +COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
> +INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
> +(INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
> +SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
> +HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, 
> +STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) 
> +ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +"""
> +DPDK Test suite.
> +
> +Set the VF max queue number when running the DPDK APP instead of
> compiling.
> +
> +"""
> +
> +import utils
> +import time
> +import re
> +
> +from test_case import TestCase
> +from settings import HEADER_SIZE
> +from pmd_output import PmdOutput
> +from settings import DRIVERS
> +
> +from project_dpdk import DPDKdut
> +from dut import Dut

These two modules are useless in the suite, please remove them.

> +from packet import Packet
> +
> +
> +class TestRuntime_Queue_Number(TestCase):
> +
> +    def set_up_all(self):
> +        """
> +        Run at the start of each test suite.
> +        Run time Queue Number Prerequistites
> +        """
> +        self.verify(self.nic in ["fortville_eagle", "fortville_spirit",
> +                                 "fortville_spirit_single",
> "fortpark_TLV"], "NIC Unsupported: " + str(self.nic))
> +

If this suite only valuable for Fortville NIC, I suggest to add this information in test_case_supportlist.json. 

> +        # Based on h/w type, choose how many ports to use
> +        self.dut_ports = self.dut.get_ports(self.nic)
> +        # Verify that enough ports are available
> +        self.verify(len(self.dut_ports) >= 1, "Insufficient ports")
> +
> +        localPort = self.tester.get_local_port(self.dut_ports[0])
> +        self.tester_intf = self.tester.get_interface(localPort)
> +        self.tester_mac = self.tester.get_mac(localPort)
> +        self.pf_interface = self.dut.ports_info[self.dut_ports[0]]['intf']
> +        self.pf_mac = self.dut.get_mac_address(0)
> +        self.pf_pci = self.dut.ports_info[self.dut_ports[0]]['pci']
> +        self.pmdout = PmdOutput(self.dut)
> +        self.cores = "1S/4C/1T"
> +
> +        self.session_secondary = self.dut.new_session()
> +        self.session_third = self.dut.new_session()
> +
> +    def set_up(self):
> +        """
> +        Run before each test case.
> +        """
> +        self.dut.kill_all()

Kill action should be in tear_down function. If previous suite has incorrect behavior, we should know that. 

> +
> +    def setup_env(self, vfs_num):
> +        """
> +        This is to set up vf environment.
> +        The pf is bound to dpdk driver.
> +        """
> +        # create two vfs
Typo here, should be "assigned number of VFs". 

> +        self.dut.generate_sriov_vfs_by_port(self.dut_ports[0], 
> + vfs_num,
> self.drivername)
> +        self.sriov_vfs_port =
> self.dut.ports_info[self.dut_ports[0]]['vfs_port']
> +
> +        try:
> +            for port in self.sriov_vfs_port:
> +                port.bind_driver(driver="vfio-pci")

"vfio-pci" may not insmoded on host. You can make sure that by call function "self.dut.setup_modules_linux()". 

> +        except Exception as e:
> +            self.destroy_env()
> +            raise Exception(e)
> +
> +    def destroy_env(self):
> +        """
> +        This is to stop testpmd and destroy vf environment.
> +        """
> +        self.session_third.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.session_secondary.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.dut.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.dut.destroy_sriov_vfs_by_port(self.dut_ports[0])
> +
> +    def verify_result(self, outstring, max_rxqn, max_txqn, cur_rxqn,
> cur_txqn):
> +        """
> +        verify the packet to the expected queue or be dropped
> +        """
> +        self.verify("Max possible RX queues: %d" % max_rxqn in 
> + outstring,
> "the vf RX max queue number is not set successfully")
> +        self.verify("Max possible TX queues: %d" % max_txqn in 
> + outstring,
> "the vf TX max queue number is not set successfully")
> +        self.verify("Current number of RX queues: %d" % cur_rxqn in
> outstring, "the vf RX queue number is not set successfully")
> +        self.verify("Current number of TX queues: %d" % cur_txqn in
> outstring, "the vf TX queue number is not set successfully")
> +
> +    def test_set_valid_vf_max_qn(self):
> +        """
> +        try to set all the valid queue number of VF
> +        """
> +        self.setup_env(2)
> +        max_qn = (1, 2, 4, 8, 16)
> +        # set all the valid queue number per VF
> +        for i in max_qn:
> +            # start testpmd on pf
> +            self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=%d --file-prefix=test1 --socket-mem 1024,1024" % 
> (self.pf_pci, i))
> +            # start testpmd on vf0
> +            self.session_secondary.send_expect("./%s/app/testpmd -c 
> + 0xf0
> -n 4 -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=%d 
> -- txq=%d" % (self.target, self.sriov_vfs_port[0].pci, i, i), 
> "testpmd>", 120)
> +            outstring = self.session_secondary.send_expect("show port
> info all", "testpmd> ", 120)
> +            self.verify_result(outstring, max_rxqn=i, max_txqn=i,
> cur_rxqn=i, cur_txqn=i)
> +            self.session_secondary.send_expect("quit", "# ")
> +            time.sleep(2)
> +            self.dut.send_expect("quit", "# ")
> +            time.sleep(2)
> +
> +    def test_set_invalid_vf_max_qn(self):
> +        """
> +        try to set several invalid queue number of VF
> +        """
> +        self.setup_env(2)
> +        max_qn = (0, 6, 17, 32)
> +        # set several invalid queue number per vf
> +        for i in max_qn:
> +            # start testpmd on pf
> +            out = self.pmdout.start_testpmd("%s" % self.cores,
> eal_param="-w %s,queue-num-per-vf=%d --file-prefix=test1 --socket-mem 
> 1024,1024" % (self.pf_pci, i))
> +            self.verify("i40e_pf_parse_vf_queue_number_handler(): 
> + Wrong
> VF queue number = %d, it must be power of 2 and equal or less than 16 
> !, Now it is kept the value = 4" % i in out, "the setting of invalid 
> vf max queue number failed.")

Check message is too long here, and this kind of error message will keep changing between releases.
You can use simplified string like "Wrong VF queue number = 6" to check this error. 

> +            # start testpmd on vf0
> +            self.session_secondary.send_expect("./%s/app/testpmd -c 
> + 0xf0
> -n 4 -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i" % 
> (self.target, self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +            outstring = self.session_secondary.send_expect("show port
> info all", "testpmd> ", 120)
> +            self.verify_result(outstring, max_rxqn=4, max_txqn=4,
> cur_rxqn=1, cur_txqn=1)
> +            self.session_secondary.send_expect("quit", "# ")
> +            time.sleep(2)
> +            self.dut.send_expect("quit", "# ")
> +            time.sleep(2)
> +
> +    def test_set_vf_qn(self):
> +        """
> +        set vf queue number with testpmd eal parameters.
> +        """
> +        self.setup_env(2)
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 
> + -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=9 
> --txq=9" % (self.target, self.sriov_vfs_port[0].pci), "# ", 120)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 
> + -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=3 
> --txq=3" % (self.target, self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +        outstring = self.session_secondary.send_expect("show port 
> + info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=3,
> cur_txqn=3)
> +
> +    def test_set_vf_qn_in_testpmd(self):
> +        """
> +        set vf queue number with testpmd command.
> +        """
> +        self.setup_env(2)
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 
> + -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i" % 
> (self.target, self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +        outstring = self.session_secondary.send_expect("show port 
> + info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=1,
> cur_txqn=1)
> +        self.session_secondary.send_expect("port stop all", "testpmd> ")
> +        self.session_secondary.send_expect("port config all rxq 8",
> "testpmd> ")
> +        self.session_secondary.send_expect("port config all txq 8",
> "testpmd> ")
> +        self.session_secondary.send_expect("port start all", "testpmd> ")
> +        outstring = self.session_secondary.send_expect("show port 
> + info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=8,
> cur_txqn=8)
> +        self.session_secondary.send_expect("port stop all", "testpmd> ")
> +        self.session_secondary.send_expect("port config all rxq 9",
> "Fail")
> +        self.session_secondary.send_expect("port config all txq 9",
> "Fail")
> +        self.session_secondary.send_expect("port start all", "testpmd> ")
> +        outstring = self.session_secondary.send_expect("show port 
> + info
> all", "testpmd> ", 120)	
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=8,
> cur_txqn=8)
> +
> +    def test_set_32vfs_1pf(self):
> +        """
> +        set max queue number when setting 32 VFs on 1 PF port.
> +        """
> +        self.setup_env(32)
> +        # failed to set VF max queue num to 16.
> +        out = self.pmdout.start_testpmd("%s" % self.cores, 
> + eal_param="-
> w %s,queue-num-per-vf=16 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.verify("Failed to allocate 577 queues, which exceeds the
> hardware maximum 384" in out, "the queue num exceeds the hardware 
> maximum
> 384")

The number 577 may be changed as default reserved queue value may be changed. Suggest to user simple check string. 

> +        self.dut.send_expect("quit", "# ")
> +        # succeed in setting VF max queue num to 8
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        # start testpmd on vf0
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 
> + -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=%d 
> --txq=%d" % (self.target, self.sriov_vfs_port[0].pci, 8, 8), 
> "testpmd>", 120)
> +        # start testpmd on vf31
> +        self.session_third.send_expect("./%s/app/testpmd -c 0xf00 -n 
> + 4 -
> w %s --file-prefix=test3 --socket-mem 1024,1024 -- -i" % (self.target, 
> self.sriov_vfs_port[31].pci), "testpmd>", 120)
> +        # check the max queue number and current queue number
> +        outstring = self.session_secondary.send_expect("show port 
> + info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=8,
> cur_txqn=8)
> +        outstring = self.session_third.send_expect("show port info 
> + all",
> "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, 
> + cur_rxqn=1,
> cur_txqn=1)
> +
> +    def test_vf_bound_to_kerneldriver(self):
> +        """
> +        set vf queue number when vf bound to kernel driver.
> +        """
> +        self.setup_env(1)
> +        # bind vf to kernel driver
> +        for port in self.sriov_vfs_port:
> +            port.bind_driver(driver="i40evf")
> +        # set vf max queue number to 8
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        # get the vf interface
> +        out = self.session_secondary.send_expect("./usertools/dpdk-
> devbind.py -s", "# ")
> +        lines = out.split("\r\n")
> +        for line in lines:
> +            line = line.strip()
> +            if len(line) != 0 and
> line.startswith(self.sriov_vfs_port[0].pci):
> +                for item in line.split():
> +                    item = item.strip()
> +                    if(item.startswith("if")):
> +                        name, vf_intf = item.split("=", 1)

Do not need to call tool in dpdk source folder, you can method get_interface_name in NIC object to retrieve interface name.

> +        # list all the vf rx/tx queues
> +        outstring = self.session_secondary.send_expect("ethtool -S 
> + %s" %
> vf_intf, "# ", 120)
> +        self.verify("rx-7.packets" in outstring, "the vf RX max queue
> number is not set successfully")
> +        self.verify("tx-7.packets" in outstring, "the vf TX max queue
> number is not set successfully")
> +        self.dut.send_expect("quit", "# ")
> +        for port in self.sriov_vfs_port:
> +            port.bind_driver(driver="vfio-pci")
> +
> +    def tear_down(self):
> +        """
> +        Run after each test case.
> +        """
> +        self.destroy_env()
> +        self.dut.kill_all()
> +
> +    def tear_down_all(self):
> +        """
> +        Run after each test suite.
> +        """
> +        self.dut.kill_all()
> +        self.dut.close_session(self.session_secondary)
> +        self.dut.close_session(self.session_third)
> --
> 2.5.0