[dts] [PATCH] tests: add TestSuite_runtime_queue_number.py

[Liu, Yong]

Yuan,
Some comments are inline.

Thanks,
Marvin

> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of Peng Yuan
> Sent: Monday, March 05, 2018 3:17 PM
> To: dts at dpdk.org
> Cc: Peng, Yuan <yuan.peng at intel.com>
> Subject: [dts] [PATCH] tests: add TestSuite_runtime_queue_number.py
> 
> Set the VF max queue number when running the DPDK APP instead of compiling.
> 
> Signed-off-by: Peng Yuan <yuan.peng at intel.com>
> 
> diff --git a/tests/TestSuite_runtime_queue_number.py
> b/tests/TestSuite_runtime_queue_number.py
> new file mode 100644
> index 0000000..11931d1
> --- /dev/null
> +++ b/tests/TestSuite_runtime_queue_number.py
> @@ -0,0 +1,257 @@
> +# BSD LICENSE
> +#
> +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> +# All rights reserved.

Should be "2010-2018". 
> +#
> +# Redistribution and use in source and binary forms, with or without
> +# modification, are permitted provided that the following conditions
> +# are met:
> +#
> +#   * Redistributions of source code must retain the above copyright
> +#     notice, this list of conditions and the following disclaimer.
> +#   * Redistributions in binary form must reproduce the above copyright
> +#     notice, this list of conditions and the following disclaimer in
> +#     the documentation and/or other materials provided with the
> +#     distribution.
> +#   * Neither the name of Intel Corporation nor the names of its
> +#     contributors may be used to endorse or promote products derived
> +#     from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +"""
> +DPDK Test suite.
> +
> +Set the VF max queue number when running the DPDK APP instead of
> compiling.
> +
> +"""
> +
> +import utils
> +import time
> +import re
> +
> +from test_case import TestCase
> +from settings import HEADER_SIZE
> +from pmd_output import PmdOutput
> +from settings import DRIVERS
> +
> +from project_dpdk import DPDKdut
> +from dut import Dut

These two modules are useless in the suite, please remove them.

> +from packet import Packet
> +
> +
> +class TestRuntime_Queue_Number(TestCase):
> +
> +    def set_up_all(self):
> +        """
> +        Run at the start of each test suite.
> +        Run time Queue Number Prerequistites
> +        """
> +        self.verify(self.nic in ["fortville_eagle", "fortville_spirit",
> +                                 "fortville_spirit_single",
> "fortpark_TLV"], "NIC Unsupported: " + str(self.nic))
> +

If this suite only valuable for Fortville NIC, I suggest to add this information in test_case_supportlist.json. 

> +        # Based on h/w type, choose how many ports to use
> +        self.dut_ports = self.dut.get_ports(self.nic)
> +        # Verify that enough ports are available
> +        self.verify(len(self.dut_ports) >= 1, "Insufficient ports")
> +
> +        localPort = self.tester.get_local_port(self.dut_ports[0])
> +        self.tester_intf = self.tester.get_interface(localPort)
> +        self.tester_mac = self.tester.get_mac(localPort)
> +        self.pf_interface = self.dut.ports_info[self.dut_ports[0]]['intf']
> +        self.pf_mac = self.dut.get_mac_address(0)
> +        self.pf_pci = self.dut.ports_info[self.dut_ports[0]]['pci']
> +        self.pmdout = PmdOutput(self.dut)
> +        self.cores = "1S/4C/1T"
> +
> +        self.session_secondary = self.dut.new_session()
> +        self.session_third = self.dut.new_session()
> +
> +    def set_up(self):
> +        """
> +        Run before each test case.
> +        """
> +        self.dut.kill_all()

Kill action should be in tear_down function. If previous suite has incorrect behavior, we should know that. 

> +
> +    def setup_env(self, vfs_num):
> +        """
> +        This is to set up vf environment.
> +        The pf is bound to dpdk driver.
> +        """
> +        # create two vfs
Typo here, should be "assigned number of VFs". 

> +        self.dut.generate_sriov_vfs_by_port(self.dut_ports[0], vfs_num,
> self.drivername)
> +        self.sriov_vfs_port =
> self.dut.ports_info[self.dut_ports[0]]['vfs_port']
> +
> +        try:
> +            for port in self.sriov_vfs_port:
> +                port.bind_driver(driver="vfio-pci")

"vfio-pci" may not insmoded on host. You can make sure that by call function "self.dut.setup_modules_linux()". 

> +        except Exception as e:
> +            self.destroy_env()
> +            raise Exception(e)
> +
> +    def destroy_env(self):
> +        """
> +        This is to stop testpmd and destroy vf environment.
> +        """
> +        self.session_third.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.session_secondary.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.dut.send_expect("quit", "# ")
> +        time.sleep(2)
> +        self.dut.destroy_sriov_vfs_by_port(self.dut_ports[0])
> +
> +    def verify_result(self, outstring, max_rxqn, max_txqn, cur_rxqn,
> cur_txqn):
> +        """
> +        verify the packet to the expected queue or be dropped
> +        """
> +        self.verify("Max possible RX queues: %d" % max_rxqn in outstring,
> "the vf RX max queue number is not set successfully")
> +        self.verify("Max possible TX queues: %d" % max_txqn in outstring,
> "the vf TX max queue number is not set successfully")
> +        self.verify("Current number of RX queues: %d" % cur_rxqn in
> outstring, "the vf RX queue number is not set successfully")
> +        self.verify("Current number of TX queues: %d" % cur_txqn in
> outstring, "the vf TX queue number is not set successfully")
> +
> +    def test_set_valid_vf_max_qn(self):
> +        """
> +        try to set all the valid queue number of VF
> +        """
> +        self.setup_env(2)
> +        max_qn = (1, 2, 4, 8, 16)
> +        # set all the valid queue number per VF
> +        for i in max_qn:
> +            # start testpmd on pf
> +            self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=%d --file-prefix=test1 --socket-mem 1024,1024" %
> (self.pf_pci, i))
> +            # start testpmd on vf0
> +            self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0
> -n 4 -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=%d --
> txq=%d" % (self.target, self.sriov_vfs_port[0].pci, i, i), "testpmd>", 120)
> +            outstring = self.session_secondary.send_expect("show port
> info all", "testpmd> ", 120)
> +            self.verify_result(outstring, max_rxqn=i, max_txqn=i,
> cur_rxqn=i, cur_txqn=i)
> +            self.session_secondary.send_expect("quit", "# ")
> +            time.sleep(2)
> +            self.dut.send_expect("quit", "# ")
> +            time.sleep(2)
> +
> +    def test_set_invalid_vf_max_qn(self):
> +        """
> +        try to set several invalid queue number of VF
> +        """
> +        self.setup_env(2)
> +        max_qn = (0, 6, 17, 32)
> +        # set several invalid queue number per vf
> +        for i in max_qn:
> +            # start testpmd on pf
> +            out = self.pmdout.start_testpmd("%s" % self.cores,
> eal_param="-w %s,queue-num-per-vf=%d --file-prefix=test1 --socket-mem
> 1024,1024" % (self.pf_pci, i))
> +            self.verify("i40e_pf_parse_vf_queue_number_handler(): Wrong
> VF queue number = %d, it must be power of 2 and equal or less than 16 !,
> Now it is kept the value = 4" % i in out, "the setting of invalid vf max
> queue number failed.")

Check message is too long here, and this kind of error message will keep changing between releases.
You can use simplified string like "Wrong VF queue number = 6" to check this error. 

> +            # start testpmd on vf0
> +            self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0
> -n 4 -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i" %
> (self.target, self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +            outstring = self.session_secondary.send_expect("show port
> info all", "testpmd> ", 120)
> +            self.verify_result(outstring, max_rxqn=4, max_txqn=4,
> cur_rxqn=1, cur_txqn=1)
> +            self.session_secondary.send_expect("quit", "# ")
> +            time.sleep(2)
> +            self.dut.send_expect("quit", "# ")
> +            time.sleep(2)
> +
> +    def test_set_vf_qn(self):
> +        """
> +        set vf queue number with testpmd eal parameters.
> +        """
> +        self.setup_env(2)
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=9 --txq=9" %
> (self.target, self.sriov_vfs_port[0].pci), "# ", 120)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=3 --txq=3" %
> (self.target, self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +        outstring = self.session_secondary.send_expect("show port info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=3,
> cur_txqn=3)
> +
> +    def test_set_vf_qn_in_testpmd(self):
> +        """
> +        set vf queue number with testpmd command.
> +        """
> +        self.setup_env(2)
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i" % (self.target,
> self.sriov_vfs_port[0].pci), "testpmd>", 120)
> +        outstring = self.session_secondary.send_expect("show port info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=1,
> cur_txqn=1)
> +        self.session_secondary.send_expect("port stop all", "testpmd> ")
> +        self.session_secondary.send_expect("port config all rxq 8",
> "testpmd> ")
> +        self.session_secondary.send_expect("port config all txq 8",
> "testpmd> ")
> +        self.session_secondary.send_expect("port start all", "testpmd> ")
> +        outstring = self.session_secondary.send_expect("show port info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=8,
> cur_txqn=8)
> +        self.session_secondary.send_expect("port stop all", "testpmd> ")
> +        self.session_secondary.send_expect("port config all rxq 9",
> "Fail")
> +        self.session_secondary.send_expect("port config all txq 9",
> "Fail")
> +        self.session_secondary.send_expect("port start all", "testpmd> ")
> +        outstring = self.session_secondary.send_expect("show port info
> all", "testpmd> ", 120)	
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=8,
> cur_txqn=8)
> +
> +    def test_set_32vfs_1pf(self):
> +        """
> +        set max queue number when setting 32 VFs on 1 PF port.
> +        """
> +        self.setup_env(32)
> +        # failed to set VF max queue num to 16.
> +        out = self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=16 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        self.verify("Failed to allocate 577 queues, which exceeds the
> hardware maximum 384" in out, "the queue num exceeds the hardware maximum
> 384")

The number 577 may be changed as default reserved queue value may be changed. Suggest to user simple check string. 

> +        self.dut.send_expect("quit", "# ")
> +        # succeed in setting VF max queue num to 8
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        # start testpmd on vf0
> +        self.session_secondary.send_expect("./%s/app/testpmd -c 0xf0 -n 4
> -w %s --file-prefix=test2 --socket-mem 1024,1024 -- -i --rxq=%d --txq=%d" %
> (self.target, self.sriov_vfs_port[0].pci, 8, 8), "testpmd>", 120)
> +        # start testpmd on vf31
> +        self.session_third.send_expect("./%s/app/testpmd -c 0xf00 -n 4 -
> w %s --file-prefix=test3 --socket-mem 1024,1024 -- -i" % (self.target,
> self.sriov_vfs_port[31].pci), "testpmd>", 120)
> +        # check the max queue number and current queue number
> +        outstring = self.session_secondary.send_expect("show port info
> all", "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=8,
> cur_txqn=8)
> +        outstring = self.session_third.send_expect("show port info all",
> "testpmd> ", 120)
> +        self.verify_result(outstring, max_rxqn=8, max_txqn=8, cur_rxqn=1,
> cur_txqn=1)
> +
> +    def test_vf_bound_to_kerneldriver(self):
> +        """
> +        set vf queue number when vf bound to kernel driver.
> +        """
> +        self.setup_env(1)
> +        # bind vf to kernel driver
> +        for port in self.sriov_vfs_port:
> +            port.bind_driver(driver="i40evf")
> +        # set vf max queue number to 8
> +        self.pmdout.start_testpmd("%s" % self.cores, eal_param="-
> w %s,queue-num-per-vf=8 --file-prefix=test1 --socket-mem 1024,1024" %
> self.pf_pci)
> +        # get the vf interface
> +        out = self.session_secondary.send_expect("./usertools/dpdk-
> devbind.py -s", "# ")
> +        lines = out.split("\r\n")
> +        for line in lines:
> +            line = line.strip()
> +            if len(line) != 0 and
> line.startswith(self.sriov_vfs_port[0].pci):
> +                for item in line.split():
> +                    item = item.strip()
> +                    if(item.startswith("if")):
> +                        name, vf_intf = item.split("=", 1)

Do not need to call tool in dpdk source folder, you can method get_interface_name in NIC object to retrieve interface name.

> +        # list all the vf rx/tx queues
> +        outstring = self.session_secondary.send_expect("ethtool -S %s" %
> vf_intf, "# ", 120)
> +        self.verify("rx-7.packets" in outstring, "the vf RX max queue
> number is not set successfully")
> +        self.verify("tx-7.packets" in outstring, "the vf TX max queue
> number is not set successfully")
> +        self.dut.send_expect("quit", "# ")
> +        for port in self.sriov_vfs_port:
> +            port.bind_driver(driver="vfio-pci")
> +
> +    def tear_down(self):
> +        """
> +        Run after each test case.
> +        """
> +        self.destroy_env()
> +        self.dut.kill_all()
> +
> +    def tear_down_all(self):
> +        """
> +        Run after each test suite.
> +        """
> +        self.dut.kill_all()
> +        self.dut.close_session(self.session_secondary)
> +        self.dut.close_session(self.session_third)
> --
> 2.5.0