[dts] [Patch V1] ramework/pktgen.py: Add packet generator framework, and integrate Trex into the framework sent this patch on behalf of chen hongli
Xu, Qian Q
qian.q.xu at intel.com
Fri Sep 22 07:45:07 CEST 2017
+ Hongli/juan in case you didn't receive it.
> -----Original Message-----
> From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of Liu, Yong
> Sent: Friday, September 22, 2017 1:42 PM
> To: Wang, FeiX Y <feix.y.wang at intel.com>; dts at dpdk.org
> Cc: Wang, FeiX Y <feix.y.wang at intel.com>
> Subject: Re: [dts] [Patch V1] ramework/pktgen.py: Add packet generator
> framework, and integrate Trex into the framework sent this patch on behalf of
> chen hongli
>
> Fei,
> Please check with pep8 first, please add description and each function.
> I think this patch is one of packet generator patch set. Please send with patch
> set.
> And also some comments are inline.
>
> Thanks,
> Marvin
>
> > -----Original Message-----
> > From: dts [mailto:dts-bounces at dpdk.org] On Behalf Of wang fei
> > Sent: Wednesday, September 20, 2017 8:09 PM
> > To: dts at dpdk.org
> > Cc: Wang, FeiX Y <feix.y.wang at intel.com>
> > Subject: [dts] [Patch V1] ramework/pktgen.py: Add packet generator
> > framework, and integrate Trex into the framework sent this patch on
> > behalf of chen hongli
> >
> > Signed-off-by: wang fei <feix.y.wang at intel.com>
> > ---
> > framework/pktgen.py | 420
> > ++++++++++++++++++++++++++++++++++++++++++++++++++++
> > 1 file changed, 420 insertions(+)
> > create mode 100644 framework/pktgen.py
> >
> > diff --git a/framework/pktgen.py b/framework/pktgen.py new file mode
> > 100644 index 0000000..03550bd
> > --- /dev/null
> > +++ b/framework/pktgen.py
> > @@ -0,0 +1,420 @@
> > +# BSD LICENSE
> > +#
> > +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
> > +# All rights reserved.
> > +#
> > +# Redistribution and use in source and binary forms, with or without
> > +# modification, are permitted provided that the following conditions
> > +# are met:
> > +#
> > +# * Redistributions of source code must retain the above copyright
> > +# notice, this list of conditions and the following disclaimer.
> > +# * Redistributions in binary form must reproduce the above copyright
> > +# notice, this list of conditions and the following disclaimer in
> > +# the documentation and/or other materials provided with the
> > +# distribution.
> > +# * Neither the name of Intel Corporation nor the names of its
> > +# contributors may be used to endorse or promote products derived
> > +# from this software without specific prior written permission.
> > +#
> > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT #
> > +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
> FOR
> > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> > +COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
> INDIRECT,
> > +INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
> > +(INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE
> GOODS OR
> > +SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
> > +HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN
> CONTRACT,
> > +STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE)
> > +ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF
> ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> > +
> > +import os
> > +import sys
> > +import re
> > +import string
> > +import time
> > +import json
> > +import argparse
> > +import IPy
> > +import logging
> > +
> > +from abc import abstractmethod
> > +#from trex_stl_lib.api import STLClient, STLStream, STLPktBuilder,
> > +Ether,
> > IP, STLTXCont
> > +from config import IxiaConf
> > +from ssh_connection import SSHConnection from settings import
> > +SCAPY2IXIA from logger import getLogger from exception import
> > +VerifyFailure from utils import create_mask from uuid import uuid4
> > +from pickletools import optimize from tester import Tester #from
> > +serializer import Serializer
> > +
> > +FORMAT = '%(message)s'
> > +logging.basicConfig(format=FORMAT)
> > +logger = logging.getLogger('TrexA')
> > +logger.setLevel(logging.INFO)
> > +# change operation directory
> > +cwd = os.getcwd()
> > +sys.path.append(cwd + '/nics')
> > +sys.path.append(cwd + '/framework')
> > +sys.path.append(cwd + '/tests')
> > +sys.path.append(cwd + '/dep')
> > +#sys.path.append("/opt/trex-core-
> > 2.26/scripts/automation/trex_control_plane/stl/trex_stl_lib")
> > +sys.path.insert(0, "/opt/trex-core-2.26/scripts/automation/"+\
> > + "trex_control_plane/stl")
>
>
> Please do not hardcoded directory, this folder should be configured in
> configuration file.
>
> > +#from api import STLClient, STLStream, STLPktBuilder, Ether, IP,
> > STLTXCont
>
> Please remove useless code.
>
> > +from trex_stl_lib.api import *
> > +from crb import Crb
> > +from config import PktgenConf, CrbsConf #from net_device import
> > +GetNicObj
> > +
> > +class PacketGenerator(object):
> > +#class PacketGenerator(Crb):
> > + """
> > + Basic class for packet generator, define basic function for each
> > kinds of
> > + generators
> > + """
> > + def __init__(self, tester):
> > + self.__streams = []
> > + self.tester = tester
> > +
> > + @abstractmethod
> > + def _check_options(self, opts={}):
> > + pass
> > +
> > + @abstractmethod
> > + def prepare_generator(self):
> > + pass
> > +
> > + @abstractmethod
> > + def _prepare_transmission(self, stream_ids=[]):
> > + pass
> > +
> > + @abstractmethod
> > + def _start_transmission(self, stream_ids, delay=50):
> > + pass
> > +
> > + @abstractmethod
> > + def _stop_transmission(self, stream_id):
> > + pass
> > +
> > + @abstractmethod
> > + def _retrieve_port_statistic(self, stream_id):
> > + pass
> > +
> > + def add_stream(self, tx_port, rx_port, pcap_file):
> > + stream_id = None
> > +
> > + stream_id = len(self.__streams)
> > + stream = {'tx_port': tx_port,
> > + 'rx_port': rx_port,
> > + 'pcap_file': pcap_file}
> > + self.__streams.append(stream)
> > +
> > + return stream_id
> > +
> > + def config_stream(self, stream_id=0, opts={}):
> > + if self._check_options(opts) is not True:
> > + self.logger.error("Failed to configure stream[%d]" %
> > stream_id)
> > + return
> > +
> > + stream = self.__streams[stream_id]
> > + stream['options'] = opts
> > +
> > + def measure_throughput(self, stream_ids=[], delay=50):
> > + """
> > + Measure throughput on each tx ports
> > + """
> > +
> > + bps_rx = []
> > + pps_rx = []
> > + self._prepare_transmission(stream_ids=stream_ids)
> > + self._start_transmission(stream_ids)
> > +
> > + time.sleep(delay)
> > + for stream_id in stream_ids:
> > + rxbps_rates, rxpps_rates =
> > self._retrieve_port_statistic(stream_id)
> > +
> > + bps_rx.append(rxbps_rates)
> > + pps_rx.append(rxpps_rates)
>
> Need to separate statistic record and summary, need another loop here.
>
> > + self._stop_transmission(stream_id)
> > + bps_rx_total = self._summary_statistic(bps_rx)
> > + pps_rx_total = self._summary_statistic(pps_rx)
> > +
> > + print "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total,
> > bps_rx_total)
> > +
> > + return bps_rx_total, pps_rx_total
> > +
> > +
> > +
> > + def _summary_statistic(self, array=[]):
> > + """
> > + Summary all values in statistic array
> > + """
> > + summary = 0.000
> > + for value in array:
> > + summary += value
> > +
> > +
> > + return summary
> > +
> > + def _get_stream(self, stream_id):
> > + return self.__streams[stream_id]
> > +
> > + def _get_generator_conf_instance(self, pktgen_type):
> > + conf_inst = PktgenConf(pktgen_type=pktgen_type)
> > + return conf_inst
> > +
> > + @abstractmethod
> > + def quit_generator(self):
> > + pass
> > +
> > +class TrexPacketGenerator(PacketGenerator):
> > + """
> > + Trex packet generator, detail usage can be seen at
> > + https://trex-tgn.cisco.com/trex/doc/trex_manual.html
> > + """
> > + def __init__(self, tester):
> > + self._conn = None
> > + self._ports = []
> > + self._traffic_ports = []
> > + self._transmit_streams = {}
> > + self.trex_app = "scripts/t-rex-64"
> > +
> > + self.conf_inst = self._get_generator_conf_instance("trex")
> > + self.conf = self.conf_inst.load_pktgen_config()
> > + self.options_keys = [ 'rate', 'ip', 'vlan']
> > + self.ip_keys = ['start', 'end','action', 'mask', 'step']
> > + self.vlan_keys = ['start', 'end', 'action', 'step', 'count']
> > + super(TrexPacketGenerator, self).__init__(tester)
> > +
> > + def connect(self):
>
> Need to check whether connection is successful, if not need raise some kind of
> errors.
>
> > + self._conn = STLClient(server=self.conf["server"])
> > + time.sleep(30)
> > + self._conn.connect()
> > + for p in self._conn.get_all_ports():
> > + self._ports.append(p)
> > +
> > + logger.debug(self._ports)
> > +
> > + def disconnect(self):
> > + self._conn.disconnect()
> > +
> > + def _check_options(self, opts={}):
>
> Could you implement the check function based on some kind of dict?
>
> Like:
> {'ip': { {'start': ip_format, 'end': ip_format, 'action': ["inc", "dec"], 'mask':
> mask_format, 'step': ip_format}
>
>
> > + for key in opts:
> > + if key in self.options_keys:
> > + if key == 'ip':
> > + ip = opts['ip']
> > + for ip_key in ip:
> > + if not ip_key in self.ip_keys:
> > + print " %s is invalid ip option" % ip_key
>
> Please use function "self.tester.logger.error" to log error.
>
> > + return False
> > + if key == 'action':
> > + if not ip[key] == 'inc' or not ip[key] ==
> > 'dec':
> > + print " %s is invalid ip action" % ip[key]
> > + return False
> > + elif key == 'vlan':
> > + vlan = opts['vlan']
> > + for vlan_key in vlan:
> > + if not vlan_key in self.vlan_keys:
> > + print " %s is invalid vlan option" % vlan_key
> > + return False
> > + if key == 'action':
> > + if not vlan[key] == 'inc' or not ip[key]
> > + ==
> > 'dec':
> > + print " %s is invalid vlan action" %
> > vlan[key]
> > + return False
> > + else:
> > + print " %s is invalid option" % key
> > + return False
> > + return True
> > +
> > + def create_vm (self, ip_src_range, ip_dst_range, action='inc',
> > step=1):
> > + if not ip_src_range and not ip_dst_range:
> > + return None
> > +
> > + vm = []
> > +
> > + if ip_src_range:
> > + vm += [STLVmFlowVar(name="src", min_value =
> > ip_src_range['start'], max_value = ip_src_range['end'], size = 4, op =
> > action),
> > + STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src")
> > + ]
> > +
> > + if ip_dst_range:
> > + vm += [STLVmFlowVar(name="dst", min_value =
> > ip_dst_range['start'], max_value = ip_dst_range['end'], size = 4, op =
> > action),
> > + STLVmWrFlowVar(fv_name="dst",pkt_offset = "IP.dst")
> > + ]
> > +
> > + vm += [STLVmFixIpv4(offset = "IP")
> > + ]
> > +
> > + return vm
> > +
> > + def prepare_generator(self):
> > + app_param_temp = "-i"
> > +
> > + for key in self.conf:
> > + #key, value = pktgen_conf
> > + if key == 'config_file':
> > + app_param_temp = app_param_temp + " --cfg " +
> > self.conf[key]
> > + elif key == 'core_num':
> > + app_param_temp = app_param_temp + " -c " +
> > + self.conf[key]
> > +
> > + app = self.conf['trex_root_path'] + os.sep + self.trex_app
> > +
> > + cmd = app + " " + app_param_temp
> > +
> > + self.tester.send_expect("cd /opt/trex-core-2.26/scripts", "#", 70)
> > + self.tester.send_expect(cmd, "", 40)
>
> If here is to start trex, need some criteria check for trex.
>
>
> > +
>
> Please do not change working folder, it may cause unexpected behaviors.
>
> > + time.sleep(15)
> > +
> > + self.connect()
> > +
> > + self.tester.send_expect("cd " + cwd, "#", 70)
> > +
> > + def _prepare_transmission(self, stream_ids=[]):
> > + # Create base packet and pad it to size
> > + streams = []
> > + ip_src_range = {}
> > + ip_dst_range = {}
> > + ip_src_range_temp = []
> > + ip_dst_range_temp = []
> > +
> > + # prepare stream configuration
> > + for stream_id in stream_ids:
> > + stream = self._get_stream(stream_id)
> > + tx_port = stream['tx_port']
> > + rx_port = stream['rx_port']
> > + rx_port_name = "port%d" % rx_port
> > + option = stream['options']
> > +
> > + #set rate
> > + rate = option['rate']
> > + ip = option['ip']
> > + mask = ip['mask']
> > + step_temp = ip['step'].split('.')
> > +
> > + #get the subnet range of src and dst ip
> > + if self.conf.has_key("ip_src"):
> > + ip_src = self.conf['ip_src']
> > + ip_src_range_string =
> > IPy.IP(IPy.IP(ip_src).make_net(mask).strNormal()).strNormal(3)
> > + ip_src_range_temp = ip_src_range_string.split('-')
> > + ip_src_range['start'] = ip_src_range_temp[0]
> > + ip_src_range['end'] = ip_src_range_temp[1]
> > +
> > + if self.conf.has_key("ip_dst"):
> > + ip_dst = self.conf['ip_dst']
> > + ip_dst_range_string =
> > IPy.IP(IPy.IP(ip_dst).make_net(mask).strNormal()).strNormal(3)
> > + ip_dst_range_temp = ip_dst_range_string.split('-')
> > + ip_dst_range['start'] = ip_dst_range_temp[0]
> > + ip_dst_range['end'] = ip_dst_range_temp[1]
> > +
> > + pcap_file = stream['pcap_file']
> > +
> > + vm = self.create_vm(ip_src_range, ip_dst_range,
> > action=ip['action'], step=step_temp[3])
> > +
>
> Here need to check whether vm is valid.
>
> > + stl_stream = STLStream(packet = STLPktBuilder(pkt =
> > + pcap_file,
> > vm=vm), mode = STLTXCont(percentage=100))
> > +
> > + self._transmit_streams[stream_id] = stl_stream
> > +
> > + def _start_transmission(self, stream_ids, delay=50):
> > + self._conn.reset(ports=self._ports)
> > + self._conn.clear_stats()
> > + self._conn.set_port_attr(self._ports, promiscuous=True)
> > + duration_int = int(self.conf["duration"])
> > + rate = "100%"
> > + warmup = 15
> > +
> > + if self.conf.has_key("warmup"):
> > + warmup = int(self.conf["warmup"])
> > +
> > + for p in self._ports:
> > + for stream_id in stream_ids:
> > + stream = self._get_stream(stream_id)
> > + if stream["tx_port"] == p:
> > +
> > self._conn.add_streams(self._transmit_streams[stream_id], ports=[p])
> > + rate = stream["options"]["rate"]
> > + self._traffic_ports.append(p)
> > +
> > + if self.conf.has_key("core_mask"):
> > + self._conn.start(ports=self._traffic_ports, mult=rate,
> > duration=warmup, core_mask=self.conf["core_mask"])
> > + self._conn.wait_on_traffic(ports=self._traffic_ports,
> > timeout=warmup+30)
> > + else:
> > + self._conn.start(ports=self._traffic_ports, mult=rate,
> > duration=warmup)
> > + self._conn.wait_on_traffic(ports=self._traffic_ports,
> > timeout=warmup+30)
> > +
> > + self._conn.clear_stats()
> > +
> > + if self.conf.has_key("core_mask"):
> > + self._conn.start(ports=self._traffic_ports, mult=rate,
> > duration=duration_int, core_mask=self.conf["core_mask"])
> > + else:
> > + self._conn.start(ports=self._traffic_ports, mult=rate,
> > duration=duration_int)
> > +
> > + if self._conn.get_warnings():
> > + for warning in self._conn.get_warnings():
> > + logger.warn(warning)
> > +
> > + def _stop_transmission(self, stream_id):
> > + self._conn.stop(ports=self._traffic_ports, rx_delay_ms=5000)
> > +
> > + def _retrieve_port_statistic(self, stream_id):
> > + stats = self._conn.get_stats()
> > + stream = self._get_stream(stream_id)
> > + port_id = stream["rx_port"]
> > + port_stats = stats[port_id]
> > + print "Port %d stats: %s " % (port_id,port_stats)
> > + rate_rx_pkts = port_stats["rx_pps"]
> > + rate_rx_bits = port_stats["rx_bps_L1"]
> > + print "rx_port: %d, rate_rx_pkts: %f, rate_rx_bits:%f " %
> > (port_id,rate_rx_pkts,rate_rx_bits)
> > + return rate_rx_bits, rate_rx_pkts
> > +
> > + def quit_generator(self):
> > + self.disconnect()
> > +
> > +def getPacketGenerator(tester, pktgen_type="trex"):
> > + """
> > + Get packet generator object
> > + """
> > +
> > + if pktgen_type == "dpdk":
> > + return DpdkPacketGenerator(tester)
> > + elif pktgen_type == "ixia":
> > + return IxiaPacketGenerator(tester)
> > + elif pktgen_type == "trex":
> > + return TrexPacketGenerator(tester)
> > +
> > +
> > +if __name__ == "__main__":
> > + # init pktgen stream options
> > + options = {
> > + 'rate' : '100%',
> > + 'ip': {'action': 'inc', 'mask' : '255.255.255.0', 'step':'0.0.0.1'}
> > + }
> > + crbsconf = CrbsConf()
> > + crb = (crbsconf.load_crbs_config())[0]
> > + tester = Tester(crb, None)
> > + # framework initial
> > + trex = getPacketGenerator(tester, pktgen_type="trex")
> > +
> > + conf_inst = trex._get_generator_conf_instance("trex")
> > + conf = conf_inst.load_pktgen_config()
> > + # prepare running environment
> > + trex.prepare_generator()
> > +
> > + #config stream and convert options into pktgen commands
> > + stream_id1 = trex.add_stream(0, 1, conf['pcap_file'])
> > + trex.config_stream(stream_id=stream_id1, opts=options)
> > + stream_id2 = trex.add_stream(1, 0, conf['pcap_file'])
> > + trex.config_stream(stream_id=stream_id2, opts=options)
> > + stream_id3 = trex.add_stream(0, 1, conf['pcap_file'])
> > + trex.config_stream(stream_id=stream_id3, opts=options)
> > + stream_id4 = trex.add_stream(1, 0, conf['pcap_file'])
> > + trex.config_stream(stream_id=stream_id4, opts=options)
> > + #pktgen.prepare_transmission(stream_ids=[stream_id])
> > +
> > trex.measure_throughput(stream_ids=[stream_id1,stream_id2,stream_id3,s
> > trea
> > m_id4], delay=5)
> > + #trex.measure_throughput(stream_ids=[stream_id1,stream_id2], delay=5)
> > + # comeback to framework
> > + trex.quit_generator()
> > --
> > 2.7.4
More information about the dts
mailing list