[dpdk-dev] [PATCH RFC] usertools: add dpdk run script

Wiles, Keith keith.wiles at intel.com
Fri Jul 14 18:19:34 CEST 2017


> On Jul 6, 2017, at 4:37 PM, Keith Wiles <keith.wiles at intel.com> wrote:
> 
> I use a script like this one with pktgen and wanted to see if DPDK
> would be interested in this application.
> 
> The following script adds support for executing applications using
> a configuration file. The configuration file is formatted as a
> python data file to be loaded by the run.py script.
> 
> Inside the configuration 'default.cfg' is the command line arguments
> needed to execute the application. Any number of configuration  files
> are allows for a single application for different execution. Normally
> the configuration file can be in the application or example directory
> or placed in a RTE_SDK/cfg directory.
> 
> The configuration file contains information about setup of the system
> and how to run the application. The run.py script can setup the system
> and/or execute the application with a simple command.
> 
> ./run.py -s default # for setup
> ./run.py default # to execute the application.
> 
> I am not a great Python coder and any suggestions/patches would be great.

No comments on this RFC, should I submit a normal patch?

> 
> Signed-off-by: Keith Wiles <keith.wiles at intel.com>
> ---
> app/test-pmd/default.cfg |  60 ++++++++
> usertools/run.py         | 355 +++++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 415 insertions(+)
> create mode 100644 app/test-pmd/default.cfg
> create mode 100755 usertools/run.py
> 
> diff --git a/app/test-pmd/default.cfg b/app/test-pmd/default.cfg
> new file mode 100644
> index 000000000..3ceabd6bd
> --- /dev/null
> +++ b/app/test-pmd/default.cfg
> @@ -0,0 +1,60 @@
> +#
> +# testpmd -l 8-15 -n 4 -w 05:00.0 -w 05:00.1 -w 06:00.0 -w 06:00.1 -- --rxq=2 --txq=2 -i
> +#
> +description = 'A testpmd default simple configuration'
> +
> +# Setup configuration
> +setup = {
> +	'devices': (
> +		'81:00.0 81:00.1',
> +		'85:00.0 85:00.1',
> +		),
> +
> +	'opts': (
> +		'-b igb_uio',
> +		)
> +	}
> +
> +# Run command and options
> +run = {
> +	'exec': (
> +		'sudo',
> +		'-E'
> +		),
> +
> +	# Application name and use app_path to help locate the app
> +	'app_name': 'testpmd',
> +
> +	# using (sdk) or (target) for specific variables
> +	# add (app_name) of the application
> +	# Each path is tested for the application
> +	'app_path': (
> +		'%(sdk)s/%(target)s/app/%(app_name)s',
> +		'./app/%(target)s/%(app_name)s',
> +		),
> +
> +	'dpdk': (
> +		'-l 8-15',
> +		'-n 4',
> +		'--proc-type auto',
> +		'--log-level 8',
> +		'--socket-mem 2048,2048'
> +		),
> +
> +	'blacklist': (
> +		#'-b 81:00.0 -b 81:00.1',
> +		#'-b 85:00.0 -b 85:00.1',
> +		'-b 81:00.2 -b 81:00.3',
> +		'-b 85:00.2 -b 85:00.3',
> +		'-b 83:00.0'
> +		),
> +
> +	'app': (
> +		'--rxq=2',
> +		'--txq=2',
> +		),
> +
> +	'misc': (
> +		'-i',
> +		)
> +	}
> diff --git a/usertools/run.py b/usertools/run.py
> new file mode 100755
> index 000000000..6ea915435
> --- /dev/null
> +++ b/usertools/run.py
> @@ -0,0 +1,355 @@
> +#! /usr/bin/env python
> +#
> +#	BSD LICENSE
> +#
> +#	Copyright(c) 2017 Intel Corporation. All rights reserved.
> +#	All rights reserved.
> +#
> +#	Redistribution and use in source and binary forms, with or without
> +#	modification, are permitted provided that the following conditions
> +#	are met:
> +#
> +#	  * Redistributions of source code must retain the above copyright
> +#		notice, this list of conditions and the following disclaimer.
> +#	  * Redistributions in binary form must reproduce the above copyright
> +#		notice, this list of conditions and the following disclaimer in
> +#		the documentation and/or other materials provided with the
> +#		distribution.
> +#	  * Neither the name of Intel Corporation nor the names of its
> +#		contributors may be used to endorse or promote products derived
> +#		from this software without specific prior written permission.
> +#
> +#	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +#	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +#	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +#	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +#	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +#	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +#	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +#	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +#	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +#	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +#	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +#
> +
> +import sys
> +import os
> +import getopt
> +import subprocess
> +import glob
> +from os.path import exists, abspath, dirname, basename
> +import imp
> +
> +def usage():
> +	'''Print usage information for the program'''
> +	argv0 = basename(sys.argv[0])
> +	print("""
> +Usage:
> +------
> +  %(argv0)s [options] [config_name]
> +
> +  Where config_name is the one of the defined configuration files if no config
> +  file is listed then the ./default.cfg file will be used. If a cfg directory
> +  is located in the current directory then it will be searched for a match.
> +
> +  The config_name is the name of the file without path and .cfg extention.
> +
> +Options:
> +--------
> +    -h, --help, -u, --usage:
> +	    Display usage information and quit
> +
> +    -l, --list:
> +	    Print a list of known configuration files
> +
> +    -v, --verbose
> +	    Print out more information
> +
> +Examples:
> +---------
> +  To display current list of configuration files:
> +	%(argv0)s --list
> +
> +  To run a config file:
> +	%(argv0)s default
> +
> +  The configuration file name is always suffixed by .cfg as in default.cfg.
> +  The search location of the configuration files is .:./cfg
> +
> +	""" % locals())  # replace items from local variables
> +	sys.exit(0)
> +
> +def err_exit(str):
> +	print(str)
> +	sys.exit(1)
> +
> +def find_file(arg, t):
> +	''' Find the first file matching the arg value '''
> +	fn = arg + '.cfg'
> +	for f in file_list('.', t):
> +		if os.path.basename(f) == fn:
> +			return f
> +	return None
> +
> +def add_run_options(s, arg_list):
> +	''' Append options to arg list '''
> +	if s in cfg.run:
> +		for a in cfg.run[s]:
> +			arg_list.extend(a.split(' '))
> +
> +def add_setup_options(s, arg_list):
> +	''' Append options to arg list '''
> +	if s in cfg.setup:
> +		for a in cfg.setup[s]:
> +			arg_list.extend(a.split(' '))
> +
> +def file_list(d, t):
> +	''' Return list of configuration files '''
> +	fileiter = (os.path.join(root, f)
> +		for root, _, files in os.walk(d)
> +			for f in files)
> +	return (f for f in fileiter if os.path.splitext(f)[1] == t)
> +
> +def load_cfg(fname):
> +	''' Load the configuration or .cfg file as a python data file '''
> +
> +	if not os.path.exists(fname):
> +		err_exit("Config file %s does not exists\n" % fname)
> +
> +	try:
> +		f = open(fname)
> +	except:
> +		err_exit("Error: unable to open file %s\n" % fname)
> +
> +	global cfg
> +	cfg = imp.load_source('cfg', '', f)
> +
> +	f.close()
> +	os.unlink('c')
> +
> +	return cfg
> +
> +def show_configs():
> +	''' Show configuration files '''
> +
> +	print("Configurations:")
> +	print("   %-16s - %s" % ("Name", "Description"))
> +	print("   %-16s   %s" % ("----", "-----------"))
> +
> +	for fname in file_list('.', '.cfg'):
> +		base = os.path.splitext(os.path.basename(fname))[0]
> +
> +		cfg = load_cfg(fname)
> +
> +		if not cfg.description:
> +			cfg.description = ""
> +		print("   %-16s - %s" % (base, cfg.description))
> +
> +		# reset the descriptoin to empty, for next loop/file
> +		cfg.description = ""
> +	sys.exit(0)
> +
> +def run_cfg(cfg_file):
> +	''' Run the configuration in the .cfg file '''
> +
> +	cfg = load_cfg(cfg_file)
> +
> +	args = []
> +	add_run_options('exec', args)
> +
> +	if not 'app_path' in cfg.run:
> +		err_exit("'app_path' variable is missing from cfg.run in config file")
> +
> +	if not 'app_name' in cfg.run:
> +		err_exit("'app_name' variable is missing from cfg.run in config file")
> +
> +	# convert the cfg.run['app_name'] into a global variable used in
> +	# the creation of the applicaiton/path. app_name must be a global variable.
> +	global app_name
> +	app_name = cfg.run['app_name']
> +
> +	# Try all of the different path versions till we find one.
> +	fname = None
> +	for app in cfg.run['app_path']:
> +		fn = app % globals()
> +		if verbose:
> +			print("Trying %s" % fn)
> +		if os.path.exists(fn):
> +			fname = fn
> +			if verbose:
> +				print("Found %s" % fn)
> +			break
> +
> +	if not fname:
> +		err_exit("Error: Unable to locate application %s" % cfg.run['app_name'])
> +
> +	args.extend([fname])
> +
> +	add_run_options('dpdk', args)
> +	add_run_options('blacklist', args)
> +	add_run_options('whitelist', args)
> +	args.extend(["--"])
> +	add_run_options('app', args)
> +	add_run_options('misc', args)
> +
> +	# Convert the args list to a single string with spaces.
> +	str = ""
> +	for a in args:
> +		str = str + "%s " % a
> +	print(str)
> +
> +	# Output the command line
> +	print(args)
> +
> +	subprocess.call(args)
> +
> +	subprocess.call(['stty', 'sane'])
> +
> +def num_sockets(hpath):
> +	''' Count the number of sockets in the system '''
> +
> +	sockets = 0
> +	for i in range(0, 8):
> +		if os.path.exists(hpath % i):
> +			sockets = sockets + 1
> +
> +	return sockets
> +
> +def setup_cfg(cfg_file):
> +	''' Setup the system by adding modules and ports to dpdk control '''
> +
> +	cfg = load_cfg(cfg_file)
> +
> +	print("Setup DPDK to run '%s' application from %s file" %
> +		   (cfg.run['app_name'], cfg_file))
> +
> +	sys_node = '/sys/devices/system/node/node%d/hugepages'
> +	hugepage_path = sys_node + '/hugepages-2048kB/nr_hugepages'
> +
> +	# calculate the number of sockets in the system.
> +	nb_sockets = int(num_sockets(hugepage_path))
> +	if nb_sockets == 0:
> +		nb_sockets = 1
> +
> +	p = subprocess.Popen(['sysctl', '-n', 'vm.nr_hugepages'],
> +						 stdout=subprocess.PIPE)
> +
> +	# split the number of hugepages between the sockets
> +	nb_hugepages = int(p.communicate()[0]) / nb_sockets
> +
> +	if verbose:
> +		print("  Hugepages per socket %d" % nb_hugepages)
> +
> +	if verbose:
> +		print("  modprobe the 'uio' required module")
> +	subprocess.call(['sudo', 'modprobe', "uio"])
> +
> +	if verbose:
> +		print("  Remove igb_uio if already installed")
> +
> +	ret = subprocess.call(['sudo', 'rmmod', 'igb_uio'])
> +	if ret > 0:
> +		print("  Remove of igb_uio, displayed an error ignore it")
> +
> +	igb_uio = ("%s/%s/kmod/igb_uio.ko" % (sdk, target))
> +	if verbose:
> +		print("  insmode the %s module" % igb_uio)
> +	subprocess.call(['sudo', 'insmod', igb_uio])
> +
> +	for i in range(0, nb_sockets):
> +		fn = (hugepage_path % i)
> +		if verbose:
> +			print("  Set %d socket to %d hugepages" % (i, nb_hugepages))
> +		subprocess.call(['sudo', '-E', 'sh', '-c', 'eval',
> +					 'echo %s > %s' % (nb_hugepages, fn)])
> +
> +	# locate the binding tool
> +	if os.path.exists("%s/usertools/dpdk-devbind.py" % sdk):
> +		script = "%s/usertools/dpdk-devbind.py" % sdk
> +	elif os.path.exits("%s/tools/dpdk_nic_bind.py" % sdk):
> +		script = "%s/tools/dpdk_nic_bind.py" % sdk
> +	else:
> +		err_exit("Error: Failed to find dpdk-devbind.py or dpdk_nic_bind.py")
> +
> +	# build up the system command line to be executed
> +	args = []
> +	add_setup_options('exec', args)
> +
> +	args.extend([script])
> +
> +	add_setup_options('opts', args)
> +	add_setup_options('devices', args)
> +
> +	if verbose:
> +		print("  Bind following devices to DPDK:")
> +		for a in cfg.setup['devices']:
> +			print("		%s" % a)
> +
> +	subprocess.call(args)
> +
> +def parse_args():
> +	''' Parse the command arguments '''
> +
> +	global run_flag, verbose
> +
> +	run_flag = True
> +	verbose = False
> +
> +	cfg_file = "./cfg/default.cfg"
> +
> +	if len(sys.argv) <= 1:
> +		print("*** Pick one of the following config files\n")
> +		show_configs()
> +
> +	try:
> +		opts, args = getopt.getopt(sys.argv[1:], "hulsv",
> +				["help", "usage", "list", "setup", "verbose", ])
> +
> +	except getopt.GetoptError as error:
> +		print(str(error))
> +		usage()
> +
> +	for opt, _ in opts:
> +		if opt == "--help" or opt == "-h":
> +			usage()
> +		if opt == "--usage" or opt == "-u":
> +			usage()
> +		if opt == "--list" or opt == "-l":
> +			show_configs()
> +		if opt == "--setup" or opt == "-s":
> +			run_flag = False
> +		if opt == "--verbose" or opt == "-v":
> +			verbose = True
> +
> +	if not args or len(args) > 1:
> +		usage()
> +
> +	fn = find_file(args[0], '.cfg')
> +	if not fn:
> +		print("*** Config file '%s' not found" % args[0])
> +		show_configs()
> +	else:
> +		cfg_file = fn
> +
> +	return cfg_file
> +
> +def main():
> +	'''program main function'''
> +
> +	global sdk, target
> +
> +	sdk = os.getenv('RTE_SDK', os.path.curdir)
> +	if sdk == '':
> +		err_exit("Set RTE_SDK environment variable")
> +
> +	target = os.getenv('RTE_TARGET', 'x86_64-native-linuxapp-gcc')
> +
> +	cfg_file = parse_args()
> +
> +	if run_flag:
> +		run_cfg(cfg_file)
> +	else:
> +		setup_cfg(cfg_file)
> +
> +if __name__ == "__main__":
> +	main()
> -- 
> 2.11.0
> 

Regards,
Keith



More information about the dev mailing list