=~=~=~=~=~=~=~=~=~=~=~= PuTTY log 2023.03.30 16:10:09 =~=~=~=~=~=~=~=~=~=~=~= login as: ndcs_hz ndcs_hz@10.108.218.22's password: Last login: Thu Mar 30 00:53:09 2023 from 10.140.58.2 ]0;ndcs_hz@localhost:~[ndcs_hz@localhost ~]$ su Password: ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# grep Huge /proc/meminfo AnonHugePages: 20480 kB ShmemHugePages: 0 kB FileHugePages: 0 kB HugePages_Total: 4 HugePages_Free: 4 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 1048576 kB Hugetlb: 4194304 kB ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# cat /rproc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-425.13.1.el8_7.x86_64+debug root=/dev/mapper/rl-root ro default_hugepagesz=1G hugepagesz=1G hugepages=4 crashkernel=auto resume=/dev/mapper/rl-swap rd.lvm.lv=rl/root rd.lvm.lv=rl/swap rhgb quiet ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# [root@localhost ndcs_hz]# cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-425.13.1.el8_7.x86_64+debug root=/dev/mapper/rl-root ro default_hugepagesz=1G hugepagesz=1G hugepages=4 crashkernel=auto resume=/dev/mapper/rl-swap rd.lvm.lv=rl/root rd.lvm.lv=rl/swap rhgb quiet ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# lscpi bash: lscpi: command not found ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 64 On-line CPU(s) list: 0-63 Thread(s) per core: 2 Core(s) per socket: 32 Socket(s): 1 NUMA node(s): 1 Vendor ID: GenuineIntel BIOS Vendor ID: Intel(R) Corporation CPU family: 6 Model: 143 Model name: Intel(R) Xeon(R) Gold 6421N BIOS Model name: Intel(R) Xeon(R) Gold 6421N Stepping: 7 CPU MHz: 1800.000 BogoMIPS: 3600.00 Virtualization: VT-x L1d cache: 48K L1i cache: 32K L2 cache: 2048K L3 cache: 61440K NUMA node0 CPU(s): 0-63 Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 invpcid_single intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hfi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr avx512_fp16 flush_l1d arch_capabilities ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# numctl --harewa--help bash: numctl: command not found ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# numactl --help numactl: unrecognized option '--help' usage: numactl [--all | -a] [--interleave= | -i ] [--preferred= | -p ] [--physcpubind= | -C ] [--cpunodebind= | -N ] [--membind= | -m ] [--localalloc | -l] command args ... numactl [--show | -s] numactl [--hardware | -H] numactl [--length | -l ] [--offset | -o ] [--shmmode | -M ] [--strict | -t] [--shmid | -I ] --shm | -S [--shmid | -I ] --file | -f [--huge | -u] [--touch | -T] memory policy | --dump | -d | --dump-nodes | -D memory policy is --interleave | -i, --preferred | -p, --membind | -m, --localalloc | -l is a comma delimited list of node numbers or A-B ranges or all. Instead of a number a node can also be: netdev:DEV the node connected to network device DEV file:PATH the node the block device of path is connected to ip:HOST the node of the network device host routes through block:PATH the node of block device path pci:[seg:]bus:dev[:func] The node of a PCI device is a comma delimited list of cpu numbers or A-B ranges or all all ranges can be inverted with ! all numbers and ranges can be made cpuset-relative with + the old --cpubind argument is deprecated. use --cpunodebind or --physcpubind instead can have g (GB), m (MB) or k (KB) suffixes ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# numactl --helphardware available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 node 0 size: 111583 MB node 0 free: 98592 MB node distances: node 0 0: 10 ]0;ndcs_hz@localhost:/home/ndcs_hz[root@localhost ndcs_hz]# cd ./root ]0;ndcs_hz@localhost:~[root@localhost ~]# cd dpdk/ ]0;ndcs_hz@localhost:~/dpdk[root@localhost dpdk]# ls buid build dpdk-22.11.1.tar dpdk-kmods dpdk-stable-22.11.1 ninja-1.11.0 re2c-1.0.3 re2c-1.0.3.tar.gz v1.11.0.tar.gz ]0;ndcs_hz@localhost:~/dpdk[root@localhost dpdk]# cd dpdk-stable-22.11.1/ ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1[root@localhost dpdk-stable-22.11.1]# ls ABI_VERSION build config doc dts kernel license Makefile meson_options.txt usertools app buildtools devtools drivers examples lib MAINTAINERS meson.build README VERSION ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1[root@localhost dpdk-stable-22.11.1]# meson --cd usertools/ ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ls cpu_layout.py dpdk-devbind.py dpdk-hugepages.py dpdk-pmdinfo.py dpdk-telemetry-client.py dpdk-telemetry.py meson.build ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# lspci -vvv | grep -i eth 01:00.0 Ethernet controller: Intel Corporation I210 Gigabit Backplane Connection (rev 03) 43:00.0 Ethernet controller: Intel Corporation Ethernet Controller E810-XXV for SFP (rev 02) Subsystem: Intel Corporation Ethernet Network Adapter E810-XXV-2 Product Name: Intel(R) Ethernet Network Adapter E810-XXVDA2 [V1] Vendor specific: Intel(R) Ethernet Network Adapter E810-XXVDA2 43:00.1 Ethernet controller: Intel Corporation Ethernet Controller E810-XXV for SFP (rev 02) Subsystem: Intel Corporation Ethernet Network Adapter E810-XXV-2 Product Name: Intel(R) Ethernet Network Adapter E810-XXVDA2 [V1] Vendor specific: Intel(R) Ethernet Network Adapter E810-XXVDA2 ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ifconfig -a enp1s0: flags=4163 mtu 1500 inet 10.108.218.22 netmask 255.255.255.128 broadcast 10.108.218.127 inet6 2a00:8a00:8000:152:de8d:8aff:fefb:ef59 prefixlen 64 scopeid 0x0 inet6 fe80::de8d:8aff:fefb:ef59 prefixlen 64 scopeid 0x20 inet6 2a00:8a00:8000:152::11c8 prefixlen 128 scopeid 0x0 ether dc:8d:8a:fb:ef:59 txqueuelen 1000 (Ethernet) RX packets 573 bytes 56735 (55.4 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 421 bytes 55321 (54.0 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 device memory 0x98100000-9817ffff ens17f0: flags=4163 mtu 1500 ether b4:96:91:ca:d3:e8 txqueuelen 1000 (Ethernet) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 ens17f1: flags=4163 mtu 1500 ether b4:96:91:ca:d3:e9 txqueuelen 1000 (Ethernet) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 lo: flags=73 mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10 loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 usb0: flags=4163 mtu 1500 ether 1e:40:0f:c7:29:74 txqueuelen 1000 (Ethernet) RX packets 10 bytes 1720 (1.6 KiB) RX errors 0 dropped 10 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ethtool -i ens17f0 driver: ice version: 4.18.0-425.13.1.el8_7.x86_64+de firmware-version: 3.10 0x8000ad4a 1.3106.0 expansion-rom-version: bus-info: 0000:43:00.0 supports-statistics: yes supports-test: yes supports-eeprom-access: yes supports-register-dump: yes supports-priv-flags: yes ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ls cpu_layout.py dpdk-devbind.py dpdk-hugepages.py dpdk-pmdinfo.py dpdk-telemetry-client.py dpdk-telemetry.py meson.build ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ./dpdk-devbind.py --status Network devices using kernel driver =================================== 0000:01:00.0 'I210 Gigabit Backplane Connection 1537' if=enp1s0 drv=igb unused= *Active* 0000:43:00.0 'Ethernet Controller E810-XXV for SFP 159b' if=ens17f0 drv=ice unused= 0000:43:00.1 'Ethernet Controller E810-XXV for SFP 159b' if=ens17f1 drv=ice unused= No 'Baseband' devices detected ============================== No 'Crypto' devices detected ============================ DMA devices using kernel driver =============================== 0000:f2:01.0 'Device 0b25' drv=idxd unused= No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== No 'Misc (rawdev)' devices detected =================================== No 'Regex' devices detected =========================== ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ls cpu_layout.py dpdk-devbind.py dpdk-hugepages.py dpdk-pmdinfo.py dpdk-telemetry-client.py dpdk-telemetry.py meson.build ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ./depdk-devbind.py --help usage: dpdk-devbind.py [-h] [-s] [--status-dev {baseband,compress,crypto,dma,event,mempool,misc,net,regex}] [-b DRIVER | -u] [--force] [DEVICE [DEVICE ...]] Utility to bind and unbind devices from Linux kernel positional arguments: DEVICE Device specified as PCI "domain:bus:slot.func" syntax or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may be referred to by interface name. optional arguments: -h, --help show this help message and exit -s, --status Print the current status of all known devices. --status-dev {baseband,compress,crypto,dma,event,mempool,misc,net,regex} Print the status of given device group. -b DRIVER, --bind DRIVER Select the driver to use or "none" to unbind the device -u, --unbind Unbind a device (equivalent to "-b none") --force Override restriction on binding devices in use by Linux" WARNING: This can lead to loss of network connection and should be used with caution. Examples: --------- To display current device status: dpdk-devbind.py --status To display current network device status: dpdk-devbind.py --status-dev net To bind eth1 from the current driver and move to use vfio-pci dpdk-devbind.py --bind=vfio-pci eth1 To unbind 0000:01:00.0 from using any driver dpdk-devbind.py -u 0000:01:00.0 To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver dpdk-devbind.py -b ixgbe 02:00.0 02:00.1 ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# dpdk-devbind.py --bind=vfio-pci ens17f0 Warning: no supported DPDK kernel modules are loaded Error: Driver 'vfio-pci' is not loaded. ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# modprobe vfio-pci ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# modprobe vfio-pcidpdk-devbind.py --bind=vfio-pci ens17f0 Error: bind failed for 0000:43:00.0 - Cannot bind to driver vfio-pci: [Errno 22] Invalid argument Error: unbind failed for 0000:43:00.0 - Cannot open /sys/bus/pci/drivers//unbind: [Errno 13] Permission denied: '/sys/bus/pci/drivers//unbind' ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# dpdk-devbind.py --bind=vfio-pci ens17f01 Error: bind failed for 0000:43:00.1 - Cannot bind to driver vfio-pci: [Errno 22] Invalid argument Error: unbind failed for 0000:43:00.1 - Cannot open /sys/bus/pci/drivers//unbind: [Errno 13] Permission denied: '/sys/bus/pci/drivers//unbind' ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ifconfig -a enp1s0: flags=4163 mtu 1500 inet 10.108.218.22 netmask 255.255.255.128 broadcast 10.108.218.127 inet6 2a00:8a00:8000:152:de8d:8aff:fefb:ef59 prefixlen 64 scopeid 0x0 inet6 fe80::de8d:8aff:fefb:ef59 prefixlen 64 scopeid 0x20 inet6 2a00:8a00:8000:152::11c8 prefixlen 128 scopeid 0x0 ether dc:8d:8a:fb:ef:59 txqueuelen 1000 (Ethernet) RX packets 823 bytes 78157 (76.3 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 588 bytes 81415 (79.5 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 device memory 0x98100000-9817ffff lo: flags=73 mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10 loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 usb0: flags=4163 mtu 1500 ether 1e:40:0f:c7:29:74 txqueuelen 1000 (Ethernet) RX packets 12 bytes 2064 (2.0 KiB) RX errors 0 dropped 12 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ip a 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enp1s0: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether dc:8d:8a:fb:ef:59 brd ff:ff:ff:ff:ff:ff inet 10.108.218.22/25 brd 10.108.218.127 scope global dynamic noprefixroute enp1s0 valid_lft 6824sec preferred_lft 6824sec inet6 2a00:8a00:8000:152::11c8/128 scope global dynamic noprefixroute valid_lft 86027sec preferred_lft 42827sec inet6 2a00:8a00:8000:152:de8d:8aff:fefb:ef59/64 scope global dynamic noprefixroute valid_lft 2591627sec preferred_lft 604427sec inet6 fe80::de8d:8aff:fefb:ef59/64 scope link noprefixroute valid_lft forever preferred_lft forever 5: usb0: mtu 1500 qdisc fq_codel state UNKNOWN group default qlen 1000 link/ether 1e:40:0f:c7:29:74 brd ff:ff:ff:ff:ff:ff ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# ip afconfig -adpdk-devbind.py --bind=vfio-pci ens17f10 modprobe vfio-pcidpdk-devbind.py --bind=vfio-pci ens17f0 ./dpdk-devbind.py --helpls./dpdk-devbind.py --status Network devices using kernel driver =================================== 0000:01:00.0 'I210 Gigabit Backplane Connection 1537' if=enp1s0 drv=igb unused=vfio-pci *Active* Other Network devices ===================== 0000:43:00.0 'Ethernet Controller E810-XXV for SFP 159b' unused=ice,vfio-pci 0000:43:00.1 'Ethernet Controller E810-XXV for SFP 159b' unused=ice,vfio-pci No 'Baseband' devices detected ============================== No 'Crypto' devices detected ============================ DMA devices using kernel driver =============================== 0000:f2:01.0 'Device 0b25' drv=idxd unused=vfio-pci No 'Eventdev' devices detected ============================== No 'Mempool' devices detected ============================= No 'Compress' devices detected ============================== No 'Misc (rawdev)' devices detected =================================== No 'Regex' devices detected =========================== ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/usertools[root@localhost usertools]# cd .. ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1[root@localhost dpdk-stable-22.11.1]# cd build ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/build[root@localhost build]# ls app buildtools config drivers lib meson-logs meson-uninstalled usertools build.ninja compile_commands.json doc examples meson-info meson-private rte_build_config.h ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/build[root@localhost build]# cd app/ ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/build/app[root@localhost app]# eth./dpdk-testpmd -c 0x00000003fff -n 4 -a 43:00.0 -a 43:00.1 --socket-mem=0,1024 -- -i -a --socket-num=0 --burst=64 --txd=1024 --rxd=1024 --mbcache=512 --rxq=2 --txq=1 --nb-cores=1 -i -a --rss-udp --record-core-cycles --record-burst-stats EAL: Detected CPU lcores: 64 EAL: Detected NUMA nodes: 1 EAL: Detected static linkage of DPDK EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'PA' EAL: VFIO support initialized EAL: Not enough memory available on socket 1! Requested: 1024MB, available: 0MB EAL: FATAL: Cannot init memory EAL: Cannot init memory EAL: Error - exiting with code: 1 Cause: Cannot init EAL: Cannot allocate memory ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/build/app[root@localhost app]# ./dpdk-testpmd -l 0-3 -n 4 -- -i --portmask=0x1 --nb-cores=2 EAL: Detected CPU lcores: 64 EAL: Detected NUMA nodes: 1 EAL: Detected static linkage of DPDK EAL: Multi-process socket /var/run/dpdk/rte/mp_socket EAL: Selected IOVA mode 'PA' EAL: VFIO support initialized TELEMETRY: No legacy callbacks, legacy socket not created testpmd: No probed ethernet devices Interactive-mode selected testpmd: create a new mbuf pool : n=171456, size=2176, socket=0 testpmd: preferred mempool ops selected: ring_mp_mc Done testpmd> start tx_first io packet forwarding - ports=0 - cores=0 - streams=0 - NUMA support enabled, MP allocation mode: native io packet forwarding packets/burst=32 nb forwarding cores=2 - nb forwarding ports=0 testpmd> show fwd stats all +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++ RX-packets: 0 RX-dropped: 0 RX-total: 0 TX-packets: 0 TX-dropped: 0 TX-total: 0 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ testpmd> quit Telling cores to stop... Waiting for lcores to finish... +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++ RX-packets: 0 RX-dropped: 0 RX-total: 0 TX-packets: 0 TX-dropped: 0 TX-total: 0 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Done. Bye... ]0;ndcs_hz@localhost:~/dpdk/dpdk-stable-22.11.1/build/app[root@localhost app]#