[v1,3/4] examples/power: allow vms to use lcores over 63
Checks
Commit Message
Extending the functionality to allow vms to power manage cores beyond 63.
Signed-off-by: David Hunt <david.hunt@intel.com>
---
examples/vm_power_manager/channel_manager.c | 59 ++++++++-------------
examples/vm_power_manager/channel_manager.h | 30 ++---------
examples/vm_power_manager/channel_monitor.c | 56 +++++++------------
examples/vm_power_manager/vm_power_cli.c | 4 +-
4 files changed, 48 insertions(+), 101 deletions(-)
Comments
On 22-Nov-18 5:02 PM, David Hunt wrote:
> Extending the functionality to allow vms to power manage cores beyond 63.
>
> Signed-off-by: David Hunt <david.hunt@intel.com>
> ---
> examples/vm_power_manager/channel_manager.c | 59 ++++++++-------------
> examples/vm_power_manager/channel_manager.h | 30 ++---------
> examples/vm_power_manager/channel_monitor.c | 56 +++++++------------
> examples/vm_power_manager/vm_power_cli.c | 4 +-
> 4 files changed, 48 insertions(+), 101 deletions(-)
>
> diff --git a/examples/vm_power_manager/channel_manager.c b/examples/vm_power_manager/channel_manager.c
> index 5af4996db..3d493c179 100644
> --- a/examples/vm_power_manager/channel_manager.c
> +++ b/examples/vm_power_manager/channel_manager.c
> @@ -49,7 +49,7 @@ static bool global_hypervisor_available;
> */
> struct virtual_machine_info {
> char name[CHANNEL_MGR_MAX_NAME_LEN];
> - rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
> + uint16_t pcpu_map[CHANNEL_CMDS_MAX_CPUS];
> struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
> char channel_mask[POWER_MGR_MAX_CPUS];
> uint8_t num_channels;
> @@ -79,7 +79,7 @@ update_pcpus_mask(struct virtual_machine_info *vm_info)
> virVcpuInfoPtr cpuinfo;
> unsigned i, j;
> int n_vcpus;
> - uint64_t mask;
> + uint16_t pcpu;
>
> memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
>
> @@ -120,26 +120,23 @@ update_pcpus_mask(struct virtual_machine_info *vm_info)
> vm_info->info.nrVirtCpu = n_vcpus;
> }
> for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
> - mask = 0;
> + pcpu = 0;
> for (j = 0; j < global_n_host_cpus; j++) {
> if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
> - mask |= 1ULL << j;
> + pcpu = j;
Not sure what this does.
Initial code goes through all CPU's, marks all that are usable into mask
(i.e. code implies there can be several), and then sets up this mask.
Now, you're going through all CPU's, store *one* arbitrary CPU index,
and set the map up with that single index.
That doesn't look like an equivalent replacement.
> }
> }
> - rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
> + vm_info->pcpu_map[i] = pcpu;
> }
> return 0;
> }
>
> int
> -set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
> +set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
See patch 1 comments - it would've made this change easier to parse if
set_pcpu was removed there, instead of here.
> {
> unsigned i = 0;
> int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
> struct virtual_machine_info *vm_info;
> - char mask[POWER_MGR_MAX_CPUS];
> -
> - memcpy(mask, core_mask, POWER_MGR_MAX_CPUS);
>
> if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
> RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
> @@ -166,17 +163,16 @@ set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
> return -1;
> }
> memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
> - for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
> - if (mask[i] == 1) {
> - VIR_USE_CPU(global_cpumaps, i);
> - if (i >= global_n_host_cpus) {
> - RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
> - "number of CPUs(%u)\n",
> - i, global_n_host_cpus);
> - return -1;
> - }
> - }
> +
> + VIR_USE_CPU(global_cpumaps, i);
> +
> + if (pcpu >= global_n_host_cpus) {
> + RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
> + "number of CPUs(%u)\n",
> + i, global_n_host_cpus);
> + return -1;
> }
Some comments on what the above code does would have been nice. Why the
check was removed?
> +
Hi Anatoly,
On 10/12/2018 1:06 PM, Burakov, Anatoly wrote:
> On 22-Nov-18 5:02 PM, David Hunt wrote:
>> Extending the functionality to allow vms to power manage cores beyond
>> 63.
>>
>> Signed-off-by: David Hunt <david.hunt@intel.com>
>> ---
>> examples/vm_power_manager/channel_manager.c | 59 ++++++++-------------
>> examples/vm_power_manager/channel_manager.h | 30 ++---------
>> examples/vm_power_manager/channel_monitor.c | 56 +++++++------------
>> examples/vm_power_manager/vm_power_cli.c | 4 +-
>> 4 files changed, 48 insertions(+), 101 deletions(-)
>>
>> diff --git a/examples/vm_power_manager/channel_manager.c
>> b/examples/vm_power_manager/channel_manager.c
>> index 5af4996db..3d493c179 100644
>> --- a/examples/vm_power_manager/channel_manager.c
>> +++ b/examples/vm_power_manager/channel_manager.c
>> @@ -49,7 +49,7 @@ static bool global_hypervisor_available;
>> */
>> struct virtual_machine_info {
>> char name[CHANNEL_MGR_MAX_NAME_LEN];
>> - rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
>> + uint16_t pcpu_map[CHANNEL_CMDS_MAX_CPUS];
>> struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
>> char channel_mask[POWER_MGR_MAX_CPUS];
>> uint8_t num_channels;
>> @@ -79,7 +79,7 @@ update_pcpus_mask(struct virtual_machine_info
>> *vm_info)
>> virVcpuInfoPtr cpuinfo;
>> unsigned i, j;
>> int n_vcpus;
>> - uint64_t mask;
>> + uint16_t pcpu;
>> memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
>> @@ -120,26 +120,23 @@ update_pcpus_mask(struct virtual_machine_info
>> *vm_info)
>> vm_info->info.nrVirtCpu = n_vcpus;
>> }
>> for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
>> - mask = 0;
>> + pcpu = 0;
>> for (j = 0; j < global_n_host_cpus; j++) {
>> if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j)
>> > 0) {
>> - mask |= 1ULL << j;
>> + pcpu = j;
>
> Not sure what this does.
>
> Initial code goes through all CPU's, marks all that are usable into
> mask (i.e. code implies there can be several), and then sets up this
> mask.
>
> Now, you're going through all CPU's, store *one* arbitrary CPU index,
> and set the map up with that single index.
>
> That doesn't look like an equivalent replacement.
>
Good catch. mask was changed to an array, so I need to set element in
the loop, not outside it. Fixed in next rev.
>> }
>> }
>> - rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
>> + vm_info->pcpu_map[i] = pcpu;
>> }
>> return 0;
>> }
>> int
>> -set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
>> +set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
>
> See patch 1 comments - it would've made this change easier to parse if
> set_pcpu was removed there, instead of here.
>
I couldn't change it back there, as the calling functions had not been
updated. Now that they are gone, I can remove it in this patch.
>> {
>> unsigned i = 0;
>> int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
>> struct virtual_machine_info *vm_info;
>> - char mask[POWER_MGR_MAX_CPUS];
>> -
>> - memcpy(mask, core_mask, POWER_MGR_MAX_CPUS);
>> if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
>> RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max
>> allowable(%d)\n",
>> @@ -166,17 +163,16 @@ set_pcpus_mask(char *vm_name, unsigned int
>> vcpu, char *core_mask)
>> return -1;
>> }
>> memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
>> - for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
>> - if (mask[i] == 1) {
>> - VIR_USE_CPU(global_cpumaps, i);
>> - if (i >= global_n_host_cpus) {
>> - RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the
>> available "
>> - "number of CPUs(%u)\n",
>> - i, global_n_host_cpus);
>> - return -1;
>> - }
>> - }
>> +
>> + VIR_USE_CPU(global_cpumaps, i);
>> +
>> + if (pcpu >= global_n_host_cpus) {
>> + RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
>> + "number of CPUs(%u)\n",
>> + i, global_n_host_cpus);
>> + return -1;
>> }
>
> Some comments on what the above code does would have been nice. Why
> the check was removed?
function changed from set_pcpus_mask to the name of what used to be the
wrapper function, and in doing so we changed the mask parameter to a
pcpu (int) parameter, so there's not need for a loop any more. However,
there was an error where I was calling VIR_USE_CPU with the incorrect
value, so that's now fixed. The check was removed because we're no
longer checking for '1's in an array, we are being passed the actual
pcpu to use.
Also, there were an addition of a spinlock around reading the pcpu
config data.
>
>> +
>
@@ -49,7 +49,7 @@ static bool global_hypervisor_available;
*/
struct virtual_machine_info {
char name[CHANNEL_MGR_MAX_NAME_LEN];
- rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
+ uint16_t pcpu_map[CHANNEL_CMDS_MAX_CPUS];
struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
char channel_mask[POWER_MGR_MAX_CPUS];
uint8_t num_channels;
@@ -79,7 +79,7 @@ update_pcpus_mask(struct virtual_machine_info *vm_info)
virVcpuInfoPtr cpuinfo;
unsigned i, j;
int n_vcpus;
- uint64_t mask;
+ uint16_t pcpu;
memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
@@ -120,26 +120,23 @@ update_pcpus_mask(struct virtual_machine_info *vm_info)
vm_info->info.nrVirtCpu = n_vcpus;
}
for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
- mask = 0;
+ pcpu = 0;
for (j = 0; j < global_n_host_cpus; j++) {
if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
- mask |= 1ULL << j;
+ pcpu = j;
}
}
- rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
+ vm_info->pcpu_map[i] = pcpu;
}
return 0;
}
int
-set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
+set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
{
unsigned i = 0;
int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
struct virtual_machine_info *vm_info;
- char mask[POWER_MGR_MAX_CPUS];
-
- memcpy(mask, core_mask, POWER_MGR_MAX_CPUS);
if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
@@ -166,17 +163,16 @@ set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
return -1;
}
memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
- for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
- if (mask[i] == 1) {
- VIR_USE_CPU(global_cpumaps, i);
- if (i >= global_n_host_cpus) {
- RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
- "number of CPUs(%u)\n",
- i, global_n_host_cpus);
- return -1;
- }
- }
+
+ VIR_USE_CPU(global_cpumaps, i);
+
+ if (pcpu >= global_n_host_cpus) {
+ RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
+ "number of CPUs(%u)\n",
+ i, global_n_host_cpus);
+ return -1;
}
+
if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
global_maplen, flags) < 0) {
RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
@@ -184,31 +180,20 @@ set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
vm_info->name);
return -1;
}
- memcpy(&vm_info->pcpu_mask[vcpu], core_mask, POWER_MGR_MAX_CPUS);
- return 0;
-
-}
-
-int
-set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
-{
- char mask[POWER_MGR_MAX_CPUS];
- memset(mask, 0, POWER_MGR_MAX_CPUS);
+ vm_info->pcpu_map[vcpu] = pcpu;
- mask[core_num] = 1;
-
- return set_pcpus_mask(vm_name, vcpu, mask);
+ return 0;
}
-uint64_t
-get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
+uint16_t
+get_pcpu(struct channel_info *chan_info, unsigned int vcpu)
{
struct virtual_machine_info *vm_info =
(struct virtual_machine_info *)chan_info->priv_info;
if (global_hypervisor_available && (vm_info != NULL))
- return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
+ return vm_info->pcpu_map[vcpu];
else
return 0;
}
@@ -776,7 +761,7 @@ get_info_vm(const char *vm_name, struct vm_info *info)
memcpy(info->name, vm_info->name, sizeof(vm_info->name));
for (i = 0; i < info->num_vcpus; i++) {
- info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
+ info->pcpu_map[i] = vm_info->pcpu_map[i];
}
return 0;
}
@@ -827,7 +812,7 @@ add_vm(const char *vm_name)
}
for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
- rte_atomic64_init(&new_domain->pcpu_mask[i]);
+ new_domain->pcpu_map[i] = 0;
}
if (update_pcpus_mask(new_domain) < 0) {
RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
@@ -40,7 +40,6 @@ struct sockaddr_un _sockaddr_un;
#define MAX_CLIENTS 64
#define MAX_VCPUS 20
-
struct libvirt_vm_info {
const char *vm_name;
unsigned int pcpus[MAX_VCPUS];
@@ -82,7 +81,7 @@ struct channel_info {
struct vm_info {
char name[CHANNEL_MGR_MAX_NAME_LEN]; /**< VM name */
enum vm_status status; /**< libvirt status */
- uint64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS]; /**< pCPU mask for each vCPU */
+ uint16_t pcpu_map[CHANNEL_CMDS_MAX_CPUS]; /**< pCPU map to vCPU */
unsigned num_vcpus; /**< number of vCPUS */
struct channel_info channels[CHANNEL_MGR_MAX_CHANNELS]; /**< Array of channel_info */
unsigned num_channels; /**< Number of channels */
@@ -115,8 +114,7 @@ int channel_manager_init(const char *path);
void channel_manager_exit(void);
/**
- * Get the Physical CPU mask for VM lcore channel(vcpu), result is assigned to
- * core_mask.
+ * Get the Physical CPU for VM lcore channel(vcpu).
* It is not thread-safe.
*
* @param chan_info
@@ -130,26 +128,7 @@ void channel_manager_exit(void);
* - 0 on error.
* - >0 on success.
*/
-uint64_t get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu);
-
-/**
- * Set the Physical CPU mask for the specified vCPU.
- * It is not thread-safe.
- *
- * @param name
- * Virtual Machine name to lookup
- *
- * @param vcpu
- * The virtual CPU to set.
- *
- * @param core_mask
- * The core mask of the physical CPU(s) to bind the vCPU
- *
- * @return
- * - 0 on success.
- * - Negative on error.
- */
-int set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask);
+uint16_t get_pcpu(struct channel_info *chan_info, unsigned int vcpu);
/**
* Set the Physical CPU for the specified vCPU.
@@ -168,7 +147,8 @@ int set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask);
* - 0 on success.
* - Negative on error.
*/
-int set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num);
+int set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu);
+
/**
* Add a VM as specified by name to the Channel Manager. The name must
* correspond to a valid libvirt domain name.
@@ -356,7 +356,6 @@ get_pcpu_to_control(struct policy *pol)
/* Convert vcpu to pcpu. */
struct vm_info info;
int pcpu, count;
- uint64_t mask_u64b;
struct core_info *ci;
ci = get_core_info();
@@ -377,13 +376,8 @@ get_pcpu_to_control(struct policy *pol)
*/
get_info_vm(pol->pkt.vm_name, &info);
for (count = 0; count < pol->pkt.num_vcpu; count++) {
- mask_u64b =
- info.pcpu_mask[pol->pkt.vcpu_to_control[count]];
- for (pcpu = 0; mask_u64b;
- mask_u64b &= ~(1ULL << pcpu++)) {
- if ((mask_u64b >> pcpu) & 1)
- pcpu_monitor(pol, ci, pcpu, count);
- }
+ pcpu = info.pcpu_map[pol->pkt.vcpu_to_control[count]];
+ pcpu_monitor(pol, ci, pcpu, count);
}
} else {
/*
@@ -636,8 +630,6 @@ apply_policy(struct policy *pol)
static int
process_request(struct channel_packet *pkt, struct channel_info *chan_info)
{
- uint64_t core_mask;
-
if (chan_info == NULL)
return -1;
@@ -646,41 +638,31 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info)
return -1;
if (pkt->command == CPU_POWER) {
- core_mask = get_pcpus_mask(chan_info, pkt->resource_id);
- if (core_mask == 0) {
- /*
- * Core mask will be 0 in the case where
- * hypervisor is not available so we're working in
- * the host, so use the core as the mask.
- */
- core_mask = 1ULL << pkt->resource_id;
- }
- if (__builtin_popcountll(core_mask) == 1) {
+ unsigned int core_num;
- unsigned core_num = __builtin_ffsll(core_mask) - 1;
+ core_num = get_pcpu(chan_info, pkt->resource_id);
- switch (pkt->unit) {
- case(CPU_POWER_SCALE_MIN):
- power_manager_scale_core_min(core_num);
+ switch (pkt->unit) {
+ case(CPU_POWER_SCALE_MIN):
+ power_manager_scale_core_min(core_num);
break;
- case(CPU_POWER_SCALE_MAX):
- power_manager_scale_core_max(core_num);
+ case(CPU_POWER_SCALE_MAX):
+ power_manager_scale_core_max(core_num);
break;
- case(CPU_POWER_SCALE_DOWN):
- power_manager_scale_core_down(core_num);
+ case(CPU_POWER_SCALE_DOWN):
+ power_manager_scale_core_down(core_num);
break;
- case(CPU_POWER_SCALE_UP):
- power_manager_scale_core_up(core_num);
+ case(CPU_POWER_SCALE_UP):
+ power_manager_scale_core_up(core_num);
break;
- case(CPU_POWER_ENABLE_TURBO):
- power_manager_enable_turbo_core(core_num);
+ case(CPU_POWER_ENABLE_TURBO):
+ power_manager_enable_turbo_core(core_num);
break;
- case(CPU_POWER_DISABLE_TURBO):
- power_manager_disable_turbo_core(core_num);
+ case(CPU_POWER_DISABLE_TURBO):
+ power_manager_disable_turbo_core(core_num);
+ break;
+ default:
break;
- default:
- break;
- }
}
}
@@ -95,8 +95,8 @@ cmd_show_vm_parsed(void *parsed_result, struct cmdline *cl,
}
cmdline_printf(cl, "Virtual CPU(s): %u\n", info.num_vcpus);
for (i = 0; i < info.num_vcpus; i++) {
- cmdline_printf(cl, " [%u]: Physical CPU Mask 0x%"PRIx64"\n", i,
- info.pcpu_mask[i]);
+ cmdline_printf(cl, " [%u]: Physical CPU %d\n", i,
+ info.pcpu_map[i]);
}
}