/* Firmware should respond within 1 milliseconds */
#define FIRMWARE_TIMEOUT (1 * NSEC_PER_MSEC)
+ ++++/*
+ ++++ * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action.
+ ++++ */
+ ++++static int acpi5;
+ ++++
+ ++++struct set_error_type_with_address {
+ ++++ u32 type;
+ ++++ u32 vendor_extension;
+ ++++ u32 flags;
+ ++++ u32 apicid;
+ ++++ u64 memory_address;
+ ++++ u64 memory_address_range;
+ ++++ u32 pcie_sbdf;
+ ++++};
+ ++++enum {
+ ++++ SETWA_FLAGS_APICID = 1,
+ ++++ SETWA_FLAGS_MEM = 2,
+ ++++ SETWA_FLAGS_PCIE_SBDF = 4,
+ ++++};
+ ++++
+ ++++/*
+ ++++ * Vendor extensions for platform specific operations
+ ++++ */
+ ++++struct vendor_error_type_extension {
+ ++++ u32 length;
+ ++++ u32 pcie_sbdf;
+ ++++ u16 vendor_id;
+ ++++ u16 device_id;
+ ++++ u8 rev_id;
+ ++++ u8 reserved[3];
+ ++++};
+ ++++
+ ++++static u32 vendor_flags;
+ ++++static struct debugfs_blob_wrapper vendor_blob;
+ ++++static char vendor_dev[64];
+ ++++
/*
* Some BIOSes allow parameters to the SET_ERROR_TYPE entries in the
* EINJ table through an unpublished extension. Use with caution as
*/
static DEFINE_MUTEX(einj_mutex);
- ----static struct einj_parameter *einj_param;
+ ++++static void *einj_param;
+ ++++
+ ++++#ifndef readq
+ ++++static inline __u64 readq(volatile void __iomem *addr)
+ ++++{
+ ++++ return ((__u64)readl(addr+4) << 32) + readl(addr);
+ ++++}
+ ++++#endif
#ifndef writeq
static inline void writeq(__u64 val, volatile void __iomem *addr)
return 0;
}
- ----static u64 einj_get_parameter_address(void)
+ ++++static void check_vendor_extension(u64 paddr,
+ ++++ struct set_error_type_with_address *v5param)
+ ++++{
+ ++++ int offset = readl(&v5param->vendor_extension);
+ ++++ struct vendor_error_type_extension *v;
+ ++++ u32 sbdf;
+ ++++
+ ++++ if (!offset)
+ ++++ return;
+ ++++ v = ioremap(paddr + offset, sizeof(*v));
+ ++++ if (!v)
+ ++++ return;
+ ++++ sbdf = readl(&v->pcie_sbdf);
+ ++++ sprintf(vendor_dev, "%x:%x:%x.%x vendor_id=%x device_id=%x rev_id=%x\n",
+ ++++ sbdf >> 24, (sbdf >> 16) & 0xff,
+ ++++ (sbdf >> 11) & 0x1f, (sbdf >> 8) & 0x7,
+ ++++ readw(&v->vendor_id), readw(&v->device_id),
+ ++++ readb(&v->rev_id));
+ ++++ iounmap(v);
+ ++++}
+ ++++
+ ++++static void *einj_get_parameter_address(void)
{
int i;
- ---- u64 paddr = 0;
+ ++++ u64 paddrv4 = 0, paddrv5 = 0;
struct acpi_whea_header *entry;
entry = EINJ_TAB_ENTRY(einj_tab);
entry->instruction == ACPI_EINJ_WRITE_REGISTER &&
entry->register_region.space_id ==
ACPI_ADR_SPACE_SYSTEM_MEMORY)
- ---- memcpy(&paddr, &entry->register_region.address,
- ---- sizeof(paddr));
+ ++++ memcpy(&paddrv4, &entry->register_region.address,
+ ++++ sizeof(paddrv4));
+ ++++ if (entry->action == ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS &&
+ ++++ entry->instruction == ACPI_EINJ_WRITE_REGISTER &&
+ ++++ entry->register_region.space_id ==
+ ++++ ACPI_ADR_SPACE_SYSTEM_MEMORY)
+ ++++ memcpy(&paddrv5, &entry->register_region.address,
+ ++++ sizeof(paddrv5));
entry++;
}
+ ++++ if (paddrv5) {
+ ++++ struct set_error_type_with_address *v5param;
+ ++++
+ ++++ v5param = ioremap(paddrv5, sizeof(*v5param));
+ ++++ if (v5param) {
+ ++++ acpi5 = 1;
+ ++++ check_vendor_extension(paddrv5, v5param);
+ ++++ return v5param;
+ ++++ }
+ ++++ }
+ ++++ if (paddrv4) {
+ ++++ struct einj_parameter *v4param;
+ ++++
+ ++++ v4param = ioremap(paddrv4, sizeof(*v4param));
+ ++++ if (!v4param)
+ ++++ return 0;
+ ++++ if (readq(&v4param->reserved1) || readq(&v4param->reserved2)) {
+ ++++ iounmap(v4param);
+ ++++ return 0;
+ ++++ }
+ ++++ return v4param;
+ ++++ }
- ---- return paddr;
+ ++++ return 0;
}
/* do sanity check to trigger table */
return 0;
}
+++++static struct acpi_generic_address *einj_get_trigger_parameter_region(
+++++ struct acpi_einj_trigger *trigger_tab, u64 param1, u64 param2)
+++++{
+++++ int i;
+++++ struct acpi_whea_header *entry;
+++++
+++++ entry = (struct acpi_whea_header *)
+++++ ((char *)trigger_tab + sizeof(struct acpi_einj_trigger));
+++++ for (i = 0; i < trigger_tab->entry_count; i++) {
+++++ if (entry->action == ACPI_EINJ_TRIGGER_ERROR &&
+++++ entry->instruction == ACPI_EINJ_WRITE_REGISTER_VALUE &&
+++++ entry->register_region.space_id ==
+++++ ACPI_ADR_SPACE_SYSTEM_MEMORY &&
+++++ (entry->register_region.address & param2) == (param1 & param2))
+++++ return &entry->register_region;
+++++ entry++;
+++++ }
+++++
+++++ return NULL;
+++++}
/* Execute instructions in trigger error action table */
-----static int __einj_error_trigger(u64 trigger_paddr)
+++++static int __einj_error_trigger(u64 trigger_paddr, u32 type,
+++++ u64 param1, u64 param2)
{
struct acpi_einj_trigger *trigger_tab = NULL;
struct apei_exec_context trigger_ctx;
struct resource *r;
u32 table_size;
int rc = -EIO;
+++++ struct acpi_generic_address *trigger_param_region = NULL;
r = request_mem_region(trigger_paddr, sizeof(*trigger_tab),
"APEI EINJ Trigger Table");
if (!r) {
pr_err(EINJ_PFX
----- "Can not request iomem region <%016llx-%016llx> for Trigger table.\n",
+++++ "Can not request [mem %#010llx-%#010llx] for Trigger table\n",
(unsigned long long)trigger_paddr,
----- (unsigned long long)trigger_paddr+sizeof(*trigger_tab));
+++++ (unsigned long long)trigger_paddr +
+++++ sizeof(*trigger_tab) - 1);
goto out;
}
trigger_tab = ioremap_cache(trigger_paddr, sizeof(*trigger_tab));
"APEI EINJ Trigger Table");
if (!r) {
pr_err(EINJ_PFX
-----"Can not request iomem region <%016llx-%016llx> for Trigger Table Entry.\n",
----- (unsigned long long)trigger_paddr+sizeof(*trigger_tab),
----- (unsigned long long)trigger_paddr + table_size);
+++++"Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n",
+++++ (unsigned long long)trigger_paddr + sizeof(*trigger_tab),
+++++ (unsigned long long)trigger_paddr + table_size - 1);
goto out_rel_header;
}
iounmap(trigger_tab);
rc = apei_resources_sub(&trigger_resources, &einj_resources);
if (rc)
goto out_fini;
+++++ /*
+++++ * Some firmware will access target address specified in
+++++ * param1 to trigger the error when injecting memory error.
+++++ * This will cause resource conflict with regular memory. So
+++++ * remove it from trigger table resources.
+++++ */
+++++ if (param_extension && (type & 0x0038) && param2) {
+++++ struct apei_resources addr_resources;
+++++ apei_resources_init(&addr_resources);
+++++ trigger_param_region = einj_get_trigger_parameter_region(
+++++ trigger_tab, param1, param2);
+++++ if (trigger_param_region) {
+++++ rc = apei_resources_add(&addr_resources,
+++++ trigger_param_region->address,
+++++ trigger_param_region->bit_width/8, true);
+++++ if (rc)
+++++ goto out_fini;
+++++ rc = apei_resources_sub(&trigger_resources,
+++++ &addr_resources);
+++++ }
+++++ apei_resources_fini(&addr_resources);
+++++ if (rc)
+++++ goto out_fini;
+++++ }
rc = apei_resources_request(&trigger_resources, "APEI EINJ Trigger");
if (rc)
goto out_fini;
if (rc)
return rc;
apei_exec_ctx_set_input(&ctx, type);
- ---- rc = apei_exec_run(&ctx, ACPI_EINJ_SET_ERROR_TYPE);
- ---- if (rc)
- ---- return rc;
- ---- if (einj_param) {
- ---- writeq(param1, &einj_param->param1);
- ---- writeq(param2, &einj_param->param2);
+ ++++ if (acpi5) {
+ ++++ struct set_error_type_with_address *v5param = einj_param;
+ ++++
+ ++++ writel(type, &v5param->type);
+ ++++ if (type & 0x80000000) {
+ ++++ switch (vendor_flags) {
+ ++++ case SETWA_FLAGS_APICID:
+ ++++ writel(param1, &v5param->apicid);
+ ++++ break;
+ ++++ case SETWA_FLAGS_MEM:
+ ++++ writeq(param1, &v5param->memory_address);
+ ++++ writeq(param2, &v5param->memory_address_range);
+ ++++ break;
+ ++++ case SETWA_FLAGS_PCIE_SBDF:
+ ++++ writel(param1, &v5param->pcie_sbdf);
+ ++++ break;
+ ++++ }
+ ++++ writel(vendor_flags, &v5param->flags);
+ ++++ } else {
+ ++++ switch (type) {
+ ++++ case ACPI_EINJ_PROCESSOR_CORRECTABLE:
+ ++++ case ACPI_EINJ_PROCESSOR_UNCORRECTABLE:
+ ++++ case ACPI_EINJ_PROCESSOR_FATAL:
+ ++++ writel(param1, &v5param->apicid);
+ ++++ writel(SETWA_FLAGS_APICID, &v5param->flags);
+ ++++ break;
+ ++++ case ACPI_EINJ_MEMORY_CORRECTABLE:
+ ++++ case ACPI_EINJ_MEMORY_UNCORRECTABLE:
+ ++++ case ACPI_EINJ_MEMORY_FATAL:
+ ++++ writeq(param1, &v5param->memory_address);
+ ++++ writeq(param2, &v5param->memory_address_range);
+ ++++ writel(SETWA_FLAGS_MEM, &v5param->flags);
+ ++++ break;
+ ++++ case ACPI_EINJ_PCIX_CORRECTABLE:
+ ++++ case ACPI_EINJ_PCIX_UNCORRECTABLE:
+ ++++ case ACPI_EINJ_PCIX_FATAL:
+ ++++ writel(param1, &v5param->pcie_sbdf);
+ ++++ writel(SETWA_FLAGS_PCIE_SBDF, &v5param->flags);
+ ++++ break;
+ ++++ }
+ ++++ }
+ ++++ } else {
+ ++++ rc = apei_exec_run(&ctx, ACPI_EINJ_SET_ERROR_TYPE);
+ ++++ if (rc)
+ ++++ return rc;
+ ++++ if (einj_param) {
+ ++++ struct einj_parameter *v4param = einj_param;
+ ++++ writeq(param1, &v4param->param1);
+ ++++ writeq(param2, &v4param->param2);
+ ++++ }
}
rc = apei_exec_run(&ctx, ACPI_EINJ_EXECUTE_OPERATION);
if (rc)
if (rc)
return rc;
trigger_paddr = apei_exec_ctx_get_output(&ctx);
----- rc = __einj_error_trigger(trigger_paddr);
+++++ rc = __einj_error_trigger(trigger_paddr, type, param1, param2);
if (rc)
return rc;
rc = apei_exec_run_optional(&ctx, ACPI_EINJ_END_OPERATION);
{
int rc;
u32 available_error_type = 0;
+ ++++ u32 tval, vendor;
+ ++++
+ ++++ /*
+ ++++ * Vendor defined types have 0x80000000 bit set, and
+ ++++ * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE
+ ++++ */
+ ++++ vendor = val & 0x80000000;
+ ++++ tval = val & 0x7fffffff;
/* Only one error type can be specified */
- ---- if (val & (val - 1))
- ---- return -EINVAL;
- ---- rc = einj_get_available_error_type(&available_error_type);
- ---- if (rc)
- ---- return rc;
- ---- if (!(val & available_error_type))
+ ++++ if (tval & (tval - 1))
return -EINVAL;
+ ++++ if (!vendor) {
+ ++++ rc = einj_get_available_error_type(&available_error_type);
+ ++++ if (rc)
+ ++++ return rc;
+ ++++ if (!(val & available_error_type))
+ ++++ return -EINVAL;
+ ++++ }
error_type = val;
return 0;
static int __init einj_init(void)
{
int rc;
- ---- u64 param_paddr;
acpi_status status;
struct dentry *fentry;
struct apei_exec_context ctx;
status = acpi_get_table(ACPI_SIG_EINJ, 0,
(struct acpi_table_header **)&einj_tab);
----- if (status == AE_NOT_FOUND) {
----- pr_info(EINJ_PFX "Table is not found!\n");
+++++ if (status == AE_NOT_FOUND)
return -ENODEV;
----- } else if (ACPI_FAILURE(status)) {
+++++ else if (ACPI_FAILURE(status)) {
const char *msg = acpi_format_exception(status);
pr_err(EINJ_PFX "Failed to get table, %s\n", msg);
return -EINVAL;
rc = apei_exec_pre_map_gars(&ctx);
if (rc)
goto err_release;
- ---- if (param_extension) {
- ---- param_paddr = einj_get_parameter_address();
- ---- if (param_paddr) {
- ---- einj_param = ioremap(param_paddr, sizeof(*einj_param));
- ---- rc = -ENOMEM;
- ---- if (!einj_param)
- ---- goto err_unmap;
- ---- fentry = debugfs_create_x64("param1", S_IRUSR | S_IWUSR,
- ---- einj_debug_dir, &error_param1);
- ---- if (!fentry)
- ---- goto err_unmap;
- ---- fentry = debugfs_create_x64("param2", S_IRUSR | S_IWUSR,
- ---- einj_debug_dir, &error_param2);
- ---- if (!fentry)
- ---- goto err_unmap;
- ---- } else
- ---- pr_warn(EINJ_PFX "Parameter extension is not supported.\n");
+ ++++
+ ++++ einj_param = einj_get_parameter_address();
+ ++++ if ((param_extension || acpi5) && einj_param) {
+ ++++ fentry = debugfs_create_x64("param1", S_IRUSR | S_IWUSR,
+ ++++ einj_debug_dir, &error_param1);
+ ++++ if (!fentry)
+ ++++ goto err_unmap;
+ ++++ fentry = debugfs_create_x64("param2", S_IRUSR | S_IWUSR,
+ ++++ einj_debug_dir, &error_param2);
+ ++++ if (!fentry)
+ ++++ goto err_unmap;
+ ++++ }
+ ++++
+ ++++ if (vendor_dev[0]) {
+ ++++ vendor_blob.data = vendor_dev;
+ ++++ vendor_blob.size = strlen(vendor_dev);
+ ++++ fentry = debugfs_create_blob("vendor", S_IRUSR,
+ ++++ einj_debug_dir, &vendor_blob);
+ ++++ if (!fentry)
+ ++++ goto err_unmap;
+ ++++ fentry = debugfs_create_x32("vendor_flags", S_IRUSR | S_IWUSR,
+ ++++ einj_debug_dir, &vendor_flags);
+ ++++ if (!fentry)
+ ++++ goto err_unmap;
}
pr_info(EINJ_PFX "Error INJection is initialized.\n");
static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */
static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
-static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state);
+static int intel_idle(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int index);
static struct cpuidle_state *cpuidle_state_table;
{ /* MWAIT C1 */
.name = "C1-NHM",
.desc = "MWAIT 0x00",
- .driver_data = (void *) 0x00,
.flags = CPUIDLE_FLAG_TIME_VALID,
.exit_latency = 3,
.target_residency = 6,
{ /* MWAIT C2 */
.name = "C3-NHM",
.desc = "MWAIT 0x10",
- .driver_data = (void *) 0x10,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 20,
.target_residency = 80,
{ /* MWAIT C3 */
.name = "C6-NHM",
.desc = "MWAIT 0x20",
- .driver_data = (void *) 0x20,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 200,
.target_residency = 800,
{ /* MWAIT C1 */
.name = "C1-SNB",
.desc = "MWAIT 0x00",
- .driver_data = (void *) 0x00,
.flags = CPUIDLE_FLAG_TIME_VALID,
.exit_latency = 1,
.target_residency = 1,
{ /* MWAIT C2 */
.name = "C3-SNB",
.desc = "MWAIT 0x10",
- .driver_data = (void *) 0x10,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 80,
.target_residency = 211,
{ /* MWAIT C3 */
.name = "C6-SNB",
.desc = "MWAIT 0x20",
- .driver_data = (void *) 0x20,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 104,
.target_residency = 345,
{ /* MWAIT C4 */
.name = "C7-SNB",
.desc = "MWAIT 0x30",
- .driver_data = (void *) 0x30,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 109,
.target_residency = 345,
{ /* MWAIT C1 */
.name = "C1-ATM",
.desc = "MWAIT 0x00",
- .driver_data = (void *) 0x00,
.flags = CPUIDLE_FLAG_TIME_VALID,
.exit_latency = 1,
.target_residency = 4,
{ /* MWAIT C2 */
.name = "C2-ATM",
.desc = "MWAIT 0x10",
- .driver_data = (void *) 0x10,
.flags = CPUIDLE_FLAG_TIME_VALID,
.exit_latency = 20,
.target_residency = 80,
{ /* MWAIT C4 */
.name = "C4-ATM",
.desc = "MWAIT 0x30",
- .driver_data = (void *) 0x30,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 100,
.target_residency = 400,
{ /* MWAIT C6 */
.name = "C6-ATM",
.desc = "MWAIT 0x52",
- .driver_data = (void *) 0x52,
.flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
.exit_latency = 140,
.target_residency = 560,
.enter = &intel_idle },
};
-- -- static int get_driver_data(int cstate)
++ +++static long get_driver_data(int cstate)
+{
+ int driver_data;
+ switch (cstate) {
+
+ case 1: /* MWAIT C1 */
+ driver_data = 0x00;
+ break;
+ case 2: /* MWAIT C2 */
+ driver_data = 0x10;
+ break;
+ case 3: /* MWAIT C3 */
+ driver_data = 0x20;
+ break;
+ case 4: /* MWAIT C4 */
+ driver_data = 0x30;
+ break;
+ case 5: /* MWAIT C5 */
+ driver_data = 0x40;
+ break;
+ case 6: /* MWAIT C6 */
+ driver_data = 0x52;
+ break;
+ default:
+ driver_data = 0x00;
+ }
+ return driver_data;
+}
+
/**
* intel_idle
* @dev: cpuidle_device
- * @state: cpuidle state
+ * @drv: cpuidle driver
+ * @index: index of cpuidle state
*
++ +++ * Must be called under local_irq_disable().
*/
-static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state)
+static int intel_idle(struct cpuidle_device *dev,
+ struct cpuidle_driver *drv, int index)
{
unsigned long ecx = 1; /* break on interrupt flag */
- unsigned long eax = (unsigned long)cpuidle_get_statedata(state);
+ struct cpuidle_state *state = &drv->states[index];
+ struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
+ unsigned long eax = (unsigned long)cpuidle_get_statedata(state_usage);
unsigned int cstate;
ktime_t kt_before, kt_after;
s64 usec_delta;
cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
-- --- local_irq_disable();
-- ---
/*
* leave_mm() to avoid costly and often unnecessary wakeups
* for flushing the user TLB's associated with the active mm.
if (!(lapic_timer_reliable_states & (1 << (cstate))))
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
- return usec_delta;
+ /* Update cpuidle counters */
+ dev->last_residency = (int)usec_delta;
+
+ return index;
}
static void __setup_broadcast_timer(void *arg)
cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
-- --- !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
++ +++ !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
++ +++ !mwait_substates)
return -ENODEV;
pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */
lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
else {
-- --- smp_call_function(__setup_broadcast_timer, (void *)true, 1);
++ +++ on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
register_cpu_notifier(&setup_broadcast_notifier);
}
return;
}
/*
- * intel_idle_cpuidle_devices_init()
+ * intel_idle_cpuidle_driver_init()
+ * allocate, initialize cpuidle_states
+ */
+static int intel_idle_cpuidle_driver_init(void)
+{
+ int cstate;
+ struct cpuidle_driver *drv = &intel_idle_driver;
+
+ drv->state_count = 1;
+
+ for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
+ int num_substates;
+
+ if (cstate > max_cstate) {
+ printk(PREFIX "max_cstate %d reached\n",
+ max_cstate);
+ break;
+ }
+
+ /* does the state exist in CPUID.MWAIT? */
+ num_substates = (mwait_substates >> ((cstate) * 4))
+ & MWAIT_SUBSTATE_MASK;
+ if (num_substates == 0)
+ continue;
+ /* is the state not enabled? */
+ if (cpuidle_state_table[cstate].enter == NULL) {
+ /* does the driver not know about the state? */
+ if (*cpuidle_state_table[cstate].name == '\0')
+ pr_debug(PREFIX "unaware of model 0x%x"
+ " MWAIT %d please"
+ " contact lenb@kernel.org",
+ boot_cpu_data.x86_model, cstate);
+ continue;
+ }
+
+ if ((cstate > 2) &&
+ !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
+ mark_tsc_unstable("TSC halts in idle"
+ " states deeper than C2");
+
+ drv->states[drv->state_count] = /* structure copy */
+ cpuidle_state_table[cstate];
+
+ drv->state_count += 1;
+ }
+
+ if (auto_demotion_disable_flags)
-- -- smp_call_function(auto_demotion_disable, NULL, 1);
++ +++ on_each_cpu(auto_demotion_disable, NULL, 1);
+
+ return 0;
+}
+
+
+/*
--- - * intel_idle_cpuidle_devices_init()
+++ ++ * intel_idle_cpu_init()
* allocate, initialize, register cpuidle_devices
+++ ++ * @cpu: cpu/core to initialize
*/
--- --static int intel_idle_cpuidle_devices_init(void)
+++ ++int intel_idle_cpu_init(int cpu)
{
--- -- int i, cstate;
+++ ++ int cstate;
struct cpuidle_device *dev;
--- -- intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
--- -- if (intel_idle_cpuidle_devices == NULL)
--- -- return -ENOMEM;
--- -
--- - for_each_online_cpu(i) {
--- - dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
+++ ++ dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
--- - dev->state_count = 1;
- for_each_online_cpu(i) {
- dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
+++ ++ dev->state_count = 1;
- dev->state_count = 1;
-
--- -- for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
--- -- int num_substates;
-
- if (cstate > max_cstate) {
- printk(PREFIX "max_cstate %d reached\n",
- max_cstate);
- break;
- }
-
- /* does the state exist in CPUID.MWAIT? */
- num_substates = (mwait_substates >> ((cstate) * 4))
- & MWAIT_SUBSTATE_MASK;
- if (num_substates == 0)
- continue;
- /* is the state not enabled? */
- if (cpuidle_state_table[cstate].enter == NULL) {
- /* does the driver not know about the state? */
- if (*cpuidle_state_table[cstate].name == '\0')
- pr_debug(PREFIX "unaware of model 0x%x"
- " MWAIT %d please"
- " contact lenb@kernel.org",
- boot_cpu_data.x86_model, cstate);
- continue;
- }
-
- if ((cstate > 2) &&
- !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
- mark_tsc_unstable("TSC halts in idle"
- " states deeper than C2");
+++ ++ for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
+++ ++ int num_substates;
--- - if (cstate > max_cstate) {
--- - printk(PREFIX "max_cstate %d reached\n",
--- - max_cstate);
--- - break;
--- - }
- dev->states[dev->state_count] = /* structure copy */
- cpuidle_state_table[cstate];
+++ ++ if (cstate > max_cstate) {
+++ ++ printk(PREFIX "max_cstate %d reached\n",
+++ ++ max_cstate);
+++ ++ break;
+++ ++ }
+
--- - /* does the state exist in CPUID.MWAIT? */
--- - num_substates = (mwait_substates >> ((cstate) * 4))
--- - & MWAIT_SUBSTATE_MASK;
--- - if (num_substates == 0)
--- - continue;
--- - /* is the state not enabled? */
--- - if (cpuidle_state_table[cstate].enter == NULL) {
--- - continue;
--- - }
+++ ++ /* does the state exist in CPUID.MWAIT? */
+++ ++ num_substates = (mwait_substates >> ((cstate) * 4))
+++ ++ & MWAIT_SUBSTATE_MASK;
+++ ++ if (num_substates == 0)
+++ ++ continue;
+++ ++ /* is the state not enabled? */
+++ ++ if (cpuidle_state_table[cstate].enter == NULL)
+++ ++ continue;
+
--- - dev->states_usage[dev->state_count].driver_data =
--- - (void *)get_driver_data(cstate);
+++ ++ dev->states_usage[dev->state_count].driver_data =
+++ ++ (void *)get_driver_data(cstate);
dev->state_count += 1;
}
+++ ++ dev->cpu = cpu;
--- -- dev->cpu = i;
--- -- if (cpuidle_register_device(dev)) {
--- -- pr_debug(PREFIX "cpuidle_register_device %d failed!\n",
--- -- i);
--- -- intel_idle_cpuidle_devices_uninit();
--- -- return -EIO;
--- -- }
+++ ++ if (cpuidle_register_device(dev)) {
+++ ++ pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu);
+++ ++ intel_idle_cpuidle_devices_uninit();
+++ ++ return -EIO;
}
- smp_call_function(auto_demotion_disable, NULL, 1);
+
+++ + if (auto_demotion_disable_flags)
+++ ++ smp_call_function_single(cpu, auto_demotion_disable, NULL, 1);
+++ +
return 0;
}
static int __init intel_idle_init(void)
{
--- -- int retval;
+++ ++ int retval, i;
/* Do not load intel_idle at all for now if idle= is passed */
if (boot_option_idle_override != IDLE_NO_OVERRIDE)
if (retval)
return retval;
+ intel_idle_cpuidle_driver_init();
retval = cpuidle_register_driver(&intel_idle_driver);
if (retval) {
printk(KERN_DEBUG PREFIX "intel_idle yielding to %s",
return retval;
}
--- -- retval = intel_idle_cpuidle_devices_init();
--- -- if (retval) {
--- -- cpuidle_unregister_driver(&intel_idle_driver);
--- -- return retval;
+++ ++ intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
+++ ++ if (intel_idle_cpuidle_devices == NULL)
+++ ++ return -ENOMEM;
+++ ++
+++ ++ for_each_online_cpu(i) {
+++ ++ retval = intel_idle_cpu_init(i);
+++ ++ if (retval) {
+++ ++ cpuidle_unregister_driver(&intel_idle_driver);
+++ ++ return retval;
+++ ++ }
}
return 0;
cpuidle_unregister_driver(&intel_idle_driver);
if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) {
-- --- smp_call_function(__setup_broadcast_timer, (void *)false, 1);
++ +++ on_each_cpu(__setup_broadcast_timer, (void *)false, 1);
unregister_cpu_notifier(&setup_broadcast_notifier);
}
void dump_cnt(struct counters *cnt)
{
- fprintf(stderr, "package: %d ", cnt->pkg);
- fprintf(stderr, "core:: %d ", cnt->core);
- fprintf(stderr, "CPU: %d ", cnt->cpu);
- fprintf(stderr, "TSC: %016llX\n", cnt->tsc);
- fprintf(stderr, "c3: %016llX\n", cnt->c3);
- fprintf(stderr, "c6: %016llX\n", cnt->c6);
- fprintf(stderr, "c7: %016llX\n", cnt->c7);
- fprintf(stderr, "aperf: %016llX\n", cnt->aperf);
- fprintf(stderr, "pc2: %016llX\n", cnt->pc2);
- fprintf(stderr, "pc3: %016llX\n", cnt->pc3);
- fprintf(stderr, "pc6: %016llX\n", cnt->pc6);
- fprintf(stderr, "pc7: %016llX\n", cnt->pc7);
- fprintf(stderr, "msr0x%x: %016llX\n", extra_msr_offset, cnt->extra_msr);
+ if (!cnt)
+ return;
+ if (cnt->pkg) fprintf(stderr, "package: %d ", cnt->pkg);
+ if (cnt->core) fprintf(stderr, "core:: %d ", cnt->core);
+ if (cnt->cpu) fprintf(stderr, "CPU: %d ", cnt->cpu);
+ if (cnt->tsc) fprintf(stderr, "TSC: %016llX\n", cnt->tsc);
+ if (cnt->c3) fprintf(stderr, "c3: %016llX\n", cnt->c3);
+ if (cnt->c6) fprintf(stderr, "c6: %016llX\n", cnt->c6);
+ if (cnt->c7) fprintf(stderr, "c7: %016llX\n", cnt->c7);
+ if (cnt->aperf) fprintf(stderr, "aperf: %016llX\n", cnt->aperf);
+ if (cnt->pc2) fprintf(stderr, "pc2: %016llX\n", cnt->pc2);
+ if (cnt->pc3) fprintf(stderr, "pc3: %016llX\n", cnt->pc3);
+ if (cnt->pc6) fprintf(stderr, "pc6: %016llX\n", cnt->pc6);
+ if (cnt->pc7) fprintf(stderr, "pc7: %016llX\n", cnt->pc7);
+ if (cnt->extra_msr) fprintf(stderr, "msr0x%x: %016llX\n", extra_msr_offset, cnt->extra_msr);
}
void dump_list(struct counters *cnt)
case 0x2C: /* Westmere EP - Gulftown */
case 0x2A: /* SNB */
case 0x2D: /* SNB Xeon */
+++++ case 0x3A: /* IVB */
+++++ case 0x3D: /* IVB Xeon */
return 1;
case 0x2E: /* Nehalem-EX Xeon - Beckton */
case 0x2F: /* Westmere-EX Xeon - Eagleton */
projects / linux.git / commitdiff