arch/arm64/kernel/cpu_errata.c

   1 /*
   2  * Contains CPU specific errata definitions
   3  *
   4  * Copyright (C) 2014 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/arm-smccc.h>
  20 #include <linux/psci.h>
  21 #include <linux/types.h>
  22 #include <asm/cpu.h>
  23 #include <asm/cputype.h>
  24 #include <asm/cpufeature.h>
  25
  26 static bool __maybe_unused
  27 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  28 {
  29         const struct arm64_midr_revidr *fix;
  30         u32 midr = read_cpuid_id(), revidr;
  31
  32         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  33         if (!is_midr_in_range(midr, &entry->midr_range))
  34                 return false;
  35
  36         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
  37         revidr = read_cpuid(REVIDR_EL1);
  38         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
  39                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
  40                         return false;
  41
  42         return true;
  43 }
  44
  45 static bool __maybe_unused
  46 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
  47                             int scope)
  48 {
  49         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  50         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
  51 }
  52
  53 static bool __maybe_unused
  54 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
  55 {
  56         u32 model;
  57
  58         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  59
  60         model = read_cpuid_id();
  61         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
  62                  MIDR_ARCHITECTURE_MASK;
  63
  64         return model == entry->midr_range.model;
  65 }
  66
  67 static bool
  68 has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
  69                           int scope)
  70 {
  71         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
  72         u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask;
  73         u64 ctr_raw, ctr_real;
  74
  75         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  76
  77         /*
  78          * We want to make sure that all the CPUs in the system expose
  79          * a consistent CTR_EL0 to make sure that applications behaves
  80          * correctly with migration.
  81          *
  82          * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 :
  83          *
  84          * 1) It is safe if the system doesn't support IDC, as CPU anyway
  85          *    reports IDC = 0, consistent with the rest.
  86          *
  87          * 2) If the system has IDC, it is still safe as we trap CTR_EL0
  88          *    access on this CPU via the ARM64_HAS_CACHE_IDC capability.
  89          *
  90          * So, we need to make sure either the raw CTR_EL0 or the effective
  91          * CTR_EL0 matches the system's copy to allow a secondary CPU to boot.
  92          */
  93         ctr_raw = read_cpuid_cachetype() & mask;
  94         ctr_real = read_cpuid_effective_cachetype() & mask;
  95
  96         return (ctr_real != sys) && (ctr_raw != sys);
  97 }
  98
  99 static void
 100 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
 101 {
 102         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
 103
 104         /* Trap CTR_EL0 access on this CPU, only if it has a mismatch */
 105         if ((read_cpuid_cachetype() & mask) !=
 106             (arm64_ftr_reg_ctrel0.sys_val & mask))
 107                 sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
 108 }
 109
 110 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
 111
 112 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 113 #include <asm/mmu_context.h>
 114 #include <asm/cacheflush.h>
 115
 116 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
 117
 118 #ifdef CONFIG_KVM_INDIRECT_VECTORS
 119 extern char __smccc_workaround_1_smc_start[];
 120 extern char __smccc_workaround_1_smc_end[];
 121
 122 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
 123                                 const char *hyp_vecs_end)
 124 {
 125         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
 126         int i;
 127
 128         for (i = 0; i < SZ_2K; i += 0x80)
 129                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
 130
 131         __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
 132 }
 133
 134 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
 135                                     const char *hyp_vecs_start,
 136                                     const char *hyp_vecs_end)
 137 {
 138         static DEFINE_RAW_SPINLOCK(bp_lock);
 139         int cpu, slot = -1;
 140
 141         /*
 142          * enable_smccc_arch_workaround_1() passes NULL for the hyp_vecs
 143          * start/end if we're a guest. Skip the hyp-vectors work.
 144          */
 145         if (!hyp_vecs_start) {
 146                 __this_cpu_write(bp_hardening_data.fn, fn);
 147                 return;
 148         }
 149
 150         raw_spin_lock(&bp_lock);
 151         for_each_possible_cpu(cpu) {
 152                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
 153                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
 154                         break;
 155                 }
 156         }
 157
 158         if (slot == -1) {
 159                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
 160                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
 161                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
 162         }
 163
 164         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
 165         __this_cpu_write(bp_hardening_data.fn, fn);
 166         raw_spin_unlock(&bp_lock);
 167 }
 168 #else
 169 #define __smccc_workaround_1_smc_start          NULL
 170 #define __smccc_workaround_1_smc_end            NULL
 171
 172 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
 173                                       const char *hyp_vecs_start,
 174                                       const char *hyp_vecs_end)
 175 {
 176         __this_cpu_write(bp_hardening_data.fn, fn);
 177 }
 178 #endif  /* CONFIG_KVM_INDIRECT_VECTORS */
 179
 180 #include <uapi/linux/psci.h>
 181 #include <linux/arm-smccc.h>
 182 #include <linux/psci.h>
 183
 184 static void call_smc_arch_workaround_1(void)
 185 {
 186         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 187 }
 188
 189 static void call_hvc_arch_workaround_1(void)
 190 {
 191         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 192 }
 193
 194 static void qcom_link_stack_sanitization(void)
 195 {
 196         u64 tmp;
 197
 198         asm volatile("mov       %0, x30         \n"
 199                      ".rept     16              \n"
 200                      "bl        . + 4           \n"
 201                      ".endr                     \n"
 202                      "mov       x30, %0         \n"
 203                      : "=&r" (tmp));
 204 }
 205
 206 static bool __nospectre_v2;
 207 static int __init parse_nospectre_v2(char *str)
 208 {
 209         __nospectre_v2 = true;
 210         return 0;
 211 }
 212 early_param("nospectre_v2", parse_nospectre_v2);
 213
 214 /*
 215  * -1: No workaround
 216  *  0: No workaround required
 217  *  1: Workaround installed
 218  */
 219 static int detect_harden_bp_fw(void)
 220 {
 221         bp_hardening_cb_t cb;
 222         void *smccc_start, *smccc_end;
 223         struct arm_smccc_res res;
 224         u32 midr = read_cpuid_id();
 225
 226         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
 227                 return -1;
 228
 229         switch (psci_ops.conduit) {
 230         case PSCI_CONDUIT_HVC:
 231                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 232                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 233                 switch ((int)res.a0) {
 234                 case 1:
 235                         /* Firmware says we're just fine */
 236                         return 0;
 237                 case 0:
 238                         cb = call_hvc_arch_workaround_1;
 239                         /* This is a guest, no need to patch KVM vectors */
 240                         smccc_start = NULL;
 241                         smccc_end = NULL;
 242                         break;
 243                 default:
 244                         return -1;
 245                 }
 246                 break;
 247
 248         case PSCI_CONDUIT_SMC:
 249                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 250                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 251                 switch ((int)res.a0) {
 252                 case 1:
 253                         /* Firmware says we're just fine */
 254                         return 0;
 255                 case 0:
 256                         cb = call_smc_arch_workaround_1;
 257                         smccc_start = __smccc_workaround_1_smc_start;
 258                         smccc_end = __smccc_workaround_1_smc_end;
 259                         break;
 260                 default:
 261                         return -1;
 262                 }
 263                 break;
 264
 265         default:
 266                 return -1;
 267         }
 268
 269         if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
 270             ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
 271                 cb = qcom_link_stack_sanitization;
 272
 273         install_bp_hardening_cb(cb, smccc_start, smccc_end);
 274
 275         return 1;
 276 }
 277 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
 278
 279 #ifdef CONFIG_ARM64_SSBD
 280 DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
 281
 282 int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
 283
 284 static const struct ssbd_options {
 285         const char      *str;
 286         int             state;
 287 } ssbd_options[] = {
 288         { "force-on",   ARM64_SSBD_FORCE_ENABLE, },
 289         { "force-off",  ARM64_SSBD_FORCE_DISABLE, },
 290         { "kernel",     ARM64_SSBD_KERNEL, },
 291 };
 292
 293 static int __init ssbd_cfg(char *buf)
 294 {
 295         int i;
 296
 297         if (!buf || !buf[0])
 298                 return -EINVAL;
 299
 300         for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
 301                 int len = strlen(ssbd_options[i].str);
 302
 303                 if (strncmp(buf, ssbd_options[i].str, len))
 304                         continue;
 305
 306                 ssbd_state = ssbd_options[i].state;
 307                 return 0;
 308         }
 309
 310         return -EINVAL;
 311 }
 312 early_param("ssbd", ssbd_cfg);
 313
 314 void __init arm64_update_smccc_conduit(struct alt_instr *alt,
 315                                        __le32 *origptr, __le32 *updptr,
 316                                        int nr_inst)
 317 {
 318         u32 insn;
 319
 320         BUG_ON(nr_inst != 1);
 321
 322         switch (psci_ops.conduit) {
 323         case PSCI_CONDUIT_HVC:
 324                 insn = aarch64_insn_get_hvc_value();
 325                 break;
 326         case PSCI_CONDUIT_SMC:
 327                 insn = aarch64_insn_get_smc_value();
 328                 break;
 329         default:
 330                 return;
 331         }
 332
 333         *updptr = cpu_to_le32(insn);
 334 }
 335
 336 void __init arm64_enable_wa2_handling(struct alt_instr *alt,
 337                                       __le32 *origptr, __le32 *updptr,
 338                                       int nr_inst)
 339 {
 340         BUG_ON(nr_inst != 1);
 341         /*
 342          * Only allow mitigation on EL1 entry/exit and guest
 343          * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
 344          * be flipped.
 345          */
 346         if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
 347                 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
 348 }
 349
 350 void arm64_set_ssbd_mitigation(bool state)
 351 {
 352         if (this_cpu_has_cap(ARM64_SSBS)) {
 353                 if (state)
 354                         asm volatile(SET_PSTATE_SSBS(0));
 355                 else
 356                         asm volatile(SET_PSTATE_SSBS(1));
 357                 return;
 358         }
 359
 360         switch (psci_ops.conduit) {
 361         case PSCI_CONDUIT_HVC:
 362                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
 363                 break;
 364
 365         case PSCI_CONDUIT_SMC:
 366                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
 367                 break;
 368
 369         default:
 370                 WARN_ON_ONCE(1);
 371                 break;
 372         }
 373 }
 374
 375 static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
 376                                     int scope)
 377 {
 378         struct arm_smccc_res res;
 379         bool required = true;
 380         s32 val;
 381
 382         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 383
 384         if (this_cpu_has_cap(ARM64_SSBS)) {
 385                 required = false;
 386                 goto out_printmsg;
 387         }
 388
 389         if (psci_ops.smccc_version == SMCCC_VERSION_1_0) {
 390                 ssbd_state = ARM64_SSBD_UNKNOWN;
 391                 return false;
 392         }
 393
 394         switch (psci_ops.conduit) {
 395         case PSCI_CONDUIT_HVC:
 396                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 397                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 398                 break;
 399
 400         case PSCI_CONDUIT_SMC:
 401                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 402                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 403                 break;
 404
 405         default:
 406                 ssbd_state = ARM64_SSBD_UNKNOWN;
 407                 return false;
 408         }
 409
 410         val = (s32)res.a0;
 411
 412         switch (val) {
 413         case SMCCC_RET_NOT_SUPPORTED:
 414                 ssbd_state = ARM64_SSBD_UNKNOWN;
 415                 return false;
 416
 417         case SMCCC_RET_NOT_REQUIRED:
 418                 pr_info_once("%s mitigation not required\n", entry->desc);
 419                 ssbd_state = ARM64_SSBD_MITIGATED;
 420                 return false;
 421
 422         case SMCCC_RET_SUCCESS:
 423                 required = true;
 424                 break;
 425
 426         case 1: /* Mitigation not required on this CPU */
 427                 required = false;
 428                 break;
 429
 430         default:
 431                 WARN_ON(1);
 432                 return false;
 433         }
 434
 435         switch (ssbd_state) {
 436         case ARM64_SSBD_FORCE_DISABLE:
 437                 arm64_set_ssbd_mitigation(false);
 438                 required = false;
 439                 break;
 440
 441         case ARM64_SSBD_KERNEL:
 442                 if (required) {
 443                         __this_cpu_write(arm64_ssbd_callback_required, 1);
 444                         arm64_set_ssbd_mitigation(true);
 445                 }
 446                 break;
 447
 448         case ARM64_SSBD_FORCE_ENABLE:
 449                 arm64_set_ssbd_mitigation(true);
 450                 required = true;
 451                 break;
 452
 453         default:
 454                 WARN_ON(1);
 455                 break;
 456         }
 457
 458 out_printmsg:
 459         switch (ssbd_state) {
 460         case ARM64_SSBD_FORCE_DISABLE:
 461                 pr_info_once("%s disabled from command-line\n", entry->desc);
 462                 break;
 463
 464         case ARM64_SSBD_FORCE_ENABLE:
 465                 pr_info_once("%s forced from command-line\n", entry->desc);
 466                 break;
 467         }
 468
 469         return required;
 470 }
 471 #endif  /* CONFIG_ARM64_SSBD */
 472
 473 static void __maybe_unused
 474 cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
 475 {
 476         sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
 477 }
 478
 479 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
 480         .matches = is_affected_midr_range,                      \
 481         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 482
 483 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
 484         .matches = is_affected_midr_range,                              \
 485         .midr_range = MIDR_ALL_VERSIONS(model)
 486
 487 #define MIDR_FIXED(rev, revidr_mask) \
 488         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
 489
 490 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
 491         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
 492         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 493
 494 #define CAP_MIDR_RANGE_LIST(list)                               \
 495         .matches = is_affected_midr_range_list,                 \
 496         .midr_range_list = list
 497
 498 /* Errata affecting a range of revisions of  given model variant */
 499 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
 500         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
 501
 502 /* Errata affecting a single variant/revision of a model */
 503 #define ERRATA_MIDR_REV(model, var, rev)        \
 504         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
 505
 506 /* Errata affecting all variants/revisions of a given a model */
 507 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
 508         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 509         CAP_MIDR_ALL_VERSIONS(model)
 510
 511 /* Errata affecting a list of midr ranges, with same work around */
 512 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
 513         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 514         CAP_MIDR_RANGE_LIST(midr_list)
 515
 516 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 517 /*
 518  * List of CPUs that do not need any Spectre-v2 mitigation at all.
 519  */
 520 static const struct midr_range spectre_v2_safe_list[] = {
 521         MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 522         MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 523         MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 524         { /* sentinel */ }
 525 };
 526
 527 static bool __maybe_unused
 528 check_branch_predictor(const struct arm64_cpu_capabilities *entry, int scope)
 529 {
 530         int need_wa;
 531
 532         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 533
 534         /* If the CPU has CSV2 set, we're safe */
 535         if (cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64PFR0_EL1),
 536                                                  ID_AA64PFR0_CSV2_SHIFT))
 537                 return false;
 538
 539         /* Alternatively, we have a list of unaffected CPUs */
 540         if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list))
 541                 return false;
 542
 543         /* Fallback to firmware detection */
 544         need_wa = detect_harden_bp_fw();
 545         if (!need_wa)
 546                 return false;
 547
 548         /* forced off */
 549         if (__nospectre_v2) {
 550                 pr_info_once("spectrev2 mitigation disabled by command line option\n");
 551                 return false;
 552         }
 553
 554         if (need_wa < 0)
 555                 pr_warn_once("ARM_SMCCC_ARCH_WORKAROUND_1 missing from firmware\n");
 556
 557         return (need_wa > 0);
 558 }
 559 #endif
 560
 561 #ifdef CONFIG_HARDEN_EL2_VECTORS
 562
 563 static const struct midr_range arm64_harden_el2_vectors[] = {
 564         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 565         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 566         {},
 567 };
 568
 569 #endif
 570
 571 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
 572
 573 static const struct midr_range arm64_repeat_tlbi_cpus[] = {
 574 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 575         MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0, 0, 0, 0),
 576 #endif
 577 #ifdef CONFIG_ARM64_ERRATUM_1286807
 578         MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
 579 #endif
 580         {},
 581 };
 582
 583 #endif
 584
 585 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 586 const struct midr_range cavium_erratum_27456_cpus[] = {
 587         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
 588         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
 589         /* Cavium ThunderX, T81 pass 1.0 */
 590         MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
 591         {},
 592 };
 593 #endif
 594
 595 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 596 static const struct midr_range cavium_erratum_30115_cpus[] = {
 597         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
 598         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 2),
 599         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
 600         MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
 601         /* Cavium ThunderX, T83 pass 1.0 */
 602         MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
 603         {},
 604 };
 605 #endif
 606
 607 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 608 static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = {
 609         {
 610                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 611         },
 612         {
 613                 .midr_range.model = MIDR_QCOM_KRYO,
 614                 .matches = is_kryo_midr,
 615         },
 616         {},
 617 };
 618 #endif
 619
 620 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 621 static const struct midr_range workaround_clean_cache[] = {
 622 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
 623         defined(CONFIG_ARM64_ERRATUM_827319) || \
 624         defined(CONFIG_ARM64_ERRATUM_824069)
 625         /* Cortex-A53 r0p[012]: ARM errata 826319, 827319, 824069 */
 626         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
 627 #endif
 628 #ifdef  CONFIG_ARM64_ERRATUM_819472
 629         /* Cortex-A53 r0p[01] : ARM errata 819472 */
 630         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
 631 #endif
 632         {},
 633 };
 634 #endif
 635
 636 const struct arm64_cpu_capabilities arm64_errata[] = {
 637 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
 638         {
 639                 .desc = "ARM errata 826319, 827319, 824069, 819472",
 640                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 641                 ERRATA_MIDR_RANGE_LIST(workaround_clean_cache),
 642                 .cpu_enable = cpu_enable_cache_maint_trap,
 643         },
 644 #endif
 645 #ifdef CONFIG_ARM64_ERRATUM_832075
 646         {
 647         /* Cortex-A57 r0p0 - r1p2 */
 648                 .desc = "ARM erratum 832075",
 649                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
 650                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 651                                   0, 0,
 652                                   1, 2),
 653         },
 654 #endif
 655 #ifdef CONFIG_ARM64_ERRATUM_834220
 656         {
 657         /* Cortex-A57 r0p0 - r1p2 */
 658                 .desc = "ARM erratum 834220",
 659                 .capability = ARM64_WORKAROUND_834220,
 660                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 661                                   0, 0,
 662                                   1, 2),
 663         },
 664 #endif
 665 #ifdef CONFIG_ARM64_ERRATUM_843419
 666         {
 667         /* Cortex-A53 r0p[01234] */
 668                 .desc = "ARM erratum 843419",
 669                 .capability = ARM64_WORKAROUND_843419,
 670                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 671                 MIDR_FIXED(0x4, BIT(8)),
 672         },
 673 #endif
 674 #ifdef CONFIG_ARM64_ERRATUM_845719
 675         {
 676         /* Cortex-A53 r0p[01234] */
 677                 .desc = "ARM erratum 845719",
 678                 .capability = ARM64_WORKAROUND_845719,
 679                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 680         },
 681 #endif
 682 #ifdef CONFIG_CAVIUM_ERRATUM_23154
 683         {
 684         /* Cavium ThunderX, pass 1.x */
 685                 .desc = "Cavium erratum 23154",
 686                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
 687                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
 688         },
 689 #endif
 690 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 691         {
 692                 .desc = "Cavium erratum 27456",
 693                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 694                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_27456_cpus),
 695         },
 696 #endif
 697 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 698         {
 699                 .desc = "Cavium erratum 30115",
 700                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 701                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_30115_cpus),
 702         },
 703 #endif
 704         {
 705                 .desc = "Mismatched cache type (CTR_EL0)",
 706                 .capability = ARM64_MISMATCHED_CACHE_TYPE,
 707                 .matches = has_mismatched_cache_type,
 708                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 709                 .cpu_enable = cpu_enable_trap_ctr_access,
 710         },
 711 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 712         {
 713                 .desc = "Qualcomm Technologies Falkor/Kryo erratum 1003",
 714                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 715                 .matches = cpucap_multi_entry_cap_matches,
 716                 .match_list = qcom_erratum_1003_list,
 717         },
 718 #endif
 719 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
 720         {
 721                 .desc = "Qualcomm erratum 1009, ARM erratum 1286807",
 722                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
 723                 ERRATA_MIDR_RANGE_LIST(arm64_repeat_tlbi_cpus),
 724         },
 725 #endif
 726 #ifdef CONFIG_ARM64_ERRATUM_858921
 727         {
 728         /* Cortex-A73 all versions */
 729                 .desc = "ARM erratum 858921",
 730                 .capability = ARM64_WORKAROUND_858921,
 731                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 732         },
 733 #endif
 734 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 735         {
 736                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
 737                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 738                 .matches = check_branch_predictor,
 739         },
 740 #endif
 741 #ifdef CONFIG_HARDEN_EL2_VECTORS
 742         {
 743                 .desc = "EL2 vector hardening",
 744                 .capability = ARM64_HARDEN_EL2_VECTORS,
 745                 ERRATA_MIDR_RANGE_LIST(arm64_harden_el2_vectors),
 746         },
 747 #endif
 748 #ifdef CONFIG_ARM64_SSBD
 749         {
 750                 .desc = "Speculative Store Bypass Disable",
 751                 .capability = ARM64_SSBD,
 752                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 753                 .matches = has_ssbd_mitigation,
 754         },
 755 #endif
 756 #ifdef CONFIG_ARM64_ERRATUM_1188873
 757         {
 758                 /* Cortex-A76 r0p0 to r2p0 */
 759                 .desc = "ARM erratum 1188873",
 760                 .capability = ARM64_WORKAROUND_1188873,
 761                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
 762         },
 763 #endif
 764 #ifdef CONFIG_ARM64_ERRATUM_1165522
 765         {
 766                 /* Cortex-A76 r0p0 to r2p0 */
 767                 .desc = "ARM erratum 1165522",
 768                 .capability = ARM64_WORKAROUND_1165522,
 769                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
 770         },
 771 #endif
 772         {
 773         }
 774 };
 775
 776 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 777                             char *buf)
 778 {
 779         return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 780 }