arch/arm64/kernel/cpu_errata.c

   1 /*
   2  * Contains CPU specific errata definitions
   3  *
   4  * Copyright (C) 2014 ARM Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/arm-smccc.h>
  20 #include <linux/psci.h>
  21 #include <linux/types.h>
  22 #include <asm/cpu.h>
  23 #include <asm/cputype.h>
  24 #include <asm/cpufeature.h>
  25
  26 static bool __maybe_unused
  27 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
  28 {
  29         const struct arm64_midr_revidr *fix;
  30         u32 midr = read_cpuid_id(), revidr;
  31
  32         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  33         if (!is_midr_in_range(midr, &entry->midr_range))
  34                 return false;
  35
  36         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
  37         revidr = read_cpuid(REVIDR_EL1);
  38         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
  39                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
  40                         return false;
  41
  42         return true;
  43 }
  44
  45 static bool __maybe_unused
  46 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
  47                             int scope)
  48 {
  49         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  50         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
  51 }
  52
  53 static bool __maybe_unused
  54 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
  55 {
  56         u32 model;
  57
  58         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  59
  60         model = read_cpuid_id();
  61         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
  62                  MIDR_ARCHITECTURE_MASK;
  63
  64         return model == entry->midr_range.model;
  65 }
  66
  67 static bool
  68 has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
  69                           int scope)
  70 {
  71         u64 mask = CTR_CACHE_MINLINE_MASK;
  72
  73         /* Skip matching the min line sizes for cache type check */
  74         if (entry->capability == ARM64_MISMATCHED_CACHE_TYPE)
  75                 mask ^= arm64_ftr_reg_ctrel0.strict_mask;
  76
  77         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
  78         return (read_cpuid_cachetype() & mask) !=
  79                (arm64_ftr_reg_ctrel0.sys_val & mask);
  80 }
  81
  82 static void
  83 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
  84 {
  85         sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
  86 }
  87
  88 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
  89
  90 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
  91 #include <asm/mmu_context.h>
  92 #include <asm/cacheflush.h>
  93
  94 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
  95
  96 #ifdef CONFIG_KVM_INDIRECT_VECTORS
  97 extern char __smccc_workaround_1_smc_start[];
  98 extern char __smccc_workaround_1_smc_end[];
  99
 100 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
 101                                 const char *hyp_vecs_end)
 102 {
 103         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
 104         int i;
 105
 106         for (i = 0; i < SZ_2K; i += 0x80)
 107                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
 108
 109         __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
 110 }
 111
 112 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 113                                       const char *hyp_vecs_start,
 114                                       const char *hyp_vecs_end)
 115 {
 116         static DEFINE_SPINLOCK(bp_lock);
 117         int cpu, slot = -1;
 118
 119         spin_lock(&bp_lock);
 120         for_each_possible_cpu(cpu) {
 121                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
 122                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
 123                         break;
 124                 }
 125         }
 126
 127         if (slot == -1) {
 128                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
 129                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
 130                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
 131         }
 132
 133         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
 134         __this_cpu_write(bp_hardening_data.fn, fn);
 135         spin_unlock(&bp_lock);
 136 }
 137 #else
 138 #define __smccc_workaround_1_smc_start          NULL
 139 #define __smccc_workaround_1_smc_end            NULL
 140
 141 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
 142                                       const char *hyp_vecs_start,
 143                                       const char *hyp_vecs_end)
 144 {
 145         __this_cpu_write(bp_hardening_data.fn, fn);
 146 }
 147 #endif  /* CONFIG_KVM_INDIRECT_VECTORS */
 148
 149 static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
 150                                      bp_hardening_cb_t fn,
 151                                      const char *hyp_vecs_start,
 152                                      const char *hyp_vecs_end)
 153 {
 154         u64 pfr0;
 155
 156         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 157                 return;
 158
 159         pfr0 = read_cpuid(ID_AA64PFR0_EL1);
 160         if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
 161                 return;
 162
 163         __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
 164 }
 165
 166 #include <uapi/linux/psci.h>
 167 #include <linux/arm-smccc.h>
 168 #include <linux/psci.h>
 169
 170 static void call_smc_arch_workaround_1(void)
 171 {
 172         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 173 }
 174
 175 static void call_hvc_arch_workaround_1(void)
 176 {
 177         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 178 }
 179
 180 static void qcom_link_stack_sanitization(void)
 181 {
 182         u64 tmp;
 183
 184         asm volatile("mov       %0, x30         \n"
 185                      ".rept     16              \n"
 186                      "bl        . + 4           \n"
 187                      ".endr                     \n"
 188                      "mov       x30, %0         \n"
 189                      : "=&r" (tmp));
 190 }
 191
 192 static void
 193 enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
 194 {
 195         bp_hardening_cb_t cb;
 196         void *smccc_start, *smccc_end;
 197         struct arm_smccc_res res;
 198         u32 midr = read_cpuid_id();
 199
 200         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
 201                 return;
 202
 203         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
 204                 return;
 205
 206         switch (psci_ops.conduit) {
 207         case PSCI_CONDUIT_HVC:
 208                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 209                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 210                 if ((int)res.a0 < 0)
 211                         return;
 212                 cb = call_hvc_arch_workaround_1;
 213                 /* This is a guest, no need to patch KVM vectors */
 214                 smccc_start = NULL;
 215                 smccc_end = NULL;
 216                 break;
 217
 218         case PSCI_CONDUIT_SMC:
 219                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 220                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 221                 if ((int)res.a0 < 0)
 222                         return;
 223                 cb = call_smc_arch_workaround_1;
 224                 smccc_start = __smccc_workaround_1_smc_start;
 225                 smccc_end = __smccc_workaround_1_smc_end;
 226                 break;
 227
 228         default:
 229                 return;
 230         }
 231
 232         if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
 233             ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
 234                 cb = qcom_link_stack_sanitization;
 235
 236         install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
 237
 238         return;
 239 }
 240 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
 241
 242 #ifdef CONFIG_ARM64_SSBD
 243 DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
 244
 245 int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
 246
 247 static const struct ssbd_options {
 248         const char      *str;
 249         int             state;
 250 } ssbd_options[] = {
 251         { "force-on",   ARM64_SSBD_FORCE_ENABLE, },
 252         { "force-off",  ARM64_SSBD_FORCE_DISABLE, },
 253         { "kernel",     ARM64_SSBD_KERNEL, },
 254 };
 255
 256 static int __init ssbd_cfg(char *buf)
 257 {
 258         int i;
 259
 260         if (!buf || !buf[0])
 261                 return -EINVAL;
 262
 263         for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
 264                 int len = strlen(ssbd_options[i].str);
 265
 266                 if (strncmp(buf, ssbd_options[i].str, len))
 267                         continue;
 268
 269                 ssbd_state = ssbd_options[i].state;
 270                 return 0;
 271         }
 272
 273         return -EINVAL;
 274 }
 275 early_param("ssbd", ssbd_cfg);
 276
 277 void __init arm64_update_smccc_conduit(struct alt_instr *alt,
 278                                        __le32 *origptr, __le32 *updptr,
 279                                        int nr_inst)
 280 {
 281         u32 insn;
 282
 283         BUG_ON(nr_inst != 1);
 284
 285         switch (psci_ops.conduit) {
 286         case PSCI_CONDUIT_HVC:
 287                 insn = aarch64_insn_get_hvc_value();
 288                 break;
 289         case PSCI_CONDUIT_SMC:
 290                 insn = aarch64_insn_get_smc_value();
 291                 break;
 292         default:
 293                 return;
 294         }
 295
 296         *updptr = cpu_to_le32(insn);
 297 }
 298
 299 void __init arm64_enable_wa2_handling(struct alt_instr *alt,
 300                                       __le32 *origptr, __le32 *updptr,
 301                                       int nr_inst)
 302 {
 303         BUG_ON(nr_inst != 1);
 304         /*
 305          * Only allow mitigation on EL1 entry/exit and guest
 306          * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
 307          * be flipped.
 308          */
 309         if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
 310                 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
 311 }
 312
 313 void arm64_set_ssbd_mitigation(bool state)
 314 {
 315         if (this_cpu_has_cap(ARM64_SSBS)) {
 316                 if (state)
 317                         asm volatile(SET_PSTATE_SSBS(0));
 318                 else
 319                         asm volatile(SET_PSTATE_SSBS(1));
 320                 return;
 321         }
 322
 323         switch (psci_ops.conduit) {
 324         case PSCI_CONDUIT_HVC:
 325                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
 326                 break;
 327
 328         case PSCI_CONDUIT_SMC:
 329                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
 330                 break;
 331
 332         default:
 333                 WARN_ON_ONCE(1);
 334                 break;
 335         }
 336 }
 337
 338 static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
 339                                     int scope)
 340 {
 341         struct arm_smccc_res res;
 342         bool required = true;
 343         s32 val;
 344
 345         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 346
 347         if (this_cpu_has_cap(ARM64_SSBS)) {
 348                 required = false;
 349                 goto out_printmsg;
 350         }
 351
 352         if (psci_ops.smccc_version == SMCCC_VERSION_1_0) {
 353                 ssbd_state = ARM64_SSBD_UNKNOWN;
 354                 return false;
 355         }
 356
 357         switch (psci_ops.conduit) {
 358         case PSCI_CONDUIT_HVC:
 359                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 360                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 361                 break;
 362
 363         case PSCI_CONDUIT_SMC:
 364                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 365                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 366                 break;
 367
 368         default:
 369                 ssbd_state = ARM64_SSBD_UNKNOWN;
 370                 return false;
 371         }
 372
 373         val = (s32)res.a0;
 374
 375         switch (val) {
 376         case SMCCC_RET_NOT_SUPPORTED:
 377                 ssbd_state = ARM64_SSBD_UNKNOWN;
 378                 return false;
 379
 380         case SMCCC_RET_NOT_REQUIRED:
 381                 pr_info_once("%s mitigation not required\n", entry->desc);
 382                 ssbd_state = ARM64_SSBD_MITIGATED;
 383                 return false;
 384
 385         case SMCCC_RET_SUCCESS:
 386                 required = true;
 387                 break;
 388
 389         case 1: /* Mitigation not required on this CPU */
 390                 required = false;
 391                 break;
 392
 393         default:
 394                 WARN_ON(1);
 395                 return false;
 396         }
 397
 398         switch (ssbd_state) {
 399         case ARM64_SSBD_FORCE_DISABLE:
 400                 arm64_set_ssbd_mitigation(false);
 401                 required = false;
 402                 break;
 403
 404         case ARM64_SSBD_KERNEL:
 405                 if (required) {
 406                         __this_cpu_write(arm64_ssbd_callback_required, 1);
 407                         arm64_set_ssbd_mitigation(true);
 408                 }
 409                 break;
 410
 411         case ARM64_SSBD_FORCE_ENABLE:
 412                 arm64_set_ssbd_mitigation(true);
 413                 required = true;
 414                 break;
 415
 416         default:
 417                 WARN_ON(1);
 418                 break;
 419         }
 420
 421 out_printmsg:
 422         switch (ssbd_state) {
 423         case ARM64_SSBD_FORCE_DISABLE:
 424                 pr_info_once("%s disabled from command-line\n", entry->desc);
 425                 break;
 426
 427         case ARM64_SSBD_FORCE_ENABLE:
 428                 pr_info_once("%s forced from command-line\n", entry->desc);
 429                 break;
 430         }
 431
 432         return required;
 433 }
 434 #endif  /* CONFIG_ARM64_SSBD */
 435
 436 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
 437         .matches = is_affected_midr_range,                      \
 438         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 439
 440 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
 441         .matches = is_affected_midr_range,                              \
 442         .midr_range = MIDR_ALL_VERSIONS(model)
 443
 444 #define MIDR_FIXED(rev, revidr_mask) \
 445         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
 446
 447 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
 448         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
 449         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
 450
 451 #define CAP_MIDR_RANGE_LIST(list)                               \
 452         .matches = is_affected_midr_range_list,                 \
 453         .midr_range_list = list
 454
 455 /* Errata affecting a range of revisions of  given model variant */
 456 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
 457         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
 458
 459 /* Errata affecting a single variant/revision of a model */
 460 #define ERRATA_MIDR_REV(model, var, rev)        \
 461         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
 462
 463 /* Errata affecting all variants/revisions of a given a model */
 464 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
 465         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 466         CAP_MIDR_ALL_VERSIONS(model)
 467
 468 /* Errata affecting a list of midr ranges, with same work around */
 469 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
 470         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
 471         CAP_MIDR_RANGE_LIST(midr_list)
 472
 473 /*
 474  * Generic helper for handling capabilties with multiple (match,enable) pairs
 475  * of call backs, sharing the same capability bit.
 476  * Iterate over each entry to see if at least one matches.
 477  */
 478 static bool __maybe_unused
 479 multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
 480 {
 481         const struct arm64_cpu_capabilities *caps;
 482
 483         for (caps = entry->match_list; caps->matches; caps++)
 484                 if (caps->matches(caps, scope))
 485                         return true;
 486
 487         return false;
 488 }
 489
 490 /*
 491  * Take appropriate action for all matching entries in the shared capability
 492  * entry.
 493  */
 494 static void __maybe_unused
 495 multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
 496 {
 497         const struct arm64_cpu_capabilities *caps;
 498
 499         for (caps = entry->match_list; caps->matches; caps++)
 500                 if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
 501                     caps->cpu_enable)
 502                         caps->cpu_enable(caps);
 503 }
 504
 505 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 506
 507 /*
 508  * List of CPUs where we need to issue a psci call to
 509  * harden the branch predictor.
 510  */
 511 static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
 512         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 513         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 514         MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 515         MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
 516         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
 517         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
 518         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
 519         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
 520         MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER),
 521         {},
 522 };
 523
 524 #endif
 525
 526 #ifdef CONFIG_HARDEN_EL2_VECTORS
 527
 528 static const struct midr_range arm64_harden_el2_vectors[] = {
 529         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 530         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 531         {},
 532 };
 533
 534 #endif
 535
 536 const struct arm64_cpu_capabilities arm64_errata[] = {
 537 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
 538         defined(CONFIG_ARM64_ERRATUM_827319) || \
 539         defined(CONFIG_ARM64_ERRATUM_824069)
 540         {
 541         /* Cortex-A53 r0p[012] */
 542                 .desc = "ARM errata 826319, 827319, 824069",
 543                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 544                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
 545                 .cpu_enable = cpu_enable_cache_maint_trap,
 546         },
 547 #endif
 548 #ifdef CONFIG_ARM64_ERRATUM_819472
 549         {
 550         /* Cortex-A53 r0p[01] */
 551                 .desc = "ARM errata 819472",
 552                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
 553                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
 554                 .cpu_enable = cpu_enable_cache_maint_trap,
 555         },
 556 #endif
 557 #ifdef CONFIG_ARM64_ERRATUM_832075
 558         {
 559         /* Cortex-A57 r0p0 - r1p2 */
 560                 .desc = "ARM erratum 832075",
 561                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
 562                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 563                                   0, 0,
 564                                   1, 2),
 565         },
 566 #endif
 567 #ifdef CONFIG_ARM64_ERRATUM_834220
 568         {
 569         /* Cortex-A57 r0p0 - r1p2 */
 570                 .desc = "ARM erratum 834220",
 571                 .capability = ARM64_WORKAROUND_834220,
 572                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
 573                                   0, 0,
 574                                   1, 2),
 575         },
 576 #endif
 577 #ifdef CONFIG_ARM64_ERRATUM_843419
 578         {
 579         /* Cortex-A53 r0p[01234] */
 580                 .desc = "ARM erratum 843419",
 581                 .capability = ARM64_WORKAROUND_843419,
 582                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 583                 MIDR_FIXED(0x4, BIT(8)),
 584         },
 585 #endif
 586 #ifdef CONFIG_ARM64_ERRATUM_845719
 587         {
 588         /* Cortex-A53 r0p[01234] */
 589                 .desc = "ARM erratum 845719",
 590                 .capability = ARM64_WORKAROUND_845719,
 591                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
 592         },
 593 #endif
 594 #ifdef CONFIG_CAVIUM_ERRATUM_23154
 595         {
 596         /* Cavium ThunderX, pass 1.x */
 597                 .desc = "Cavium erratum 23154",
 598                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
 599                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
 600         },
 601 #endif
 602 #ifdef CONFIG_CAVIUM_ERRATUM_27456
 603         {
 604         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
 605                 .desc = "Cavium erratum 27456",
 606                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 607                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 608                                   0, 0,
 609                                   1, 1),
 610         },
 611         {
 612         /* Cavium ThunderX, T81 pass 1.0 */
 613                 .desc = "Cavium erratum 27456",
 614                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
 615                 ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
 616         },
 617 #endif
 618 #ifdef CONFIG_CAVIUM_ERRATUM_30115
 619         {
 620         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
 621                 .desc = "Cavium erratum 30115",
 622                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 623                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
 624                                       0, 0,
 625                                       1, 2),
 626         },
 627         {
 628         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
 629                 .desc = "Cavium erratum 30115",
 630                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 631                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
 632         },
 633         {
 634         /* Cavium ThunderX, T83 pass 1.0 */
 635                 .desc = "Cavium erratum 30115",
 636                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
 637                 ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
 638         },
 639 #endif
 640         {
 641                 .desc = "Mismatched cache line size",
 642                 .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
 643                 .matches = has_mismatched_cache_type,
 644                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 645                 .cpu_enable = cpu_enable_trap_ctr_access,
 646         },
 647         {
 648                 .desc = "Mismatched cache type",
 649                 .capability = ARM64_MISMATCHED_CACHE_TYPE,
 650                 .matches = has_mismatched_cache_type,
 651                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 652                 .cpu_enable = cpu_enable_trap_ctr_access,
 653         },
 654 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
 655         {
 656                 .desc = "Qualcomm Technologies Falkor erratum 1003",
 657                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 658                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 659         },
 660         {
 661                 .desc = "Qualcomm Technologies Kryo erratum 1003",
 662                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
 663                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 664                 .midr_range.model = MIDR_QCOM_KRYO,
 665                 .matches = is_kryo_midr,
 666         },
 667 #endif
 668 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 669         {
 670                 .desc = "Qualcomm Technologies Falkor erratum 1009",
 671                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
 672                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
 673         },
 674 #endif
 675 #ifdef CONFIG_ARM64_ERRATUM_858921
 676         {
 677         /* Cortex-A73 all versions */
 678                 .desc = "ARM erratum 858921",
 679                 .capability = ARM64_WORKAROUND_858921,
 680                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 681         },
 682 #endif
 683 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
 684         {
 685                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
 686                 .cpu_enable = enable_smccc_arch_workaround_1,
 687                 ERRATA_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
 688         },
 689 #endif
 690 #ifdef CONFIG_HARDEN_EL2_VECTORS
 691         {
 692                 .desc = "EL2 vector hardening",
 693                 .capability = ARM64_HARDEN_EL2_VECTORS,
 694                 ERRATA_MIDR_RANGE_LIST(arm64_harden_el2_vectors),
 695         },
 696 #endif
 697 #ifdef CONFIG_ARM64_SSBD
 698         {
 699                 .desc = "Speculative Store Bypass Disable",
 700                 .capability = ARM64_SSBD,
 701                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 702                 .matches = has_ssbd_mitigation,
 703         },
 704 #endif
 705         {
 706         }
 707 };