src/target/armv8_cache.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2
   3 /***************************************************************************
   4  *   Copyright (C) 2016 by Matthias Welwarsky                              *
   5  *   matthias.welwarsky@sysgo.com                                          *
   6  ***************************************************************************/
   7
   8 #ifdef HAVE_CONFIG_H
   9 #include "config.h"
  10 #endif
  11
  12 #include "armv8_cache.h"
  13 #include "armv8_dpm.h"
  14 #include "armv8_opcodes.h"
  15 #include "smp.h"
  16
  17 /* CLIDR cache types */
  18 #define CACHE_LEVEL_HAS_UNIFIED_CACHE   0x4
  19 #define CACHE_LEVEL_HAS_D_CACHE         0x2
  20 #define CACHE_LEVEL_HAS_I_CACHE         0x1
  21
  22 static int armv8_d_cache_sanity_check(struct armv8_common *armv8)
  23 {
  24         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
  25
  26         if (armv8_cache->d_u_cache_enabled)
  27                 return ERROR_OK;
  28
  29         return ERROR_TARGET_INVALID;
  30 }
  31
  32 static int armv8_i_cache_sanity_check(struct armv8_common *armv8)
  33 {
  34         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
  35
  36         if (armv8_cache->i_cache_enabled)
  37                 return ERROR_OK;
  38
  39         return ERROR_TARGET_INVALID;
  40 }
  41
  42 static int armv8_cache_d_inner_flush_level(struct armv8_common *armv8, struct armv8_cachesize *size, int cl)
  43 {
  44         struct arm_dpm *dpm = armv8->arm.dpm;
  45         int retval = ERROR_OK;
  46         int32_t c_way, c_index = size->index;
  47
  48         LOG_DEBUG("cl %" PRId32, cl);
  49         do {
  50                 c_way = size->way;
  51                 do {
  52                         uint32_t value = (c_index << size->index_shift)
  53                                 | (c_way << size->way_shift) | (cl << 1);
  54                         /*
  55                          * DC CISW - Clean and invalidate data cache
  56                          * line by Set/Way.
  57                          */
  58                         retval = dpm->instr_write_data_r0(dpm,
  59                                         armv8_opcode(armv8, ARMV8_OPC_DCCISW), value);
  60                         if (retval != ERROR_OK)
  61                                 goto done;
  62                         c_way -= 1;
  63                 } while (c_way >= 0);
  64                 c_index -= 1;
  65         } while (c_index >= 0);
  66
  67  done:
  68         return retval;
  69 }
  70
  71 static int armv8_cache_d_inner_clean_inval_all(struct armv8_common *armv8)
  72 {
  73         struct armv8_cache_common *cache = &(armv8->armv8_mmu.armv8_cache);
  74         struct arm_dpm *dpm = armv8->arm.dpm;
  75         int cl;
  76         int retval;
  77
  78         retval = armv8_d_cache_sanity_check(armv8);
  79         if (retval != ERROR_OK)
  80                 return retval;
  81
  82         retval = dpm->prepare(dpm);
  83         if (retval != ERROR_OK)
  84                 goto done;
  85
  86         for (cl = 0; cl < cache->loc; cl++) {
  87                 /* skip i-only caches */
  88                 if (cache->arch[cl].ctype < CACHE_LEVEL_HAS_D_CACHE)
  89                         continue;
  90
  91                 armv8_cache_d_inner_flush_level(armv8, &cache->arch[cl].d_u_size, cl);
  92         }
  93
  94         retval = dpm->finish(dpm);
  95         return retval;
  96
  97 done:
  98         LOG_ERROR("clean invalidate failed");
  99         dpm->finish(dpm);
 100
 101         return retval;
 102 }
 103
 104 int armv8_cache_d_inner_flush_virt(struct armv8_common *armv8, target_addr_t va, size_t size)
 105 {
 106         struct arm_dpm *dpm = armv8->arm.dpm;
 107         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
 108         uint64_t linelen = armv8_cache->dminline;
 109         target_addr_t va_line, va_end;
 110         int retval;
 111
 112         retval = armv8_d_cache_sanity_check(armv8);
 113         if (retval != ERROR_OK)
 114                 return retval;
 115
 116         retval = dpm->prepare(dpm);
 117         if (retval != ERROR_OK)
 118                 goto done;
 119
 120         va_line = va & (-linelen);
 121         va_end = va + size;
 122
 123         while (va_line < va_end) {
 124                 /* DC CIVAC */
 125                 /* Aarch32: DCCIMVAC: ARMV4_5_MCR(15, 0, 0, 7, 14, 1) */
 126                 retval = dpm->instr_write_data_r0_64(dpm,
 127                                 armv8_opcode(armv8, ARMV8_OPC_DCCIVAC), va_line);
 128                 if (retval != ERROR_OK)
 129                         goto done;
 130                 va_line += linelen;
 131         }
 132
 133         dpm->finish(dpm);
 134         return retval;
 135
 136 done:
 137         LOG_ERROR("d-cache invalidate failed");
 138         dpm->finish(dpm);
 139
 140         return retval;
 141 }
 142
 143 int armv8_cache_i_inner_inval_virt(struct armv8_common *armv8, target_addr_t va, size_t size)
 144 {
 145         struct arm_dpm *dpm = armv8->arm.dpm;
 146         struct armv8_cache_common *armv8_cache = &armv8->armv8_mmu.armv8_cache;
 147         uint64_t linelen = armv8_cache->iminline;
 148         target_addr_t va_line, va_end;
 149         int retval;
 150
 151         retval = armv8_i_cache_sanity_check(armv8);
 152         if (retval != ERROR_OK)
 153                 return retval;
 154
 155         retval = dpm->prepare(dpm);
 156         if (retval != ERROR_OK)
 157                 goto done;
 158
 159         va_line = va & (-linelen);
 160         va_end = va + size;
 161
 162         while (va_line < va_end) {
 163                 /* IC IVAU - Invalidate instruction cache by VA to PoU. */
 164                 retval = dpm->instr_write_data_r0_64(dpm,
 165                                 armv8_opcode(armv8, ARMV8_OPC_ICIVAU), va_line);
 166                 if (retval != ERROR_OK)
 167                         goto done;
 168                 va_line += linelen;
 169         }
 170
 171         dpm->finish(dpm);
 172         return retval;
 173
 174 done:
 175         LOG_ERROR("d-cache invalidate failed");
 176         dpm->finish(dpm);
 177
 178         return retval;
 179 }
 180
 181 static int armv8_handle_inner_cache_info_command(struct command_invocation *cmd,
 182         struct armv8_cache_common *armv8_cache)
 183 {
 184         int cl;
 185
 186         if (armv8_cache->info == -1) {
 187                 command_print(cmd, "cache not yet identified");
 188                 return ERROR_OK;
 189         }
 190
 191         for (cl = 0; cl < armv8_cache->loc; cl++) {
 192                 struct armv8_arch_cache *arch = &(armv8_cache->arch[cl]);
 193
 194                 if (arch->ctype & 1) {
 195                         command_print(cmd,
 196                                 "L%d I-Cache: linelen %" PRIu32
 197                                 ", associativity %" PRIu32
 198                                 ", nsets %" PRIu32
 199                                 ", cachesize %" PRIu32 " KBytes",
 200                                 cl+1,
 201                                 arch->i_size.linelen,
 202                                 arch->i_size.associativity,
 203                                 arch->i_size.nsets,
 204                                 arch->i_size.cachesize);
 205                 }
 206
 207                 if (arch->ctype >= 2) {
 208                         command_print(cmd,
 209                                 "L%d D-Cache: linelen %" PRIu32
 210                                 ", associativity %" PRIu32
 211                                 ", nsets %" PRIu32
 212                                 ", cachesize %" PRIu32 " KBytes",
 213                                 cl+1,
 214                                 arch->d_u_size.linelen,
 215                                 arch->d_u_size.associativity,
 216                                 arch->d_u_size.nsets,
 217                                 arch->d_u_size.cachesize);
 218                 }
 219         }
 220
 221         return ERROR_OK;
 222 }
 223
 224 static int _armv8_flush_all_data(struct target *target)
 225 {
 226         return armv8_cache_d_inner_clean_inval_all(target_to_armv8(target));
 227 }
 228
 229 static int  armv8_flush_all_data(struct target *target)
 230 {
 231         int retval = ERROR_FAIL;
 232         /*  check that armv8_cache is correctly identify */
 233         struct armv8_common *armv8 = target_to_armv8(target);
 234         if (armv8->armv8_mmu.armv8_cache.info == -1) {
 235                 LOG_ERROR("trying to flush un-identified cache");
 236                 return retval;
 237         }
 238
 239         if (target->smp) {
 240                 /*  look if all the other target have been flushed in order to flush level
 241                  *  2 */
 242                 struct target_list *head;
 243                 foreach_smp_target(head, target->smp_targets) {
 244                         struct target *curr = head->target;
 245                         if (curr->state == TARGET_HALTED) {
 246                                 LOG_INFO("Wait flushing data l1 on core %" PRId32, curr->coreid);
 247                                 retval = _armv8_flush_all_data(curr);
 248                         }
 249                 }
 250         } else
 251                 retval = _armv8_flush_all_data(target);
 252         return retval;
 253 }
 254
 255 static int get_cache_info(struct arm_dpm *dpm, int cl, int ct, uint32_t *cache_reg)
 256 {
 257         struct armv8_common *armv8 = dpm->arm->arch_info;
 258         int retval = ERROR_OK;
 259
 260         /*  select cache level */
 261         retval = dpm->instr_write_data_r0(dpm,
 262                         armv8_opcode(armv8, WRITE_REG_CSSELR),
 263                         (cl << 1) | (ct == 1 ? 1 : 0));
 264         if (retval != ERROR_OK)
 265                 goto done;
 266
 267         retval = dpm->instr_read_data_r0(dpm,
 268                         armv8_opcode(armv8, READ_REG_CCSIDR),
 269                         cache_reg);
 270  done:
 271         return retval;
 272 }
 273
 274 static struct armv8_cachesize decode_cache_reg(uint32_t cache_reg)
 275 {
 276         struct armv8_cachesize size;
 277         int i = 0;
 278
 279         size.linelen = 16 << (cache_reg & 0x7);
 280         size.associativity = ((cache_reg >> 3) & 0x3ff) + 1;
 281         size.nsets = ((cache_reg >> 13) & 0x7fff) + 1;
 282         size.cachesize = size.linelen * size.associativity * size.nsets / 1024;
 283
 284         /*  compute info for set way operation on cache */
 285         size.index_shift = (cache_reg & 0x7) + 4;
 286         size.index = (cache_reg >> 13) & 0x7fff;
 287         size.way = ((cache_reg >> 3) & 0x3ff);
 288
 289         while (((size.way << i) & 0x80000000) == 0)
 290                 i++;
 291         size.way_shift = i;
 292
 293         return size;
 294 }
 295
 296 int armv8_identify_cache(struct armv8_common *armv8)
 297 {
 298         /*  read cache descriptor */
 299         int retval = ERROR_FAIL;
 300         struct arm *arm = &armv8->arm;
 301         struct arm_dpm *dpm = armv8->arm.dpm;
 302         uint32_t csselr, clidr, ctr;
 303         uint32_t cache_reg;
 304         int cl, ctype;
 305         struct armv8_cache_common *cache = &(armv8->armv8_mmu.armv8_cache);
 306
 307         retval = dpm->prepare(dpm);
 308         if (retval != ERROR_OK)
 309                 goto done;
 310
 311         /* check if we're in an unprivileged mode */
 312         if (armv8_curel_from_core_mode(arm->core_mode) < SYSTEM_CUREL_EL1) {
 313                 retval = armv8_dpm_modeswitch(dpm, ARMV8_64_EL1H);
 314                 if (retval != ERROR_OK)
 315                         return retval;
 316         }
 317
 318         /* retrieve CTR */
 319         retval = dpm->instr_read_data_r0(dpm,
 320                         armv8_opcode(armv8, READ_REG_CTR), &ctr);
 321         if (retval != ERROR_OK)
 322                 goto done;
 323
 324         cache->iminline = 4UL << (ctr & 0xf);
 325         cache->dminline = 4UL << ((ctr & 0xf0000) >> 16);
 326         LOG_DEBUG("ctr %" PRIx32 " ctr.iminline %" PRIu32 " ctr.dminline %" PRIu32,
 327                  ctr, cache->iminline, cache->dminline);
 328
 329         /*  retrieve CLIDR */
 330         retval = dpm->instr_read_data_r0(dpm,
 331                         armv8_opcode(armv8, READ_REG_CLIDR), &clidr);
 332         if (retval != ERROR_OK)
 333                 goto done;
 334
 335         cache->loc = (clidr & 0x7000000) >> 24;
 336         LOG_DEBUG("Number of cache levels to PoC %" PRId32, cache->loc);
 337
 338         /*  retrieve selected cache for later restore
 339          *  MRC p15, 2,<Rd>, c0, c0, 0; Read CSSELR */
 340         retval = dpm->instr_read_data_r0(dpm,
 341                         armv8_opcode(armv8, READ_REG_CSSELR), &csselr);
 342         if (retval != ERROR_OK)
 343                 goto done;
 344
 345         /* retrieve all available inner caches */
 346         for (cl = 0; cl < cache->loc; clidr >>= 3, cl++) {
 347
 348                 /* isolate cache type at current level */
 349                 ctype = clidr & 7;
 350
 351                 /* skip reserved values */
 352                 if (ctype > CACHE_LEVEL_HAS_UNIFIED_CACHE)
 353                         continue;
 354
 355                 /* separate d or unified d/i cache at this level ? */
 356                 if (ctype & (CACHE_LEVEL_HAS_UNIFIED_CACHE | CACHE_LEVEL_HAS_D_CACHE)) {
 357                         /* retrieve d-cache info */
 358                         retval = get_cache_info(dpm, cl, 0, &cache_reg);
 359                         if (retval != ERROR_OK)
 360                                 goto done;
 361                         cache->arch[cl].d_u_size = decode_cache_reg(cache_reg);
 362
 363                         LOG_DEBUG("data/unified cache index %" PRIu32 " << %" PRIu32 ", way %" PRIu32 " << %" PRIu32,
 364                                         cache->arch[cl].d_u_size.index,
 365                                         cache->arch[cl].d_u_size.index_shift,
 366                                         cache->arch[cl].d_u_size.way,
 367                                         cache->arch[cl].d_u_size.way_shift);
 368
 369                         LOG_DEBUG("cacheline %" PRIu32 " bytes %" PRIu32 " KBytes asso %" PRIu32 " ways",
 370                                         cache->arch[cl].d_u_size.linelen,
 371                                         cache->arch[cl].d_u_size.cachesize,
 372                                         cache->arch[cl].d_u_size.associativity);
 373                 }
 374
 375                 /* separate i-cache at this level ? */
 376                 if (ctype & CACHE_LEVEL_HAS_I_CACHE) {
 377                         /* retrieve i-cache info */
 378                         retval = get_cache_info(dpm, cl, 1, &cache_reg);
 379                         if (retval != ERROR_OK)
 380                                 goto done;
 381                         cache->arch[cl].i_size = decode_cache_reg(cache_reg);
 382
 383                         LOG_DEBUG("instruction cache index %" PRIu32 " << %" PRIu32 ", way %" PRIu32 " << %" PRIu32,
 384                                         cache->arch[cl].i_size.index,
 385                                         cache->arch[cl].i_size.index_shift,
 386                                         cache->arch[cl].i_size.way,
 387                                         cache->arch[cl].i_size.way_shift);
 388
 389                         LOG_DEBUG("cacheline %" PRIu32 " bytes %" PRIu32 " KBytes asso %" PRIu32 " ways",
 390                                         cache->arch[cl].i_size.linelen,
 391                                         cache->arch[cl].i_size.cachesize,
 392                                         cache->arch[cl].i_size.associativity);
 393                 }
 394
 395                 cache->arch[cl].ctype = ctype;
 396         }
 397
 398         /*  restore selected cache  */
 399         dpm->instr_write_data_r0(dpm,
 400                         armv8_opcode(armv8, WRITE_REG_CSSELR), csselr);
 401         if (retval != ERROR_OK)
 402                 goto done;
 403
 404         armv8->armv8_mmu.armv8_cache.info = 1;
 405
 406         /*  if no l2 cache initialize l1 data cache flush function function */
 407         if (!armv8->armv8_mmu.armv8_cache.flush_all_data_cache) {
 408                 armv8->armv8_mmu.armv8_cache.display_cache_info =
 409                         armv8_handle_inner_cache_info_command;
 410                 armv8->armv8_mmu.armv8_cache.flush_all_data_cache =
 411                         armv8_flush_all_data;
 412         }
 413
 414 done:
 415         armv8_dpm_modeswitch(dpm, ARM_MODE_ANY);
 416         dpm->finish(dpm);
 417         return retval;
 418
 419 }