include/asm-mips/r4kcache.h

   1 /*
   2  * r4kcache.h: Inline assembly cache operations.
   3  *
   4  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   5  *
   6  * $Id: r4kcache.h,v 1.7 1997/12/18 13:00:45 ralf Exp $
   7  *
   8  * FIXME: Handle split L2 caches.
   9  */
  10 #ifndef _MIPS_R4KCACHE_H
  11 #define _MIPS_R4KCACHE_H
  12
  13 #include <asm/asm.h>
  14 #include <asm/cacheops.h>
  15
  16 extern inline void flush_icache_line_indexed(unsigned long addr)
  17 {
  18         __asm__ __volatile__(
  19                 ".set noreorder\n\t"
  20                 ".set mips3\n\t"
  21                 "cache %1, (%0)\n\t"
  22                 ".set mips0\n\t"
  23                 ".set reorder"
  24                 :
  25                 : "r" (addr),
  26                   "i" (Index_Invalidate_I));
  27 }
  28
  29 extern inline void flush_dcache_line_indexed(unsigned long addr)
  30 {
  31         __asm__ __volatile__(
  32                 ".set noreorder\n\t"
  33                 ".set mips3\n\t"
  34                 "cache %1, (%0)\n\t"
  35                 ".set mips0\n\t"
  36                 ".set reorder"
  37                 :
  38                 : "r" (addr),
  39                   "i" (Index_Writeback_Inv_D));
  40 }
  41
  42 extern inline void flush_scache_line_indexed(unsigned long addr)
  43 {
  44         __asm__ __volatile__(
  45                 ".set noreorder\n\t"
  46                 ".set mips3\n\t"
  47                 "cache %1, (%0)\n\t"
  48                 ".set mips0\n\t"
  49                 ".set reorder"
  50                 :
  51                 : "r" (addr),
  52                   "i" (Index_Writeback_Inv_SD));
  53 }
  54
  55 extern inline void flush_icache_line(unsigned long addr)
  56 {
  57         __asm__ __volatile__(
  58                 ".set noreorder\n\t"
  59                 ".set mips3\n\t"
  60                 "cache %1, (%0)\n\t"
  61                 ".set mips0\n\t"
  62                 ".set reorder"
  63                 :
  64                 : "r" (addr),
  65                   "i" (Hit_Invalidate_I));
  66 }
  67
  68 extern inline void flush_dcache_line(unsigned long addr)
  69 {
  70         __asm__ __volatile__(
  71                 ".set noreorder\n\t"
  72                 ".set mips3\n\t"
  73                 "cache %1, (%0)\n\t"
  74                 ".set mips0\n\t"
  75                 ".set reorder"
  76                 :
  77                 : "r" (addr),
  78                   "i" (Hit_Writeback_Inv_D));
  79 }
  80
  81 extern inline void invalidate_dcache_line(unsigned long addr)
  82 {
  83         __asm__ __volatile__(
  84                 ".set noreorder\n\t"
  85                 ".set mips3\n\t"
  86                 "cache %1, (%0)\n\t"
  87                 ".set mips0\n\t"
  88                 ".set reorder"
  89                 :
  90                 : "r" (addr),
  91                   "i" (Hit_Invalidate_D));
  92 }
  93
  94 extern inline void invalidate_scache_line(unsigned long addr)
  95 {
  96         __asm__ __volatile__(
  97                 ".set noreorder\n\t"
  98                 ".set mips3\n\t"
  99                 "cache %1, (%0)\n\t"
 100                 ".set mips0\n\t"
 101                 ".set reorder"
 102                 :
 103                 : "r" (addr),
 104                   "i" (Hit_Invalidate_SD));
 105 }
 106
 107 extern inline void flush_scache_line(unsigned long addr)
 108 {
 109         __asm__ __volatile__(
 110                 ".set noreorder\n\t"
 111                 ".set mips3\n\t"
 112                 "cache %1, (%0)\n\t"
 113                 ".set mips0\n\t"
 114                 ".set reorder"
 115                 :
 116                 : "r" (addr),
 117                   "i" (Hit_Writeback_Inv_SD));
 118 }
 119
 120 /*
 121  * The next two are for badland addresses like signal trampolines.
 122  */
 123 extern inline void protected_flush_icache_line(unsigned long addr)
 124 {
 125         __asm__ __volatile__(
 126                 ".set noreorder\n\t"
 127                 ".set mips3\n"
 128                 "1:\tcache %1,(%0)\n"
 129                 "2:\t.set mips0\n\t"
 130                 ".set reorder\n\t"
 131                 ".section\t__ex_table,\"a\"\n\t"
 132                 STR(PTR)"\t1b,2b\n\t"
 133                 ".previous"
 134                 :
 135                 : "r" (addr),
 136                   "i" (Hit_Invalidate_I));
 137 }
 138
 139 extern inline void protected_writeback_dcache_line(unsigned long addr)
 140 {
 141         __asm__ __volatile__(
 142                 ".set noreorder\n\t"
 143                 ".set mips3\n"
 144                 "1:\tcache %1,(%0)\n"
 145                 "2:\t.set mips0\n\t"
 146                 ".set reorder\n\t"
 147                 ".section\t__ex_table,\"a\"\n\t"
 148                 STR(PTR)"\t1b,2b\n\t"
 149                 ".previous"
 150                 :
 151                 : "r" (addr),
 152                   "i" (Hit_Writeback_D));
 153 }
 154
 155 #define cache16_unroll32(base,op)                               \
 156         __asm__ __volatile__("                                  \
 157                 .set noreorder;                                 \
 158                 .set mips3;                                     \
 159                 cache %1, 0x000(%0); cache %1, 0x010(%0);       \
 160                 cache %1, 0x020(%0); cache %1, 0x030(%0);       \
 161                 cache %1, 0x040(%0); cache %1, 0x050(%0);       \
 162                 cache %1, 0x060(%0); cache %1, 0x070(%0);       \
 163                 cache %1, 0x080(%0); cache %1, 0x090(%0);       \
 164                 cache %1, 0x0a0(%0); cache %1, 0x0b0(%0);       \
 165                 cache %1, 0x0c0(%0); cache %1, 0x0d0(%0);       \
 166                 cache %1, 0x0e0(%0); cache %1, 0x0f0(%0);       \
 167                 cache %1, 0x100(%0); cache %1, 0x110(%0);       \
 168                 cache %1, 0x120(%0); cache %1, 0x130(%0);       \
 169                 cache %1, 0x140(%0); cache %1, 0x150(%0);       \
 170                 cache %1, 0x160(%0); cache %1, 0x170(%0);       \
 171                 cache %1, 0x180(%0); cache %1, 0x190(%0);       \
 172                 cache %1, 0x1a0(%0); cache %1, 0x1b0(%0);       \
 173                 cache %1, 0x1c0(%0); cache %1, 0x1d0(%0);       \
 174                 cache %1, 0x1e0(%0); cache %1, 0x1f0(%0);       \
 175                 .set mips0;                                     \
 176                 .set reorder"                                   \
 177                 :                                               \
 178                 : "r" (base),                                   \
 179                   "i" (op));
 180
 181 extern inline void blast_dcache16(void)
 182 {
 183         unsigned long start = KSEG0;
 184         unsigned long end = (start + dcache_size);
 185
 186         while(start < end) {
 187                 cache16_unroll32(start,Index_Writeback_Inv_D);
 188                 start += 0x200;
 189         }
 190 }
 191
 192 extern inline void blast_dcache16_page(unsigned long page)
 193 {
 194         unsigned long start = page;
 195         unsigned long end = (start + PAGE_SIZE);
 196
 197         while(start < end) {
 198                 cache16_unroll32(start,Hit_Writeback_Inv_D);
 199                 start += 0x200;
 200         }
 201 }
 202
 203 extern inline void blast_dcache16_page_indexed(unsigned long page)
 204 {
 205         unsigned long start = page;
 206         unsigned long end = (start + PAGE_SIZE);
 207
 208         while(start < end) {
 209                 cache16_unroll32(start,Index_Writeback_Inv_D);
 210                 start += 0x200;
 211         }
 212 }
 213
 214 extern inline void blast_icache16(void)
 215 {
 216         unsigned long start = KSEG0;
 217         unsigned long end = (start + icache_size);
 218
 219         while(start < end) {
 220                 cache16_unroll32(start,Index_Invalidate_I);
 221                 start += 0x200;
 222         }
 223 }
 224
 225 extern inline void blast_icache16_page(unsigned long page)
 226 {
 227         unsigned long start = page;
 228         unsigned long end = (start + PAGE_SIZE);
 229
 230         while(start < end) {
 231                 cache16_unroll32(start,Hit_Invalidate_I);
 232                 start += 0x200;
 233         }
 234 }
 235
 236 extern inline void blast_icache16_page_indexed(unsigned long page)
 237 {
 238         unsigned long start = page;
 239         unsigned long end = (start + PAGE_SIZE);
 240
 241         while(start < end) {
 242                 cache16_unroll32(start,Index_Invalidate_I);
 243                 start += 0x200;
 244         }
 245 }
 246
 247 extern inline void blast_scache16(void)
 248 {
 249         unsigned long start = KSEG0;
 250         unsigned long end = KSEG0 + scache_size;
 251
 252         while(start < end) {
 253                 cache16_unroll32(start,Index_Writeback_Inv_SD);
 254                 start += 0x200;
 255         }
 256 }
 257
 258 extern inline void blast_scache16_page(unsigned long page)
 259 {
 260         unsigned long start = page;
 261         unsigned long end = page + PAGE_SIZE;
 262
 263         while(start < end) {
 264                 cache16_unroll32(start,Hit_Writeback_Inv_SD);
 265                 start += 0x200;
 266         }
 267 }
 268
 269 extern inline void blast_scache16_page_indexed(unsigned long page)
 270 {
 271         unsigned long start = page;
 272         unsigned long end = page + PAGE_SIZE;
 273
 274         while(start < end) {
 275                 cache16_unroll32(start,Index_Writeback_Inv_SD);
 276                 start += 0x200;
 277         }
 278 }
 279
 280 #define cache32_unroll32(base,op)                               \
 281         __asm__ __volatile__("                                  \
 282                 .set noreorder;                                 \
 283                 .set mips3;                                     \
 284                 cache %1, 0x000(%0); cache %1, 0x020(%0);       \
 285                 cache %1, 0x040(%0); cache %1, 0x060(%0);       \
 286                 cache %1, 0x080(%0); cache %1, 0x0a0(%0);       \
 287                 cache %1, 0x0c0(%0); cache %1, 0x0e0(%0);       \
 288                 cache %1, 0x100(%0); cache %1, 0x120(%0);       \
 289                 cache %1, 0x140(%0); cache %1, 0x160(%0);       \
 290                 cache %1, 0x180(%0); cache %1, 0x1a0(%0);       \
 291                 cache %1, 0x1c0(%0); cache %1, 0x1e0(%0);       \
 292                 cache %1, 0x200(%0); cache %1, 0x220(%0);       \
 293                 cache %1, 0x240(%0); cache %1, 0x260(%0);       \
 294                 cache %1, 0x280(%0); cache %1, 0x2a0(%0);       \
 295                 cache %1, 0x2c0(%0); cache %1, 0x2e0(%0);       \
 296                 cache %1, 0x300(%0); cache %1, 0x320(%0);       \
 297                 cache %1, 0x340(%0); cache %1, 0x360(%0);       \
 298                 cache %1, 0x380(%0); cache %1, 0x3a0(%0);       \
 299                 cache %1, 0x3c0(%0); cache %1, 0x3e0(%0);       \
 300                 .set mips0;                                     \
 301                 .set reorder"                                   \
 302                 :                                               \
 303                 : "r" (base),                                   \
 304                   "i" (op));
 305
 306 extern inline void blast_dcache32(void)
 307 {
 308         unsigned long start = KSEG0;
 309         unsigned long end = (start + dcache_size);
 310
 311         while(start < end) {
 312                 cache32_unroll32(start,Index_Writeback_Inv_D);
 313                 start += 0x400;
 314         }
 315 }
 316
 317 /*
 318  * Call this function only with interrupts disabled or R4600 V2.0 may blow
 319  * up on you.
 320  *
 321  * R4600 v2.0 bug: "The CACHE instructions Hit_Writeback_Inv_D,
 322  * Hit_Writeback_D, Hit_Invalidate_D and Create_Dirty_Excl_D will only
 323  * operate correctly if the internal data cache refill buffer is empty.  These
 324  * CACHE instructions should be separated from any potential data cache miss
 325  * by a load instruction to an uncached address to empty the response buffer."
 326  * (Revision 2.0 device errata from IDT available on http://www.idt.com/
 327  * in .pdf format.)
 328  */
 329 extern inline void blast_dcache32_page(unsigned long page)
 330 {
 331         unsigned long start = page;
 332         unsigned long end = (start + PAGE_SIZE);
 333
 334         /*
 335          * Sigh ... workaround for R4600 v1.7 bug.  Explanation see above.
 336          */
 337         *(volatile unsigned long *)KSEG1;
 338
 339         __asm__ __volatile__("nop;nop;nop;nop");
 340         while(start < end) {
 341                 cache32_unroll32(start,Hit_Writeback_Inv_D);
 342                 start += 0x400;
 343         }
 344 }
 345
 346 extern inline void blast_dcache32_page_indexed(unsigned long page)
 347 {
 348         unsigned long start = page;
 349         unsigned long end = (start + PAGE_SIZE);
 350
 351         while(start < end) {
 352                 cache32_unroll32(start,Index_Writeback_Inv_D);
 353                 start += 0x400;
 354         }
 355 }
 356
 357 extern inline void blast_icache32(void)
 358 {
 359         unsigned long start = KSEG0;
 360         unsigned long end = (start + icache_size);
 361
 362         while(start < end) {
 363                 cache32_unroll32(start,Index_Invalidate_I);
 364                 start += 0x400;
 365         }
 366 }
 367
 368 extern inline void blast_icache32_page(unsigned long page)
 369 {
 370         unsigned long start = page;
 371         unsigned long end = (start + PAGE_SIZE);
 372
 373         while(start < end) {
 374                 cache32_unroll32(start,Hit_Invalidate_I);
 375                 start += 0x400;
 376         }
 377 }
 378
 379 extern inline void blast_icache32_page_indexed(unsigned long page)
 380 {
 381         unsigned long start = page;
 382         unsigned long end = (start + PAGE_SIZE);
 383
 384         while(start < end) {
 385                 cache32_unroll32(start,Index_Invalidate_I);
 386                 start += 0x400;
 387         }
 388 }
 389
 390 extern inline void blast_scache32(void)
 391 {
 392         unsigned long start = KSEG0;
 393         unsigned long end = KSEG0 + scache_size;
 394
 395         while(start < end) {
 396                 cache32_unroll32(start,Index_Writeback_Inv_SD);
 397                 start += 0x400;
 398         }
 399 }
 400
 401 extern inline void blast_scache32_page(unsigned long page)
 402 {
 403         unsigned long start = page;
 404         unsigned long end = page + PAGE_SIZE;
 405
 406         while(start < end) {
 407                 cache32_unroll32(start,Hit_Writeback_Inv_SD);
 408                 start += 0x400;
 409         }
 410 }
 411
 412 extern inline void blast_scache32_page_indexed(unsigned long page)
 413 {
 414         unsigned long start = page;
 415         unsigned long end = page + PAGE_SIZE;
 416
 417         while(start < end) {
 418                 cache32_unroll32(start,Index_Writeback_Inv_SD);
 419                 start += 0x400;
 420         }
 421 }
 422
 423 #define cache64_unroll32(base,op)                               \
 424         __asm__ __volatile__("                                  \
 425                 .set noreorder;                                 \
 426                 .set mips3;                                     \
 427                 cache %1, 0x000(%0); cache %1, 0x040(%0);       \
 428                 cache %1, 0x080(%0); cache %1, 0x0c0(%0);       \
 429                 cache %1, 0x100(%0); cache %1, 0x140(%0);       \
 430                 cache %1, 0x180(%0); cache %1, 0x1c0(%0);       \
 431                 cache %1, 0x200(%0); cache %1, 0x240(%0);       \
 432                 cache %1, 0x280(%0); cache %1, 0x2c0(%0);       \
 433                 cache %1, 0x300(%0); cache %1, 0x340(%0);       \
 434                 cache %1, 0x380(%0); cache %1, 0x3c0(%0);       \
 435                 cache %1, 0x400(%0); cache %1, 0x440(%0);       \
 436                 cache %1, 0x480(%0); cache %1, 0x4c0(%0);       \
 437                 cache %1, 0x500(%0); cache %1, 0x540(%0);       \
 438                 cache %1, 0x580(%0); cache %1, 0x5c0(%0);       \
 439                 cache %1, 0x600(%0); cache %1, 0x640(%0);       \
 440                 cache %1, 0x680(%0); cache %1, 0x6c0(%0);       \
 441                 cache %1, 0x700(%0); cache %1, 0x740(%0);       \
 442                 cache %1, 0x780(%0); cache %1, 0x7c0(%0);       \
 443                 .set mips0;                                     \
 444                 .set reorder"                                   \
 445                 :                                               \
 446                 : "r" (base),                                   \
 447                   "i" (op));
 448
 449 extern inline void blast_scache64(void)
 450 {
 451         unsigned long start = KSEG0;
 452         unsigned long end = KSEG0 + scache_size;
 453
 454         while(start < end) {
 455                 cache64_unroll32(start,Index_Writeback_Inv_SD);
 456                 start += 0x800;
 457         }
 458 }
 459
 460 extern inline void blast_scache64_page(unsigned long page)
 461 {
 462         unsigned long start = page;
 463         unsigned long end = page + PAGE_SIZE;
 464
 465         while(start < end) {
 466                 cache64_unroll32(start,Hit_Writeback_Inv_SD);
 467                 start += 0x800;
 468         }
 469 }
 470
 471 extern inline void blast_scache64_page_indexed(unsigned long page)
 472 {
 473         unsigned long start = page;
 474         unsigned long end = page + PAGE_SIZE;
 475
 476         while(start < end) {
 477                 cache64_unroll32(start,Index_Writeback_Inv_SD);
 478                 start += 0x800;
 479         }
 480 }
 481
 482 #define cache128_unroll32(base,op)                              \
 483         __asm__ __volatile__("                                  \
 484                 .set noreorder;                                 \
 485                 .set mips3;                                     \
 486                 cache %1, 0x000(%0); cache %1, 0x080(%0);       \
 487                 cache %1, 0x100(%0); cache %1, 0x180(%0);       \
 488                 cache %1, 0x200(%0); cache %1, 0x280(%0);       \
 489                 cache %1, 0x300(%0); cache %1, 0x380(%0);       \
 490                 cache %1, 0x400(%0); cache %1, 0x480(%0);       \
 491                 cache %1, 0x500(%0); cache %1, 0x580(%0);       \
 492                 cache %1, 0x600(%0); cache %1, 0x680(%0);       \
 493                 cache %1, 0x700(%0); cache %1, 0x780(%0);       \
 494                 cache %1, 0x800(%0); cache %1, 0x880(%0);       \
 495                 cache %1, 0x900(%0); cache %1, 0x980(%0);       \
 496                 cache %1, 0xa00(%0); cache %1, 0xa80(%0);       \
 497                 cache %1, 0xb00(%0); cache %1, 0xb80(%0);       \
 498                 cache %1, 0xc00(%0); cache %1, 0xc80(%0);       \
 499                 cache %1, 0xd00(%0); cache %1, 0xd80(%0);       \
 500                 cache %1, 0xe00(%0); cache %1, 0xe80(%0);       \
 501                 cache %1, 0xf00(%0); cache %1, 0xf80(%0);       \
 502                 .set mips0;                                     \
 503                 .set reorder"                                   \
 504                 :                                               \
 505                 : "r" (base),                                   \
 506                   "i" (op));
 507
 508 extern inline void blast_scache128(void)
 509 {
 510         unsigned long start = KSEG0;
 511         unsigned long end = KSEG0 + scache_size;
 512
 513         while(start < end) {
 514                 cache128_unroll32(start,Index_Writeback_Inv_SD);
 515                 start += 0x1000;
 516         }
 517 }
 518
 519 extern inline void blast_scache128_page(unsigned long page)
 520 {
 521         cache128_unroll32(page,Hit_Writeback_Inv_SD);
 522 }
 523
 524 extern inline void blast_scache128_page_indexed(unsigned long page)
 525 {
 526         cache128_unroll32(page,Index_Writeback_Inv_SD);
 527 }
 528
 529 #endif /* !(_MIPS_R4KCACHE_H) */