arch/blackfin/kernel/ipipe.c

   1 /* -*- linux-c -*-
   2  * linux/arch/blackfin/kernel/ipipe.c
   3  *
   4  * Copyright (C) 2005-2007 Philippe Gerum.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
   9  * USA; either version 2 of the License, or (at your option) any later
  10  * version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  20  *
  21  * Architecture-dependent I-pipe support for the Blackfin.
  22  */
  23
  24 #include <linux/kernel.h>
  25 #include <linux/sched.h>
  26 #include <linux/module.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/percpu.h>
  29 #include <linux/bitops.h>
  30 #include <linux/errno.h>
  31 #include <linux/kthread.h>
  32 #include <linux/unistd.h>
  33 #include <linux/io.h>
  34 #include <asm/system.h>
  35 #include <asm/atomic.h>
  36
  37 DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
  38
  39 asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
  40
  41 static void __ipipe_no_irqtail(void);
  42
  43 unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
  44 EXPORT_SYMBOL(__ipipe_irq_tail_hook);
  45
  46 unsigned long __ipipe_core_clock;
  47 EXPORT_SYMBOL(__ipipe_core_clock);
  48
  49 unsigned long __ipipe_freq_scale;
  50 EXPORT_SYMBOL(__ipipe_freq_scale);
  51
  52 atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
  53
  54 unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
  55 EXPORT_SYMBOL(__ipipe_irq_lvmask);
  56
  57 static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
  58 {
  59         desc->ipipe_ack(irq, desc);
  60 }
  61
  62 /*
  63  * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
  64  * interrupts are off, and secondary CPUs are still lost in space.
  65  */
  66 void __ipipe_enable_pipeline(void)
  67 {
  68         unsigned irq;
  69
  70         __ipipe_core_clock = get_cclk(); /* Fetch this once. */
  71         __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
  72
  73         for (irq = 0; irq < NR_IRQS; ++irq)
  74                 ipipe_virtualize_irq(ipipe_root_domain,
  75                                      irq,
  76                                      (ipipe_irq_handler_t)&asm_do_IRQ,
  77                                      NULL,
  78                                      &__ipipe_ack_irq,
  79                                      IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
  80 }
  81
  82 /*
  83  * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
  84  * interrupt protection log is maintained here for each domain. Hw
  85  * interrupts are masked on entry.
  86  */
  87 void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
  88 {
  89         struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
  90         struct ipipe_domain *this_domain, *next_domain;
  91         struct list_head *head, *pos;
  92         struct ipipe_irqdesc *idesc;
  93         int m_ack, s = -1;
  94
  95         /*
  96          * Software-triggered IRQs do not need any ack.  The contents
  97          * of the register frame should only be used when processing
  98          * the timer interrupt, but not for handling any other
  99          * interrupt.
 100          */
 101         m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
 102         this_domain = __ipipe_current_domain;
 103         idesc = &this_domain->irqs[irq];
 104
 105         if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control)))
 106                 head = &this_domain->p_link;
 107         else {
 108                 head = __ipipe_pipeline.next;
 109                 next_domain = list_entry(head, struct ipipe_domain, p_link);
 110                 idesc = &next_domain->irqs[irq];
 111                 if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) {
 112                         if (!m_ack && idesc->acknowledge != NULL)
 113                                 idesc->acknowledge(irq, irq_to_desc(irq));
 114                         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
 115                                 s = __test_and_set_bit(IPIPE_STALL_FLAG,
 116                                                        &p->status);
 117                         __ipipe_dispatch_wired(next_domain, irq);
 118                         goto out;
 119                 }
 120         }
 121
 122         /* Ack the interrupt. */
 123
 124         pos = head;
 125         while (pos != &__ipipe_pipeline) {
 126                 next_domain = list_entry(pos, struct ipipe_domain, p_link);
 127                 idesc = &next_domain->irqs[irq];
 128                 if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) {
 129                         __ipipe_set_irq_pending(next_domain, irq);
 130                         if (!m_ack && idesc->acknowledge != NULL) {
 131                                 idesc->acknowledge(irq, irq_to_desc(irq));
 132                                 m_ack = 1;
 133                         }
 134                 }
 135                 if (!test_bit(IPIPE_PASS_FLAG, &idesc->control))
 136                         break;
 137                 pos = next_domain->p_link.next;
 138         }
 139
 140         /*
 141          * Now walk the pipeline, yielding control to the highest
 142          * priority domain that has pending interrupt(s) or
 143          * immediately to the current domain if the interrupt has been
 144          * marked as 'sticky'. This search does not go beyond the
 145          * current domain in the pipeline. We also enforce the
 146          * additional root stage lock (blackfin-specific).
 147          */
 148         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
 149                 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
 150
 151         /*
 152          * If the interrupt preempted the head domain, then do not
 153          * even try to walk the pipeline, unless an interrupt is
 154          * pending for it.
 155          */
 156         if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
 157             ipipe_head_cpudom_var(irqpend_himask) == 0)
 158                 goto out;
 159
 160         __ipipe_walk_pipeline(head);
 161 out:
 162         if (!s)
 163                 __clear_bit(IPIPE_STALL_FLAG, &p->status);
 164 }
 165
 166 void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 167 {
 168         struct irq_desc *desc = irq_to_desc(irq);
 169         int prio = __ipipe_get_irq_priority(irq);
 170
 171         desc->depth = 0;
 172         if (ipd != &ipipe_root &&
 173             atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
 174                 __set_bit(prio, &__ipipe_irq_lvmask);
 175 }
 176 EXPORT_SYMBOL(__ipipe_enable_irqdesc);
 177
 178 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 179 {
 180         int prio = __ipipe_get_irq_priority(irq);
 181
 182         if (ipd != &ipipe_root &&
 183             atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
 184                 __clear_bit(prio, &__ipipe_irq_lvmask);
 185 }
 186 EXPORT_SYMBOL(__ipipe_disable_irqdesc);
 187
 188 int __ipipe_syscall_root(struct pt_regs *regs)
 189 {
 190         struct ipipe_percpu_domain_data *p;
 191         unsigned long flags;
 192         int ret;
 193
 194         /*
 195          * We need to run the IRQ tail hook whenever we don't
 196          * propagate a syscall to higher domains, because we know that
 197          * important operations might be pending there (e.g. Xenomai
 198          * deferred rescheduling).
 199          */
 200
 201         if (regs->orig_p0 < NR_syscalls) {
 202                 void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
 203                 hook();
 204                 if ((current->flags & PF_EVNOTIFY) == 0)
 205                         return 0;
 206         }
 207
 208         /*
 209          * This routine either returns:
 210          * 0 -- if the syscall is to be passed to Linux;
 211          * >0 -- if the syscall should not be passed to Linux, and no
 212          * tail work should be performed;
 213          * <0 -- if the syscall should not be passed to Linux but the
 214          * tail work has to be performed (for handling signals etc).
 215          */
 216
 217         if (!__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
 218                 return 0;
 219
 220         ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
 221
 222         local_irq_save_hw(flags);
 223
 224         if (!__ipipe_root_domain_p) {
 225                 local_irq_restore_hw(flags);
 226                 return 1;
 227         }
 228
 229         p = ipipe_root_cpudom_ptr();
 230         if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0)
 231                 __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
 232
 233         local_irq_restore_hw(flags);
 234
 235         return -ret;
 236 }
 237
 238 unsigned long ipipe_critical_enter(void (*syncfn) (void))
 239 {
 240         unsigned long flags;
 241
 242         local_irq_save_hw(flags);
 243
 244         return flags;
 245 }
 246
 247 void ipipe_critical_exit(unsigned long flags)
 248 {
 249         local_irq_restore_hw(flags);
 250 }
 251
 252 static void __ipipe_no_irqtail(void)
 253 {
 254 }
 255
 256 int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
 257 {
 258         info->ncpus = num_online_cpus();
 259         info->cpufreq = ipipe_cpu_freq();
 260         info->archdep.tmirq = IPIPE_TIMER_IRQ;
 261         info->archdep.tmfreq = info->cpufreq;
 262
 263         return 0;
 264 }
 265
 266 /*
 267  * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
 268  * just like if it has been actually received from a hw source. Also
 269  * works for virtual interrupts.
 270  */
 271 int ipipe_trigger_irq(unsigned irq)
 272 {
 273         unsigned long flags;
 274
 275 #ifdef CONFIG_IPIPE_DEBUG
 276         if (irq >= IPIPE_NR_IRQS ||
 277             (ipipe_virtual_irq_p(irq)
 278              && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
 279                 return -EINVAL;
 280 #endif
 281
 282         local_irq_save_hw(flags);
 283         __ipipe_handle_irq(irq, NULL);
 284         local_irq_restore_hw(flags);
 285
 286         return 1;
 287 }
 288
 289 asmlinkage void __ipipe_sync_root(void)
 290 {
 291         void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
 292         unsigned long flags;
 293
 294         BUG_ON(irqs_disabled());
 295
 296         local_irq_save_hw(flags);
 297
 298         if (irq_tail_hook)
 299                 irq_tail_hook();
 300
 301         clear_thread_flag(TIF_IRQ_SYNC);
 302
 303         if (ipipe_root_cpudom_var(irqpend_himask) != 0)
 304                 __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
 305
 306         local_irq_restore_hw(flags);
 307 }
 308
 309 void ___ipipe_sync_pipeline(unsigned long syncmask)
 310 {
 311         if (__ipipe_root_domain_p &&
 312             test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
 313                 return;
 314
 315         __ipipe_sync_stage(syncmask);
 316 }
 317
 318 void __ipipe_disable_root_irqs_hw(void)
 319 {
 320         /*
 321          * This code is called by the ins{bwl} routines (see
 322          * arch/blackfin/lib/ins.S), which are heavily used by the
 323          * network stack. It masks all interrupts but those handled by
 324          * non-root domains, so that we keep decent network transfer
 325          * rates for Linux without inducing pathological jitter for
 326          * the real-time domain.
 327          */
 328         bfin_sti(__ipipe_irq_lvmask);
 329         __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
 330 }
 331
 332 void __ipipe_enable_root_irqs_hw(void)
 333 {
 334         __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
 335         bfin_sti(bfin_irq_flags);
 336 }
 337
 338 /*
 339  * We could use standard atomic bitops in the following root status
 340  * manipulation routines, but let's prepare for SMP support in the
 341  * same move, preventing CPU migration as required.
 342  */
 343 void __ipipe_stall_root(void)
 344 {
 345         unsigned long *p, flags;
 346
 347         local_irq_save_hw(flags);
 348         p = &__ipipe_root_status;
 349         __set_bit(IPIPE_STALL_FLAG, p);
 350         local_irq_restore_hw(flags);
 351 }
 352 EXPORT_SYMBOL(__ipipe_stall_root);
 353
 354 unsigned long __ipipe_test_and_stall_root(void)
 355 {
 356         unsigned long *p, flags;
 357         int x;
 358
 359         local_irq_save_hw(flags);
 360         p = &__ipipe_root_status;
 361         x = __test_and_set_bit(IPIPE_STALL_FLAG, p);
 362         local_irq_restore_hw(flags);
 363
 364         return x;
 365 }
 366 EXPORT_SYMBOL(__ipipe_test_and_stall_root);
 367
 368 unsigned long __ipipe_test_root(void)
 369 {
 370         const unsigned long *p;
 371         unsigned long flags;
 372         int x;
 373
 374         local_irq_save_hw_smp(flags);
 375         p = &__ipipe_root_status;
 376         x = test_bit(IPIPE_STALL_FLAG, p);
 377         local_irq_restore_hw_smp(flags);
 378
 379         return x;
 380 }
 381 EXPORT_SYMBOL(__ipipe_test_root);
 382
 383 void __ipipe_lock_root(void)
 384 {
 385         unsigned long *p, flags;
 386
 387         local_irq_save_hw(flags);
 388         p = &__ipipe_root_status;
 389         __set_bit(IPIPE_SYNCDEFER_FLAG, p);
 390         local_irq_restore_hw(flags);
 391 }
 392 EXPORT_SYMBOL(__ipipe_lock_root);
 393
 394 void __ipipe_unlock_root(void)
 395 {
 396         unsigned long *p, flags;
 397
 398         local_irq_save_hw(flags);
 399         p = &__ipipe_root_status;
 400         __clear_bit(IPIPE_SYNCDEFER_FLAG, p);
 401         local_irq_restore_hw(flags);
 402 }
 403 EXPORT_SYMBOL(__ipipe_unlock_root);