summaryrefslogtreecommitdiff
path: root/arch/sparc64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc64')
-rw-r--r--arch/sparc64/Kconfig18
-rw-r--r--arch/sparc64/kernel/auxio.c2
-rw-r--r--arch/sparc64/kernel/entry.S137
-rw-r--r--arch/sparc64/kernel/irq.c752
-rw-r--r--arch/sparc64/kernel/kprobes.c5
-rw-r--r--arch/sparc64/kernel/pci_psycho.c3
-rw-r--r--arch/sparc64/kernel/pci_sabre.c46
-rw-r--r--arch/sparc64/kernel/pci_schizo.c78
-rw-r--r--arch/sparc64/kernel/semaphore.c12
-rw-r--r--arch/sparc64/kernel/sparc64_ksyms.c1
-rw-r--r--arch/sparc64/kernel/time.c2
-rw-r--r--arch/sparc64/kernel/trampoline.S3
-rw-r--r--arch/sparc64/lib/U1memcpy.S103
-rw-r--r--arch/sparc64/lib/VISsave.S15
-rw-r--r--arch/sparc64/lib/atomic.S42
-rw-r--r--arch/sparc64/lib/bitops.S31
-rw-r--r--arch/sparc64/lib/debuglocks.c6
-rw-r--r--arch/sparc64/lib/dec_and_lock.S6
-rw-r--r--arch/sparc64/lib/rwsem.S15
-rw-r--r--arch/sparc64/mm/init.c6
-rw-r--r--arch/sparc64/mm/ultra.S8
21 files changed, 515 insertions, 776 deletions
diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig
index e2b050eb3b96..d78bc13ebbb9 100644
--- a/arch/sparc64/Kconfig
+++ b/arch/sparc64/Kconfig
@@ -444,6 +444,24 @@ config PRINTER
If you have more than 8 printers, you need to increase the LP_NO
macro in lp.c and the PARPORT_MAX macro in parport.h.
+config PPDEV
+ tristate "Support for user-space parallel port device drivers"
+ depends on PARPORT
+ ---help---
+ Saying Y to this adds support for /dev/parport device nodes. This
+ is needed for programs that want portable access to the parallel
+ port, for instance deviceid (which displays Plug-and-Play device
+ IDs).
+
+ This is the parallel port equivalent of SCSI generic support (sg).
+ It is safe to say N to this -- it is not needed for normal printing
+ or parallel port CD-ROM/disk support.
+
+ To compile this driver as a module, choose M here: the
+ module will be called ppdev.
+
+ If unsure, say N.
+
config ENVCTRL
tristate "SUNW, envctrl support"
depends on PCI
diff --git a/arch/sparc64/kernel/auxio.c b/arch/sparc64/kernel/auxio.c
index a0716ccc2f4a..8852c20c8d99 100644
--- a/arch/sparc64/kernel/auxio.c
+++ b/arch/sparc64/kernel/auxio.c
@@ -16,7 +16,7 @@
#include <asm/ebus.h>
#include <asm/auxio.h>
-/* This cannot be static, as it is referenced in entry.S */
+/* This cannot be static, as it is referenced in irq.c */
void __iomem *auxio_register = NULL;
enum auxio_type {
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index a47f2d0b1a29..d3973d8a7195 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -271,8 +271,9 @@ cplus_fptrap_insn_1:
fmuld %f0, %f2, %f26
faddd %f0, %f2, %f28
fmuld %f0, %f2, %f30
+ membar #Sync
b,pt %xcc, fpdis_exit
- membar #Sync
+ nop
2: andcc %g5, FPRS_DU, %g0
bne,pt %icc, 3f
fzero %f32
@@ -301,8 +302,9 @@ cplus_fptrap_insn_2:
fmuld %f32, %f34, %f58
faddd %f32, %f34, %f60
fmuld %f32, %f34, %f62
+ membar #Sync
ba,pt %xcc, fpdis_exit
- membar #Sync
+ nop
3: mov SECONDARY_CONTEXT, %g3
add %g6, TI_FPREGS, %g1
ldxa [%g3] ASI_DMMU, %g5
@@ -551,13 +553,11 @@ do_ivec:
sllx %g3, 5, %g3
or %g2, %lo(ivector_table), %g2
add %g2, %g3, %g3
- ldx [%g3 + 0x08], %g2 /* irq_info */
ldub [%g3 + 0x04], %g4 /* pil */
- brz,pn %g2, do_ivec_spurious
- mov 1, %g2
-
+ mov 1, %g2
sllx %g2, %g4, %g2
sllx %g4, 2, %g4
+
lduw [%g6 + %g4], %g5 /* g5 = irq_work(cpu, pil) */
stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */
stw %g3, [%g6 + %g4] /* irq_work(cpu, pil) = bucket */
@@ -565,9 +565,9 @@ do_ivec:
retry
do_ivec_xcall:
mov 0x50, %g1
-
ldxa [%g1 + %g0] ASI_INTR_R, %g1
srl %g3, 0, %g3
+
mov 0x60, %g7
ldxa [%g7 + %g0] ASI_INTR_R, %g7
stxa %g0, [%g0] ASI_INTR_RECEIVE
@@ -579,19 +579,6 @@ do_ivec_xcall:
1: jmpl %g3, %g0
nop
-do_ivec_spurious:
- stw %g3, [%g6 + 0x00] /* irq_work(cpu, 0) = bucket */
- rdpr %pstate, %g5
-
- wrpr %g5, PSTATE_IG | PSTATE_AG, %pstate
- sethi %hi(109f), %g7
- ba,pt %xcc, etrap
-109: or %g7, %lo(109b), %g7
- call catch_disabled_ivec
- add %sp, PTREGS_OFF, %o0
- ba,pt %xcc, rtrap
- clr %l6
-
.globl save_alternate_globals
save_alternate_globals: /* %o0 = save_area */
rdpr %pstate, %o5
@@ -699,116 +686,6 @@ utrap_ill:
ba,pt %xcc, rtrap
clr %l6
-#ifdef CONFIG_BLK_DEV_FD
- .globl floppy_hardint
-floppy_hardint:
- wr %g0, (1 << 11), %clear_softint
- sethi %hi(doing_pdma), %g1
- ld [%g1 + %lo(doing_pdma)], %g2
- brz,pn %g2, floppy_dosoftint
- sethi %hi(fdc_status), %g3
- ldx [%g3 + %lo(fdc_status)], %g3
- sethi %hi(pdma_vaddr), %g5
- ldx [%g5 + %lo(pdma_vaddr)], %g4
- sethi %hi(pdma_size), %g5
- ldx [%g5 + %lo(pdma_size)], %g5
-
-next_byte:
- lduba [%g3] ASI_PHYS_BYPASS_EC_E, %g7
- andcc %g7, 0x80, %g0
- be,pn %icc, floppy_fifo_emptied
- andcc %g7, 0x20, %g0
- be,pn %icc, floppy_overrun
- andcc %g7, 0x40, %g0
- be,pn %icc, floppy_write
- sub %g5, 1, %g5
-
- inc %g3
- lduba [%g3] ASI_PHYS_BYPASS_EC_E, %g7
- dec %g3
- orcc %g0, %g5, %g0
- stb %g7, [%g4]
- bne,pn %xcc, next_byte
- add %g4, 1, %g4
-
- b,pt %xcc, floppy_tdone
- nop
-
-floppy_write:
- ldub [%g4], %g7
- orcc %g0, %g5, %g0
- inc %g3
- stba %g7, [%g3] ASI_PHYS_BYPASS_EC_E
- dec %g3
- bne,pn %xcc, next_byte
- add %g4, 1, %g4
-
-floppy_tdone:
- sethi %hi(pdma_vaddr), %g1
- stx %g4, [%g1 + %lo(pdma_vaddr)]
- sethi %hi(pdma_size), %g1
- stx %g5, [%g1 + %lo(pdma_size)]
- sethi %hi(auxio_register), %g1
- ldx [%g1 + %lo(auxio_register)], %g7
- lduba [%g7] ASI_PHYS_BYPASS_EC_E, %g5
- or %g5, AUXIO_AUX1_FTCNT, %g5
-/* andn %g5, AUXIO_AUX1_MASK, %g5 */
- stba %g5, [%g7] ASI_PHYS_BYPASS_EC_E
- andn %g5, AUXIO_AUX1_FTCNT, %g5
-/* andn %g5, AUXIO_AUX1_MASK, %g5 */
-
- nop; nop; nop; nop; nop; nop;
- nop; nop; nop; nop; nop; nop;
-
- stba %g5, [%g7] ASI_PHYS_BYPASS_EC_E
- sethi %hi(doing_pdma), %g1
- b,pt %xcc, floppy_dosoftint
- st %g0, [%g1 + %lo(doing_pdma)]
-
-floppy_fifo_emptied:
- sethi %hi(pdma_vaddr), %g1
- stx %g4, [%g1 + %lo(pdma_vaddr)]
- sethi %hi(pdma_size), %g1
- stx %g5, [%g1 + %lo(pdma_size)]
- sethi %hi(irq_action), %g1
- or %g1, %lo(irq_action), %g1
- ldx [%g1 + (11 << 3)], %g3 ! irqaction[floppy_irq]
- ldx [%g3 + 0x08], %g4 ! action->flags>>48==ino
- sethi %hi(ivector_table), %g3
- srlx %g4, 48, %g4
- or %g3, %lo(ivector_table), %g3
- sllx %g4, 5, %g4
- ldx [%g3 + %g4], %g4 ! &ivector_table[ino]
- ldx [%g4 + 0x10], %g4 ! bucket->iclr
- stwa %g0, [%g4] ASI_PHYS_BYPASS_EC_E ! ICLR_IDLE
- membar #Sync ! probably not needed...
- retry
-
-floppy_overrun:
- sethi %hi(pdma_vaddr), %g1
- stx %g4, [%g1 + %lo(pdma_vaddr)]
- sethi %hi(pdma_size), %g1
- stx %g5, [%g1 + %lo(pdma_size)]
- sethi %hi(doing_pdma), %g1
- st %g0, [%g1 + %lo(doing_pdma)]
-
-floppy_dosoftint:
- rdpr %pil, %g2
- wrpr %g0, 15, %pil
- sethi %hi(109f), %g7
- b,pt %xcc, etrap_irq
-109: or %g7, %lo(109b), %g7
-
- mov 11, %o0
- mov 0, %o1
- call sparc_floppy_irq
- add %sp, PTREGS_OFF, %o2
-
- b,pt %xcc, rtrap_irq
- nop
-
-#endif /* CONFIG_BLK_DEV_FD */
-
/* XXX Here is stuff we still need to write... -DaveM XXX */
.globl netbsd_syscall
netbsd_syscall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 4dcb8af94090..74a2e0808cbc 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -37,6 +37,7 @@
#include <asm/uaccess.h>
#include <asm/cache.h>
#include <asm/cpudata.h>
+#include <asm/auxio.h>
#ifdef CONFIG_SMP
static void distribute_irqs(void);
@@ -70,31 +71,7 @@ struct irq_work_struct {
struct irq_work_struct __irq_work[NR_CPUS];
#define irq_work(__cpu, __pil) &(__irq_work[(__cpu)].irq_worklists[(__pil)])
-#ifdef CONFIG_PCI
-/* This is a table of physical addresses used to deal with IBF_DMA_SYNC.
- * It is used for PCI only to synchronize DMA transfers with IRQ delivery
- * for devices behind busses other than APB on Sabre systems.
- *
- * Currently these physical addresses are just config space accesses
- * to the command register for that device.
- */
-unsigned long pci_dma_wsync;
-unsigned long dma_sync_reg_table[256];
-unsigned char dma_sync_reg_table_entry = 0;
-#endif
-
-/* This is based upon code in the 32-bit Sparc kernel written mostly by
- * David Redman (djhr@tadpole.co.uk).
- */
-#define MAX_STATIC_ALLOC 4
-static struct irqaction static_irqaction[MAX_STATIC_ALLOC];
-static int static_irq_count;
-
-/* This is exported so that fast IRQ handlers can get at it... -DaveM */
-struct irqaction *irq_action[NR_IRQS+1] = {
- NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL
-};
+static struct irqaction *irq_action[NR_IRQS+1];
/* This only synchronizes entities which modify IRQ handler
* state and some selected user-level spots that want to
@@ -240,17 +217,22 @@ void disable_irq(unsigned int irq)
* the CPU %tick register and not by some normal vectored interrupt
* source. To handle this special case, we use this dummy INO bucket.
*/
+static struct irq_desc pil0_dummy_desc;
static struct ino_bucket pil0_dummy_bucket = {
- 0, /* irq_chain */
- 0, /* pil */
- 0, /* pending */
- 0, /* flags */
- 0, /* __unused */
- NULL, /* irq_info */
- 0UL, /* iclr */
- 0UL, /* imap */
+ .irq_info = &pil0_dummy_desc,
};
+static void build_irq_error(const char *msg, unsigned int ino, int pil, int inofixup,
+ unsigned long iclr, unsigned long imap,
+ struct ino_bucket *bucket)
+{
+ prom_printf("IRQ: INO %04x (%d:%016lx:%016lx) --> "
+ "(%d:%d:%016lx:%016lx), halting...\n",
+ ino, bucket->pil, bucket->iclr, bucket->imap,
+ pil, inofixup, iclr, imap);
+ prom_halt();
+}
+
unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap)
{
struct ino_bucket *bucket;
@@ -279,28 +261,35 @@ unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long
prom_halt();
}
- /* Ok, looks good, set it up. Don't touch the irq_chain or
- * the pending flag.
- */
bucket = &ivector_table[ino];
- if ((bucket->flags & IBF_ACTIVE) ||
- (bucket->irq_info != NULL)) {
- /* This is a gross fatal error if it happens here. */
- prom_printf("IRQ: Trying to reinit INO bucket, fatal error.\n");
- prom_printf("IRQ: Request INO %04x (%d:%d:%016lx:%016lx)\n",
- ino, pil, inofixup, iclr, imap);
- prom_printf("IRQ: Existing (%d:%016lx:%016lx)\n",
- bucket->pil, bucket->iclr, bucket->imap);
- prom_printf("IRQ: Cannot continue, halting...\n");
+ if (bucket->flags & IBF_ACTIVE)
+ build_irq_error("IRQ: Trying to build active INO bucket.\n",
+ ino, pil, inofixup, iclr, imap, bucket);
+
+ if (bucket->irq_info) {
+ if (bucket->imap != imap || bucket->iclr != iclr)
+ build_irq_error("IRQ: Trying to reinit INO bucket.\n",
+ ino, pil, inofixup, iclr, imap, bucket);
+
+ goto out;
+ }
+
+ bucket->irq_info = kmalloc(sizeof(struct irq_desc), GFP_ATOMIC);
+ if (!bucket->irq_info) {
+ prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
prom_halt();
}
+ memset(bucket->irq_info, 0, sizeof(struct irq_desc));
+
+ /* Ok, looks good, set it up. Don't touch the irq_chain or
+ * the pending flag.
+ */
bucket->imap = imap;
bucket->iclr = iclr;
bucket->pil = pil;
bucket->flags = 0;
- bucket->irq_info = NULL;
-
+out:
return __irq(bucket);
}
@@ -318,26 +307,65 @@ static void atomic_bucket_insert(struct ino_bucket *bucket)
__asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
}
+static int check_irq_sharing(int pil, unsigned long irqflags)
+{
+ struct irqaction *action, *tmp;
+
+ action = *(irq_action + pil);
+ if (action) {
+ if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) {
+ for (tmp = action; tmp->next; tmp = tmp->next)
+ ;
+ } else {
+ return -EBUSY;
+ }
+ }
+ return 0;
+}
+
+static void append_irq_action(int pil, struct irqaction *action)
+{
+ struct irqaction **pp = irq_action + pil;
+
+ while (*pp)
+ pp = &((*pp)->next);
+ *pp = action;
+}
+
+static struct irqaction *get_action_slot(struct ino_bucket *bucket)
+{
+ struct irq_desc *desc = bucket->irq_info;
+ int max_irq, i;
+
+ max_irq = 1;
+ if (bucket->flags & IBF_PCI)
+ max_irq = MAX_IRQ_DESC_ACTION;
+ for (i = 0; i < max_irq; i++) {
+ struct irqaction *p = &desc->action[i];
+ u32 mask = (1 << i);
+
+ if (desc->action_active_mask & mask)
+ continue;
+
+ desc->action_active_mask |= mask;
+ return p;
+ }
+ return NULL;
+}
+
int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *),
unsigned long irqflags, const char *name, void *dev_id)
{
- struct irqaction *action, *tmp = NULL;
+ struct irqaction *action;
struct ino_bucket *bucket = __bucket(irq);
unsigned long flags;
int pending = 0;
- if ((bucket != &pil0_dummy_bucket) &&
- (bucket < &ivector_table[0] ||
- bucket >= &ivector_table[NUM_IVECS])) {
- unsigned int *caller;
-
- __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
- printk(KERN_CRIT "request_irq: Old style IRQ registry attempt "
- "from %p, irq %08x.\n", caller, irq);
+ if (unlikely(!handler))
return -EINVAL;
- }
- if (!handler)
- return -EINVAL;
+
+ if (unlikely(!bucket->irq_info))
+ return -ENODEV;
if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) {
/*
@@ -355,93 +383,20 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
spin_lock_irqsave(&irq_action_lock, flags);
- action = *(bucket->pil + irq_action);
- if (action) {
- if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ))
- for (tmp = action; tmp->next; tmp = tmp->next)
- ;
- else {
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return -EBUSY;
- }
- action = NULL; /* Or else! */
+ if (check_irq_sharing(bucket->pil, irqflags)) {
+ spin_unlock_irqrestore(&irq_action_lock, flags);
+ return -EBUSY;
}
- /* If this is flagged as statically allocated then we use our
- * private struct which is never freed.
- */
- if (irqflags & SA_STATIC_ALLOC) {
- if (static_irq_count < MAX_STATIC_ALLOC)
- action = &static_irqaction[static_irq_count++];
- else
- printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
- "using kmalloc\n", irq, name);
- }
- if (action == NULL)
- action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
- GFP_ATOMIC);
-
+ action = get_action_slot(bucket);
if (!action) {
spin_unlock_irqrestore(&irq_action_lock, flags);
return -ENOMEM;
}
- if (bucket == &pil0_dummy_bucket) {
- bucket->irq_info = action;
- bucket->flags |= IBF_ACTIVE;
- } else {
- if ((bucket->flags & IBF_ACTIVE) != 0) {
- void *orig = bucket->irq_info;
- void **vector = NULL;
-
- if ((bucket->flags & IBF_PCI) == 0) {
- printk("IRQ: Trying to share non-PCI bucket.\n");
- goto free_and_ebusy;
- }
- if ((bucket->flags & IBF_MULTI) == 0) {
- vector = kmalloc(sizeof(void *) * 4, GFP_ATOMIC);
- if (vector == NULL)
- goto free_and_enomem;
-
- /* We might have slept. */
- if ((bucket->flags & IBF_MULTI) != 0) {
- int ent;
-
- kfree(vector);
- vector = (void **)bucket->irq_info;
- for(ent = 0; ent < 4; ent++) {
- if (vector[ent] == NULL) {
- vector[ent] = action;
- break;
- }
- }
- if (ent == 4)
- goto free_and_ebusy;
- } else {
- vector[0] = orig;
- vector[1] = action;
- vector[2] = NULL;
- vector[3] = NULL;
- bucket->irq_info = vector;
- bucket->flags |= IBF_MULTI;
- }
- } else {
- int ent;
-
- vector = (void **)orig;
- for (ent = 0; ent < 4; ent++) {
- if (vector[ent] == NULL) {
- vector[ent] = action;
- break;
- }
- }
- if (ent == 4)
- goto free_and_ebusy;
- }
- } else {
- bucket->irq_info = action;
- bucket->flags |= IBF_ACTIVE;
- }
+ bucket->flags |= IBF_ACTIVE;
+ pending = 0;
+ if (bucket != &pil0_dummy_bucket) {
pending = bucket->pending;
if (pending)
bucket->pending = 0;
@@ -455,10 +410,7 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
put_ino_in_irqaction(action, irq);
put_smpaff_in_irqaction(action, CPU_MASK_NONE);
- if (tmp)
- tmp->next = action;
- else
- *(bucket->pil + irq_action) = action;
+ append_irq_action(bucket->pil, action);
enable_irq(irq);
@@ -467,147 +419,103 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
atomic_bucket_insert(bucket);
set_softint(1 << bucket->pil);
}
+
spin_unlock_irqrestore(&irq_action_lock, flags);
- if ((bucket != &pil0_dummy_bucket) && (!(irqflags & SA_STATIC_ALLOC)))
+
+ if (bucket != &pil0_dummy_bucket)
register_irq_proc(__irq_ino(irq));
#ifdef CONFIG_SMP
distribute_irqs();
#endif
return 0;
-
-free_and_ebusy:
- kfree(action);
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return -EBUSY;
-
-free_and_enomem:
- kfree(action);
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return -ENOMEM;
}
EXPORT_SYMBOL(request_irq);
-void free_irq(unsigned int irq, void *dev_id)
+static struct irqaction *unlink_irq_action(unsigned int irq, void *dev_id)
{
- struct irqaction *action;
- struct irqaction *tmp = NULL;
- unsigned long flags;
- struct ino_bucket *bucket = __bucket(irq), *bp;
+ struct ino_bucket *bucket = __bucket(irq);
+ struct irqaction *action, **pp;
- if ((bucket != &pil0_dummy_bucket) &&
- (bucket < &ivector_table[0] ||
- bucket >= &ivector_table[NUM_IVECS])) {
- unsigned int *caller;
+ pp = irq_action + bucket->pil;
+ action = *pp;
+ if (unlikely(!action))
+ return NULL;
- __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
- printk(KERN_CRIT "free_irq: Old style IRQ removal attempt "
- "from %p, irq %08x.\n", caller, irq);
- return;
- }
-
- spin_lock_irqsave(&irq_action_lock, flags);
-
- action = *(bucket->pil + irq_action);
- if (!action->handler) {
+ if (unlikely(!action->handler)) {
printk("Freeing free IRQ %d\n", bucket->pil);
- return;
- }
- if (dev_id) {
- for ( ; action; action = action->next) {
- if (action->dev_id == dev_id)
- break;
- tmp = action;
- }
- if (!action) {
- printk("Trying to free free shared IRQ %d\n", bucket->pil);
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return;
- }
- } else if (action->flags & SA_SHIRQ) {
- printk("Trying to free shared IRQ %d with NULL device ID\n", bucket->pil);
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return;
+ return NULL;
}
- if (action->flags & SA_STATIC_ALLOC) {
- printk("Attempt to free statically allocated IRQ %d (%s)\n",
- bucket->pil, action->name);
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return;
+ while (action && action->dev_id != dev_id) {
+ pp = &action->next;
+ action = *pp;
}
- if (action && tmp)
- tmp->next = action->next;
- else
- *(bucket->pil + irq_action) = action->next;
+ if (likely(action))
+ *pp = action->next;
+
+ return action;
+}
+
+void free_irq(unsigned int irq, void *dev_id)
+{
+ struct irqaction *action;
+ struct ino_bucket *bucket;
+ unsigned long flags;
+
+ spin_lock_irqsave(&irq_action_lock, flags);
+
+ action = unlink_irq_action(irq, dev_id);
spin_unlock_irqrestore(&irq_action_lock, flags);
+ if (unlikely(!action))
+ return;
+
synchronize_irq(irq);
spin_lock_irqsave(&irq_action_lock, flags);
+ bucket = __bucket(irq);
if (bucket != &pil0_dummy_bucket) {
+ struct irq_desc *desc = bucket->irq_info;
unsigned long imap = bucket->imap;
- void **vector, *orig;
- int ent;
-
- orig = bucket->irq_info;
- vector = (void **)orig;
-
- if ((bucket->flags & IBF_MULTI) != 0) {
- int other = 0;
- void *orphan = NULL;
- for (ent = 0; ent < 4; ent++) {
- if (vector[ent] == action)
- vector[ent] = NULL;
- else if (vector[ent] != NULL) {
- orphan = vector[ent];
- other++;
- }
- }
+ int ent, i;
- /* Only free when no other shared irq
- * uses this bucket.
- */
- if (other) {
- if (other == 1) {
- /* Convert back to non-shared bucket. */
- bucket->irq_info = orphan;
- bucket->flags &= ~(IBF_MULTI);
- kfree(vector);
- }
- goto out;
+ for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
+ struct irqaction *p = &desc->action[i];
+
+ if (p == action) {
+ desc->action_active_mask &= ~(1 << i);
+ break;
}
- } else {
- bucket->irq_info = NULL;
}
- /* This unique interrupt source is now inactive. */
- bucket->flags &= ~IBF_ACTIVE;
+ if (!desc->action_active_mask) {
+ /* This unique interrupt source is now inactive. */
+ bucket->flags &= ~IBF_ACTIVE;
- /* See if any other buckets share this bucket's IMAP
- * and are still active.
- */
- for (ent = 0; ent < NUM_IVECS; ent++) {
- bp = &ivector_table[ent];
- if (bp != bucket &&
- bp->imap == imap &&
- (bp->flags & IBF_ACTIVE) != 0)
- break;
- }
+ /* See if any other buckets share this bucket's IMAP
+ * and are still active.
+ */
+ for (ent = 0; ent < NUM_IVECS; ent++) {
+ struct ino_bucket *bp = &ivector_table[ent];
+ if (bp != bucket &&
+ bp->imap == imap &&
+ (bp->flags & IBF_ACTIVE) != 0)
+ break;
+ }
- /* Only disable when no other sub-irq levels of
- * the same IMAP are active.
- */
- if (ent == NUM_IVECS)
- disable_irq(irq);
+ /* Only disable when no other sub-irq levels of
+ * the same IMAP are active.
+ */
+ if (ent == NUM_IVECS)
+ disable_irq(irq);
+ }
}
-out:
- kfree(action);
spin_unlock_irqrestore(&irq_action_lock, flags);
}
@@ -646,99 +554,55 @@ void synchronize_irq(unsigned int irq)
}
#endif /* CONFIG_SMP */
-void catch_disabled_ivec(struct pt_regs *regs)
-{
- int cpu = smp_processor_id();
- struct ino_bucket *bucket = __bucket(*irq_work(cpu, 0));
-
- /* We can actually see this on Ultra/PCI PCI cards, which are bridges
- * to other devices. Here a single IMAP enabled potentially multiple
- * unique interrupt sources (which each do have a unique ICLR register.
- *
- * So what we do is just register that the IVEC arrived, when registered
- * for real the request_irq() code will check the bit and signal
- * a local CPU interrupt for it.
- */
-#if 0
- printk("IVEC: Spurious interrupt vector (%x) received at (%016lx)\n",
- bucket - &ivector_table[0], regs->tpc);
-#endif
- *irq_work(cpu, 0) = 0;
- bucket->pending = 1;
-}
-
-/* Tune this... */
-#define FORWARD_VOLUME 12
-
-#ifdef CONFIG_SMP
-
-static inline void redirect_intr(int cpu, struct ino_bucket *bp)
+static void process_bucket(int irq, struct ino_bucket *bp, struct pt_regs *regs)
{
- /* Ok, here is what is going on:
- * 1) Retargeting IRQs on Starfire is very
- * expensive so just forget about it on them.
- * 2) Moving around very high priority interrupts
- * is a losing game.
- * 3) If the current cpu is idle, interrupts are
- * useful work, so keep them here. But do not
- * pass to our neighbour if he is not very idle.
- * 4) If sysadmin explicitly asks for directed intrs,
- * Just Do It.
- */
- struct irqaction *ap = bp->irq_info;
- cpumask_t cpu_mask;
- unsigned int buddy, ticks;
+ struct irq_desc *desc = bp->irq_info;
+ unsigned char flags = bp->flags;
+ u32 action_mask, i;
+ int random;
- cpu_mask = get_smpaff_in_irqaction(ap);
- cpus_and(cpu_mask, cpu_mask, cpu_online_map);
- if (cpus_empty(cpu_mask))
- cpu_mask = cpu_online_map;
+ bp->flags |= IBF_INPROGRESS;
- if (this_is_starfire != 0 ||
- bp->pil >= 10 || current->pid == 0)
+ if (unlikely(!(flags & IBF_ACTIVE))) {
+ bp->pending = 1;
goto out;
-
- /* 'cpu' is the MID (ie. UPAID), calculate the MID
- * of our buddy.
- */
- buddy = cpu + 1;
- if (buddy >= NR_CPUS)
- buddy = 0;
-
- ticks = 0;
- while (!cpu_isset(buddy, cpu_mask)) {
- if (++buddy >= NR_CPUS)
- buddy = 0;
- if (++ticks > NR_CPUS) {
- put_smpaff_in_irqaction(ap, CPU_MASK_NONE);
- goto out;
- }
}
- if (buddy == cpu)
- goto out;
+ if (desc->pre_handler)
+ desc->pre_handler(bp,
+ desc->pre_handler_arg1,
+ desc->pre_handler_arg2);
- /* Voo-doo programming. */
- if (cpu_data(buddy).idle_volume < FORWARD_VOLUME)
- goto out;
+ action_mask = desc->action_active_mask;
+ random = 0;
+ for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
+ struct irqaction *p = &desc->action[i];
+ u32 mask = (1 << i);
- /* This just so happens to be correct on Cheetah
- * at the moment.
- */
- buddy <<= 26;
+ if (!(action_mask & mask))
+ continue;
- /* Push it to our buddy. */
- upa_writel(buddy | IMAP_VALID, bp->imap);
+ action_mask &= ~mask;
+ if (p->handler(__irq(bp), p->dev_id, regs) == IRQ_HANDLED)
+ random |= p->flags;
+
+ if (!action_mask)
+ break;
+ }
+ if (bp->pil != 0) {
+ upa_writel(ICLR_IDLE, bp->iclr);
+ /* Test and add entropy */
+ if (random & SA_SAMPLE_RANDOM)
+ add_interrupt_randomness(irq);
+ }
out:
- return;
+ bp->flags &= ~IBF_INPROGRESS;
}
-#endif
-
void handler_irq(int irq, struct pt_regs *regs)
{
- struct ino_bucket *bp, *nbp;
+ struct ino_bucket *bp;
int cpu = smp_processor_id();
#ifndef CONFIG_SMP
@@ -756,8 +620,6 @@ void handler_irq(int irq, struct pt_regs *regs)
clear_softint(clr_mask);
}
#else
- int should_forward = 0;
-
clear_softint(1 << irq);
#endif
@@ -772,199 +634,76 @@ void handler_irq(int irq, struct pt_regs *regs)
#else
bp = __bucket(xchg32(irq_work(cpu, irq), 0));
#endif
- for ( ; bp != NULL; bp = nbp) {
- unsigned char flags = bp->flags;
- unsigned char random = 0;
+ while (bp) {
+ struct ino_bucket *nbp = __bucket(bp->irq_chain);
- nbp = __bucket(bp->irq_chain);
bp->irq_chain = 0;
-
- bp->flags |= IBF_INPROGRESS;
-
- if ((flags & IBF_ACTIVE) != 0) {
-#ifdef CONFIG_PCI
- if ((flags & IBF_DMA_SYNC) != 0) {
- upa_readl(dma_sync_reg_table[bp->synctab_ent]);
- upa_readq(pci_dma_wsync);
- }
-#endif
- if ((flags & IBF_MULTI) == 0) {
- struct irqaction *ap = bp->irq_info;
- int ret;
-
- ret = ap->handler(__irq(bp), ap->dev_id, regs);
- if (ret == IRQ_HANDLED)
- random |= ap->flags;
- } else {
- void **vector = (void **)bp->irq_info;
- int ent;
- for (ent = 0; ent < 4; ent++) {
- struct irqaction *ap = vector[ent];
- if (ap != NULL) {
- int ret;
-
- ret = ap->handler(__irq(bp),
- ap->dev_id,
- regs);
- if (ret == IRQ_HANDLED)
- random |= ap->flags;
- }
- }
- }
- /* Only the dummy bucket lacks IMAP/ICLR. */
- if (bp->pil != 0) {
-#ifdef CONFIG_SMP
- if (should_forward) {
- redirect_intr(cpu, bp);
- should_forward = 0;
- }
-#endif
- upa_writel(ICLR_IDLE, bp->iclr);
-
- /* Test and add entropy */
- if (random & SA_SAMPLE_RANDOM)
- add_interrupt_randomness(irq);
- }
- } else
- bp->pending = 1;
-
- bp->flags &= ~IBF_INPROGRESS;
+ process_bucket(irq, bp, regs);
+ bp = nbp;
}
irq_exit();
}
#ifdef CONFIG_BLK_DEV_FD
-extern void floppy_interrupt(int irq, void *dev_cookie, struct pt_regs *regs);
-
-void sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
-{
- struct irqaction *action = *(irq + irq_action);
- struct ino_bucket *bucket;
- int cpu = smp_processor_id();
-
- irq_enter();
- kstat_this_cpu.irqs[irq]++;
-
- *(irq_work(cpu, irq)) = 0;
- bucket = get_ino_in_irqaction(action) + ivector_table;
-
- bucket->flags |= IBF_INPROGRESS;
-
- floppy_interrupt(irq, dev_cookie, regs);
- upa_writel(ICLR_IDLE, bucket->iclr);
+extern irqreturn_t floppy_interrupt(int, void *, struct pt_regs *);;
- bucket->flags &= ~IBF_INPROGRESS;
+/* XXX No easy way to include asm/floppy.h XXX */
+extern unsigned char *pdma_vaddr;
+extern unsigned long pdma_size;
+extern volatile int doing_pdma;
+extern unsigned long fdc_status;
- irq_exit();
-}
-#endif
-
-/* The following assumes that the branch lies before the place we
- * are branching to. This is the case for a trap vector...
- * You have been warned.
- */
-#define SPARC_BRANCH(dest_addr, inst_addr) \
- (0x10800000 | ((((dest_addr)-(inst_addr))>>2)&0x3fffff))
-
-#define SPARC_NOP (0x01000000)
-
-static void install_fast_irq(unsigned int cpu_irq,
- irqreturn_t (*handler)(int, void *, struct pt_regs *))
+irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
{
- extern unsigned long sparc64_ttable_tl0;
- unsigned long ttent = (unsigned long) &sparc64_ttable_tl0;
- unsigned int *insns;
-
- ttent += 0x820;
- ttent += (cpu_irq - 1) << 5;
- insns = (unsigned int *) ttent;
- insns[0] = SPARC_BRANCH(((unsigned long) handler),
- ((unsigned long)&insns[0]));
- insns[1] = SPARC_NOP;
- __asm__ __volatile__("membar #StoreStore; flush %0" : : "r" (ttent));
-}
-
-int request_fast_irq(unsigned int irq,
- irqreturn_t (*handler)(int, void *, struct pt_regs *),
- unsigned long irqflags, const char *name, void *dev_id)
-{
- struct irqaction *action;
- struct ino_bucket *bucket = __bucket(irq);
- unsigned long flags;
-
- /* No pil0 dummy buckets allowed here. */
- if (bucket < &ivector_table[0] ||
- bucket >= &ivector_table[NUM_IVECS]) {
- unsigned int *caller;
-
- __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
- printk(KERN_CRIT "request_fast_irq: Old style IRQ registry attempt "
- "from %p, irq %08x.\n", caller, irq);
- return -EINVAL;
- }
-
- if (!handler)
- return -EINVAL;
+ if (likely(doing_pdma)) {
+ void __iomem *stat = (void __iomem *) fdc_status;
+ unsigned char *vaddr = pdma_vaddr;
+ unsigned long size = pdma_size;
+ u8 val;
+
+ while (size) {
+ val = readb(stat);
+ if (unlikely(!(val & 0x80))) {
+ pdma_vaddr = vaddr;
+ pdma_size = size;
+ return IRQ_HANDLED;
+ }
+ if (unlikely(!(val & 0x20))) {
+ pdma_vaddr = vaddr;
+ pdma_size = size;
+ doing_pdma = 0;
+ goto main_interrupt;
+ }
+ if (val & 0x40) {
+ /* read */
+ *vaddr++ = readb(stat + 1);
+ } else {
+ unsigned char data = *vaddr++;
- if ((bucket->pil == 0) || (bucket->pil == 14)) {
- printk("request_fast_irq: Trying to register shared IRQ 0 or 14.\n");
- return -EBUSY;
- }
+ /* write */
+ writeb(data, stat + 1);
+ }
+ size--;
+ }
- spin_lock_irqsave(&irq_action_lock, flags);
+ pdma_vaddr = vaddr;
+ pdma_size = size;
- action = *(bucket->pil + irq_action);
- if (action) {
- if (action->flags & SA_SHIRQ)
- panic("Trying to register fast irq when already shared.\n");
- if (irqflags & SA_SHIRQ)
- panic("Trying to register fast irq as shared.\n");
- printk("request_fast_irq: Trying to register yet already owned.\n");
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return -EBUSY;
- }
+ /* Send Terminal Count pulse to floppy controller. */
+ val = readb(auxio_register);
+ val |= AUXIO_AUX1_FTCNT;
+ writeb(val, auxio_register);
+ val &= AUXIO_AUX1_FTCNT;
+ writeb(val, auxio_register);
- /*
- * We do not check for SA_SAMPLE_RANDOM in this path. Neither do we
- * support smp intr affinity in this path.
- */
- if (irqflags & SA_STATIC_ALLOC) {
- if (static_irq_count < MAX_STATIC_ALLOC)
- action = &static_irqaction[static_irq_count++];
- else
- printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
- "using kmalloc\n", bucket->pil, name);
- }
- if (action == NULL)
- action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
- GFP_ATOMIC);
- if (!action) {
- spin_unlock_irqrestore(&irq_action_lock, flags);
- return -ENOMEM;
+ doing_pdma = 0;
}
- install_fast_irq(bucket->pil, handler);
-
- bucket->irq_info = action;
- bucket->flags |= IBF_ACTIVE;
-
- action->handler = handler;
- action->flags = irqflags;
- action->dev_id = NULL;
- action->name = name;
- action->next = NULL;
- put_ino_in_irqaction(action, irq);
- put_smpaff_in_irqaction(action, CPU_MASK_NONE);
-
- *(bucket->pil + irq_action) = action;
- enable_irq(irq);
- spin_unlock_irqrestore(&irq_action_lock, flags);
-
-#ifdef CONFIG_SMP
- distribute_irqs();
-#endif
- return 0;
+main_interrupt:
+ return floppy_interrupt(irq, dev_cookie, regs);
}
+EXPORT_SYMBOL(sparc_floppy_irq);
+#endif
/* We really don't need these at all on the Sparc. We only have
* stubs here because they are exported to modules.
@@ -1030,7 +769,10 @@ static void distribute_irqs(void)
*/
for (level = 1; level < NR_IRQS; level++) {
struct irqaction *p = irq_action[level];
- if (level == 12) continue;
+
+ if (level == 12)
+ continue;
+
while(p) {
cpu = retarget_one_irq(p, cpu);
p = p->next;
diff --git a/arch/sparc64/kernel/kprobes.c b/arch/sparc64/kernel/kprobes.c
index bdac631cf011..bbf11f85dab1 100644
--- a/arch/sparc64/kernel/kprobes.c
+++ b/arch/sparc64/kernel/kprobes.c
@@ -433,3 +433,8 @@ int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
return 0;
}
+/* architecture specific initialization */
+int arch_init_kprobes(void)
+{
+ return 0;
+}
diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
index 534320ef0db2..91ab466d6c66 100644
--- a/arch/sparc64/kernel/pci_psycho.c
+++ b/arch/sparc64/kernel/pci_psycho.c
@@ -1303,8 +1303,7 @@ static void psycho_controller_hwinit(struct pci_controller_info *p)
{
u64 tmp;
- /* PROM sets the IRQ retry value too low, increase it. */
- psycho_write(p->pbm_A.controller_regs + PSYCHO_IRQ_RETRY, 0xff);
+ psycho_write(p->pbm_A.controller_regs + PSYCHO_IRQ_RETRY, 5);
/* Enable arbiter for all PCI slots. */
tmp = psycho_read(p->pbm_A.controller_regs + PSYCHO_PCIA_CTRL);
diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
index 53d333b4a4e8..52bf3431a422 100644
--- a/arch/sparc64/kernel/pci_sabre.c
+++ b/arch/sparc64/kernel/pci_sabre.c
@@ -595,6 +595,23 @@ static int __init sabre_ino_to_pil(struct pci_dev *pdev, unsigned int ino)
return ret;
}
+/* When a device lives behind a bridge deeper in the PCI bus topology
+ * than APB, a special sequence must run to make sure all pending DMA
+ * transfers at the time of IRQ delivery are visible in the coherency
+ * domain by the cpu. This sequence is to perform a read on the far
+ * side of the non-APB bridge, then perform a read of Sabre's DMA
+ * write-sync register.
+ */
+static void sabre_wsync_handler(struct ino_bucket *bucket, void *_arg1, void *_arg2)
+{
+ struct pci_dev *pdev = _arg1;
+ unsigned long sync_reg = (unsigned long) _arg2;
+ u16 _unused;
+
+ pci_read_config_word(pdev, PCI_VENDOR_ID, &_unused);
+ sabre_read(sync_reg);
+}
+
static unsigned int __init sabre_irq_build(struct pci_pbm_info *pbm,
struct pci_dev *pdev,
unsigned int ino)
@@ -639,24 +656,14 @@ static unsigned int __init sabre_irq_build(struct pci_pbm_info *pbm,
if (pdev) {
struct pcidev_cookie *pcp = pdev->sysdata;
- /* When a device lives behind a bridge deeper in the
- * PCI bus topology than APB, a special sequence must
- * run to make sure all pending DMA transfers at the
- * time of IRQ delivery are visible in the coherency
- * domain by the cpu. This sequence is to perform
- * a read on the far side of the non-APB bridge, then
- * perform a read of Sabre's DMA write-sync register.
- *
- * Currently, the PCI_CONFIG register for the device
- * is used for this read from the far side of the bridge.
- */
if (pdev->bus->number != pcp->pbm->pci_first_busno) {
- bucket->flags |= IBF_DMA_SYNC;
- bucket->synctab_ent = dma_sync_reg_table_entry++;
- dma_sync_reg_table[bucket->synctab_ent] =
- (unsigned long) sabre_pci_config_mkaddr(
- pcp->pbm,
- pdev->bus->number, pdev->devfn, PCI_COMMAND);
+ struct pci_controller_info *p = pcp->pbm->parent;
+ struct irq_desc *d = bucket->irq_info;
+
+ d->pre_handler = sabre_wsync_handler;
+ d->pre_handler_arg1 = pdev;
+ d->pre_handler_arg2 = (void *)
+ p->pbm_A.controller_regs + SABRE_WRSYNC;
}
}
return __irq(bucket);
@@ -1626,10 +1633,9 @@ void __init sabre_init(int pnode, char *model_name)
*/
p->pbm_A.controller_regs = pr_regs[0].phys_addr;
p->pbm_B.controller_regs = pr_regs[0].phys_addr;
- pci_dma_wsync = p->pbm_A.controller_regs + SABRE_WRSYNC;
- printk("PCI: Found SABRE, main regs at %016lx, wsync at %016lx\n",
- p->pbm_A.controller_regs, pci_dma_wsync);
+ printk("PCI: Found SABRE, main regs at %016lx\n",
+ p->pbm_A.controller_regs);
/* Clear interrupts */
diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c
index 5753175b94e6..6a182bb66281 100644
--- a/arch/sparc64/kernel/pci_schizo.c
+++ b/arch/sparc64/kernel/pci_schizo.c
@@ -15,6 +15,7 @@
#include <asm/iommu.h>
#include <asm/irq.h>
#include <asm/upa.h>
+#include <asm/pstate.h>
#include "pci_impl.h"
#include "iommu_common.h"
@@ -326,6 +327,44 @@ static int __init schizo_ino_to_pil(struct pci_dev *pdev, unsigned int ino)
return ret;
}
+static void tomatillo_wsync_handler(struct ino_bucket *bucket, void *_arg1, void *_arg2)
+{
+ unsigned long sync_reg = (unsigned long) _arg2;
+ u64 mask = 1 << (__irq_ino(__irq(bucket)) & IMAP_INO);
+ u64 val;
+ int limit;
+
+ schizo_write(sync_reg, mask);
+
+ limit = 100000;
+ val = 0;
+ while (--limit) {
+ val = schizo_read(sync_reg);
+ if (!(val & mask))
+ break;
+ }
+ if (limit <= 0) {
+ printk("tomatillo_wsync_handler: DMA won't sync [%lx:%lx]\n",
+ val, mask);
+ }
+
+ if (_arg1) {
+ static unsigned char cacheline[64]
+ __attribute__ ((aligned (64)));
+
+ __asm__ __volatile__("rd %%fprs, %0\n\t"
+ "or %0, %4, %1\n\t"
+ "wr %1, 0x0, %%fprs\n\t"
+ "stda %%f0, [%5] %6\n\t"
+ "wr %0, 0x0, %%fprs\n\t"
+ "membar #Sync"
+ : "=&r" (mask), "=&r" (val)
+ : "0" (mask), "1" (val),
+ "i" (FPRS_FEF), "r" (&cacheline[0]),
+ "i" (ASI_BLK_COMMIT_P));
+ }
+}
+
static unsigned int schizo_irq_build(struct pci_pbm_info *pbm,
struct pci_dev *pdev,
unsigned int ino)
@@ -369,6 +408,15 @@ static unsigned int schizo_irq_build(struct pci_pbm_info *pbm,
bucket = __bucket(build_irq(pil, ign_fixup, iclr, imap));
bucket->flags |= IBF_PCI;
+ if (pdev && pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) {
+ struct irq_desc *p = bucket->irq_info;
+
+ p->pre_handler = tomatillo_wsync_handler;
+ p->pre_handler_arg1 = ((pbm->chip_version <= 4) ?
+ (void *) 1 : (void *) 0);
+ p->pre_handler_arg2 = (void *) pbm->sync_reg;
+ }
+
return __irq(bucket);
}
@@ -885,6 +933,7 @@ static irqreturn_t schizo_ce_intr(int irq, void *dev_id, struct pt_regs *regs)
#define SCHIZO_PCI_CTRL (0x2000UL)
#define SCHIZO_PCICTRL_BUS_UNUS (1UL << 63UL) /* Safari */
+#define SCHIZO_PCICTRL_DTO_INT (1UL << 61UL) /* Tomatillo */
#define SCHIZO_PCICTRL_ARB_PRIO (0x1ff << 52UL) /* Tomatillo */
#define SCHIZO_PCICTRL_ESLCK (1UL << 51UL) /* Safari */
#define SCHIZO_PCICTRL_ERRSLOT (7UL << 48UL) /* Safari */
@@ -1887,37 +1936,27 @@ static void __init schizo_pbm_hw_init(struct pci_pbm_info *pbm)
{
u64 tmp;
- /* Set IRQ retry to infinity. */
- schizo_write(pbm->pbm_regs + SCHIZO_PCI_IRQ_RETRY,
- SCHIZO_IRQ_RETRY_INF);
+ schizo_write(pbm->pbm_regs + SCHIZO_PCI_IRQ_RETRY, 5);
- /* Enable arbiter for all PCI slots. Also, disable PCI interval
- * timer so that DTO (Discard TimeOuts) are not reported because
- * some Schizo revisions report them erroneously.
- */
tmp = schizo_read(pbm->pbm_regs + SCHIZO_PCI_CTRL);
- if (pbm->chip_type == PBM_CHIP_TYPE_SCHIZO_PLUS &&
- pbm->chip_version == 0x5 &&
- pbm->chip_revision == 0x1)
- tmp |= 0x0f;
- else
- tmp |= 0xff;
- tmp &= ~SCHIZO_PCICTRL_PTO;
+ /* Enable arbiter for all PCI slots. */
+ tmp |= 0xff;
+
if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO &&
pbm->chip_version >= 0x2)
tmp |= 0x3UL << SCHIZO_PCICTRL_PTO_SHIFT;
- else
- tmp |= 0x1UL << SCHIZO_PCICTRL_PTO_SHIFT;
if (!prom_getbool(pbm->prom_node, "no-bus-parking"))
tmp |= SCHIZO_PCICTRL_PARK;
+ else
+ tmp &= ~SCHIZO_PCICTRL_PARK;
if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO &&
pbm->chip_version <= 0x1)
- tmp |= (1UL << 61);
+ tmp |= SCHIZO_PCICTRL_DTO_INT;
else
- tmp &= ~(1UL << 61);
+ tmp &= ~SCHIZO_PCICTRL_DTO_INT;
if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO)
tmp |= (SCHIZO_PCICTRL_MRM_PREF |
@@ -2015,6 +2054,9 @@ static void __init schizo_pbm_init(struct pci_controller_info *p,
pbm->pbm_regs = pr_regs[0].phys_addr;
pbm->controller_regs = pr_regs[1].phys_addr - 0x10000UL;
+ if (chip_type == PBM_CHIP_TYPE_TOMATILLO)
+ pbm->sync_reg = pr_regs[3].phys_addr + 0x1a18UL;
+
sprintf(pbm->name,
(chip_type == PBM_CHIP_TYPE_TOMATILLO ?
"TOMATILLO%d PBM%c" :
diff --git a/arch/sparc64/kernel/semaphore.c b/arch/sparc64/kernel/semaphore.c
index 63496c43fe17..a809e63f03ef 100644
--- a/arch/sparc64/kernel/semaphore.c
+++ b/arch/sparc64/kernel/semaphore.c
@@ -32,8 +32,9 @@ static __inline__ int __sem_update_count(struct semaphore *sem, int incr)
" add %1, %4, %1\n"
" cas [%3], %0, %1\n"
" cmp %0, %1\n"
+" membar #StoreLoad | #StoreStore\n"
" bne,pn %%icc, 1b\n"
-" membar #StoreLoad | #StoreStore\n"
+" nop\n"
: "=&r" (old_count), "=&r" (tmp), "=m" (sem->count)
: "r" (&sem->count), "r" (incr), "m" (sem->count)
: "cc");
@@ -71,8 +72,9 @@ void up(struct semaphore *sem)
" cmp %%g1, %%g7\n"
" bne,pn %%icc, 1b\n"
" addcc %%g7, 1, %%g0\n"
+" membar #StoreLoad | #StoreStore\n"
" ble,pn %%icc, 3f\n"
-" membar #StoreLoad | #StoreStore\n"
+" nop\n"
"2:\n"
" .subsection 2\n"
"3: mov %0, %%g1\n"
@@ -128,8 +130,9 @@ void __sched down(struct semaphore *sem)
" cmp %%g1, %%g7\n"
" bne,pn %%icc, 1b\n"
" cmp %%g7, 1\n"
+" membar #StoreLoad | #StoreStore\n"
" bl,pn %%icc, 3f\n"
-" membar #StoreLoad | #StoreStore\n"
+" nop\n"
"2:\n"
" .subsection 2\n"
"3: mov %0, %%g1\n"
@@ -233,8 +236,9 @@ int __sched down_interruptible(struct semaphore *sem)
" cmp %%g1, %%g7\n"
" bne,pn %%icc, 1b\n"
" cmp %%g7, 1\n"
+" membar #StoreLoad | #StoreStore\n"
" bl,pn %%icc, 3f\n"
-" membar #StoreLoad | #StoreStore\n"
+" nop\n"
"2:\n"
" .subsection 2\n"
"3: mov %2, %%g1\n"
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index e78cc53594fa..56cd96f4a5cd 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -227,7 +227,6 @@ EXPORT_SYMBOL(__flush_dcache_range);
EXPORT_SYMBOL(mostek_lock);
EXPORT_SYMBOL(mstk48t02_regs);
-EXPORT_SYMBOL(request_fast_irq);
#ifdef CONFIG_SUN_AUXIO
EXPORT_SYMBOL(auxio_set_led);
EXPORT_SYMBOL(auxio_set_lte);
diff --git a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c
index 71b4e3807694..b40db389f90b 100644
--- a/arch/sparc64/kernel/time.c
+++ b/arch/sparc64/kernel/time.c
@@ -973,7 +973,7 @@ static void sparc64_start_timers(irqreturn_t (*cfunc)(int, void *, struct pt_reg
int err;
/* Register IRQ handler. */
- err = request_irq(build_irq(0, 0, 0UL, 0UL), cfunc, SA_STATIC_ALLOC,
+ err = request_irq(build_irq(0, 0, 0UL, 0UL), cfunc, 0,
"timer", NULL);
if (err) {
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 2c8f9344b4ee..3a145fc39cf2 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -98,8 +98,9 @@ startup_continue:
sethi %hi(prom_entry_lock), %g2
1: ldstub [%g2 + %lo(prom_entry_lock)], %g1
+ membar #StoreLoad | #StoreStore
brnz,pn %g1, 1b
- membar #StoreLoad | #StoreStore
+ nop
sethi %hi(p1275buf), %g2
or %g2, %lo(p1275buf), %g2
diff --git a/arch/sparc64/lib/U1memcpy.S b/arch/sparc64/lib/U1memcpy.S
index da9b520c7189..bafd2fc07acb 100644
--- a/arch/sparc64/lib/U1memcpy.S
+++ b/arch/sparc64/lib/U1memcpy.S
@@ -87,14 +87,17 @@
#define LOOP_CHUNK3(src, dest, len, branch_dest) \
MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
+#define DO_SYNC membar #Sync;
#define STORE_SYNC(dest, fsrc) \
EX_ST(STORE_BLK(%fsrc, %dest)); \
- add %dest, 0x40, %dest;
+ add %dest, 0x40, %dest; \
+ DO_SYNC
#define STORE_JUMP(dest, fsrc, target) \
EX_ST(STORE_BLK(%fsrc, %dest)); \
add %dest, 0x40, %dest; \
- ba,pt %xcc, target;
+ ba,pt %xcc, target; \
+ nop;
#define FINISH_VISCHUNK(dest, f0, f1, left) \
subcc %left, 8, %left;\
@@ -239,17 +242,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f0, %f2, %f48
1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
- STORE_JUMP(o0, f48, 40f) membar #Sync
+ STORE_JUMP(o0, f48, 40f)
2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
- STORE_JUMP(o0, f48, 48f) membar #Sync
+ STORE_JUMP(o0, f48, 48f)
3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
- STORE_JUMP(o0, f48, 56f) membar #Sync
+ STORE_JUMP(o0, f48, 56f)
1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -260,17 +263,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f2, %f4, %f48
1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
- STORE_JUMP(o0, f48, 41f) membar #Sync
+ STORE_JUMP(o0, f48, 41f)
2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
- STORE_JUMP(o0, f48, 49f) membar #Sync
+ STORE_JUMP(o0, f48, 49f)
3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
- STORE_JUMP(o0, f48, 57f) membar #Sync
+ STORE_JUMP(o0, f48, 57f)
1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -281,17 +284,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f4, %f6, %f48
1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
- STORE_JUMP(o0, f48, 42f) membar #Sync
+ STORE_JUMP(o0, f48, 42f)
2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
- STORE_JUMP(o0, f48, 50f) membar #Sync
+ STORE_JUMP(o0, f48, 50f)
3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
- STORE_JUMP(o0, f48, 58f) membar #Sync
+ STORE_JUMP(o0, f48, 58f)
1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -302,17 +305,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f6, %f8, %f48
1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
- STORE_JUMP(o0, f48, 43f) membar #Sync
+ STORE_JUMP(o0, f48, 43f)
2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
- STORE_JUMP(o0, f48, 51f) membar #Sync
+ STORE_JUMP(o0, f48, 51f)
3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
- STORE_JUMP(o0, f48, 59f) membar #Sync
+ STORE_JUMP(o0, f48, 59f)
1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -323,17 +326,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f8, %f10, %f48
1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
- STORE_JUMP(o0, f48, 44f) membar #Sync
+ STORE_JUMP(o0, f48, 44f)
2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
- STORE_JUMP(o0, f48, 52f) membar #Sync
+ STORE_JUMP(o0, f48, 52f)
3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
- STORE_JUMP(o0, f48, 60f) membar #Sync
+ STORE_JUMP(o0, f48, 60f)
1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -344,17 +347,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f10, %f12, %f48
1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
- STORE_JUMP(o0, f48, 45f) membar #Sync
+ STORE_JUMP(o0, f48, 45f)
2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
- STORE_JUMP(o0, f48, 53f) membar #Sync
+ STORE_JUMP(o0, f48, 53f)
3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
- STORE_JUMP(o0, f48, 61f) membar #Sync
+ STORE_JUMP(o0, f48, 61f)
1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -365,17 +368,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f12, %f14, %f48
1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
- STORE_JUMP(o0, f48, 46f) membar #Sync
+ STORE_JUMP(o0, f48, 46f)
2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
- STORE_JUMP(o0, f48, 54f) membar #Sync
+ STORE_JUMP(o0, f48, 54f)
3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
- STORE_JUMP(o0, f48, 62f) membar #Sync
+ STORE_JUMP(o0, f48, 62f)
1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -386,17 +389,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
ba,pt %xcc, 1b+4
faligndata %f14, %f16, %f48
1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
- STORE_JUMP(o0, f48, 47f) membar #Sync
+ STORE_JUMP(o0, f48, 47f)
2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
- STORE_JUMP(o0, f48, 55f) membar #Sync
+ STORE_JUMP(o0, f48, 55f)
3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
- STORE_SYNC(o0, f48) membar #Sync
+ STORE_SYNC(o0, f48)
FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
- STORE_JUMP(o0, f48, 63f) membar #Sync
+ STORE_JUMP(o0, f48, 63f)
40: FINISH_VISCHUNK(o0, f0, f2, g3)
41: FINISH_VISCHUNK(o0, f2, f4, g3)
diff --git a/arch/sparc64/lib/VISsave.S b/arch/sparc64/lib/VISsave.S
index 65e328d600a8..4e18989bd602 100644
--- a/arch/sparc64/lib/VISsave.S
+++ b/arch/sparc64/lib/VISsave.S
@@ -72,7 +72,11 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3
stda %f48, [%g3 + %g1] ASI_BLK_P
5: membar #Sync
- jmpl %g7 + %g0, %g0
+ ba,pt %xcc, 80f
+ nop
+
+ .align 32
+80: jmpl %g7 + %g0, %g0
nop
6: ldub [%g3 + TI_FPSAVED], %o5
@@ -87,8 +91,11 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3
stda %f32, [%g2 + %g1] ASI_BLK_P
stda %f48, [%g3 + %g1] ASI_BLK_P
membar #Sync
- jmpl %g7 + %g0, %g0
+ ba,pt %xcc, 80f
+ nop
+ .align 32
+80: jmpl %g7 + %g0, %g0
nop
.align 32
@@ -126,6 +133,10 @@ VISenterhalf:
stda %f0, [%g2 + %g1] ASI_BLK_P
stda %f16, [%g3 + %g1] ASI_BLK_P
membar #Sync
+ ba,pt %xcc, 4f
+ nop
+
+ .align 32
4: and %o5, FPRS_DU, %o5
jmpl %g7 + %g0, %g0
wr %o5, FPRS_FEF, %fprs
diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
index e528b8d1a3e6..faf87c31598b 100644
--- a/arch/sparc64/lib/atomic.S
+++ b/arch/sparc64/lib/atomic.S
@@ -7,18 +7,6 @@
#include <linux/config.h>
#include <asm/asi.h>
- /* On SMP we need to use memory barriers to ensure
- * correct memory operation ordering, nop these out
- * for uniprocessor.
- */
-#ifdef CONFIG_SMP
-#define ATOMIC_PRE_BARRIER membar #StoreLoad | #LoadLoad
-#define ATOMIC_POST_BARRIER membar #StoreLoad | #StoreStore
-#else
-#define ATOMIC_PRE_BARRIER nop
-#define ATOMIC_POST_BARRIER nop
-#endif
-
.text
/* Two versions of the atomic routines, one that
@@ -52,6 +40,24 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
nop
.size atomic_sub, .-atomic_sub
+ /* On SMP we need to use memory barriers to ensure
+ * correct memory operation ordering, nop these out
+ * for uniprocessor.
+ */
+#ifdef CONFIG_SMP
+
+#define ATOMIC_PRE_BARRIER membar #StoreLoad | #LoadLoad;
+#define ATOMIC_POST_BARRIER \
+ ba,pt %xcc, 80b; \
+ membar #StoreLoad | #StoreStore
+
+80: retl
+ nop
+#else
+#define ATOMIC_PRE_BARRIER
+#define ATOMIC_POST_BARRIER
+#endif
+
.globl atomic_add_ret
.type atomic_add_ret,#function
atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
@@ -62,9 +68,10 @@ atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
cmp %g1, %g7
bne,pn %icc, 1b
add %g7, %o0, %g7
+ sra %g7, 0, %o0
ATOMIC_POST_BARRIER
retl
- sra %g7, 0, %o0
+ nop
.size atomic_add_ret, .-atomic_add_ret
.globl atomic_sub_ret
@@ -77,9 +84,10 @@ atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
cmp %g1, %g7
bne,pn %icc, 1b
sub %g7, %o0, %g7
+ sra %g7, 0, %o0
ATOMIC_POST_BARRIER
retl
- sra %g7, 0, %o0
+ nop
.size atomic_sub_ret, .-atomic_sub_ret
.globl atomic64_add
@@ -118,9 +126,10 @@ atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
cmp %g1, %g7
bne,pn %xcc, 1b
add %g7, %o0, %g7
+ mov %g7, %o0
ATOMIC_POST_BARRIER
retl
- mov %g7, %o0
+ nop
.size atomic64_add_ret, .-atomic64_add_ret
.globl atomic64_sub_ret
@@ -133,7 +142,8 @@ atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
cmp %g1, %g7
bne,pn %xcc, 1b
sub %g7, %o0, %g7
+ mov %g7, %o0
ATOMIC_POST_BARRIER
retl
- mov %g7, %o0
+ nop
.size atomic64_sub_ret, .-atomic64_sub_ret
diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 886dcd2b376a..31afbfe6c1e8 100644
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -7,20 +7,26 @@
#include <linux/config.h>
#include <asm/asi.h>
+ .text
+
/* On SMP we need to use memory barriers to ensure
* correct memory operation ordering, nop these out
* for uniprocessor.
*/
+
#ifdef CONFIG_SMP
#define BITOP_PRE_BARRIER membar #StoreLoad | #LoadLoad
-#define BITOP_POST_BARRIER membar #StoreLoad | #StoreStore
+#define BITOP_POST_BARRIER \
+ ba,pt %xcc, 80b; \
+ membar #StoreLoad | #StoreStore
+
+80: retl
+ nop
#else
-#define BITOP_PRE_BARRIER nop
-#define BITOP_POST_BARRIER nop
+#define BITOP_PRE_BARRIER
+#define BITOP_POST_BARRIER
#endif
- .text
-
.globl test_and_set_bit
.type test_and_set_bit,#function
test_and_set_bit: /* %o0=nr, %o1=addr */
@@ -37,10 +43,11 @@ test_and_set_bit: /* %o0=nr, %o1=addr */
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %o2, %g2
- BITOP_POST_BARRIER
clr %o0
+ movrne %g2, 1, %o0
+ BITOP_POST_BARRIER
retl
- movrne %g2, 1, %o0
+ nop
.size test_and_set_bit, .-test_and_set_bit
.globl test_and_clear_bit
@@ -59,10 +66,11 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %o2, %g2
- BITOP_POST_BARRIER
clr %o0
+ movrne %g2, 1, %o0
+ BITOP_POST_BARRIER
retl
- movrne %g2, 1, %o0
+ nop
.size test_and_clear_bit, .-test_and_clear_bit
.globl test_and_change_bit
@@ -81,10 +89,11 @@ test_and_change_bit: /* %o0=nr, %o1=addr */
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %o2, %g2
- BITOP_POST_BARRIER
clr %o0
+ movrne %g2, 1, %o0
+ BITOP_POST_BARRIER
retl
- movrne %g2, 1, %o0
+ nop
.size test_and_change_bit, .-test_and_change_bit
.globl set_bit
diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
index c421e0c65325..f03344cf784e 100644
--- a/arch/sparc64/lib/debuglocks.c
+++ b/arch/sparc64/lib/debuglocks.c
@@ -252,8 +252,9 @@ wlock_again:
" andn %%g1, %%g3, %%g7\n"
" casx [%0], %%g1, %%g7\n"
" cmp %%g1, %%g7\n"
+" membar #StoreLoad | #StoreStore\n"
" bne,pn %%xcc, 1b\n"
-" membar #StoreLoad | #StoreStore"
+" nop"
: /* no outputs */
: "r" (&(rw->lock))
: "g3", "g1", "g7", "cc", "memory");
@@ -351,8 +352,9 @@ int _do_write_trylock (rwlock_t *rw, char *str)
" andn %%g1, %%g3, %%g7\n"
" casx [%0], %%g1, %%g7\n"
" cmp %%g1, %%g7\n"
+" membar #StoreLoad | #StoreStore\n"
" bne,pn %%xcc, 1b\n"
-" membar #StoreLoad | #StoreStore"
+" nop"
: /* no outputs */
: "r" (&(rw->lock))
: "g3", "g1", "g7", "cc", "memory");
diff --git a/arch/sparc64/lib/dec_and_lock.S b/arch/sparc64/lib/dec_and_lock.S
index 7e6fdaebedba..8ee288dd0afc 100644
--- a/arch/sparc64/lib/dec_and_lock.S
+++ b/arch/sparc64/lib/dec_and_lock.S
@@ -48,8 +48,9 @@ start_to_zero:
#endif
to_zero:
ldstub [%o1], %g3
+ membar #StoreLoad | #StoreStore
brnz,pn %g3, spin_on_lock
- membar #StoreLoad | #StoreStore
+ nop
loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
cmp %g2, %g7
@@ -71,8 +72,9 @@ loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
nop
spin_on_lock:
ldub [%o1], %g3
+ membar #LoadLoad
brnz,pt %g3, spin_on_lock
- membar #LoadLoad
+ nop
ba,pt %xcc, to_zero
nop
nop
diff --git a/arch/sparc64/lib/rwsem.S b/arch/sparc64/lib/rwsem.S
index 174ff7b9164c..75f0e6b951d6 100644
--- a/arch/sparc64/lib/rwsem.S
+++ b/arch/sparc64/lib/rwsem.S
@@ -17,8 +17,9 @@ __down_read:
bne,pn %icc, 1b
add %g7, 1, %g7
cmp %g7, 0
+ membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
- membar #StoreLoad | #StoreStore
+ nop
2:
retl
nop
@@ -57,8 +58,9 @@ __down_write:
cmp %g3, %g7
bne,pn %icc, 1b
cmp %g7, 0
+ membar #StoreLoad | #StoreStore
bne,pn %icc, 3f
- membar #StoreLoad | #StoreStore
+ nop
2: retl
nop
3:
@@ -97,8 +99,9 @@ __up_read:
cmp %g1, %g7
bne,pn %icc, 1b
cmp %g7, 0
+ membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
- membar #StoreLoad | #StoreStore
+ nop
2: retl
nop
3: sethi %hi(RWSEM_ACTIVE_MASK), %g1
@@ -126,8 +129,9 @@ __up_write:
bne,pn %icc, 1b
sub %g7, %g1, %g7
cmp %g7, 0
+ membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
- membar #StoreLoad | #StoreStore
+ nop
2:
retl
nop
@@ -151,8 +155,9 @@ __downgrade_write:
bne,pn %icc, 1b
sub %g7, %g1, %g7
cmp %g7, 0
+ membar #StoreLoad | #StoreStore
bl,pn %icc, 3f
- membar #StoreLoad | #StoreStore
+ nop
2:
retl
nop
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 9c5222075da9..8fc413cb6acd 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -136,8 +136,9 @@ static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
"or %%g1, %0, %%g1\n\t"
"casx [%2], %%g7, %%g1\n\t"
"cmp %%g7, %%g1\n\t"
+ "membar #StoreLoad | #StoreStore\n\t"
"bne,pn %%xcc, 1b\n\t"
- " membar #StoreLoad | #StoreStore"
+ " nop"
: /* no outputs */
: "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
: "g1", "g7");
@@ -157,8 +158,9 @@ static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long c
" andn %%g7, %1, %%g1\n\t"
"casx [%2], %%g7, %%g1\n\t"
"cmp %%g7, %%g1\n\t"
+ "membar #StoreLoad | #StoreStore\n\t"
"bne,pn %%xcc, 1b\n\t"
- " membar #StoreLoad | #StoreStore\n"
+ " nop\n"
"2:"
: /* no outputs */
: "r" (cpu), "r" (mask), "r" (&page->flags),
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index 7a0934321010..363770893797 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -72,6 +72,7 @@ __flush_tlb_pending:
flush %g6
retl
wrpr %g7, 0x0, %pstate
+ nop
.align 32
.globl __flush_tlb_kernel_range
@@ -249,7 +250,7 @@ __cheetah_flush_tlb_mm: /* 15 insns */
retl
wrpr %g7, 0x0, %pstate
-__cheetah_flush_tlb_pending: /* 22 insns */
+__cheetah_flush_tlb_pending: /* 23 insns */
/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
rdpr %pstate, %g7
sllx %o1, 3, %o1
@@ -266,8 +267,9 @@ __cheetah_flush_tlb_pending: /* 22 insns */
andn %o3, 1, %o3
stxa %g0, [%o3] ASI_IMMU_DEMAP
2: stxa %g0, [%o3] ASI_DMMU_DEMAP
+ membar #Sync
brnz,pt %o1, 1b
- membar #Sync
+ nop
stxa %g2, [%o4] ASI_DMMU
flush %g6
wrpr %g0, 0, %tl
@@ -316,7 +318,7 @@ cheetah_patch_cachetlbops:
sethi %hi(__cheetah_flush_tlb_pending), %o1
or %o1, %lo(__cheetah_flush_tlb_pending), %o1
call cheetah_patch_one
- mov 22, %o2
+ mov 23, %o2
#ifdef DCACHE_ALIASING_POSSIBLE
sethi %hi(__flush_dcache_page), %o0
generated by cgit v1.2.3 (git 2.39.0) at 2025-01-05 05:08:23 +0300