// SPDX-License-Identifier: GPL-2.0-or-later /* * Cell Broadband Engine OProfile Support * * (C) Copyright IBM Corporation 2006 * * Author: Maynard Johnson */ /* The purpose of this file is to handle SPU event task switching * and to record SPU context information into the OProfile * event buffer. * * Additionally, the spu_sync_buffer function is provided as a helper * for recoding actual SPU program counter samples to the event buffer. */ #include #include #include #include #include #include #include #include #include #include #include #include "pr_util.h" #define RELEASE_ALL 9999 static DEFINE_SPINLOCK(buffer_lock); static DEFINE_SPINLOCK(cache_lock); static int num_spu_nodes; static int spu_prof_num_nodes; struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE]; struct delayed_work spu_work; static unsigned max_spu_buff; static void spu_buff_add(unsigned long int value, int spu) { /* spu buff is a circular buffer. Add entries to the * head. Head is the index to store the next value. * The buffer is full when there is one available entry * in the queue, i.e. head and tail can't be equal. * That way we can tell the difference between the * buffer being full versus empty. * * ASSUMPTION: the buffer_lock is held when this function * is called to lock the buffer, head and tail. */ int full = 1; if (spu_buff[spu].head >= spu_buff[spu].tail) { if ((spu_buff[spu].head - spu_buff[spu].tail) < (max_spu_buff - 1)) full = 0; } else if (spu_buff[spu].tail > spu_buff[spu].head) { if ((spu_buff[spu].tail - spu_buff[spu].head) > 1) full = 0; } if (!full) { spu_buff[spu].buff[spu_buff[spu].head] = value; spu_buff[spu].head++; if (spu_buff[spu].head >= max_spu_buff) spu_buff[spu].head = 0; } else { /* From the user's perspective make the SPU buffer * size management/overflow look like we are using * per cpu buffers. The user uses the same * per cpu parameter to adjust the SPU buffer size. * Increment the sample_lost_overflow to inform * the user the buffer size needs to be increased. */ oprofile_cpu_buffer_inc_smpl_lost(); } } /* This function copies the per SPU buffers to the * OProfile kernel buffer. */ static void sync_spu_buff(void) { int spu; unsigned long flags; int curr_head; for (spu = 0; spu < num_spu_nodes; spu++) { /* In case there was an issue and the buffer didn't * get created skip it. */ if (spu_buff[spu].buff == NULL) continue; /* Hold the lock to make sure the head/tail * doesn't change while spu_buff_add() is * deciding if the buffer is full or not. * Being a little paranoid. */ spin_lock_irqsave(&buffer_lock, flags); curr_head = spu_buff[spu].head; spin_unlock_irqrestore(&buffer_lock, flags); /* Transfer the current contents to the kernel buffer. * data can still be added to the head of the buffer. */ oprofile_put_buff(spu_buff[spu].buff, spu_buff[spu].tail, curr_head, max_spu_buff); spin_lock_irqsave(&buffer_lock, flags); spu_buff[spu].tail = curr_head; spin_unlock_irqrestore(&buffer_lock, flags); } } static void wq_sync_spu_buff(struct work_struct *work) { /* move data from spu buffers to kernel buffer */ sync_spu_buff(); /* only reschedule if profiling is not done */ if (spu_prof_running) schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE); } /* Container for caching information about an active SPU task. */ struct cached_info { struct vma_to_fileoffset_map *map; struct spu *the_spu; /* needed to access pointer to local_store */ struct kref cache_ref; }; static struct cached_info *spu_info[MAX_NUMNODES * 8]; static void destroy_cached_info(struct kref *kref) { struct cached_info *info; info = container_of(kref, struct cached_info, cache_ref); vma_map_free(info->map); kfree(info); module_put(THIS_MODULE); } /* Return the cached_info for the passed SPU number. * ATTENTION: Callers are responsible for obtaining the * cache_lock if needed prior to invoking this function. */ static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num) { struct kref *ref; struct cached_info *ret_info; if (spu_num >= num_spu_nodes) { printk(KERN_ERR "SPU_PROF: " "%s, line %d: Invalid index %d into spu info cache\n", __func__, __LINE__, spu_num); ret_info = NULL; goto out; } if (!spu_info[spu_num] && the_spu) { ref = spu_get_profile_private_kref(the_spu->ctx); if (ref) { spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref); kref_get(&spu_info[spu_num]->cache_ref); } } ret_info = spu_info[spu_num]; out: return ret_info; } /* Looks for cached info for the passed spu. If not found, the * cached info is created for the passed spu. * Returns 0 for success; otherwise, -1 for error. */ static int prepare_cached_spu_info(struct spu *spu, unsigned long objectId) { unsigned long flags; struct vma_to_fileoffset_map *new_map; int retval = 0; struct cached_info *info; /* We won't bother getting cache_lock here since * don't do anything with the cached_info that's returned. */ info = get_cached_info(spu, spu->number); if (info) { pr_debug("Found cached SPU info.\n"); goto out; } /* Create cached_info and set spu_info[spu->number] to point to it. * spu->number is a system-wide value, not a per-node value. */ info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) { printk(KERN_ERR "SPU_PROF: " "%s, line %d: create vma_map failed\n", __func__, __LINE__); retval = -ENOMEM; goto err_alloc; } new_map = create_vma_map(spu, objectId); if (!new_map) { printk(KERN_ERR "SPU_PROF: " "%s, line %d: create vma_map failed\n", __func__, __LINE__); retval = -ENOMEM; goto err_alloc; } pr_debug("Created vma_map\n"); info->map = new_map; info->the_spu = spu; kref_init(&info->cache_ref); spin_lock_irqsave(&cache_lock, flags); spu_info[spu->number] = info; /* Increment count before passing off ref to SPUFS. */ kref_get(&info->cache_ref); /* We increment the module refcount here since SPUFS is * responsible for the final destruction of the cached_info, * and it must be able to access the destroy_cached_info() * function defined in the OProfile module. We decrement * the module refcount in destroy_cached_info. */ try_module_get(THIS_MODULE); spu_set_profile_private_kref(spu->ctx, &info->cache_ref, destroy_cached_info); spin_unlock_irqrestore(&cache_lock, flags); goto out; err_alloc: kfree(info); out: return retval; } /* * NOTE: The caller is responsible for locking the * cache_lock prior to calling this function. */ static int release_cached_info(int spu_index) { int index, end; if (spu_index == RELEASE_ALL) { end = num_spu_nodes; index = 0; } else { if (spu_index >= num_spu_nodes) { printk(KERN_ERR "SPU_PROF: " "%s, line %d: " "Invalid index %d into spu info cache\n", __func__, __LINE__, spu_index); goto out; } end = spu_index + 1; index = spu_index; } for (; index < end; index++) { if (spu_info[index]) { kref_put(&spu_info[index]->cache_ref, destroy_cached_info); spu_info[index] = NULL; } } out: return 0; } /* The source code for fast_get_dcookie was "borrowed" * from drivers/oprofile/buffer_sync.c. */ /* Optimisation. We can manage without taking the dcookie sem * because we cannot reach this code without at least one * dcookie user still being registered (namely, the reader * of the event buffer). */ static inline unsigned long fast_get_dcookie(const struct path *path) { unsigned long cookie; if (path->dentry->d_flags & DCACHE_COOKIE) return (unsigned long)path->dentry; get_dcookie(path, &cookie); return cookie; } /* Look up the dcookie for the task's mm->exe_file, * which corresponds loosely to "application name". Also, determine * the offset for the SPU ELF object. If computed offset is * non-zero, it implies an embedded SPU object; otherwise, it's a * separate SPU binary, in which case we retrieve it's dcookie. * For the embedded case, we must determine if SPU ELF is embedded * in the executable application or another file (i.e., shared lib). * If embedded in a shared lib, we must get the dcookie and return * that to the caller. */ static unsigned long get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, unsigned long *spu_bin_dcookie, unsigned long spu_ref) { unsigned long app_cookie = 0; unsigned int my_offset = 0; struct vm_area_struct *vma; struct file *exe_file; struct mm_struct *mm = spu->mm; if (!mm) goto out; exe_file = get_mm_exe_file(mm); if (exe_file) { app_cookie = fast_get_dcookie(&exe_file->f_path); pr_debug("got dcookie for %pD\n", exe_file); fput(exe_file); } mmap_read_lock(mm); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) continue; my_offset = spu_ref - vma->vm_start; if (!vma->vm_file) goto fail_no_image_cookie; pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n", my_offset, spu_ref, vma->vm_file); *offsetp = my_offset; break; } *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path); pr_debug("got dcookie for %pD\n", vma->vm_file); mmap_read_unlock(mm); out: return app_cookie; fail_no_image_cookie: mmap_read_unlock(mm); printk(KERN_ERR "SPU_PROF: " "%s, line %d: Cannot find dcookie for SPU binary\n", __func__, __LINE__); goto out; } /* This function finds or creates cached context information for the * passed SPU and records SPU context information into the OProfile * event buffer. */ static int process_context_switch(struct spu *spu, unsigned long objectId) { unsigned long flags; int retval; unsigned int offset = 0; unsigned long spu_cookie = 0, app_dcookie; retval = prepare_cached_spu_info(spu, objectId); if (retval) goto out; /* Get dcookie first because a mutex_lock is taken in that * code path, so interrupts must not be disabled. */ app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId); if (!app_dcookie || !spu_cookie) { retval = -ENOENT; goto out; } /* Record context info in event buffer */ spin_lock_irqsave(&buffer_lock, flags); spu_buff_add(ESCAPE_CODE, spu->number); spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number); spu_buff_add(spu->number, spu->number); spu_buff_add(spu->pid, spu->number); spu_buff_add(spu->tgid, spu->number); spu_buff_add(app_dcookie, spu->number); spu_buff_add(spu_cookie, spu->number); spu_buff_add(offset, spu->number); /* Set flag to indicate SPU PC data can now be written out. If * the SPU program counter data is seen before an SPU context * record is seen, the postprocessing will fail. */ spu_buff[spu->number].ctx_sw_seen = 1; spin_unlock_irqrestore(&buffer_lock, flags); smp_wmb(); /* insure spu event buffer updates are written */ /* don't want entries intermingled... */ out: return retval; } /* * This function is invoked on either a bind_context or unbind_context. * If called for an unbind_context, the val arg is 0; otherwise, * it is the object-id value for the spu context. * The data arg is of type 'struct spu *'. */ static int spu_active_notify(struct notifier_block *self, unsigned long val, void *data) { int retval; unsigned long flags; struct spu *the_spu = data; pr_debug("SPU event notification arrived\n"); if (!val) { spin_lock_irqsave(&cache_lock, flags); retval = release_cached_info(the_spu->number); spin_unlock_irqrestore(&cache_lock, flags); } else { retval = process_context_switch(the_spu, val); } return retval; } static struct notifier_block spu_active = { .notifier_call = spu_active_notify, }; static int number_of_online_nodes(void) { u32 cpu; u32 tmp; int nodes = 0; for_each_online_cpu(cpu) { tmp = cbe_cpu_to_node(cpu) + 1; if (tmp > nodes) nodes++; } return nodes; } static int oprofile_spu_buff_create(void) { int spu; max_spu_buff = oprofile_get_cpu_buffer_size(); for (spu = 0; spu < num_spu_nodes; spu++) { /* create circular buffers to store the data in. * use locks to manage accessing the buffers */ spu_buff[spu].head = 0; spu_buff[spu].tail = 0; /* * Create a buffer for each SPU. Can't reliably * create a single buffer for all spus due to not * enough contiguous kernel memory. */ spu_buff[spu].buff = kzalloc((max_spu_buff * sizeof(unsigned long)), GFP_KERNEL); if (!spu_buff[spu].buff) { printk(KERN_ERR "SPU_PROF: " "%s, line %d: oprofile_spu_buff_create " "failed to allocate spu buffer %d.\n", __func__, __LINE__, spu); /* release the spu buffers that have been allocated */ while (spu >= 0) { kfree(spu_buff[spu].buff); spu_buff[spu].buff = 0; spu--; } return -ENOMEM; } } return 0; } /* The main purpose of this function is to synchronize * OProfile with SPUFS by registering to be notified of * SPU task switches. * * NOTE: When profiling SPUs, we must ensure that only * spu_sync_start is invoked and not the generic sync_start * in drivers/oprofile/oprof.c. A return value of * SKIP_GENERIC_SYNC or SYNC_START_ERROR will * accomplish this. */ int spu_sync_start(void) { int spu; int ret = SKIP_GENERIC_SYNC; int register_ret; unsigned long flags = 0; spu_prof_num_nodes = number_of_online_nodes(); num_spu_nodes = spu_prof_num_nodes * 8; INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff); /* create buffer for storing the SPU data to put in * the kernel buffer. */ ret = oprofile_spu_buff_create(); if (ret) goto out; spin_lock_irqsave(&buffer_lock, flags); for (spu = 0; spu < num_spu_nodes; spu++) { spu_buff_add(ESCAPE_CODE, spu); spu_buff_add(SPU_PROFILING_CODE, spu); spu_buff_add(num_spu_nodes, spu); } spin_unlock_irqrestore(&buffer_lock, flags); for (spu = 0; spu < num_spu_nodes; spu++) { spu_buff[spu].ctx_sw_seen = 0; spu_buff[spu].last_guard_val = 0; } /* Register for SPU events */ register_ret = spu_switch_event_register(&spu_active); if (register_ret) { ret = SYNC_START_ERROR; goto out; } pr_debug("spu_sync_start -- running.\n"); out: return ret; } /* Record SPU program counter samples to the oprofile event buffer. */ void spu_sync_buffer(int spu_num, unsigned int *samples, int num_samples) { unsigned long long file_offset; unsigned long flags; int i; struct vma_to_fileoffset_map *map; struct spu *the_spu; unsigned long long spu_num_ll = spu_num; unsigned long long spu_num_shifted = spu_num_ll << 32; struct cached_info *c_info; /* We need to obtain the cache_lock here because it's * possible that after getting the cached_info, the SPU job * corresponding to this cached_info may end, thus resulting * in the destruction of the cached_info. */ spin_lock_irqsave(&cache_lock, flags); c_info = get_cached_info(NULL, spu_num); if (!c_info) { /* This legitimately happens when the SPU task ends before all * samples are recorded. * No big deal -- so we just drop a few samples. */ pr_debug("SPU_PROF: No cached SPU contex " "for SPU #%d. Dropping samples.\n", spu_num); goto out; } map = c_info->map; the_spu = c_info->the_spu; spin_lock(&buffer_lock); for (i = 0; i < num_samples; i++) { unsigned int sample = *(samples+i); int grd_val = 0; file_offset = 0; if (sample == 0) continue; file_offset = vma_map_lookup( map, sample, the_spu, &grd_val); /* If overlays are used by this SPU application, the guard * value is non-zero, indicating which overlay section is in * use. We need to discard samples taken during the time * period which an overlay occurs (i.e., guard value changes). */ if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) { spu_buff[spu_num].last_guard_val = grd_val; /* Drop the rest of the samples. */ break; } /* We must ensure that the SPU context switch has been written * out before samples for the SPU. Otherwise, the SPU context * information is not available and the postprocessing of the * SPU PC will fail with no available anonymous map information. */ if (spu_buff[spu_num].ctx_sw_seen) spu_buff_add((file_offset | spu_num_shifted), spu_num); } spin_unlock(&buffer_lock); out: spin_unlock_irqrestore(&cache_lock, flags); } int spu_sync_stop(void) { unsigned long flags = 0; int ret; int k; ret = spu_switch_event_unregister(&spu_active); if (ret) printk(KERN_ERR "SPU_PROF: " "%s, line %d: spu_switch_event_unregister " \ "returned %d\n", __func__, __LINE__, ret); /* flush any remaining data in the per SPU buffers */ sync_spu_buff(); spin_lock_irqsave(&cache_lock, flags); ret = release_cached_info(RELEASE_ALL); spin_unlock_irqrestore(&cache_lock, flags); /* remove scheduled work queue item rather then waiting * for every queued entry to execute. Then flush pending * system wide buffer to event buffer. */ cancel_delayed_work(&spu_work); for (k = 0; k < num_spu_nodes; k++) { spu_buff[k].ctx_sw_seen = 0; /* * spu_sys_buff will be null if there was a problem * allocating the buffer. Only delete if it exists. */ kfree(spu_buff[k].buff); spu_buff[k].buff = 0; } pr_debug("spu_sync_stop -- done.\n"); return ret; }