text: /*
   * Cell Broadband Engine OProfile Support
   *
   * (C) Copyright IBM Corporation 2006
   *
   * Author: Maynard Johnson <maynardj@us.ibm.com>
   *
   * This program is free software; you can redistribute it and/or
   * modify it under the terms of the GNU General Public License
   * as published by the Free Software Foundation; either version
   * 2 of the License, or (at your option) any later version.
   */
  
  /* The purpose of this file is to handle SPU event task switching
   * and to record SPU context information into the OProfile
   * event buffer.
   *
   * Additionally, the spu_sync_buffer function is provided as a helper
   * for recoding actual SPU program counter samples to the event buffer.
   */
  #include <linux/dcookies.h>
  #include <linux/kref.h>
  #include <linux/mm.h>
  #include <linux/fs.h>
  #include <linux/file.h>
  #include <linux/module.h>
  #include <linux/notifier.h>
  #include <linux/numa.h>
  #include <linux/oprofile.h>
  #include <linux/slab.h>
  #include <linux/spinlock.h>
  #include "pr_util.h"
  
  #define RELEASE_ALL 9999
  
  static DEFINE_SPINLOCK(buffer_lock);
  static DEFINE_SPINLOCK(cache_lock);
  static int num_spu_nodes;
  static int spu_prof_num_nodes;
  
  struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
  struct delayed_work spu_work;
  static unsigned max_spu_buff;
  
  static void spu_buff_add(unsigned long int value, int spu)
  {
  	/* spu buff is a circular buffer.  Add entries to the
  	 * head.  Head is the index to store the next value.
  	 * The buffer is full when there is one available entry
  	 * in the queue, i.e. head and tail can't be equal.
  	 * That way we can tell the difference between the
  	 * buffer being full versus empty.
  	 *
  	 *  ASSUMPTION: the buffer_lock is held when this function
  	 *             is called to lock the buffer, head and tail.
  	 */
  	int full = 1;
  
  	if (spu_buff[spu].head >= spu_buff[spu].tail) {
  		if ((spu_buff[spu].head - spu_buff[spu].tail)
  		    <  (max_spu_buff - 1))
  			full = 0;
  
  	} else if (spu_buff[spu].tail > spu_buff[spu].head) {
  		if ((spu_buff[spu].tail - spu_buff[spu].head)
  		    > 1)
  			full = 0;
  	}
  
  	if (!full) {
  		spu_buff[spu].buff[spu_buff[spu].head] = value;
  		spu_buff[spu].head++;
  
  		if (spu_buff[spu].head >= max_spu_buff)
  			spu_buff[spu].head = 0;
  	} else {
  		/* From the user's perspective make the SPU buffer
  		 * size management/overflow look like we are using
  		 * per cpu buffers.  The user uses the same
  		 * per cpu parameter to adjust the SPU buffer size.
  		 * Increment the sample_lost_overflow to inform
  		 * the user the buffer size needs to be increased.
  		 */
  		oprofile_cpu_buffer_inc_smpl_lost();
  	}
  }
  
  /* This function copies the per SPU buffers to the
   * OProfile kernel buffer.
   */
  static void sync_spu_buff(void)
  {
  	int spu;
  	unsigned long flags;
  	int curr_head;
  
  	for (spu = 0; spu < num_spu_nodes; spu++) {
  		/* In case there was an issue and the buffer didn't
  		 * get created skip it.
  		 */
  		if (spu_buff[spu].buff == NULL)
  			continue;
  
  		/* Hold the lock to make sure the head/tail
  		 * doesn't change while spu_buff_add() is
  		 * deciding if the buffer is full or not.
  		 * Being a little paranoid.
  		 */
  		spin_lock_irqsave(&buffer_lock, flags);
  		curr_head = spu_buff[spu].head;
  		spin_unlock_irqrestore(&buffer_lock, flags);
  
  		/* Transfer the current contents to the kernel buffer.
  		 * data can still be added to the head of the buffer.
  		 */
  		oprofile_put_buff(spu_buff[spu].buff,
  				  spu_buff[spu].tail,
  				  curr_head, max_spu_buff);
  
  		spin_lock_irqsave(&buffer_lock, flags);
  		spu_buff[spu].tail = curr_head;
  		spin_unlock_irqrestore(&buffer_lock, flags);
  	}
  
  }
  
  static void wq_sync_spu_buff(struct work_struct *work)
  {
  	/* move data from spu buffers to kernel buffer */
  	sync_spu_buff();
  
  	/* only reschedule if profiling is not done */
  	if (spu_prof_running)
  		schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
  }
  
  /* Container for caching information about an active SPU task. */
  struct cached_info {
  	struct vma_to_fileoffset_map *map;
  	struct spu *the_spu;	/* needed to access pointer to local_store */
  	struct kref cache_ref;
  };
  
  static struct cached_info *spu_info[MAX_NUMNODES * 8];
  
  static void destroy_cached_info(struct kref *kref)
  {
  	struct cached_info *info;
  
  	info = container_of(kref, struct cached_info, cache_ref);
  	vma_map_free(info->map);
  	kfree(info);
  	module_put(THIS_MODULE);
  }
  
  /* Return the cached_info for the passed SPU number.
   * ATTENTION:  Callers are responsible for obtaining the
   *	       cache_lock if needed prior to invoking this function.
   */
  static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
  {
  	struct kref *ref;
  	struct cached_info *ret_info;
  
  	if (spu_num >= num_spu_nodes) {
  		printk(KERN_ERR "SPU_PROF: "
  		       "%s, line %d: Invalid index %d into spu info cache\n",
  		       __func__, __LINE__, spu_num);
  		ret_info = NULL;
  		goto out;
  	}
  	if (!spu_info[spu_num] && the_spu) {
  		ref = spu_get_profile_private_kref(the_spu->ctx);
  		if (ref) {
  			spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
  			kref_get(&spu_info[spu_num]->cache_ref);
  		}
  	}
  
  	ret_info = spu_info[spu_num];
   out:
  	return ret_info;
  }
  
  
  /* Looks for cached info for the passed spu.  If not found, the
   * cached info is created for the passed spu.
   * Returns 0 for success; otherwise, -1 for error.
   */
  static int
  prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
  {
  	unsigned long flags;
  	struct vma_to_fileoffset_map *new_map;
  	int retval = 0;
  	struct cached_info *info;
  
  	/* We won't bother getting cache_lock here since
  	 * don't do anything with the cached_info that's returned.
  	 */
  	info = get_cached_info(spu, spu->number);
  
  	if (info) {
  		pr_debug("Found cached SPU info.\n");
  		goto out;
  	}
  
  	/* Create cached_info and set spu_info[spu->number] to point to it.
  	 * spu->number is a system-wide value, not a per-node value.
  	 */
  	info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
  	if (!info) {
  		printk(KERN_ERR "SPU_PROF: "
  		       "%s, line %d: create vma_map failed\n",
  		       __func__, __LINE__);
  		retval = -ENOMEM;
  		goto err_alloc;
  	}
  	new_map = create_vma_map(spu, objectId);
  	if (!new_map) {
  		printk(KERN_ERR "SPU_PROF: "
  		       "%s, line %d: create vma_map failed\n",
  		       __func__, __LINE__);
  		retval = -ENOMEM;
  		goto err_alloc;
  	}
  
  	pr_debug("Created vma_map\n");
  	info->map = new_map;
  	info->the_spu = spu;
  	kref_init(&info->cache_ref);
  	spin_lock_irqsave(&cache_lock, flags);
  	spu_info[spu->number] = info;
  	/* Increment count before passing off ref to SPUFS. */
  	kref_get(&info->cache_ref);
  
  	/* We increment the module refcount here since SPUFS is
  	 * responsible for the final destruction of the cached_info,
  	 * and it must be able to access the destroy_cached_info()
  	 * function defined in the OProfile module.  We decrement
  	 * the module refcount in destroy_cached_info.
  	 */
  	try_module_get(THIS_MODULE);
  	spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
  				destroy_cached_info);
  	spin_unlock_irqrestore(&cache_lock, flags);
  	goto out;
  
  err_alloc:
  	kfree(info);
  out:
  	return retval;
  }
  
  /*
   * NOTE:  The caller is responsible for locking the
   *	  cache_lock prior to calling this function.
   */
  static int release_cached_info(int spu_index)
  {
  	int index, end;
  
  	if (spu_index == RELEASE_ALL) {
  		end = num_spu_nodes;
  		index = 0;
  	} else {
  		if (spu_index >= num_spu_nodes) {
  			printk(KERN_ERR "SPU_PROF: "
  				"%s, line %d: "
  				"Invalid index %d into spu info cache\n",
  				__func__, __LINE__, spu_index);
  			goto out;
  		}
  		end = spu_index + 1;
  		index = spu_index;
  	}
  	for (; index < end; index++) {
  		if (spu_info[index]) {
  			kref_put(&spu_info[index]->cache_ref,
  				 destroy_cached_info);
  			spu_info[index] = NULL;
  		}
  	}
  
  out:
  	return 0;
  }
  
  /* The source code for fast_get_dcookie was "borrowed"
   * from drivers/oprofile/buffer_sync.c.
   */
  
  /* Optimisation. We can manage without taking the dcookie sem
   * because we cannot reach this code without at least one
   * dcookie user still being registered (namely, the reader
   * of the event buffer).
   */
  static inline unsigned long fast_get_dcookie(const struct path *path)
  {
  	unsigned long cookie;
  
  	if (path->dentry->d_flags & DCACHE_COOKIE)
  		return (unsigned long)path->dentry;
  	get_dcookie(path, &cookie);
  	return cookie;
  }
  
  /* Look up the dcookie for the task's mm->exe_file,
   * which corresponds loosely to "application name". Also, determine
   * the offset for the SPU ELF object.  If computed offset is
   * non-zero, it implies an embedded SPU object; otherwise, it's a
   * separate SPU binary, in which case we retrieve it's dcookie.
   * For the embedded case, we must determine if SPU ELF is embedded
   * in the executable application or another file (i.e., shared lib).
   * If embedded in a shared lib, we must get the dcookie and return
   * that to the caller.
   */
  static unsigned long
  get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
  			    unsigned long *spu_bin_dcookie,
  			    unsigned long spu_ref)
  {
  	unsigned long app_cookie = 0;
  	unsigned int my_offset = 0;
  	struct vm_area_struct *vma;
  	struct file *exe_file;
  	struct mm_struct *mm = spu->mm;
  
  	if (!mm)
  		goto out;
  
  	exe_file = get_mm_exe_file(mm);
  	if (exe_file) {
  		app_cookie = fast_get_dcookie(&exe_file->f_path);
  		pr_debug("got dcookie for %pD\n", exe_file);
  		fput(exe_file);
  	}
  
  	down_read(&mm->mmap_sem);
  	for (vma = mm->mmap; vma; vma = vma->vm_next) {
  		if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
  			continue;
  		my_offset = spu_ref - vma->vm_start;
  		if (!vma->vm_file)
  			goto fail_no_image_cookie;
  
  		pr_debug("Found spu ELF at %X(object-id:%lx) for file %pD\n",
  			 my_offset, spu_ref, vma->vm_file);
  		*offsetp = my_offset;
  		break;
  	}
  
  	*spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
  	pr_debug("got dcookie for %pD\n", vma->vm_file);
  
  	up_read(&mm->mmap_sem);
  
  out:
  	return app_cookie;
  
  fail_no_image_cookie:
  	up_read(&mm->mmap_sem);
  
  	printk(KERN_ERR "SPU_PROF: "
  		"%s, line %d: Cannot find dcookie for SPU binary\n",
  		__func__, __LINE__);
  	goto out;
  }
  
  
  
  /* This function finds or creates cached context information for the
   * passed SPU and records SPU context information into the OProfile
   * event buffer.
   */
  static int process_context_switch(struct spu *spu, unsigned long objectId)
  {
  	unsigned long flags;
  	int retval;
  	unsigned int offset = 0;
  	unsigned long spu_cookie = 0, app_dcookie;
  
  	retval = prepare_cached_spu_info(spu, objectId);
  	if (retval)
  		goto out;
  
  	/* Get dcookie first because a mutex_lock is taken in that
  	 * code path, so interrupts must not be disabled.
  	 */
  	app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
  	if (!app_dcookie || !spu_cookie) {
  		retval  = -ENOENT;
  		goto out;
  	}
  
  	/* Record context info in event buffer */
  	spin_lock_irqsave(&buffer_lock, flags);
  	spu_buff_add(ESCAPE_CODE, spu->number);
  	spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
  	spu_buff_add(spu->number, spu->number);
  	spu_buff_add(spu->pid, spu->number);
  	spu_buff_add(spu->tgid, spu->number);
  	spu_buff_add(app_dcookie, spu->number);
  	spu_buff_add(spu_cookie, spu->number);
  	spu_buff_add(offset, spu->number);
  
  	/* Set flag to indicate SPU PC data can now be written out.  If
  	 * the SPU program counter data is seen before an SPU context
  	 * record is seen, the postprocessing will fail.
  	 */
  	spu_buff[spu->number].ctx_sw_seen = 1;
  
  	spin_unlock_irqrestore(&buffer_lock, flags);
  	smp_wmb();	/* insure spu event buffer updates are written */
  			/* don't want entries intermingled... */
  out:
  	return retval;
  }
  
  /*
   * This function is invoked on either a bind_context or unbind_context.
   * If called for an unbind_context, the val arg is 0; otherwise,
   * it is the object-id value for the spu context.
   * The data arg is of type 'struct spu *'.
   */
  static int spu_active_notify(struct notifier_block *self, unsigned long val,
  				void *data)
  {
  	int retval;
  	unsigned long flags;
  	struct spu *the_spu = data;
  
  	pr_debug("SPU event notification arrived\n");
  	if (!val) {
  		spin_lock_irqsave(&cache_lock, flags);
  		retval = release_cached_info(the_spu->number);
  		spin_unlock_irqrestore(&cache_lock, flags);
  	} else {
  		retval = process_context_switch(the_spu, val);
  	}
  	return retval;
  }
  
  static struct notifier_block spu_active = {
  	.notifier_call = spu_active_notify,
  };
  
  static int number_of_online_nodes(void)
  {
          u32 cpu; u32 tmp;
          int nodes = 0;
          for_each_online_cpu(cpu) {
                  tmp = cbe_cpu_to_node(cpu) + 1;
                  if (tmp > nodes)
                          nodes++;
          }
          return nodes;
  }
  
  static int oprofile_spu_buff_create(void)
  {
  	int spu;
  
  	max_spu_buff = oprofile_get_cpu_buffer_size();
  
  	for (spu = 0; spu < num_spu_nodes; spu++) {
  		/* create circular buffers to store the data in.
  		 * use locks to manage accessing the buffers
  		 */
  		spu_buff[spu].head = 0;
  		spu_buff[spu].tail = 0;
  
  		/*
  		 * Create a buffer for each SPU.  Can't reliably
  		 * create a single buffer for all spus due to not
  		 * enough contiguous kernel memory.
  		 */
  
  		spu_buff[spu].buff = kzalloc((max_spu_buff
  					      * sizeof(unsigned long)),
  					     GFP_KERNEL);
  
  		if (!spu_buff[spu].buff) {
  			printk(KERN_ERR "SPU_PROF: "
  			       "%s, line %d:  oprofile_spu_buff_create "
  		       "failed to allocate spu buffer %d.\n",
  			       __func__, __LINE__, spu);
  
  			/* release the spu buffers that have been allocated */
  			while (spu >= 0) {
  				kfree(spu_buff[spu].buff);
  				spu_buff[spu].buff = 0;
  				spu--;
  			}
  			return -ENOMEM;
  		}
  	}
  	return 0;
  }
  
  /* The main purpose of this function is to synchronize
   * OProfile with SPUFS by registering to be notified of
   * SPU task switches.
   *
   * NOTE: When profiling SPUs, we must ensure that only
   * spu_sync_start is invoked and not the generic sync_start
   * in drivers/oprofile/oprof.c.	 A return value of
   * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
   * accomplish this.
   */
  int spu_sync_start(void)
  {
  	int spu;
  	int ret = SKIP_GENERIC_SYNC;
  	int register_ret;
  	unsigned long flags = 0;
  
  	spu_prof_num_nodes = number_of_online_nodes();
  	num_spu_nodes = spu_prof_num_nodes * 8;
  	INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);
  
  	/* create buffer for storing the SPU data to put in
  	 * the kernel buffer.
  	 */
  	ret = oprofile_spu_buff_create();
  	if (ret)
  		goto out;
  
  	spin_lock_irqsave(&buffer_lock, flags);
  	for (spu = 0; spu < num_spu_nodes; spu++) {
  		spu_buff_add(ESCAPE_CODE, spu);
  		spu_buff_add(SPU_PROFILING_CODE, spu);
  		spu_buff_add(num_spu_nodes, spu);
  	}
  	spin_unlock_irqrestore(&buffer_lock, flags);
  
  	for (spu = 0; spu < num_spu_nodes; spu++) {
  		spu_buff[spu].ctx_sw_seen = 0;
  		spu_buff[spu].last_guard_val = 0;
  	}
  
  	/* Register for SPU events  */
  	register_ret = spu_switch_event_register(&spu_active);
  	if (register_ret) {
  		ret = SYNC_START_ERROR;
  		goto out;
  	}
  
  	pr_debug("spu_sync_start -- running.\n");
  out:
  	return ret;
  }
  
  /* Record SPU program counter samples to the oprofile event buffer. */
  void spu_sync_buffer(int spu_num, unsigned int *samples,
  		     int num_samples)
  {
  	unsigned long long file_offset;
  	unsigned long flags;
  	int i;
  	struct vma_to_fileoffset_map *map;
  	struct spu *the_spu;
  	unsigned long long spu_num_ll = spu_num;
  	unsigned long long spu_num_shifted = spu_num_ll << 32;
  	struct cached_info *c_info;
  
  	/* We need to obtain the cache_lock here because it's
  	 * possible that after getting the cached_info, the SPU job
  	 * corresponding to this cached_info may end, thus resulting
  	 * in the destruction of the cached_info.
  	 */
  	spin_lock_irqsave(&cache_lock, flags);
  	c_info = get_cached_info(NULL, spu_num);
  	if (!c_info) {
  		/* This legitimately happens when the SPU task ends before all
  		 * samples are recorded.
  		 * No big deal -- so we just drop a few samples.
  		 */
  		pr_debug("SPU_PROF: No cached SPU contex "
  			  "for SPU #%d. Dropping samples.\n", spu_num);
  		goto out;
  	}
  
  	map = c_info->map;
  	the_spu = c_info->the_spu;
  	spin_lock(&buffer_lock);
  	for (i = 0; i < num_samples; i++) {
  		unsigned int sample = *(samples+i);
  		int grd_val = 0;
  		file_offset = 0;
  		if (sample == 0)
  			continue;
  		file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);
  
  		/* If overlays are used by this SPU application, the guard
  		 * value is non-zero, indicating which overlay section is in
  		 * use.	 We need to discard samples taken during the time
  		 * period which an overlay occurs (i.e., guard value changes).
  		 */
  		if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
  			spu_buff[spu_num].last_guard_val = grd_val;
  			/* Drop the rest of the samples. */
  			break;
  		}
  
  		/* We must ensure that the SPU context switch has been written
  		 * out before samples for the SPU.  Otherwise, the SPU context
  		 * information is not available and the postprocessing of the
  		 * SPU PC will fail with no available anonymous map information.
  		 */
  		if (spu_buff[spu_num].ctx_sw_seen)
  			spu_buff_add((file_offset | spu_num_shifted),
  					 spu_num);
  	}
  	spin_unlock(&buffer_lock);
  out:
  	spin_unlock_irqrestore(&cache_lock, flags);
  }
  
  
  int spu_sync_stop(void)
  {
  	unsigned long flags = 0;
  	int ret;
  	int k;
  
  	ret = spu_switch_event_unregister(&spu_active);
  
  	if (ret)
  		printk(KERN_ERR "SPU_PROF: "
  		       "%s, line %d: spu_switch_event_unregister "	\
  		       "returned %d\n",
  		       __func__, __LINE__, ret);
  
  	/* flush any remaining data in the per SPU buffers */
  	sync_spu_buff();
  
  	spin_lock_irqsave(&cache_lock, flags);
  	ret = release_cached_info(RELEASE_ALL);
  	spin_unlock_irqrestore(&cache_lock, flags);
  
  	/* remove scheduled work queue item rather then waiting
  	 * for every queued entry to execute.  Then flush pending
  	 * system wide buffer to event buffer.
  	 */
  	cancel_delayed_work(&spu_work);
  
  	for (k = 0; k < num_spu_nodes; k++) {
  		spu_buff[k].ctx_sw_seen = 0;
  
  		/*
  		 * spu_sys_buff will be null if there was a problem
  		 * allocating the buffer.  Only delete if it exists.
  		 */
  		kfree(spu_buff[k].buff);
  		spu_buff[k].buff = 0;
  	}
  	pr_debug("spu_sync_stop -- done.\n");
  	return ret;
  }
  
  