text: /*
   * Linux performance counter support for MIPS.
   *
   * Copyright (C) 2010 MIPS Technologies, Inc.
   * Author: Deng-Cheng Zhu
   *
   * This code is based on the implementation for ARM, which is in turn
   * based on the sparc64 perf event code and the x86 code. Performance
   * counter access is based on the MIPS Oprofile code. And the callchain
   * support references the code of MIPS stacktrace.c.
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
   */
  
  #include <linux/cpumask.h>
  #include <linux/interrupt.h>
  #include <linux/smp.h>
  #include <linux/kernel.h>
  #include <linux/perf_event.h>
  #include <linux/uaccess.h>
  
  #include <asm/irq.h>
  #include <asm/irq_regs.h>
  #include <asm/stacktrace.h>
  #include <asm/time.h> /* For perf_irq */
  
  /* These are for 32bit counters. For 64bit ones, define them accordingly. */
  #define MAX_PERIOD	((1ULL << 32) - 1)
  #define VALID_COUNT	0x7fffffff
  #define TOTAL_BITS	32
  #define HIGHEST_BIT	31
  
  #define MIPS_MAX_HWEVENTS 4
  
  struct cpu_hw_events {
  	/* Array of events on this cpu. */
  	struct perf_event	*events[MIPS_MAX_HWEVENTS];
  
  	/*
  	 * Set the bit (indexed by the counter number) when the counter
  	 * is used for an event.
  	 */
  	unsigned long		used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
  
  	/*
  	 * The borrowed MSB for the performance counter. A MIPS performance
  	 * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
  	 * counters) as a factor of determining whether a counter overflow
  	 * should be signaled. So here we use a separate MSB for each
  	 * counter to make things easy.
  	 */
  	unsigned long		msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
  
  	/*
  	 * Software copy of the control register for each performance counter.
  	 * MIPS CPUs vary in performance counters. They use this differently,
  	 * and even may not use it.
  	 */
  	unsigned int		saved_ctrl[MIPS_MAX_HWEVENTS];
  };
  DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
  	.saved_ctrl = {0},
  };
  
  /* The description of MIPS performance events. */
  struct mips_perf_event {
  	unsigned int event_id;
  	/*
  	 * MIPS performance counters are indexed starting from 0.
  	 * CNTR_EVEN indicates the indexes of the counters to be used are
  	 * even numbers.
  	 */
  	unsigned int cntr_mask;
  	#define CNTR_EVEN	0x55555555
  	#define CNTR_ODD	0xaaaaaaaa
  #ifdef CONFIG_MIPS_MT_SMP
  	enum {
  		T  = 0,
  		V  = 1,
  		P  = 2,
  	} range;
  #else
  	#define T
  	#define V
  	#define P
  #endif
  };
  
  static struct mips_perf_event raw_event;
  static DEFINE_MUTEX(raw_event_mutex);
  
  #define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
  #define C(x) PERF_COUNT_HW_CACHE_##x
  
  struct mips_pmu {
  	const char	*name;
  	int		irq;
  	irqreturn_t	(*handle_irq)(int irq, void *dev);
  	int		(*handle_shared_irq)(void);
  	void		(*start)(void);
  	void		(*stop)(void);
  	int		(*alloc_counter)(struct cpu_hw_events *cpuc,
  					struct hw_perf_event *hwc);
  	u64		(*read_counter)(unsigned int idx);
  	void		(*write_counter)(unsigned int idx, u64 val);
  	void		(*enable_event)(struct hw_perf_event *evt, int idx);
  	void		(*disable_event)(int idx);
  	const struct mips_perf_event *(*map_raw_event)(u64 config);
  	const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
  	const struct mips_perf_event (*cache_event_map)
  				[PERF_COUNT_HW_CACHE_MAX]
  				[PERF_COUNT_HW_CACHE_OP_MAX]
  				[PERF_COUNT_HW_CACHE_RESULT_MAX];
  	unsigned int	num_counters;
  };
  
  static const struct mips_pmu *mipspmu;
  
  static int
  mipspmu_event_set_period(struct perf_event *event,
  			struct hw_perf_event *hwc,
  			int idx)
  {
  	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  	s64 left = local64_read(&hwc->period_left);
  	s64 period = hwc->sample_period;
  	int ret = 0;
  	u64 uleft;
  	unsigned long flags;
  
  	if (unlikely(left <= -period)) {
  		left = period;
  		local64_set(&hwc->period_left, left);
  		hwc->last_period = period;
  		ret = 1;
  	}
  
  	if (unlikely(left <= 0)) {
  		left += period;
  		local64_set(&hwc->period_left, left);
  		hwc->last_period = period;
  		ret = 1;
  	}
  
  	if (left > (s64)MAX_PERIOD)
  		left = MAX_PERIOD;
  
  	local64_set(&hwc->prev_count, (u64)-left);
  
  	local_irq_save(flags);
  	uleft = (u64)(-left) & MAX_PERIOD;
  	uleft > VALID_COUNT ?
  		set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
  	mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
  	local_irq_restore(flags);
  
  	perf_event_update_userpage(event);
  
  	return ret;
  }
  
  static void mipspmu_event_update(struct perf_event *event,
  			struct hw_perf_event *hwc,
  			int idx)
  {
  	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  	unsigned long flags;
  	int shift = 64 - TOTAL_BITS;
  	s64 prev_raw_count, new_raw_count;
  	u64 delta;
  
  again:
  	prev_raw_count = local64_read(&hwc->prev_count);
  	local_irq_save(flags);
  	/* Make the counter value be a "real" one. */
  	new_raw_count = mipspmu->read_counter(idx);
  	if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
  		new_raw_count &= VALID_COUNT;
  		clear_bit(idx, cpuc->msbs);
  	} else
  		new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
  	local_irq_restore(flags);
  
  	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  				new_raw_count) != prev_raw_count)
  		goto again;
  
  	delta = (new_raw_count << shift) - (prev_raw_count << shift);
  	delta >>= shift;
  
  	local64_add(delta, &event->count);
  	local64_sub(delta, &hwc->period_left);
  
  	return;
  }
  
  static void mipspmu_start(struct perf_event *event, int flags)
  {
  	struct hw_perf_event *hwc = &event->hw;
  
  	if (!mipspmu)
  		return;
  
  	if (flags & PERF_EF_RELOAD)
  		WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
  
  	hwc->state = 0;
  
  	/* Set the period for the event. */
  	mipspmu_event_set_period(event, hwc, hwc->idx);
  
  	/* Enable the event. */
  	mipspmu->enable_event(hwc, hwc->idx);
  }
  
  static void mipspmu_stop(struct perf_event *event, int flags)
  {
  	struct hw_perf_event *hwc = &event->hw;
  
  	if (!mipspmu)
  		return;
  
  	if (!(hwc->state & PERF_HES_STOPPED)) {
  		/* We are working on a local event. */
  		mipspmu->disable_event(hwc->idx);
  		barrier();
  		mipspmu_event_update(event, hwc, hwc->idx);
  		hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  	}
  }
  
  static int mipspmu_add(struct perf_event *event, int flags)
  {
  	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  	struct hw_perf_event *hwc = &event->hw;
  	int idx;
  	int err = 0;
  
  	perf_pmu_disable(event->pmu);
  
  	/* To look for a free counter for this event. */
  	idx = mipspmu->alloc_counter(cpuc, hwc);
  	if (idx < 0) {
  		err = idx;
  		goto out;
  	}
  
  	/*
  	 * If there is an event in the counter we are going to use then
  	 * make sure it is disabled.
  	 */
  	event->hw.idx = idx;
  	mipspmu->disable_event(idx);
  	cpuc->events[idx] = event;
  
  	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  	if (flags & PERF_EF_START)
  		mipspmu_start(event, PERF_EF_RELOAD);
  
  	/* Propagate our changes to the userspace mapping. */
  	perf_event_update_userpage(event);
  
  out:
  	perf_pmu_enable(event->pmu);
  	return err;
  }
  
  static void mipspmu_del(struct perf_event *event, int flags)
  {
  	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  	struct hw_perf_event *hwc = &event->hw;
  	int idx = hwc->idx;
  
  	WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
  
  	mipspmu_stop(event, PERF_EF_UPDATE);
  	cpuc->events[idx] = NULL;
  	clear_bit(idx, cpuc->used_mask);
  
  	perf_event_update_userpage(event);
  }
  
  static void mipspmu_read(struct perf_event *event)
  {
  	struct hw_perf_event *hwc = &event->hw;
  
  	/* Don't read disabled counters! */
  	if (hwc->idx < 0)
  		return;
  
  	mipspmu_event_update(event, hwc, hwc->idx);
  }
  
  static void mipspmu_enable(struct pmu *pmu)
  {
  	if (mipspmu)
  		mipspmu->start();
  }
  
  static void mipspmu_disable(struct pmu *pmu)
  {
  	if (mipspmu)
  		mipspmu->stop();
  }
  
  static atomic_t active_events = ATOMIC_INIT(0);
  static DEFINE_MUTEX(pmu_reserve_mutex);
  static int (*save_perf_irq)(void);
  
  static int mipspmu_get_irq(void)
  {
  	int err;
  
  	if (mipspmu->irq >= 0) {
  		/* Request my own irq handler. */
  		err = request_irq(mipspmu->irq, mipspmu->handle_irq,
  			IRQF_DISABLED | IRQF_NOBALANCING,
  			"mips_perf_pmu", NULL);
  		if (err) {
  			pr_warning("Unable to request IRQ%d for MIPS "
  			   "performance counters!\n", mipspmu->irq);
  		}
  	} else if (cp0_perfcount_irq < 0) {
  		/*
  		 * We are sharing the irq number with the timer interrupt.
  		 */
  		save_perf_irq = perf_irq;
  		perf_irq = mipspmu->handle_shared_irq;
  		err = 0;
  	} else {
  		pr_warning("The platform hasn't properly defined its "
  			"interrupt controller.\n");
  		err = -ENOENT;
  	}
  
  	return err;
  }
  
  static void mipspmu_free_irq(void)
  {
  	if (mipspmu->irq >= 0)
  		free_irq(mipspmu->irq, NULL);
  	else if (cp0_perfcount_irq < 0)
  		perf_irq = save_perf_irq;
  }
  
  /*
   * mipsxx/rm9000/loongson2 have different performance counters, they have
   * specific low-level init routines.
   */
  static void reset_counters(void *arg);
  static int __hw_perf_event_init(struct perf_event *event);
  
  static void hw_perf_event_destroy(struct perf_event *event)
  {
  	if (atomic_dec_and_mutex_lock(&active_events,
  				&pmu_reserve_mutex)) {
  		/*
  		 * We must not call the destroy function with interrupts
  		 * disabled.
  		 */
  		on_each_cpu(reset_counters,
  			(void *)(long)mipspmu->num_counters, 1);
  		mipspmu_free_irq();
  		mutex_unlock(&pmu_reserve_mutex);
  	}
  }
  
  static int mipspmu_event_init(struct perf_event *event)
  {
  	int err = 0;
  
  	switch (event->attr.type) {
  	case PERF_TYPE_RAW:
  	case PERF_TYPE_HARDWARE:
  	case PERF_TYPE_HW_CACHE:
  		break;
  
  	default:
  		return -ENOENT;
  	}
  
  	if (!mipspmu || event->cpu >= nr_cpumask_bits ||
  		(event->cpu >= 0 && !cpu_online(event->cpu)))
  		return -ENODEV;
  
  	if (!atomic_inc_not_zero(&active_events)) {
  		if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
  			atomic_dec(&active_events);
  			return -ENOSPC;
  		}
  
  		mutex_lock(&pmu_reserve_mutex);
  		if (atomic_read(&active_events) == 0)
  			err = mipspmu_get_irq();
  
  		if (!err)
  			atomic_inc(&active_events);
  		mutex_unlock(&pmu_reserve_mutex);
  	}
  
  	if (err)
  		return err;
  
  	err = __hw_perf_event_init(event);
  	if (err)
  		hw_perf_event_destroy(event);
  
  	return err;
  }
  
  static struct pmu pmu = {
  	.pmu_enable	= mipspmu_enable,
  	.pmu_disable	= mipspmu_disable,
  	.event_init	= mipspmu_event_init,
  	.add		= mipspmu_add,
  	.del		= mipspmu_del,
  	.start		= mipspmu_start,
  	.stop		= mipspmu_stop,
  	.read		= mipspmu_read,
  };
  
  static inline unsigned int
  mipspmu_perf_event_encode(const struct mips_perf_event *pev)
  {
  /*
   * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
   * event_id.
   */
  #ifdef CONFIG_MIPS_MT_SMP
  	return ((unsigned int)pev->range << 24) |
  		(pev->cntr_mask & 0xffff00) |
  		(pev->event_id & 0xff);
  #else
  	return (pev->cntr_mask & 0xffff00) |
  		(pev->event_id & 0xff);
  #endif
  }
  
  static const struct mips_perf_event *
  mipspmu_map_general_event(int idx)
  {
  	const struct mips_perf_event *pev;
  
  	pev = ((*mipspmu->general_event_map)[idx].event_id ==
  		UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
  		&(*mipspmu->general_event_map)[idx]);
  
  	return pev;
  }
  
  static const struct mips_perf_event *
  mipspmu_map_cache_event(u64 config)
  {
  	unsigned int cache_type, cache_op, cache_result;
  	const struct mips_perf_event *pev;
  
  	cache_type = (config >> 0) & 0xff;
  	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  		return ERR_PTR(-EINVAL);
  
  	cache_op = (config >> 8) & 0xff;
  	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  		return ERR_PTR(-EINVAL);
  
  	cache_result = (config >> 16) & 0xff;
  	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  		return ERR_PTR(-EINVAL);
  
  	pev = &((*mipspmu->cache_event_map)
  					[cache_type]
  					[cache_op]
  					[cache_result]);
  
  	if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
  		return ERR_PTR(-EOPNOTSUPP);
  
  	return pev;
  
  }
  
  static int validate_event(struct cpu_hw_events *cpuc,
  	       struct perf_event *event)
  {
  	struct hw_perf_event fake_hwc = event->hw;
  
  	/* Allow mixed event group. So return 1 to pass validation. */
  	if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF)
  		return 1;
  
  	return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
  }
  
  static int validate_group(struct perf_event *event)
  {
  	struct perf_event *sibling, *leader = event->group_leader;
  	struct cpu_hw_events fake_cpuc;
  
  	memset(&fake_cpuc, 0, sizeof(fake_cpuc));
  
  	if (!validate_event(&fake_cpuc, leader))
  		return -ENOSPC;
  
  	list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
  		if (!validate_event(&fake_cpuc, sibling))
  			return -ENOSPC;
  	}
  
  	if (!validate_event(&fake_cpuc, event))
  		return -ENOSPC;
  
  	return 0;
  }
  
  /* This is needed by specific irq handlers in perf_event_*.c */
  static void
  handle_associated_event(struct cpu_hw_events *cpuc,
  	int idx, struct perf_sample_data *data, struct pt_regs *regs)
  {
  	struct perf_event *event = cpuc->events[idx];
  	struct hw_perf_event *hwc = &event->hw;
  
  	mipspmu_event_update(event, hwc, idx);
  	data->period = event->hw.last_period;
  	if (!mipspmu_event_set_period(event, hwc, idx))
  		return;
  
  	if (perf_event_overflow(event, 0, data, regs))
  		mipspmu->disable_event(idx);
  }
  
  #include "perf_event_mipsxx.c"
  
  /* Callchain handling code. */
  
  /*
   * Leave userspace callchain empty for now. When we find a way to trace
   * the user stack callchains, we add here.
   */
  void perf_callchain_user(struct perf_callchain_entry *entry,
  		    struct pt_regs *regs)
  {
  }
  
  static void save_raw_perf_callchain(struct perf_callchain_entry *entry,
  	unsigned long reg29)
  {
  	unsigned long *sp = (unsigned long *)reg29;
  	unsigned long addr;
  
  	while (!kstack_end(sp)) {
  		addr = *sp++;
  		if (__kernel_text_address(addr)) {
  			perf_callchain_store(entry, addr);
  			if (entry->nr >= PERF_MAX_STACK_DEPTH)
  				break;
  		}
  	}
  }
  
  void perf_callchain_kernel(struct perf_callchain_entry *entry,
  		      struct pt_regs *regs)
  {
  	unsigned long sp = regs->regs[29];
  #ifdef CONFIG_KALLSYMS
  	unsigned long ra = regs->regs[31];
  	unsigned long pc = regs->cp0_epc;
  
  	if (raw_show_trace || !__kernel_text_address(pc)) {
  		unsigned long stack_page =
  			(unsigned long)task_stack_page(current);
  		if (stack_page && sp >= stack_page &&
  		    sp <= stack_page + THREAD_SIZE - 32)
  			save_raw_perf_callchain(entry, sp);
  		return;
  	}
  	do {
  		perf_callchain_store(entry, pc);
  		if (entry->nr >= PERF_MAX_STACK_DEPTH)
  			break;
  		pc = unwind_stack(current, &sp, pc, &ra);
  	} while (pc);
  #else
  	save_raw_perf_callchain(entry, sp);
  #endif
  }
  