text: /*
   * BTS PMU driver for perf
   * Copyright (c) 2013-2014, Intel Corporation.
   *
   * This program is free software; you can redistribute it and/or modify it
   * under the terms and conditions of the GNU General Public License,
   * version 2, as published by the Free Software Foundation.
   *
   * This program is distributed in the hope it will be useful, but WITHOUT
   * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
   * more details.
   */
  
  #undef DEBUG
  
  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  
  #include <linux/bitops.h>
  #include <linux/types.h>
  #include <linux/slab.h>
  #include <linux/debugfs.h>
  #include <linux/device.h>
  #include <linux/coredump.h>
  
  #include <asm-generic/sizes.h>
  #include <asm/perf_event.h>
  
  #include "perf_event.h"
  
  struct bts_ctx {
  	struct perf_output_handle	handle;
  	struct debug_store		ds_back;
  	int				started;
  };
  
  static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  
  #define BTS_RECORD_SIZE		24
  #define BTS_SAFETY_MARGIN	4080
  
  struct bts_phys {
  	struct page	*page;
  	unsigned long	size;
  	unsigned long	offset;
  	unsigned long	displacement;
  };
  
  struct bts_buffer {
  	size_t		real_size;	/* multiple of BTS_RECORD_SIZE */
  	unsigned int	nr_pages;
  	unsigned int	nr_bufs;
  	unsigned int	cur_buf;
  	bool		snapshot;
  	local_t		data_size;
  	local_t		lost;
  	local_t		head;
  	unsigned long	end;
  	void		**data_pages;
  	struct bts_phys	buf[0];
  };
  
  struct pmu bts_pmu;
  
  static size_t buf_size(struct page *page)
  {
  	return 1 << (PAGE_SHIFT + page_private(page));
  }
  
  static void *
  bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
  {
  	struct bts_buffer *buf;
  	struct page *page;
  	int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  	unsigned long offset;
  	size_t size = nr_pages << PAGE_SHIFT;
  	int pg, nbuf, pad;
  
  	/* count all the high order buffers */
  	for (pg = 0, nbuf = 0; pg < nr_pages;) {
  		page = virt_to_page(pages[pg]);
  		if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  			return NULL;
  		pg += 1 << page_private(page);
  		nbuf++;
  	}
  
  	/*
  	 * to avoid interrupts in overwrite mode, only allow one physical
  	 */
  	if (overwrite && nbuf > 1)
  		return NULL;
  
  	buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
  	if (!buf)
  		return NULL;
  
  	buf->nr_pages = nr_pages;
  	buf->nr_bufs = nbuf;
  	buf->snapshot = overwrite;
  	buf->data_pages = pages;
  	buf->real_size = size - size % BTS_RECORD_SIZE;
  
  	for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
  		unsigned int __nr_pages;
  
  		page = virt_to_page(pages[pg]);
  		__nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
  		buf->buf[nbuf].page = page;
  		buf->buf[nbuf].offset = offset;
  		buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
  		buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
  		pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
  		buf->buf[nbuf].size -= pad;
  
  		pg += __nr_pages;
  		offset += __nr_pages << PAGE_SHIFT;
  	}
  
  	return buf;
  }
  
  static void bts_buffer_free_aux(void *data)
  {
  	kfree(data);
  }
  
  static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
  {
  	return buf->buf[idx].offset + buf->buf[idx].displacement;
  }
  
  static void
  bts_config_buffer(struct bts_buffer *buf)
  {
  	int cpu = raw_smp_processor_id();
  	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  	struct bts_phys *phys = &buf->buf[buf->cur_buf];
  	unsigned long index, thresh = 0, end = phys->size;
  	struct page *page = phys->page;
  
  	index = local_read(&buf->head);
  
  	if (!buf->snapshot) {
  		if (buf->end < phys->offset + buf_size(page))
  			end = buf->end - phys->offset - phys->displacement;
  
  		index -= phys->offset + phys->displacement;
  
  		if (end - index > BTS_SAFETY_MARGIN)
  			thresh = end - BTS_SAFETY_MARGIN;
  		else if (end - index > BTS_RECORD_SIZE)
  			thresh = end - BTS_RECORD_SIZE;
  		else
  			thresh = end;
  	}
  
  	ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
  	ds->bts_index = ds->bts_buffer_base + index;
  	ds->bts_absolute_maximum = ds->bts_buffer_base + end;
  	ds->bts_interrupt_threshold = !buf->snapshot
  		? ds->bts_buffer_base + thresh
  		: ds->bts_absolute_maximum + BTS_RECORD_SIZE;
  }
  
  static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
  {
  	unsigned long index = head - phys->offset;
  
  	memset(page_address(phys->page) + index, 0, phys->size - index);
  }
  
  static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)
  {
  	if (buf->snapshot)
  		return false;
  
  	if (local_read(&buf->data_size) >= bts->handle.size ||
  	    bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
  		return true;
  
  	return false;
  }
  
  static void bts_update(struct bts_ctx *bts)
  {
  	int cpu = raw_smp_processor_id();
  	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
  	struct bts_buffer *buf = perf_get_aux(&bts->handle);
  	unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
  
  	if (!buf)
  		return;
  
  	head = index + bts_buffer_offset(buf, buf->cur_buf);
  	old = local_xchg(&buf->head, head);
  
  	if (!buf->snapshot) {
  		if (old == head)
  			return;
  
  		if (ds->bts_index >= ds->bts_absolute_maximum)
  			local_inc(&buf->lost);
  
  		/*
  		 * old and head are always in the same physical buffer, so we
  		 * can subtract them to get the data size.
  		 */
  		local_add(head - old, &buf->data_size);
  	} else {
  		local_set(&buf->data_size, head);
  	}
  }
  
  static void __bts_event_start(struct perf_event *event)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  	struct bts_buffer *buf = perf_get_aux(&bts->handle);
  	u64 config = 0;
  
  	if (!buf || bts_buffer_is_full(buf, bts))
  		return;
  
  	event->hw.itrace_started = 1;
  	event->hw.state = 0;
  
  	if (!buf->snapshot)
  		config |= ARCH_PERFMON_EVENTSEL_INT;
  	if (!event->attr.exclude_kernel)
  		config |= ARCH_PERFMON_EVENTSEL_OS;
  	if (!event->attr.exclude_user)
  		config |= ARCH_PERFMON_EVENTSEL_USR;
  
  	bts_config_buffer(buf);
  
  	/*
  	 * local barrier to make sure that ds configuration made it
  	 * before we enable BTS
  	 */
  	wmb();
  
  	intel_pmu_enable_bts(config);
  }
  
  static void bts_event_start(struct perf_event *event, int flags)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  
  	__bts_event_start(event);
  
  	/* PMI handler: this counter is running and likely generating PMIs */
  	ACCESS_ONCE(bts->started) = 1;
  }
  
  static void __bts_event_stop(struct perf_event *event)
  {
  	/*
  	 * No extra synchronization is mandated by the documentation to have
  	 * BTS data stores globally visible.
  	 */
  	intel_pmu_disable_bts();
  
  	if (event->hw.state & PERF_HES_STOPPED)
  		return;
  
  	ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
  }
  
  static void bts_event_stop(struct perf_event *event, int flags)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  
  	/* PMI handler: don't restart this counter */
  	ACCESS_ONCE(bts->started) = 0;
  
  	__bts_event_stop(event);
  
  	if (flags & PERF_EF_UPDATE)
  		bts_update(bts);
  }
  
  void intel_bts_enable_local(void)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  
  	if (bts->handle.event && bts->started)
  		__bts_event_start(bts->handle.event);
  }
  
  void intel_bts_disable_local(void)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  
  	if (bts->handle.event)
  		__bts_event_stop(bts->handle.event);
  }
  
  static int
  bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
  {
  	unsigned long head, space, next_space, pad, gap, skip, wakeup;
  	unsigned int next_buf;
  	struct bts_phys *phys, *next_phys;
  	int ret;
  
  	if (buf->snapshot)
  		return 0;
  
  	head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
  	if (WARN_ON_ONCE(head != local_read(&buf->head)))
  		return -EINVAL;
  
  	phys = &buf->buf[buf->cur_buf];
  	space = phys->offset + phys->displacement + phys->size - head;
  	pad = space;
  	if (space > handle->size) {
  		space = handle->size;
  		space -= space % BTS_RECORD_SIZE;
  	}
  	if (space <= BTS_SAFETY_MARGIN) {
  		/* See if next phys buffer has more space */
  		next_buf = buf->cur_buf + 1;
  		if (next_buf >= buf->nr_bufs)
  			next_buf = 0;
  		next_phys = &buf->buf[next_buf];
  		gap = buf_size(phys->page) - phys->displacement - phys->size +
  		      next_phys->displacement;
  		skip = pad + gap;
  		if (handle->size >= skip) {
  			next_space = next_phys->size;
  			if (next_space + skip > handle->size) {
  				next_space = handle->size - skip;
  				next_space -= next_space % BTS_RECORD_SIZE;
  			}
  			if (next_space > space || !space) {
  				if (pad)
  					bts_buffer_pad_out(phys, head);
  				ret = perf_aux_output_skip(handle, skip);
  				if (ret)
  					return ret;
  				/* Advance to next phys buffer */
  				phys = next_phys;
  				space = next_space;
  				head = phys->offset + phys->displacement;
  				/*
  				 * After this, cur_buf and head won't match ds
  				 * anymore, so we must not be racing with
  				 * bts_update().
  				 */
  				buf->cur_buf = next_buf;
  				local_set(&buf->head, head);
  			}
  		}
  	}
  
  	/* Don't go far beyond wakeup watermark */
  	wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
  		 handle->head;
  	if (space > wakeup) {
  		space = wakeup;
  		space -= space % BTS_RECORD_SIZE;
  	}
  
  	buf->end = head + space;
  
  	/*
  	 * If we have no space, the lost notification would have been sent when
  	 * we hit absolute_maximum - see bts_update()
  	 */
  	if (!space)
  		return -ENOSPC;
  
  	return 0;
  }
  
  int intel_bts_interrupt(void)
  {
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  	struct perf_event *event = bts->handle.event;
  	struct bts_buffer *buf;
  	s64 old_head;
  	int err;
  
  	if (!event || !bts->started)
  		return 0;
  
  	buf = perf_get_aux(&bts->handle);
  	/*
  	 * Skip snapshot counters: they don't use the interrupt, but
  	 * there's no other way of telling, because the pointer will
  	 * keep moving
  	 */
  	if (!buf || buf->snapshot)
  		return 0;
  
  	old_head = local_read(&buf->head);
  	bts_update(bts);
  
  	/* no new data */
  	if (old_head == local_read(&buf->head))
  		return 0;
  
  	perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
  			    !!local_xchg(&buf->lost, 0));
  
  	buf = perf_aux_output_begin(&bts->handle, event);
  	if (!buf)
  		return 1;
  
  	err = bts_buffer_reset(buf, &bts->handle);
  	if (err)
  		perf_aux_output_end(&bts->handle, 0, false);
  
  	return 1;
  }
  
  static void bts_event_del(struct perf_event *event, int mode)
  {
  	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  	struct bts_buffer *buf = perf_get_aux(&bts->handle);
  
  	bts_event_stop(event, PERF_EF_UPDATE);
  
  	if (buf) {
  		if (buf->snapshot)
  			bts->handle.head =
  				local_xchg(&buf->data_size,
  					   buf->nr_pages << PAGE_SHIFT);
  		perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
  				    !!local_xchg(&buf->lost, 0));
  	}
  
  	cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
  	cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
  	cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
  	cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
  }
  
  static int bts_event_add(struct perf_event *event, int mode)
  {
  	struct bts_buffer *buf;
  	struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
  	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  	struct hw_perf_event *hwc = &event->hw;
  	int ret = -EBUSY;
  
  	event->hw.state = PERF_HES_STOPPED;
  
  	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
  		return -EBUSY;
  
  	if (bts->handle.event)
  		return -EBUSY;
  
  	buf = perf_aux_output_begin(&bts->handle, event);
  	if (!buf)
  		return -EINVAL;
  
  	ret = bts_buffer_reset(buf, &bts->handle);
  	if (ret) {
  		perf_aux_output_end(&bts->handle, 0, false);
  		return ret;
  	}
  
  	bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
  	bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
  	bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
  
  	if (mode & PERF_EF_START) {
  		bts_event_start(event, 0);
  		if (hwc->state & PERF_HES_STOPPED) {
  			bts_event_del(event, 0);
  			return -EBUSY;
  		}
  	}
  
  	return 0;
  }
  
  static void bts_event_destroy(struct perf_event *event)
  {
  	x86_release_hardware();
  	x86_del_exclusive(x86_lbr_exclusive_bts);
  }
  
  static int bts_event_init(struct perf_event *event)
  {
  	int ret;
  
  	if (event->attr.type != bts_pmu.type)
  		return -ENOENT;
  
  	if (x86_add_exclusive(x86_lbr_exclusive_bts))
  		return -EBUSY;
  
  	ret = x86_reserve_hardware();
  	if (ret) {
  		x86_del_exclusive(x86_lbr_exclusive_bts);
  		return ret;
  	}
  
  	event->destroy = bts_event_destroy;
  
  	return 0;
  }
  
  static void bts_event_read(struct perf_event *event)
  {
  }
  
  static __init int bts_init(void)
  {
  	if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
  		return -ENODEV;
  
  	bts_pmu.capabilities	= PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
  	bts_pmu.task_ctx_nr	= perf_sw_context;
  	bts_pmu.event_init	= bts_event_init;
  	bts_pmu.add		= bts_event_add;
  	bts_pmu.del		= bts_event_del;
  	bts_pmu.start		= bts_event_start;
  	bts_pmu.stop		= bts_event_stop;
  	bts_pmu.read		= bts_event_read;
  	bts_pmu.setup_aux	= bts_buffer_setup_aux;
  	bts_pmu.free_aux	= bts_buffer_free_aux;
  
  	return perf_pmu_register(&bts_pmu, "intel_bts", -1);
  }
  arch_initcall(bts_init);
  