text: /* SPDX-License-Identifier: GPL-2.0 */
  #ifndef _ASM_X86_BITOPS_H
  #define _ASM_X86_BITOPS_H
  
  /*
   * Copyright 1992, Linus Torvalds.
   *
   * Note: inlines with more than a single statement should be marked
   * __always_inline to avoid problems with older gcc's inlining heuristics.
   */
  
  #ifndef _LINUX_BITOPS_H
  #error only <linux/bitops.h> can be included directly
  #endif
  
  #include <linux/compiler.h>
  #include <asm/alternative.h>
  #include <asm/rmwcc.h>
  #include <asm/barrier.h>
  
  #if BITS_PER_LONG == 32
  # define _BITOPS_LONG_SHIFT 5
  #elif BITS_PER_LONG == 64
  # define _BITOPS_LONG_SHIFT 6
  #else
  # error "Unexpected BITS_PER_LONG"
  #endif
  
  #define BIT_64(n)			(U64_C(1) << (n))
  
  /*
   * These have to be done with inline assembly: that way the bit-setting
   * is guaranteed to be atomic. All bit operations return 0 if the bit
   * was cleared before the operation and != 0 if it was not.
   *
   * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
   */
  
  #define RLONG_ADDR(x)			 "m" (*(volatile long *) (x))
  #define WBYTE_ADDR(x)			"+m" (*(volatile char *) (x))
  
  #define ADDR				RLONG_ADDR(addr)
  
  /*
   * We do the locked ops that don't return the old value as
   * a mask operation on a byte.
   */
  #define CONST_MASK_ADDR(nr, addr)	WBYTE_ADDR((void *)(addr) + ((nr)>>3))
  #define CONST_MASK(nr)			(1 << ((nr) & 7))
  
  static __always_inline void
  arch_set_bit(long nr, volatile unsigned long *addr)
  {
  	if (__builtin_constant_p(nr)) {
  		asm volatile(LOCK_PREFIX "orb %1,%0"
  			: CONST_MASK_ADDR(nr, addr)
  			: "iq" ((u8)CONST_MASK(nr))
  			: "memory");
  	} else {
  		asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0"
  			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
  	}
  }
  
  static __always_inline void
  arch___set_bit(long nr, volatile unsigned long *addr)
  {
  	asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
  }
  
  static __always_inline void
  arch_clear_bit(long nr, volatile unsigned long *addr)
  {
  	if (__builtin_constant_p(nr)) {
  		asm volatile(LOCK_PREFIX "andb %1,%0"
  			: CONST_MASK_ADDR(nr, addr)
  			: "iq" ((u8)~CONST_MASK(nr)));
  	} else {
  		asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0"
  			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
  	}
  }
  
  static __always_inline void
  arch_clear_bit_unlock(long nr, volatile unsigned long *addr)
  {
  	barrier();
  	arch_clear_bit(nr, addr);
  }
  
  static __always_inline void
  arch___clear_bit(long nr, volatile unsigned long *addr)
  {
  	asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
  }
  
  static __always_inline bool
  arch_clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
  {
  	bool negative;
  	asm volatile(LOCK_PREFIX "andb %2,%1"
  		CC_SET(s)
  		: CC_OUT(s) (negative), WBYTE_ADDR(addr)
  		: "ir" ((char) ~(1 << nr)) : "memory");
  	return negative;
  }
  #define arch_clear_bit_unlock_is_negative_byte                                 \
  	arch_clear_bit_unlock_is_negative_byte
  
  static __always_inline void
  arch___clear_bit_unlock(long nr, volatile unsigned long *addr)
  {
  	arch___clear_bit(nr, addr);
  }
  
  static __always_inline void
  arch___change_bit(long nr, volatile unsigned long *addr)
  {
  	asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
  }
  
  static __always_inline void
  arch_change_bit(long nr, volatile unsigned long *addr)
  {
  	if (__builtin_constant_p(nr)) {
  		asm volatile(LOCK_PREFIX "xorb %1,%0"
  			: CONST_MASK_ADDR(nr, addr)
  			: "iq" ((u8)CONST_MASK(nr)));
  	} else {
  		asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
  			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");
  	}
  }
  
  static __always_inline bool
  arch_test_and_set_bit(long nr, volatile unsigned long *addr)
  {
  	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), *addr, c, "Ir", nr);
  }
  
  static __always_inline bool
  arch_test_and_set_bit_lock(long nr, volatile unsigned long *addr)
  {
  	return arch_test_and_set_bit(nr, addr);
  }
  
  static __always_inline bool
  arch___test_and_set_bit(long nr, volatile unsigned long *addr)
  {
  	bool oldbit;
  
  	asm(__ASM_SIZE(bts) " %2,%1"
  	    CC_SET(c)
  	    : CC_OUT(c) (oldbit)
  	    : ADDR, "Ir" (nr) : "memory");
  	return oldbit;
  }
  
  static __always_inline bool
  arch_test_and_clear_bit(long nr, volatile unsigned long *addr)
  {
  	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), *addr, c, "Ir", nr);
  }
  
  /*
   * Note: the operation is performed atomically with respect to
   * the local CPU, but not other CPUs. Portable code should not
   * rely on this behaviour.
   * KVM relies on this behaviour on x86 for modifying memory that is also
   * accessed from a hypervisor on the same CPU if running in a VM: don't change
   * this without also updating arch/x86/kernel/kvm.c
   */
  static __always_inline bool
  arch___test_and_clear_bit(long nr, volatile unsigned long *addr)
  {
  	bool oldbit;
  
  	asm volatile(__ASM_SIZE(btr) " %2,%1"
  		     CC_SET(c)
  		     : CC_OUT(c) (oldbit)
  		     : ADDR, "Ir" (nr) : "memory");
  	return oldbit;
  }
  
  static __always_inline bool
  arch___test_and_change_bit(long nr, volatile unsigned long *addr)
  {
  	bool oldbit;
  
  	asm volatile(__ASM_SIZE(btc) " %2,%1"
  		     CC_SET(c)
  		     : CC_OUT(c) (oldbit)
  		     : ADDR, "Ir" (nr) : "memory");
  
  	return oldbit;
  }
  
  static __always_inline bool
  arch_test_and_change_bit(long nr, volatile unsigned long *addr)
  {
  	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr);
  }
  
  static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
  {
  	return ((1UL << (nr & (BITS_PER_LONG-1))) &
  		(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
  }
  
  static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)
  {
  	bool oldbit;
  
  	asm volatile(__ASM_SIZE(bt) " %2,%1"
  		     CC_SET(c)
  		     : CC_OUT(c) (oldbit)
  		     : "m" (*(unsigned long *)addr), "Ir" (nr) : "memory");
  
  	return oldbit;
  }
  
  #define arch_test_bit(nr, addr)			\
  	(__builtin_constant_p((nr))		\
  	 ? constant_test_bit((nr), (addr))	\
  	 : variable_test_bit((nr), (addr)))
  
  /**
   * __ffs - find first set bit in word
   * @word: The word to search
   *
   * Undefined if no bit exists, so code should check against 0 first.
   */
  static __always_inline unsigned long __ffs(unsigned long word)
  {
  	asm("rep; bsf %1,%0"
  		: "=r" (word)
  		: "rm" (word));
  	return word;
  }
  
  /**
   * ffz - find first zero bit in word
   * @word: The word to search
   *
   * Undefined if no zero exists, so code should check against ~0UL first.
   */
  static __always_inline unsigned long ffz(unsigned long word)
  {
  	asm("rep; bsf %1,%0"
  		: "=r" (word)
  		: "r" (~word));
  	return word;
  }
  
  /*
   * __fls: find last set bit in word
   * @word: The word to search
   *
   * Undefined if no set bit exists, so code should check against 0 first.
   */
  static __always_inline unsigned long __fls(unsigned long word)
  {
  	asm("bsr %1,%0"
  	    : "=r" (word)
  	    : "rm" (word));
  	return word;
  }
  
  #undef ADDR
  
  #ifdef __KERNEL__
  /**
   * ffs - find first set bit in word
   * @x: the word to search
   *
   * This is defined the same way as the libc and compiler builtin ffs
   * routines, therefore differs in spirit from the other bitops.
   *
   * ffs(value) returns 0 if value is 0 or the position of the first
   * set bit if value is nonzero. The first (least significant) bit
   * is at position 1.
   */
  static __always_inline int ffs(int x)
  {
  	int r;
  
  #ifdef CONFIG_X86_64
  	/*
  	 * AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
  	 * dest reg is undefined if x==0, but their CPU architect says its
  	 * value is written to set it to the same as before, except that the
  	 * top 32 bits will be cleared.
  	 *
  	 * We cannot do this on 32 bits because at the very least some
  	 * 486 CPUs did not behave this way.
  	 */
  	asm("bsfl %1,%0"
  	    : "=r" (r)
  	    : "rm" (x), "0" (-1));
  #elif defined(CONFIG_X86_CMOV)
  	asm("bsfl %1,%0\n\t"
  	    "cmovzl %2,%0"
  	    : "=&r" (r) : "rm" (x), "r" (-1));
  #else
  	asm("bsfl %1,%0\n\t"
  	    "jnz 1f\n\t"
  	    "movl $-1,%0\n"
  	    "1:" : "=r" (r) : "rm" (x));
  #endif
  	return r + 1;
  }
  
  /**
   * fls - find last set bit in word
   * @x: the word to search
   *
   * This is defined in a similar way as the libc and compiler builtin
   * ffs, but returns the position of the most significant set bit.
   *
   * fls(value) returns 0 if value is 0 or the position of the last
   * set bit if value is nonzero. The last (most significant) bit is
   * at position 32.
   */
  static __always_inline int fls(unsigned int x)
  {
  	int r;
  
  #ifdef CONFIG_X86_64
  	/*
  	 * AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the
  	 * dest reg is undefined if x==0, but their CPU architect says its
  	 * value is written to set it to the same as before, except that the
  	 * top 32 bits will be cleared.
  	 *
  	 * We cannot do this on 32 bits because at the very least some
  	 * 486 CPUs did not behave this way.
  	 */
  	asm("bsrl %1,%0"
  	    : "=r" (r)
  	    : "rm" (x), "0" (-1));
  #elif defined(CONFIG_X86_CMOV)
  	asm("bsrl %1,%0\n\t"
  	    "cmovzl %2,%0"
  	    : "=&r" (r) : "rm" (x), "rm" (-1));
  #else
  	asm("bsrl %1,%0\n\t"
  	    "jnz 1f\n\t"
  	    "movl $-1,%0\n"
  	    "1:" : "=r" (r) : "rm" (x));
  #endif
  	return r + 1;
  }
  
  /**
   * fls64 - find last set bit in a 64-bit word
   * @x: the word to search
   *
   * This is defined in a similar way as the libc and compiler builtin
   * ffsll, but returns the position of the most significant set bit.
   *
   * fls64(value) returns 0 if value is 0 or the position of the last
   * set bit if value is nonzero. The last (most significant) bit is
   * at position 64.
   */
  #ifdef CONFIG_X86_64
  static __always_inline int fls64(__u64 x)
  {
  	int bitpos = -1;
  	/*
  	 * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
  	 * dest reg is undefined if x==0, but their CPU architect says its
  	 * value is written to set it to the same as before.
  	 */
  	asm("bsrq %1,%q0"
  	    : "+r" (bitpos)
  	    : "rm" (x));
  	return bitpos + 1;
  }
  #else
  #include <asm-generic/bitops/fls64.h>
  #endif
  
  #include <asm-generic/bitops/find.h>
  
  #include <asm-generic/bitops/sched.h>
  
  #include <asm/arch_hweight.h>
  
  #include <asm-generic/bitops/const_hweight.h>
  
  #include <asm-generic/bitops/instrumented-atomic.h>
  #include <asm-generic/bitops/instrumented-non-atomic.h>
  #include <asm-generic/bitops/instrumented-lock.h>
  
  #include <asm-generic/bitops/le.h>
  
  #include <asm-generic/bitops/ext2-atomic-setbit.h>
  
  #endif /* __KERNEL__ */
  #endif /* _ASM_X86_BITOPS_H */
  