text: // Copyright 2016 The Fuchsia Authors
  //
  // Use of this source code is governed by a MIT-style
  // license that can be found in the LICENSE file or at
  // https://opensource.org/licenses/MIT
  
  #ifndef ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_DISPATCHER_H_
  #define ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_DISPATCHER_H_
  
  #include <stdint.h>
  #include <string.h>
  #include <zircon/compiler.h>
  #include <zircon/errors.h>
  #include <zircon/syscalls/object.h>
  #include <zircon/types.h>
  
  #include <fbl/auto_lock.h>
  #include <fbl/canary.h>
  #include <fbl/intrusive_double_list.h>
  #include <fbl/intrusive_single_list.h>
  #include <fbl/recycler.h>
  #include <fbl/ref_counted.h>
  #include <fbl/ref_counted_upgradeable.h>
  #include <fbl/ref_ptr.h>
  #include <kernel/lockdep.h>
  #include <kernel/mutex.h>
  #include <kernel/spinlock.h>
  #include <ktl/move.h>
  #include <ktl/type_traits.h>
  #include <ktl/unique_ptr.h>
  #include <object/handle.h>
  #include <object/signal_observer.h>
  
  template <typename T>
  struct DispatchTag;
  template <typename T>
  struct CanaryTag;
  
  #define DECLARE_DISPTAG(T, E, M)                     \
    class T;                                           \
    template <>                                        \
    struct DispatchTag<T> {                            \
      static constexpr zx_obj_type_t ID = E;           \
    };                                                 \
    template <>                                        \
    struct CanaryTag<T> {                              \
      static constexpr uint32_t magic = fbl::magic(M); \
    };
  
  DECLARE_DISPTAG(ProcessDispatcher, ZX_OBJ_TYPE_PROCESS, "PROC")
  DECLARE_DISPTAG(ThreadDispatcher, ZX_OBJ_TYPE_THREAD, "THRD")
  DECLARE_DISPTAG(VmObjectDispatcher, ZX_OBJ_TYPE_VMO, "VMOD")
  DECLARE_DISPTAG(ChannelDispatcher, ZX_OBJ_TYPE_CHANNEL, "CHAN")
  DECLARE_DISPTAG(EventDispatcher, ZX_OBJ_TYPE_EVENT, "EVTD")
  DECLARE_DISPTAG(PortDispatcher, ZX_OBJ_TYPE_PORT, "PORT")
  DECLARE_DISPTAG(InterruptDispatcher, ZX_OBJ_TYPE_INTERRUPT, "INTD")
  DECLARE_DISPTAG(PciDeviceDispatcher, ZX_OBJ_TYPE_PCI_DEVICE, "PCID")
  DECLARE_DISPTAG(LogDispatcher, ZX_OBJ_TYPE_LOG, "LOGD")
  DECLARE_DISPTAG(SocketDispatcher, ZX_OBJ_TYPE_SOCKET, "SOCK")
  DECLARE_DISPTAG(ResourceDispatcher, ZX_OBJ_TYPE_RESOURCE, "RSRD")
  DECLARE_DISPTAG(EventPairDispatcher, ZX_OBJ_TYPE_EVENTPAIR, "EPAI")
  DECLARE_DISPTAG(JobDispatcher, ZX_OBJ_TYPE_JOB, "JOBD")
  DECLARE_DISPTAG(VmAddressRegionDispatcher, ZX_OBJ_TYPE_VMAR, "VARD")
  DECLARE_DISPTAG(FifoDispatcher, ZX_OBJ_TYPE_FIFO, "FIFO")
  DECLARE_DISPTAG(GuestDispatcher, ZX_OBJ_TYPE_GUEST, "GSTD")
  DECLARE_DISPTAG(VcpuDispatcher, ZX_OBJ_TYPE_VCPU, "VCPU")
  DECLARE_DISPTAG(TimerDispatcher, ZX_OBJ_TYPE_TIMER, "TIMR")
  DECLARE_DISPTAG(IommuDispatcher, ZX_OBJ_TYPE_IOMMU, "IOMM")
  DECLARE_DISPTAG(BusTransactionInitiatorDispatcher, ZX_OBJ_TYPE_BTI, "BTID")
  DECLARE_DISPTAG(ProfileDispatcher, ZX_OBJ_TYPE_PROFILE, "PROF")
  DECLARE_DISPTAG(PinnedMemoryTokenDispatcher, ZX_OBJ_TYPE_PMT, "PIMT")
  DECLARE_DISPTAG(SuspendTokenDispatcher, ZX_OBJ_TYPE_SUSPEND_TOKEN, "SUTD")
  DECLARE_DISPTAG(PagerDispatcher, ZX_OBJ_TYPE_PAGER, "PGRD")
  DECLARE_DISPTAG(ExceptionDispatcher, ZX_OBJ_TYPE_EXCEPTION, "EXCD")
  DECLARE_DISPTAG(ClockDispatcher, ZX_OBJ_TYPE_CLOCK, "CLOK")
  DECLARE_DISPTAG(StreamDispatcher, ZX_OBJ_TYPE_STREAM, "STRM")
  DECLARE_DISPTAG(MsiAllocationDispatcher, ZX_OBJ_TYPE_MSI_ALLOCATION, "MSIA")
  DECLARE_DISPTAG(MsiDispatcher, ZX_OBJ_TYPE_MSI_INTERRUPT, "MSII")
  
  #undef DECLARE_DISPTAG
  
  // Base class for all kernel objects that can be exposed to user-mode via
  // the syscall API and referenced by handles.
  //
  // It implements RefCounted because handles are abstractions to a multiple
  // references from user mode or kernel mode that control the lifetime of
  // the object.
  //
  // It implements Recyclable because upon final Release() on the RefPtr
  // it might be necessary to implement a destruction pattern that avoids
  // deep recursion since the kernel stack is very limited.
  //
  // You don't derive directly from this class; instead derive
  // from SoloDispatcher or PeeredDispatcher.
  class Dispatcher : private fbl::RefCountedUpgradeable<Dispatcher>,
                     private fbl::Recyclable<Dispatcher> {
   public:
    using fbl::RefCountedUpgradeable<Dispatcher>::AddRef;
    using fbl::RefCountedUpgradeable<Dispatcher>::Release;
    using fbl::RefCountedUpgradeable<Dispatcher>::Adopt;
    using fbl::RefCountedUpgradeable<Dispatcher>::AddRefMaybeInDestructor;
  
    // Dispatchers are either Solo or Peered. They handle refcounting
    // and locking differently.
    virtual ~Dispatcher();
  
    zx_koid_t get_koid() const { return koid_; }
  
    void increment_handle_count() {
      // As this function does not return anything actionable, not even something implicit like "you
      // now have the lock", there are no correct assumptions the caller can make about orderings
      // of this increment and any other memory access. As such it can just be relaxed.
      handle_count_.fetch_add(1, ktl::memory_order_relaxed);
    }
  
    // Returns true exactly when the handle count goes to zero.
    bool decrement_handle_count() {
      if (handle_count_.fetch_sub(1, ktl::memory_order_release) == 1u) {
        // The decrement operation above synchronizes with the fence below.  This ensures that changes
        // to the object prior to its handle count reaching 0 will be visible to the thread that
        // ultimately drops the count to 0.  This is similar to what's done in
        // |fbl::RefCountedInternal|.
        ktl::atomic_thread_fence(ktl::memory_order_acquire);
        return true;
      }
      return false;
    }
  
    uint32_t current_handle_count() const {
      // Requesting the count is fundamentally racy with other users of the dispatcher. A typical
      // reference count implementation might place an acquire here for the scenario where you then
      // run an object destructor without acquiring any locks. As a handle count is not a refcount
      // and a low handle count does not imply any ownership of the dispatcher (which has its own
      // refcount), this can just be relaxed.
      return handle_count_.load(ktl::memory_order_relaxed);
    }
  
    enum class TriggerMode : uint32_t {
      Level = 0,
      Edge,
    };
  
    // Add a observer which will be triggered when any |signal| becomes active
    // or cancelled when |handle| is destroyed.
    //
    // |observer| must be non-null, and |is_waitable| must report true.
    //
    // Be sure to |RemoveObserver| before the Dispatcher is destroyed.
    //
    // If |trigger_mode| is set to Edge, the signal state is not checked
    // on entry and the observer is only triggered if a signal subsequently
    // becomes active.
    zx_status_t AddObserver(SignalObserver* observer, const Handle* handle, zx_signals_t signals,
                            TriggerMode trigger_mode = TriggerMode::Level);
  
    // Remove an observer.
    //
    // Returns true if the method removed |observer|, otherwise returns false. If
    // provided, |signals| will be given the current state of the dispatcher's
    // signals when the observer was removed.
    //
    // This method may return false if the observer was never added or has already been removed in
    // preparation for its destruction.
    //
    // It is an error to call this method with an observer that's observing some other Dispatcher.
    //
    // May only be called when |is_waitable| reports true.
    bool RemoveObserver(SignalObserver* observer, zx_signals_t* signals = nullptr);
  
    // Cancel observers of this object's state (e.g., waits on the object).
    // Should be called when a handle to this dispatcher is being destroyed.
    //
    // May only be called when |is_waitable| reports true.
    void Cancel(const Handle* handle);
  
    // Like Cancel() but issued via via zx_port_cancel().
    //
    // Returns true if an observer was canceled.
    //
    // May only be called when |is_waitable| reports true.
    bool CancelByKey(const Handle* handle, const void* port, uint64_t key);
  
    // Interface for derived classes.
  
    virtual zx_obj_type_t get_type() const = 0;
  
    virtual zx_status_t user_signal_self(uint32_t clear_mask, uint32_t set_mask) = 0;
    virtual zx_status_t user_signal_peer(uint32_t clear_mask, uint32_t set_mask) = 0;
  
    virtual void on_zero_handles() {}
  
    virtual zx_koid_t get_related_koid() const = 0;
    virtual bool is_waitable() const = 0;
  
    // get_name() will return a null-terminated name of ZX_MAX_NAME_LEN - 1 or fewer
    // characters.  For objects that don't have names it will be "".
    virtual void get_name(char (&out_name)[ZX_MAX_NAME_LEN]) const {
      memset(out_name, 0, ZX_MAX_NAME_LEN);
    }
  
    // set_name() will truncate to ZX_MAX_NAME_LEN - 1 and ensure there is a
    // terminating null
    virtual zx_status_t set_name(const char* name, size_t len) { return ZX_ERR_NOT_SUPPORTED; }
  
    struct DeleterListTraits {
      static fbl::SinglyLinkedListNodeState<Dispatcher*>& node_state(Dispatcher& obj) {
        return obj.deleter_ll_;
      }
    };
  
    // Called whenever the object is bound to a new process. The |new_owner| is
    // the koid of the new process. It is only overridden for objects where a single
    // owner makes sense.
    virtual void set_owner(zx_koid_t new_owner) {}
  
    // Poll the currently active signals on this object.
    //
    // By the time the result of the function is inspected, the signals may have already
    // changed. Typically should only be used for tests or logging.
    zx_signals_t PollSignals() const TA_EXCL(get_lock());
  
   protected:
    // At construction, the object is asserting |signals|.
    explicit Dispatcher(zx_signals_t signals);
  
    // Notify others of a change in signals (possibly waking them). (Clearing satisfied signals or
    // setting satisfiable signals should not wake anyone.)
    //
    // May only be called when |is_waitable| reports true.
    void UpdateState(zx_signals_t clear_mask, zx_signals_t set_mask) TA_EXCL(get_lock());
    void UpdateStateLocked(zx_signals_t clear_mask, zx_signals_t set_mask) TA_REQ(get_lock());
  
    zx_signals_t GetSignalsStateLocked() const TA_REQ(get_lock()) { return signals_; }
  
    // Dispatcher subtypes should use this lock to protect their internal state.
    virtual Lock<Mutex>* get_lock() const = 0;
  
   private:
    friend class fbl::Recyclable<Dispatcher>;
    void fbl_recycle();
  
    fbl::Canary<fbl::magic("DISP")> canary_;
  
    const zx_koid_t koid_;
    ktl::atomic<uint32_t> handle_count_;
  
    zx_signals_t signals_ TA_GUARDED(get_lock());
  
    // List of observers watching for changes in signals on this dispatcher.
    fbl::DoublyLinkedList<SignalObserver*> observers_ TA_GUARDED(get_lock());
  
    // Used to store this dispatcher on the dispatcher deleter list.
    fbl::SinglyLinkedListNodeState<Dispatcher*> deleter_ll_;
  };
  
  // SoloDispatchers stand alone. Since they have no peer to coordinate with, they
  // directly contain their state lock. This is a CRTP template type to permit
  // the lock validator to distinguish between locks in different subclasses of
  // SoloDispatcher.
  template <typename Self, zx_rights_t def_rights, zx_signals_t extra_signals = 0u,
            lockdep::LockFlags Flags = lockdep::LockFlagsNone>
  class SoloDispatcher : public Dispatcher {
   public:
    static constexpr zx_rights_t default_rights() { return def_rights; }
  
    // At construction, the object is asserting |signals|.
    explicit SoloDispatcher(zx_signals_t signals = 0u) : Dispatcher(signals) {}
  
    // Related koid is overridden by subclasses, like thread and process.
    zx_koid_t get_related_koid() const override TA_REQ(get_lock()) { return 0ULL; }
    bool is_waitable() const final { return default_rights() & ZX_RIGHT_WAIT; }
  
    zx_status_t user_signal_self(uint32_t clear_mask, uint32_t set_mask) final {
      if (!is_waitable())
        return ZX_ERR_NOT_SUPPORTED;
      // Generic objects can set all USER_SIGNALs. Particular object
      // types (events and eventpairs) may be able to set more.
      auto allowed_signals = ZX_USER_SIGNAL_ALL | extra_signals;
      if ((set_mask & ~allowed_signals) || (clear_mask & ~allowed_signals))
        return ZX_ERR_INVALID_ARGS;
  
      UpdateState(clear_mask, set_mask);
      return ZX_OK;
    }
  
    zx_status_t user_signal_peer(uint32_t clear_mask, uint32_t set_mask) final {
      return ZX_ERR_NOT_SUPPORTED;
    }
  
   protected:
    Lock<Mutex>* get_lock() const final { return &lock_; }
  
    const fbl::Canary<CanaryTag<Self>::magic> canary_;
    mutable DECLARE_MUTEX(SoloDispatcher, Flags) lock_;
  };
  
  // PeeredDispatchers have opposing endpoints to coordinate state
  // with. For example, writing into one endpoint of a Channel needs to
  // modify zx_signals_t state (for the readability bit) on the opposite
  // side. To coordinate their state, they share a mutex, which is held
  // by the PeerHolder. Both endpoints have a RefPtr back to the
  // PeerHolder; no one else ever does.
  
  // Thus creating a pair of peered objects will typically look
  // something like
  //     // Make the two RefPtrs for each endpoint's handle to the mutex.
  //     auto holder0 = AdoptRef(new PeerHolder<Foo>(...));
  //     auto holder1 = peer_holder0;
  //     // Create the opposing sides.
  //     auto foo0 = AdoptRef(new Foo(ktl::move(holder0, ...));
  //     auto foo1 = AdoptRef(new Foo(ktl::move(holder1, ...));
  //     // Initialize the opposing sides, teaching them about each other.
  //     foo0->Init(&foo1);
  //     foo1->Init(&foo0);
  
  // A PeeredDispatcher object, in its |on_zero_handles| call must clear
  // out its peer's |peer_| field. This is needed to avoid leaks, and to
  // ensure that |user_signal| can correctly report ZX_ERR_PEER_CLOSED.
  
  // TODO(kulakowski) We should investigate turning this into one
  // allocation. This would mean PeerHolder would have two EndPoint
  // members, and that PeeredDispatcher would have custom refcounting.
  template <typename Endpoint>
  class PeerHolder : public fbl::RefCounted<PeerHolder<Endpoint>> {
   public:
    PeerHolder() = default;
    ~PeerHolder() = default;
  
    Lock<Mutex>* get_lock() const { return &lock_; }
  
    mutable DECLARE_MUTEX(PeerHolder) lock_;
  };
  
  template <typename Self, zx_rights_t def_rights, zx_signals_t extra_signals = 0u>
  class PeeredDispatcher : public Dispatcher {
   public:
    static constexpr zx_rights_t default_rights() { return def_rights; }
  
    // At construction, the object is asserting |signals|.
    explicit PeeredDispatcher(fbl::RefPtr<PeerHolder<Self>> holder, zx_signals_t signals = 0u)
        : Dispatcher(signals), holder_(ktl::move(holder)) {}
    virtual ~PeeredDispatcher() = default;
  
    zx_koid_t get_related_koid() const final TA_REQ(get_lock()) { return peer_koid_; }
    bool is_waitable() const final { return default_rights() & ZX_RIGHT_WAIT; }
  
    zx_status_t user_signal_self(uint32_t clear_mask,
                                 uint32_t set_mask) final TA_NO_THREAD_SAFETY_ANALYSIS {
      auto allowed_signals = ZX_USER_SIGNAL_ALL | extra_signals;
      if ((set_mask & ~allowed_signals) || (clear_mask & ~allowed_signals))
        return ZX_ERR_INVALID_ARGS;
  
      Guard<Mutex> guard{get_lock()};
  
      UpdateStateLocked(clear_mask, set_mask);
      return ZX_OK;
    }
  
    zx_status_t user_signal_peer(uint32_t clear_mask,
                                 uint32_t set_mask) final TA_NO_THREAD_SAFETY_ANALYSIS {
      auto allowed_signals = ZX_USER_SIGNAL_ALL | extra_signals;
      if ((set_mask & ~allowed_signals) || (clear_mask & ~allowed_signals))
        return ZX_ERR_INVALID_ARGS;
  
      Guard<Mutex> guard{get_lock()};
      // object_signal() may race with handle_close() on another thread.
      if (!peer_)
        return ZX_ERR_PEER_CLOSED;
      peer_->UpdateStateLocked(clear_mask, set_mask);
      return ZX_OK;
    }
  
    // All subclasses of PeeredDispatcher must implement a public
    // |void on_zero_handles_locked()|. The peer lifetime management
    // (i.e. the peer zeroing) is centralized here.
    void on_zero_handles() final TA_NO_THREAD_SAFETY_ANALYSIS {
      Guard<Mutex> guard{get_lock()};
      auto peer = ktl::move(peer_);
      static_cast<Self*>(this)->on_zero_handles_locked();
  
      // This is needed to avoid leaks, and to ensure that
      // |user_signal| can correctly report ZX_ERR_PEER_CLOSED.
      if (peer != nullptr) {
        // This defeats the lock analysis in the usual way: it
        // can't reason that the peers' get_lock() calls alias.
        peer->peer_.reset();
        static_cast<Self*>(peer.get())->OnPeerZeroHandlesLocked();
      }
    }
  
    // Returns true if the peer has closed. Once the peer has closed it
    // will never re-open.
    bool PeerHasClosed() const {
      Guard<Mutex> guard{get_lock()};
      return peer_ == nullptr;
    }
  
    Lock<Mutex>* get_lock() const final { return holder_->get_lock(); }
  
   protected:
    const fbl::Canary<CanaryTag<Self>::magic> canary_;
  
    zx_koid_t peer_koid_ = 0u;
    fbl::RefPtr<Self> peer_ TA_GUARDED(get_lock());
  
   private:
    const fbl::RefPtr<PeerHolder<Self>> holder_;
  };
  
  // DownCastDispatcher checks if a RefPtr<Dispatcher> points to a
  // dispatcher of a given dispatcher subclass T and, if so, moves the
  // reference to a RefPtr<T>, otherwise it leaves the
  // RefPtr<Dispatcher> alone.  Must be called with a pointer to a valid
  // (non-null) dispatcher.
  
  // Note that the Dispatcher -> Dispatcher versions come up in generic
  // code, and so aren't totally vacuous.
  
  // Dispatcher -> FooDispatcher
  template <typename T>
  fbl::RefPtr<T> DownCastDispatcher(fbl::RefPtr<Dispatcher>* disp) {
    return (likely(DispatchTag<T>::ID == (*disp)->get_type()))
               ? fbl::RefPtr<T>::Downcast(ktl::move(*disp))
               : nullptr;
  }
  
  // Dispatcher -> Dispatcher
  template <>
  inline fbl::RefPtr<Dispatcher> DownCastDispatcher(fbl::RefPtr<Dispatcher>* disp) {
    return ktl::move(*disp);
  }
  
  // const Dispatcher -> const FooDispatcher
  template <typename T>
  fbl::RefPtr<T> DownCastDispatcher(fbl::RefPtr<const Dispatcher>* disp) {
    static_assert(ktl::is_const<T>::value, "");
    return (likely(DispatchTag<typename ktl::remove_const<T>::type>::ID == (*disp)->get_type()))
               ? fbl::RefPtr<T>::Downcast(ktl::move(*disp))
               : nullptr;
  }
  
  // const Dispatcher -> const Dispatcher
  template <>
  inline fbl::RefPtr<const Dispatcher> DownCastDispatcher(fbl::RefPtr<const Dispatcher>* disp) {
    return ktl::move(*disp);
  }
  
  // The same, but for Dispatcher* and FooDispatcher* instead of RefPtr.
  
  // Dispatcher -> FooDispatcher
  template <typename T>
  T* DownCastDispatcher(Dispatcher* disp) {
    return (likely(DispatchTag<T>::ID == disp->get_type())) ? static_cast<T*>(disp) : nullptr;
  }
  
  // Dispatcher -> Dispatcher
  template <>
  inline Dispatcher* DownCastDispatcher(Dispatcher* disp) {
    return disp;
  }
  
  // const Dispatcher -> const FooDispatcher
  template <typename T>
  const T* DownCastDispatcher(const Dispatcher* disp) {
    static_assert(ktl::is_const<T>::value, "");
    return (likely(DispatchTag<typename ktl::remove_const<T>::type>::ID == disp->get_type()))
               ? static_cast<const T*>(disp)
               : nullptr;
  }
  
  // const Dispatcher -> const Dispatcher
  template <>
  inline const Dispatcher* DownCastDispatcher(const Dispatcher* disp) {
    return disp;
  }
  
  #endif  // ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_DISPATCHER_H_
  