text: // Copyright 2019 The Fuchsia Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  
  #ifndef SRC_DEVICES_USB_DRIVERS_XHCI_REWRITE_USB_XHCI_H_
  #define SRC_DEVICES_USB_DRIVERS_XHCI_REWRITE_USB_XHCI_H_
  
  #include <lib/async-loop/cpp/loop.h>
  #include <lib/async/cpp/executor.h>
  #include <lib/device-protocol/pci.h>
  #include <lib/device-protocol/pdev.h>
  #include <lib/fit/function.h>
  #include <lib/fit/single_threaded_executor.h>
  #include <lib/mmio/mmio.h>
  #include <lib/sync/completion.h>
  #include <lib/synchronous-executor/executor.h>
  #include <lib/zx/profile.h>
  #include <threads.h>
  #include <unistd.h>
  
  #include <algorithm>
  #include <atomic>
  #include <memory>
  #include <thread>
  
  #include <ddktl/device.h>
  #include <ddktl/protocol/composite.h>
  #include <ddktl/protocol/pci.h>
  #include <ddktl/protocol/platform/device.h>
  #include <ddktl/protocol/usb/bus.h>
  #include <ddktl/protocol/usb/hci.h>
  #include <ddktl/protocol/usb/phy.h>
  #include <fbl/auto_lock.h>
  #include <fbl/mutex.h>
  
  #include "lib/dma-buffer/buffer.h"
  #include "registers.h"
  #include "xhci-context.h"
  #include "xhci-device-state.h"
  #include "xhci-enumeration.h"
  #include "xhci-event-ring.h"
  #include "xhci-hub.h"
  #include "xhci-interrupter.h"
  #include "xhci-port-state.h"
  #include "xhci-transfer-ring.h"
  
  namespace usb_xhci {
  
  inline void InvalidatePageCache(void* addr, uint32_t options) {
    uintptr_t page = reinterpret_cast<uintptr_t>(addr);
    page = fbl::round_down(page, static_cast<uintptr_t>(PAGE_SIZE));
    zx_cache_flush(reinterpret_cast<void*>(page), PAGE_SIZE, options);
  }
  
  // This is the main class for the USB XHCI host controller driver.
  // Refer to 3.1 for general architectural information on xHCI.
  class UsbXhci;
  using UsbXhciType = ddk::Device<UsbXhci, ddk::Initializable, ddk::Suspendable, ddk::Unbindable>;
  class UsbXhci : public UsbXhciType, public ddk::UsbHciProtocol<UsbXhci, ddk::base_protocol> {
   public:
    explicit UsbXhci(zx_device_t* parent, std::unique_ptr<dma_buffer::BufferFactory> buffer_factory)
        : UsbXhciType(parent),
          pci_(parent),
          pdev_(parent),
          composite_(parent),
          buffer_factory_(std::move(buffer_factory)),
          ddk_interaction_loop_(&kAsyncLoopConfigNeverAttachToThread),
          ddk_interaction_executor_(ddk_interaction_loop_.dispatcher()) {}
  
    // Constructor for unit testing (to allow interception of MMIO read/write)
    explicit UsbXhci(zx_device_t* parent, ddk::MmioBuffer buffer)
        : UsbXhciType(parent),
          ddk_interaction_loop_(&kAsyncLoopConfigNeverAttachToThread),
          ddk_interaction_executor_(ddk_interaction_loop_.dispatcher()) {}
  
    static zx_status_t Create(void* ctx, zx_device_t* parent);
  
    // Forces an immediate shutdown of the HCI
    // This should only be called for critical errors that cannot
    // be recovered from.
    void Shutdown(zx_status_t status);
    // Device protocol implementation.
    void DdkInit(ddk::InitTxn txn);
    void DdkSuspend(ddk::SuspendTxn txn);
    void DdkUnbind(ddk::UnbindTxn txn);
  
    void DdkRelease();
  
    // Queues a control request
    void UsbHciControlRequestQueue(Request request);
  
    // Queues a normal request
    void UsbHciNormalRequestQueue(Request request);
  
    // USB HCI protocol implementation.
    void UsbHciRequestQueue(usb_request_t* usb_request, const usb_request_complete_t* complete_cb);
  
    // Queues a USB request (compatibility shim for usb::CallbackRequest in unit test)
    void RequestQueue(usb_request_t* usb_request, const usb_request_complete_t* complete_cb) {
      UsbHciRequestQueue(usb_request, complete_cb);
    }
  
    // Queues a request and returns a promise
    fit::promise<OwnedRequest, void> UsbHciRequestQueue(OwnedRequest usb_request);
  
    void UsbHciSetBusInterface(const usb_bus_interface_protocol_t* bus_intf);
  
    // Retrieves the max number of device slots supported by this host controller
    size_t UsbHciGetMaxDeviceCount();
  
    zx_status_t UsbHciEnableEndpoint(uint32_t device_id, const usb_endpoint_descriptor_t* ep_desc,
                                     const usb_ss_ep_comp_descriptor_t* ss_com_desc, bool enable);
  
    TRBPromise UsbHciEnableEndpoint(uint32_t device_id, const usb_endpoint_descriptor_t* ep_desc,
                                    const usb_ss_ep_comp_descriptor_t* ss_com_desc);
  
    TRBPromise UsbHciDisableEndpoint(uint32_t device_id, const usb_endpoint_descriptor_t* ep_desc,
                                     const usb_ss_ep_comp_descriptor_t* ss_com_desc);
  
    uint64_t UsbHciGetCurrentFrame();
  
    zx_status_t UsbHciConfigureHub(uint32_t device_id, usb_speed_t speed,
                                   const usb_hub_descriptor_t* desc, bool multi_tt);
  
    zx_status_t UsbHciHubDeviceAdded(uint32_t device_id, uint32_t port, usb_speed_t speed);
  
    zx_status_t UsbHciHubDeviceRemoved(uint32_t device_id, uint32_t port);
  
    zx_status_t UsbHciHubDeviceReset(uint32_t device_id, uint32_t port);
  
    zx_status_t UsbHciResetEndpoint(uint32_t device_id, uint8_t ep_address);
  
    TRBPromise UsbHciResetEndpointAsync(uint32_t device_id, uint8_t ep_address);
  
    bool Running() const { return running_; }
  
    zx_status_t UsbHciResetDevice(uint32_t hub_address, uint32_t device_id);
  
    size_t UsbHciGetMaxTransferSize(uint32_t device_id, uint8_t ep_address);
  
    zx_status_t UsbHciCancelAll(uint32_t device_id, uint8_t ep_address);
  
    TRBPromise UsbHciCancelAllAsync(uint32_t device_id, uint8_t ep_address);
  
    size_t UsbHciGetRequestSize();
  
    // Offlines a device slot, removing its device node from the topology.
    TRBPromise DeviceOffline(uint32_t slot, TRB* continuation);
  
    // Onlines a device, publishing a device node in the DDK.
    zx_status_t DeviceOnline(uint32_t slot, uint16_t port, usb_speed_t speed);
  
    // Returns whether or not a device is connected to the root hub.
    // Always returns true for devices attached via a hub.
    bool IsDeviceConnected(uint8_t slot) {
      auto& state = device_state_[slot - 1];
      fbl::AutoLock _(&state.transaction_lock());
      return !state.IsDisconnecting();
    }
  
    // Disables a slot
    TRBPromise DisableSlotCommand(uint32_t slot_id);
  
    TRBPromise EnableSlotCommand();
  
    TRBPromise AddressDeviceCommand(uint8_t slot_id, uint8_t port_id, std::optional<HubInfo> hub_info,
                                    bool bsr);
  
    TRBPromise AddressDeviceCommand(uint8_t slot_id);
  
    TRBPromise SetMaxPacketSizeCommand(uint8_t slot_id, uint8_t bMaxPacketSize0);
  
    usb_speed_t GetDeviceSpeed(uint8_t slot_id);
  
    uint8_t GetPortSpeed(uint8_t port_id) const;
  
    // Returns the CSZ bit from HCCPARAMS1
    bool CSZ() const { return hcc_.CSZ(); }
  
    // Returns the value in the CAPLENGTH register
    uint8_t CapLength() const { return cap_length_; }
  
    static uint8_t DeviceIdToSlotId(uint8_t device_id) { return static_cast<uint8_t>(device_id + 1); }
  
    static uint8_t SlotIdToDeviceId(uint8_t slot_id) { return static_cast<uint8_t>(slot_id - 1); }
  
    void SetDeviceInformation(uint8_t slot, uint8_t port, const std::optional<HubInfo>& hub);
  
    // MfIndex wrapper handler. The previous driver used this to increment
    // the mfindex wrap value. This caused race conditions that resulted
    // in incorrect values for the mfindex wrap value.
    // This function is left empty as a placeholder
    // for future use of the MFIndex wrap event. It is unclear at the moment
    // what, if anything this callback should be used for.
    void MfIndexWrapped() {}
  
    zx::profile& get_profile() { return profile_; }
  
    template <typename T>
    void PostCallback(T&& callback) {
      ddk_interaction_executor_.schedule_task(
          fit::make_ok_promise().then([this, cb = std::forward<T>(callback)](
                                          fit::result<void, void>& result) mutable { cb(bus_); }));
    }
  
    uint8_t get_port_count() { return static_cast<uint8_t>(params_.MaxPorts()); }
  
    TRBPromise UsbHciHubDeviceAddedAsync(uint32_t device_id, uint32_t port, usb_speed_t speed);
  
    TRBPromise ConfigureHubAsync(uint32_t device_id, usb_speed_t speed,
                                 const usb_hub_descriptor_t* desc, bool multi_tt);
  
    // Resets a port. Not to be confused with ResetDevice.
    void ResetPort(uint16_t port);
  
    // Waits for xHCI bringup to complete
    void WaitForBringup() { sync_completion_wait(&bringup_, ZX_TIME_INFINITE); }
  
    CommandRing* get_command_ring() { return &command_ring_; }
  
    DeviceState* get_device_state() { return device_state_.get(); }
  
    PortState* get_port_state() { return port_state_.get(); }
    // Indicates whether or not the controller supports cache coherency
    // for transfers.
    bool HasCoherentCache() const { return has_coherent_cache_; }
    // Indicates whether or not the controller has a cache coherent state.
    // Currently, this is the same as HasCoherentCache, but the spec
    // leaves open the possibility that a controller may have a coherent cache,
    // but not a coherent state.
    bool HasCoherentState() const { return HasCoherentCache(); }
    // Returns whether or not we are running in Qemu. Quirks need to be applied
    // where the emulated controller violates the xHCI specification.
    bool IsQemu() { return qemu_quirk_; }
  
    // Schedules a promise for execution on the executor
    void ScheduleTask(TRBPromise promise) {
      interrupters_[0].ring().ScheduleTask(std::move(promise));
    }
  
    // Schedules the promise for execution and synchronously waits for it to complete
    zx_status_t RunSynchronously(fit::promise<TRB*, zx_status_t> promise) {
      sync_completion_t completion;
      zx_status_t completion_code;
      auto continuation = promise.then([&](fit::result<TRB*, zx_status_t>& result) {
        if (result.is_ok()) {
          completion_code = ZX_OK;
          sync_completion_signal(&completion);
        } else {
          completion_code = result.error();
          sync_completion_signal(&completion);
        }
        return result;
      });
      ScheduleTask(continuation.box());
      RunUntilIdle();
      sync_completion_wait(&completion, ZX_TIME_INFINITE);
      return completion_code;
    }
  
    // Creates a promise that resolves after a timeout
    TRBPromise Timeout(zx::time deadline) { return interrupters_[0].Timeout(deadline); }
  
    // Provides a barrier for promises.
    // After this method is invoked,
    // all pending promises will be flushed.
    void RunUntilIdle() { interrupters_[0].ring().RunUntilIdle(); }
  
    // Initialization thread method. This is invoked from a separate detached thread
    // when xHCI binds
    zx_status_t InitThread();
  
    // Resets the xHCI controller. This should only be called during initialization.
    void ResetController();
  
    // Initializes PCI
    zx_status_t InitPci();
  
    // Initializes MMIO
    zx_status_t InitMmio();
  
    // Performs the handoff from the BIOS to the xHCI driver
    void BiosHandoff();
  
    // Performs platform-specific initialization functions
    void InitQuirks();
  
    zx_status_t HciFinalize();
  
    const zx::bti& bti() const { return bti_; }
  
    size_t get_page_size() const { return page_size_; }
  
    bool get_is_32_bit_controller() const { return is_32bit_; }
  
    // Asynchronously submits a command to the command queue.
    TRBPromise SubmitCommand(const TRB& command, std::unique_ptr<TRBContext> trb_context);
  
    // Retrieves the current test harness
    void* get_test_harness() const { return test_harness_; }
  
    // Sets the test harness
    void set_test_harness(void* harness) { test_harness_ = harness; }
  
    dma_buffer::BufferFactory& buffer_factory() const { return *buffer_factory_; }
  
   private:
    DISALLOW_COPY_ASSIGN_AND_MOVE(UsbXhci);
  
    struct UsbRequestState;
  
    // Waits for a time interval when it is suitable to schedule an isochronous transfer
    void WaitForIsochronousReady(UsbRequestState* state);
  
    // Starts a normal transfer
    void StartNormalTransaction(UsbRequestState* state);
  
    // Continues a normal transfer
    void ContinueNormalTransaction(UsbRequestState* state);
  
    // Commits a normal transfer
    void CommitNormalTransaction(UsbRequestState* state);
  
    // Performs the allocation phase of the control request
    // (allocates TRBs for the request)
    void ControlRequestAllocationPhase(UsbRequestState* state);
  
    // Performs the status phase of a control request
    static void ControlRequestStatusPhase(UsbRequestState* state);
  
    // Performs the data transfer phase of a control request
    void ControlRequestDataPhase(UsbRequestState* state);
  
    // Performs the setup phase of a control request
    static void ControlRequestSetupPhase(UsbRequestState* state);
  
    // Starts the transfer of a control request
    void ControlRequestCommit(UsbRequestState* state);
  
    // Global scheduler lock. This should be held when adding or removing
    // interrupters, and; eventually dynamically assigning transfer rings
    // to interrupters.
    fbl::Mutex scheduler_lock_;
  
    // This is a high-priority profile used for increasing the priority
    // of the interrupt thread. This is currently necessary to mitigate
    // fxbug.dev/34507, and can be removed once the scheduling problem is fixed.
    zx::profile profile_;
    // Performs the initialization sequence defined in section
    // 4.2 of the xHCI specification.
    zx_status_t Init();
  
    // PCI protocol client (if x86)
    ddk::Pci pci_;
  
    // PDev (if ARM)
    ddk::PDev pdev_;
  
    // Composite device protocol client used for communicating with the USB PHY
    // on certain ARM boards which support USB OTG.
    ddk::CompositeProtocolClient composite_;
  
    // MMIO buffer for communicating with the physical hardware
    // Must be optional to allow for asynchronous initialization,
    // since an MmioBuffer has no default constructor.
    std::optional<ddk::MmioBuffer> mmio_;
  
    // The number of IRQs supported by the HCI
    uint32_t irq_count_;
  
    // Array of interrupters, which service interrupts from the HCI
    std::unique_ptr<Interrupter[]> interrupters_;
  
    // Pointer to the start of the device context base address array
    // See xHCI section 6.1 for more information.
    uint64_t* dcbaa_;
  
    // IO buffer for the device context base address array
    std::unique_ptr<dma_buffer::PagedBuffer> dcbaa_buffer_;
  
    // BTI for retrieving physical memory addresses from IO buffers.
    zx::bti bti_;
  
    // xHCI scratchpad buffers (see xHCI section 4.20)
    std::unique_ptr<std::unique_ptr<dma_buffer::ContiguousBuffer>[]> scratchpad_buffers_;
  
    // IO buffer for the scratchpad buffer array
    std::unique_ptr<dma_buffer::PagedBuffer> scratchpad_buffer_array_;
  
    std::unique_ptr<dma_buffer::BufferFactory> buffer_factory_;
  
    // Page size of the HCI
    size_t page_size_;
  
    // xHCI command ring (see xHCI section 4.6.1)
    CommandRing command_ring_;
  
    // Whether or not the host controller is 32 bit
    bool is_32bit_ = false;
  
    // Whether or not the HCI's cache is coherent with the CPU
    bool has_coherent_cache_ = false;
  
    // Offset to the doorbells. See xHCI section 5.3.7
    DoorbellOffset doorbell_offset_;
  
    // The number of enabled interrupters
    uint32_t active_interrupters_ __TA_GUARDED(scheduler_lock_);
  
    // The value in the CAPLENGTH register (see xHCI section 5.3.1)
    uint8_t cap_length_;
  
    // The last recorded MFINDEX value
    std::atomic<uint32_t> last_mfindex_ = 0;
  
    // Runtime register offset (see xHCI section 5.3.8)
    RuntimeRegisterOffset runtime_offset_;
  
    // Status information on connected devices
    std::unique_ptr<DeviceState[]> device_state_;
  
    // Status information for each port in the system
    std::unique_ptr<PortState[]> port_state_;
  
    // Completion which is signalled when the bus interface is bound
    sync_completion_t bus_completion;
  
    // Completion which is signalled when xHCI enters an operational state
    sync_completion_t bringup_;
  
    // HCSPARAMS1 register (see xHCI section 5.3.3)
    HCSPARAMS1 params_;
  
    // HCCPARAMS1 register (see xHCI section 5.3.6)
    HCCPARAMS1 hcc_;
  
    // Number of slots supported by the HCI
    size_t max_slots_;
  
    // Whether or not we are running on Qemu
    bool qemu_quirk_ = false;
  
    // Number of times the MFINDEX has wrapped
    std::atomic_uint64_t wrap_count_ = 0;
  
    // USB bus protocol client
    ddk::UsbBusInterfaceProtocolClient bus_;
  
    async::Loop ddk_interaction_loop_;
  
    // Pending DDK callbacks that need to be ran on the dedicated DDK interaction thread
    async::Executor ddk_interaction_executor_;
  
    // Thread for interacting with the Devhost thread (main event loop)
    std::optional<thrd_t> ddk_interaction_thread_;
  
    // Whether or not the HCI instance is currently active
    std::atomic_bool running_ = true;
  
    // PHY protocol
    ddk::UsbPhyProtocolClient phy_;
  
    // Pointer to the test harness when being called from a unit test
    // This is an opaque pointer that is managed by the test.
    void* test_harness_;
  
    // Completion event which is signalled when driver initialization finishes
    sync_completion_t init_complete_;
  
    std::optional<thrd_t> init_thread_;
  
    std::optional<ddk::InitTxn> init_txn_;
  };
  
  }  // namespace usb_xhci
  
  #endif  // SRC_DEVICES_USB_DRIVERS_XHCI_REWRITE_USB_XHCI_H_
  