A query to find patients with any 4 visits within a 365 day period
Here is a possibility for you to look at.  It uses the Lag function with the offset option set.  It doesn't show what all the visits were or when they occurred, but it does only show the most recent visit date for each patient and then when the fourth prior visit occurred within the year.

Test Data

IF OBJECT_ID('tempdb..#TestData') IS NOT NULL
BEGIN
    DROP TABLE #TestData;
END

CREATE TABLE #TestData
    (
    TestDataID INT IDENTITY(1,1) PRIMARY KEY NOT NULL
    , PatientName VARCHAR(40) NOT NULL
    , VisitDate DATETIME NOT NULL
    );

INSERT INTO #TestData
(PatientName, VisitDate)
VALUES ('Smith', '11/30/2017')
    , ('Smith', '1/1/2018')
    , ('Smith', '12/5/2017')
    , ('Smith', '2/1/2015')
    , ('Smith', '9/5/2017')
    , ('Doe', '9/8/2017')
    , ('Smith', '9/12/2017');


Query

/** CTE_Visitor
    Limits data to only where all visits happened in 2017
    Determines if most recent (MostRecentDesc) will be 1 for most recent.
    Determines the fourth prior visit ID by using LAG with an offset.
    **/
;WITH CTE_Visitor AS
    (
    SELECT S.TestDataID
        , S.PatientName
        , S.VisitDate
        , FourVisitsAgo = LAG(S.TestDataID, 3) OVER (PARTITION BY S.PatientName ORDER BY S.VisitDate)
        , MostRecentDesc = ROW_NUMBER() OVER (PARTITION BY S.PatientName ORDER BY S.VisitDate DESC)
    FROM #TestData AS S
    WHERE YEAR(S.VisitDate) &gt;= 2017
    )
/** Joins test data back on to the FourVistsAgo to get the date.
    - Only displays for the most recent visit and the fourth prior visit.
    - Must have had at least four visits in the past
    - Fourth prior visit must be within a year of most recent visit
    **/
SELECT S.TestDataID
    , S.PatientName
    , S.VisitDate 
    , Prev.VisitDate AS DateFourVisitsAgo
    , DiffDays = DATEDIFF(DAY, Prev.VisitDate, S.VisitDate)
FROM CTE_Visitor AS S
    INNER JOIN #TestData AS Prev ON Prev.TestDataID = S.FourVisitsAgo
WHERE MostRecentDesc = 1
    AND FourVisitsAgo IS NOT NULL
    AND DATEDIFF(DAY, Prev.VisitDate, S.VisitDate) &lt;= 365;