title: Programming to query 100000 users in AD
body: <p>I am currently working on a project to query AD, I have a script that do that, but the script is failing after 1000 users, while the users I am querying is around 150.000 users.</p>
  
  <p>Here is my code:</p>
  
  <p>below is my script , can you tell me on which line </p>
  
  <pre><code>Sub UserSynchQuery(ByRef res As APIResult, ByRef oRespDS As APIDataSet, ByRef sLDAPServer As String, ByRef sLDAPPort As String, ByRef sLDAPBase As String, ByRef sUserName As String, ByRef sPassword As String, ByRef sSLPPrimary As String, ByRef sSLPSecondary As String, ByRef sExtension As String, ByRef sConfiggroup As String, ByRef sFilter As String )
  
  Trace( "Called UserSynchQuery Entered" )
  
  Dim oDSP As Object
  Dim oDSRS As Object
  
  On Error Resume Next
  Set oDSP = CreateObject("ADODB.Connection") 
  oDSP.Provider = "ADSDSOObject" 
  
  oDSP.Open( "Ads Provider", sUserName, Demung( sPassword ))  
  
  If Err.Number &lt;&gt; 0 Then
          Trace("ERROR: Failed to instantiate ADO Object. " &amp; Err.Number &amp; " " &amp; Err.Description)
          res.Code = "FAILED"
          res.Reason = "Failed to instantiate ADO Object"
          Exit Sub
  End If
  
  On Error Goto 0
  
      Dim sRoot      'Holds the root of the LDAP object
      sRoot = "LDAP://" &amp; sLDAPServer &amp; ":" &amp; sLDAPPort &amp; "/" &amp; sLDAPBase
  
  Dim sQuery As String
  Dim sSelect As String
  
  sSelect = ADS_COLUMN_DN &amp; "," &amp; ADS_COLUMN_USERNAME &amp; "," &amp; ADS_COLUMN_LASTNAME &amp; "," &amp; ADS_COLUMN_FIRSTNAME &amp; "," &amp; ADS_COLUMN_EMAIL &amp; ","
  
  If Len(sSLPPrimary) &gt; 0 Then
      sSelect = sSelect &amp; sSLPPrimary &amp; ","
  End If
  If Len(sSLPSecondary) &gt; 0 Then
      sSelect = sSelect &amp; sSLPSecondary &amp; ","
  End If
  If Len(sExtension) &gt; 0 Then
      sSelect = sSelect &amp; sExtension &amp; ","
  End If
  If Len(sConfiggroup) &gt; 0 Then
      sSelect = sSelect &amp; sConfiggroup &amp; ","
  End If
  
  sSelect = sSelect &amp; ADS_COLUMN_MEMBEROF
  
  sQuery = "SELECT " &amp; sSelect &amp; " FROM '" &amp; sRoot &amp; "' WHERE " &amp; sFilter
  
  Trace( "Query String: " &amp; sQuery )
  
  On Error Resume Next
  Set oDSRS = oDSP.Execute(sQuery)  
  
  If Err.Number &lt;&gt; 0 Then
          Trace("ERROR: Query Failed. " &amp; Err.Number &amp; " " &amp; Err.Description)
          res.Code = "FAILED"
          res.Reason = "Query Failed"
          Exit Sub
  End If
  
  On Error Goto 0
  
  '// before you can fill in the dataset, you must initialize it with the 
  '// number of columns
  oRespDS.Initialize(MSG_USER_QUERY_RESP_NUM_COLS)
  
  
      Dim nRow
      Dim sRSUserName
      Dim sRSLastName
      Dim sRSFirstName
      Dim sRSEmail
      Dim sRSDN
      Dim sRSSLPPrimary
      Dim sRSSLPSecondary
      Dim sRSExtension
      Dim sRSConfiggroup
  
      nRow = 0
  
      Do Until oDSRS.EOF
  
      sRSUserName = oDSRS.Fields(ADS_COLUMN_USERNAME).Value
      sRSLastName = oDSRS.Fields(ADS_COLUMN_LASTNAME).Value
      sRSFirstName = oDSRS.Fields(ADS_COLUMN_FIRSTNAME).Value
      sRSEmail = oDSRS.Fields(ADS_COLUMN_EMAIL).Value
      sRSDN = oDSRS.Fields(ADS_COLUMN_DN).Value
  
  
      Trace("----------- Found User -----------")
      Trace("Username: " &amp; sRSUserName)
      Trace("Last Name: " &amp; sRSLastName)
      Trace("First Name: " &amp; sRSFirstName)
      Trace("Email: " &amp; sRSEmail)
      Trace("DN: " &amp; sRSDN)
      If Len(sSLPPrimary) &gt; 0 Then
          sRSSLPPrimary = oDSRS.Fields(sSLPPrimary).Value
          Trace("sSLPPrimary: " &amp; sRSSLPPrimary)
      End If
      If Len(sSLPSecondary) &gt; 0 Then
          sRSSLPSecondary = oDSRS.Fields(sSLPSecondary).Value
          Trace("sSLPSecondary: " &amp; sRSSLPSecondary)
      End If
      If Len(sExtension) &gt; 0 Then
          sRSExtension = oDSRS.Fields(sExtension).Value
          Trace("sExtension: " &amp; sRSExtension)
      End If
      If Len(sConfiggroup) &gt; 0 Then
          sRSConfiggroup = oDSRS.Fields(sConfiggroup).Value
          Trace("sConfiggroup: " &amp; sRSConfiggroup)
      End If
  
      If( IsNull( sRSUserNamme ) Or IsNull( sRSLastName ) Or IsNull( sRSFirstName ) Or IsNull( sRSDN ) ) Then
          Trace( "Error: Ignoring user due to missing information" )
      Else
          'We need to build up the list of groups which needs
          'to include any indirect group membership which could
          'be the result of assigning a group to be a member of
          'another group.
  
          Dim arrGroups
          Dim dictGroupNamesByDN
  
          Set dictGroupNamesByDN = CreateObject("Scripting.Dictionary")
  
          arrGroups = oDSRS.Fields(ADS_COLUMN_MEMBEROF).Value
  
          if IsNull( arrGroups )  Then
              Trace("---&gt;No groups found")
          Else
              ProcessGroupMembership( dictGroupNamesByDN, arrGroups )             
          End If
  
  
  
          'Now assing the roles to the user based on
          'the nested groups that we just retrieved.
  
          Dim sApplications As String
          sApplications = ""
  
          'We also use this opportunity to build the
          'workgroup membership up.
  
          Dim sWorkgroup As String
          sWorkgroups = ""
  
          Dim sCN As String
          Dim sDN As String
  
          Dim keys
          keys = dictGroupNamesByDN.Keys
  
          For Each key in keys
              sDN = key
              sCN = dictGroupNamesByDN.Item(key)
  
              sWorkgroups = sWorkgroups &amp; sCN &amp; ";"
  
              If sCN = CIM_AGENT_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "AGENT;"
              End If
  
              If sCN = CIM_RESMAN_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "RESMAN;"
              End If
  
              If sCN = CIM_CONFIGMAN_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "CONMAN;"
              End If
  
              If sCN = CIM_IVAULT_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "IVAULT;"
              End If
  
              If sCN = CIM_DECMAN_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "DMWEB;"
              End If
  
              If sCN = CIM_QIM_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "QIM;"
              End If
  
              If sCN = CIM_SYSMAN_APPLICATION_GROUP_NAME Then
                  sApplications = sApplications &amp; "SYSMAN;"
              End If
          Next
  
  
          Trace("Roles: " &amp; sApplications)
          Trace("Workgroups: " &amp; sWorkgroups)
  
  
          oRespDS.AddRow
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_USERNAME, sRSUserName)
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_LASTNAME, sRSLastName)
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_FIRSTNAME, sRSFirstName)
  
          If Not IsNull(sRSEMail) Then
              oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_EMAIL, sRSEmail)
          End If
  
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_DN, sRSDN)
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_APPLICATIONS, sApplications)
          oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_WORKGROUPS, sWorkgroups)
  
          If Len(sSLPPrimary) &gt; 0 Then
              If IsNull( sRSSLPPrimary ) Then
                  Trace("Warning: " &amp; sSLPPrimary &amp; " value not populated")
              Else
                  oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_SLPPRIMARY, sRSSLPPrimary)
              End If
          End If
  
          If Len(sSLPSecondary) &gt; 0 Then
              If IsNull( sRSSLPSecondary ) Then
                  Trace("Warning: " &amp; sSLPSecondary &amp; " value not populated")
              Else
                  oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_SLPSECONDARY, sRSSLPSecondary)
              End If
          End If
  
          If Len(sExtension) &gt; 0 Then
              If IsNull( sRSExtension ) Then
                  Trace("Warning: " &amp; sExtension &amp; " value not populated")
              Else
                  oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_EXTENSION, sRSExtension)
              End If
          End If
  
          If Len(sConfiggroup) &gt; 0 Then
              If IsNull( sRSConfiggroup ) Then
                  Trace("Warning: " &amp; sConfiggroup &amp; " value not populated")
              Else
                  oRespDS.SetField(nRow,MSG_USER_QUERY_RESP_CONFIGGROUPS, sRSConfiggroup)
              End If
          End If
  
          nRow = nRow + 1
      End If
  
      oDSRS.MoveNext
      Loop
  
  
      'Clean up
  On Error Resume Next
  
      oDSP = Nothing
      oDSRS = Nothing
  
      On Error Goto 0
  End Sub
  </code></pre>
  
  <p>The variable of LDAP SERVER, LDAP PORT, Username, Password and search base for both user and group is entered via application and it is working so far.</p>
  
  <p>Error what I have is once it reaches 1000 users:</p>
  
  <blockquote>
    <p>The size limit for this request was exceeded.</p>
  </blockquote>
  
  <p>If I removed the line <code>oDSRS.MoveNext</code> it would give an "Overflow" error.</p>
  
  <p>I did some reading and <a href="http://www.computing.net/answers/programming/do-until-eof-will-not-finish/28417.html" rel="nofollow">this</a> is the closest I could come up with.</p>