boolean function ( ) { return validRobots ; }
if true then valid robots.txt information has been retrieved . if false either no attempt has been made to fetch robots.txt or the attempt failed . concode_field_sep String server concode_elem_sep ObjectIdentityCache<?> cache concode_elem_sep long MIN_ROBOTS_RETRIES concode_elem_sep Robotstxt robotstxt concode_elem_sep Set<Credential> credentials concode_elem_sep Logger logger concode_elem_sep long serialVersionUID concode_elem_sep int consecutiveConnectionErrors concode_elem_sep boolean validRobots concode_elem_sep Map<String,String> httpAuthChallenges concode_elem_sep int port concode_elem_sep FetchStats substats concode_elem_sep long robotsFetched concode_elem_sep long ROBOTS_NOT_FETCHED concode_field_sep void updateRobots concode_elem_sep String getKey concode_elem_sep void resetConsecutiveConnectionErrors concode_elem_sep String getName concode_elem_sep void incrementConsecutiveConnectionErrors concode_elem_sep Map<String,String> getHttpAuthChallenges concode_elem_sep void setIdentityCache concode_elem_sep Set<Credential> getCredentials concode_elem_sep boolean hasCredentials concode_elem_sep void addCredential concode_elem_sep String getServerKey concode_elem_sep boolean isRobotsExpired concode_elem_sep int getPort concode_elem_sep void autoregisterTo concode_elem_sep FetchStats getSubstats concode_elem_sep int hashCode concode_elem_sep boolean equals concode_elem_sep String toString concode_elem_sep void makeDirty concode_elem_sep void setHttpAuthChallenges concode_elem_sep Robotstxt getRobotstxt