def sites_google_site_proxyfree4u(self):
        proxies = []
        url = "https://sites.google.com/site/proxyfree4u/proxy-list?offset="
        # fetch the latest 10 pages
        for i in range(0, 100, 10):
            # print url + str(i)
            soup = BeautifulSoup(self.getpage(url + str(i)))
            http_client = HTTPClient()
            for link in soup.find_all('a'):
                fetch_url = link.get('href')
                #get the correct URL
                if fetch_url == None:
                    continue
                if fetch_url.find("&single=true&gid=0&output=txt") != -1:
                    request = HTTPRequest(
                                          url=fetch_url,
                                          connect_timeout=30,
                                          request_timeout=30,
                                          follow_redirects=False,
                                          use_gzip=True,
                                          user_agent=Proxy_Miner.User_agent
                                          )
                    #  sometime during tests the response was 599.
                    #  re-sending the packet 4 times
                    for times in range(0, 4):
                        try:
                            response = http_client.fetch(request)
                        except HTTPError as e:
                            if e.code in [408, 599]:
                                continue
                        #getting the cookies. In order to get the proxy list 2 cookies are needed
                        first_redirect = e.response.headers['Location']
                        cookie = e.response.headers['Set-Cookie']
                        cookie_headers = HTTPHeaders()
                        cookie_headers.add("Cookie", cookie.split(";")[0])
                        req2 = HTTPRequest(
                                           url=first_redirect,
                                           connect_timeout=30.0,
                                           request_timeout=30.0,
                                           follow_redirects=False,
                                           use_gzip=True,
                                           headers=cookie_headers,
                                           user_agent=Proxy_Miner.User_agent
                                           )
                        try:
                            http_client.fetch(req2)
                        except __HOLE__ as e2:
                            second_redirect = e2.response.headers['Location']
                            # get the second cookie
                            cookie2 = e2.response.headers['Set-Cookie']
                            cookie_headers.add("Cookie", cookie2.split(";")[0])
                            req3 = HTTPRequest(
                                               url=second_redirect,
                                               connect_timeout=30.0,
                                               request_timeout=30.0,
                                               follow_redirects=True,
                                               use_gzip=True,
                                               headers=cookie_headers,
                                               user_agent=Proxy_Miner.User_agent
                                               )
                            resp3 = http_client.fetch(req3)
                            # print resp3.body
                            lines = resp3.body.split("\n")
                            counter = 0
                            for j in range(1, len(lines)):
                                proxy = lines[j].split(":")
                                if self.check_proxy(proxy):
                                    proxies.append(proxy)
                                # if the list contains non valid proxies
                                else:
                                    counter += 1
                                    if counter == 15:
                                        break
                        break

        return proxies
#---------------------------------------------------------------
dataset/ETHPy150Open owtf/owtf/framework/http/proxy/outbound_proxyminer.py/Proxy_Miner.sites_google_site_proxyfree4u