title: Invalid URL '': No schema supplied. Perhaps you meant http://?
body: <p>I want code a python3 script for split URLs and check a string with requests module in python</p>
  <p>But some of URLs redirected By refresh meta tag i want python follow this link and a other problem is that :
  I when use for loop for(!) send request to URLs i receive</p>
  <blockquote>
  <p>&lt;Invalid URL '': No schema supplied. Perhaps you meant http://?&gt; ERROR</p>
  </blockquote>
  <p>if you look at the code below you'll find out better my mean</p>
  <pre><code>        fopen2 = open(&quot;clean url.txt&quot;, &quot;r&quot;)
      splurl = fopen2.read().split('\n')
      urlcln = []
      urlcln2 = []
      print(splurl)
      for i in splurl:
          getthis = requests.get(i)
          parserres = BeautifulSoup(getthis.text, 'html.parser')
          print(parserres)
          if &quot;&lt;title&gt;&quot; in str(parserres):
              print('yes')
          else:
              print('no')
  </code></pre>
  <p>this is full of the source code...</p>
  <pre><code>import time
  import sys
  import re
  from tqdm.auto import tqdm
  import requests
  from bs4 import BeautifulSoup
  # from colorama import Fore, Back, Style
  
  try:
  
      def finish():
          print(&quot;&quot;&quot;
  
          ==============================
  
          Duplicated URLs Remover With it Fathers!
  
          Developed Version ~ 1.0.3 By Ehsan Abafat
           _____ _                          _    _            __       _
          | ____| |__  ___  __ _ _ __      / \  | |__   __ _ / _| __ _| |_
          |  _| | '_ \/ __|/ _` | '_ \    / _ \ | '_ \ / _` | |_ / _` | __|
          | |___| | | \__ \ (_| | | | |  / ___ \| |_) | (_| |  _| (_| | |_
          |_____|_| |_|___/\__,_|_| |_| /_/   \_\_.__/ \__,_|_|  \__,_|\__|
  
          =============================
  
          clean URLs successfully!
  
          time that spent for this proccess:
  
          &quot;&quot;&quot;)
          print(toc - tic, &quot;S&quot;)
  
      if sys.argv[1] == '-h':
          print(100*&quot;*&quot;)
          print('''
  
              =&gt; ~ Usage : Put dirty URLs in &quot;old url.txt&quot; file and call python3 source.py -?
  
                      -s : Fully Clean URLs
                      -d : Clean URLs with keeping &quot;/&quot;
                      -f : Clean Duplicated URLs Example : 1 2 3 2 =&gt; 1 2 3
  
             =&gt; ~ Developed Version ~ 1.0.3 By Ehsan Abafat
  
          ''')
          print(100*&quot;*&quot;)
      elif sys.argv[1] == '-s':
  
          tic = time.time()
          f = open(&quot;old url.txt&quot;, &quot;r&quot;)
          flisted = f.read().lower().replace('https', 'http').replace(
              'http://', '').replace('www.', '')
          SmartRemover = re.sub(&quot;/(\w+)?&quot;, &quot;&quot;, flisted).split('\n')
          listurl = []
          dupurl = [&quot;\n&quot;]
          fullclean = []
          print('\n Getting Lines... \n')
          for i in tqdm(SmartRemover):
              if i in listurl:
                  dupurl.append(i.strip())
              else:
                  listurl.append(i.strip())
          print('\n Cleaning... \n')
          for i in tqdm(listurl):
              if i not in dupurl:
                  fullclean.append(i)
          f.close()
  
          flast = open(&quot;clean url.txt&quot;, &quot;w&quot;)
          for i in fullclean:
              if(i != '\n' and i != '\s' and i != '' and len(i) &gt; 2):
                  flast.write('http://'+str(i)+'\n')
          toc = time.time()
          finish()
          flast.close()
      elif sys.argv[1] == '-d':
  
          tic = time.time()
          f = open(&quot;old url.txt&quot;, &quot;r&quot;)
          flisted = f.read().lower().replace('https', 'http').replace(
              'http://', '').replace('www.', '').split()
          listurl = []
          dupurl = [&quot;\n&quot;]
          fullclean = []
          print('\n Getting Lines... \n')
          for i in tqdm(flisted):
              if i in listurl:
                  dupurl.append(i.strip())
              else:
                  listurl.append(i.strip())
          print('\n Cleaning... \n')
          for i in tqdm(listurl):
              if i not in dupurl:
                  fullclean.append(i)
          f.close()
  
          flast = open(&quot;clean url.txt&quot;, &quot;w&quot;)
          for i in fullclean:
              if(i != '\n' and i != '\s' and i != '' and len(i) &gt; 2):
                  flast.write('http://'+str(i)+'\n')
          toc = time.time()
          finish()
          flast.close()
      elif sys.argv[1] == '-f':
  
          tic = time.time()
          f = open(&quot;old url.txt&quot;, &quot;r&quot;)
          flisted = f.read().lower().replace('https', 'http').replace(
              'http://', '').replace('www.', '').split()
          listurl = []
          dupurl = [&quot;\n&quot;]
          fullclean = []
          print('\n Getting Lines... \n')
          for i in tqdm(flisted):
              if i in listurl:
                  dupurl.append(i.strip())
              else:
                  listurl.append(i.strip())
          print('\n Cleaning... \n')
          flast = open(&quot;clean url.txt&quot;, &quot;w&quot;)
          for i in tqdm(listurl):
              print(i)
              if(i != '\n' and i != '\s' and i != '' and len(i) &gt; 2):
                  flast.write('http://'+str(i)+'\n')
          f.close()
  
          toc = time.time()
          finish()
          flast.close()
      else:
          print('unknown command! use python3 source.py -h')
  
      if len(sys.argv) == 2:
          pass
      elif len(sys.argv) == 3:
          telerikuiVul = '{ &quot;message&quot; : &quot;RadAsyncUpload handler is registered succesfully, however, it may not be accessed directly.&quot; }'
          telerikBugCheckADR = &quot;/Telerik.Web.UI.WebResource.axd?type=rau&quot;
          fopen2 = open(&quot;clean url.txt&quot;, &quot;r&quot;)
          splurl = fopen2.read().split('\n')
          urlcln = []
          urlcln2 = []
          print(splurl)
          for i in splurl:
              getthis = requests.get(i)
              parserres = BeautifulSoup(getthis.text, 'html.parser')
              print(parserres)
              if &quot;&lt;title&gt;&quot; in str(parserres):
                  print('yes')
              else:
                  print('no')
      else:
          if(sys.argv[1] != '-h'):
              print(&quot;use '&lt;Python3 source.py -h&gt;' command&quot;)
          else:
              print('You are see Usage of This Script!')
  except Exception as e:
      print(e)
  </code></pre>