text: """
  .. function:: diavgeiaget(url, verb, metadataPrefix,...)
  
      Fetches data from an OAIPMH service, using resumption tokens to fetch large datasets.
  
      - If no *verb* is provided then *verb* is assumed to be 'ListRecords'.
      - If no *metadataPrefix* is provided then *verb* is assumed to be 'ListMetadataFormats', which will list
        all metadata formats.
  
  :Returned table schema:
      Column C1 as text
  
  Examples:
  
      >>> sql("select * from diavgeiaget('verb:ListRecords', 'metadataPrefix:ctxo')")    # doctest:+ELLIPSIS +NORMALIZE_WHITESPACE
      Traceback (most recent call last):
      ...
      OperatorError: Madis SQLError:
      Operator OAIGET: An OAIPMH URL should be provided
  
      >>> sql("select * from (diavgeiaget verb:ListRecords metadataPrefix:ctxo 'http://oaiurl' )")    # doctest:+ELLIPSIS +NORMALIZE_WHITESPACE
      Traceback (most recent call last):
      ...
      OperatorError: Madis SQLError:
      Operator OAIGET: <urlopen error [Errno -2] Name or service not known>
  
  """
  import functions
  import time
  from functions.vtable import vtbase
  
  registered = True
  external_stream = True
  
  
  class diavgeiaget(vtbase.VT):
      def VTiter(self, *parsedArgs, **envars):
  
          def buildURL(baseurl, opts):
              return '?'.join([baseurl, '&'.join([x + '=' + unicode(y) for x, y in opts if y != None])])
  
          import urllib2
          import re
  
          opts = self.full_parse(parsedArgs)[1]
  
          yield ('c1', 'text')
  
          if 'datefrom' not in opts:
              opts['datefrom'] = '01-01-1000'
          if 'output' not in opts:
              opts['output'] = 'full'
          if 'order' not in opts:
              opts['order'] = 'asc'
          if 'http' not in opts:
              opts['http'] = '//opendata.diavgeia.gov.gr/api/decisions'
  
          baseurl = 'http:' + opts['http']
  
          findcount = re.compile(r"""<count>[^\d]*?(\d+)[^\d]*?</count>""", re.DOTALL | re.UNICODE)
          findtotal = re.compile(r"""<total>[^\d]*?(\d+)[^\d]*?</total>""", re.DOTALL | re.UNICODE)
          findfrom = re.compile(r"""<from>[^\d]*?(\d+)[^\d]*?</from>""", re.DOTALL | re.UNICODE)
  
          count = total = fromv = lastfromv = None
          firsttime = True
  
          del (opts['http'])
          opts = list(opts.iteritems())
          url = buildURL(baseurl, opts)
  
          def buildopener():
              o = urllib2.build_opener()
              o.addheaders = [
                  ('Accept', '*/*'),
                  ('Connection', 'Keep-Alive'),
                  ('Content-type', 'text/xml')
              ]
              return o
  
          opener = buildopener()
  
          errorcount = 0
          while True:
              try:
                  for i in opener.open(url, timeout=1200):
                      if count == None:
                          t = findcount.search(i)
                          if t:
                              count = int(t.groups()[0])
                      if total == None:
                          t = findtotal.search(i)
                          if t:
                              errorcount = 0
                              total = int(t.groups()[0])
                      if fromv == None:
                          t = findfrom.search(i)
                          if t:
                              errorcount = 0
                              fromv = int(t.groups()[0])
                      yield (unicode(i.rstrip("\n"), 'utf-8'),)
                  if count == None or total == None or fromv == None:
                      break
                  fromv = fromv + count
                  if fromv > total:
                      break
                  url = buildURL(baseurl, opts + [('from', fromv)])
                  lastfromv = fromv
                  count = total = fromv = None
                  firsttime = False
              except Exception, e:
                  if errorcount < 10 and not firsttime:
                      time.sleep(2 ** errorcount)
                      errorcount += 1
                  else:
                      if lastfromv == None:
                          raise functions.OperatorError(__name__.rsplit('.')[-1], e)
                      else:
                          raise functions.OperatorError(__name__.rsplit('.')[-1], str(
                              e) + '\n' + 'To continue, use the following "from" parameter:\n' + str(lastfromv))
  
  
  def Source():
      return vtbase.VTGenerator(diavgeiaget)
  
  
  if not ('.' in __name__):
      """
      This is needed to be able to test the function, put it at the end of every
      new function you create
      """
      import sys
      from functions import *
  
      testfunction()
      if __name__ == "__main__":
          reload(sys)
          sys.setdefaultencoding('utf-8')
          import doctest
  
          doctest.testmod()
  