Python xml : list all elements in item
Question: I need to list all elements in my `<product>` item, because the elements of
`<product>` is variable.

XML file :

    
    
    <catalog>
       <product>
          <element1>text 1</element1>
          <element2>text 2</element2>
          <element..>text ..</element..>
       </produc>
    </catalog>
    

Python parser : I use fast_iter because my xml file is large...

    
    
    import lxml.etree as etree
    import configs.application as configs
    
    myfile = configs.application.tmp + '/xml_hug_file.xml'
    
    def fast_iter(context, func, *args, **kwargs):
        for event, elem in context:
            func(elem, *args, **kwargs)
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]
        del context
    
    def process_element(catalog):
        print("List all element of <product>")
    
    context = etree.iterparse(myfile, tag='catalog', events = ('end', ))
    fast_iter(context, process_element)
    


Answer: You could use the XPath `'product/*[starts-with(local-name(),"element")]'`:

* * *
    
    
    import lxml.etree as ET
    import io
    
    content = '''\
    <catalog>
       <product>
          <element1>text 1</element1>
          <element2>text 2</element2>
          <element3>text ..</element3>
       </product>
    </catalog>'''
    
    def fast_iter(context, func, *args, **kwargs):
        """
        http://www.ibm.com/developerworks/xml/library/x-hiperfparse/
        Author: Liza Daly
        See also http://effbot.org/zone/element-iterparse.htm
        """
        for event, elem in context:
            func(elem, *args, **kwargs)
            # It's safe to call clear() here because no descendants will be
            # accessed
            elem.clear()
            # Also eliminate now-empty references from the root node to elem
            for ancestor in elem.xpath('ancestor-or-self::*'):
                while ancestor.getprevious() is not None:
                    del ancestor.getparent()[0]
        del context
    
    
    def process_element(catalog):
        for elt in catalog.xpath('product/*[starts-with(local-name(),"element")]'):
            print(elt)
    
    context = ET.iterparse(io.BytesIO(content), tag='catalog', events = ('end', ))
    fast_iter(context, process_element)
    

yields

    
    
    <Element element1 at 0xb7449374>
    <Element element2 at 0xb744939c>
    <Element element3 at 0xb74493c4>
    

* * *

By the way, I made an alteration to Liz Daly's fast_iter, which will delete
more elements as they become unused. This should reduce memory requirements
when parsing large XML files.

Here is a example which shows how the modified `fast_iter` above removes more
elements than the original `fast_iter`:

    
    
    import logging
    import textwrap
    import lxml.etree as ET
    import io
    
    logger = logging.getLogger(__name__)
    level = logging.INFO
    # level = logging.DEBUG  # uncomment to see more debugging information
    logging.basicConfig(level=level)
    
    def fast_iter(context, func, *args, **kwargs):
        """
        http://www.ibm.com/developerworks/xml/library/x-hiperfparse/
        Author: Liza Daly
        See also http://effbot.org/zone/element-iterparse.htm
        """
        for event, elem in context:
            logger.debug('Processing {e}'.format(e=ET.tostring(elem)))
            func(elem, *args, **kwargs)
            # It's safe to call clear() here because no descendants will be
            # accessed
            logger.debug('Clearing {e}'.format(e=ET.tostring(elem)))
            elem.clear()
            # Also eliminate now-empty references from the root node to elem
            for ancestor in elem.xpath('ancestor-or-self::*'):
                logger.debug('Checking ancestor: {a}'.format(a=ancestor.tag))
                while ancestor.getprevious() is not None:
                    logger.info('Deleting {p}'.format(
                        p=(ancestor.getparent()[0]).tag))
                    del ancestor.getparent()[0]
        del context
    
    def orig_fast_iter(context, func, *args, **kwargs):
        for event, elem in context:
            logger.debug('Processing {e}'.format(e=ET.tostring(elem)))
            func(elem, *args, **kwargs)
            logger.debug('Clearing {e}'.format(e=ET.tostring(elem)))
            elem.clear()
            while elem.getprevious() is not None:
                logger.info('Deleting {p}'.format(
                    p=(elem.getparent()[0]).tag))                
                del elem.getparent()[0]
        del context
    
    def setup_ABC():
        content = textwrap.dedent('''\
          <root>
            <A1>
              <B1></B1>
              <C>1<D1></D1></C>
              <E1></E1>
            </A1>
            <A2>
              <B2></B2>
              <C>2<D></D></C>
              <E2></E2>
            </A2>
          </root>
            ''')
        return content
    
    content = setup_ABC()
    context = ET.iterparse(io.BytesIO(content), events=('end', ), tag='C')
    orig_fast_iter(context, lambda elem: None)
    # DEBUG:__main__:Deleting B1
    # DEBUG:__main__:Deleting B2
    
    print('-'*80)
    """
    The improved fast_iter deletes A1. The original fast_iter does not.
    """
    content = setup_ABC()
    context = ET.iterparse(io.BytesIO(content), events=('end', ), tag='C')
    fast_iter(context, lambda elem: None)
    # DEBUG:__main__:Deleting B1
    # DEBUG:__main__:Deleting A1
    # DEBUG:__main__:Deleting B2
    

Thus you see the modified `fast_iter` manages to delete the `A1` element
because it is not needed by the time the second `C` element is processed. The
original `fast_iter` only deletes parents of `C` elements (i.e. `B` elements).
You could imagine things like `A1` could be quite large in a large XML file,
and there could be many such elements. So the modified `fast_iter` will allow
reclamation of a lot of memory that the original `fast_iter` does not free.

