Code transformation from BS4 to lxml parser
Question: I am working on a project to extract specific information from locally stored
HTML files by using BS4. As I do have a considerably big amount of files (>1
Million) speed and performance is the key for having a useful code browsing
through all files. Until now I am working with BS4 as I was working before on
a web crawler and I thought BS4 is pretty easy and handy. However if it comes
to big data, BS4 is way to slow. I read about the `lxml parser` and
`html.parser` , that seems to be the most easy and fastest parsers among
python for HTML documents.

So my code right now looks like:

    
    
    from bs4 import BeautifulSoup
    import glob
    import os
    import re
    import contextlib
    
    
    @contextlib.contextmanager
    def stdout2file(fname):
        import sys
        f = open(fname, 'w')
        sys.stdout = f
        yield
        sys.stdout = sys.__stdout__
        f.close()
    
    
    def trade_spider():
        os.chdir(r"C:\Users\XXX")
        with stdout2file("output.txt"):
            for file in glob.iglob('**/*.html', recursive=True):
                with open(file, encoding="utf8") as f:
                    contents = f.read()
                    soup = BeautifulSoup(contents, "html.parser")
                    for item in soup.findAll("ix:nonfraction"):
                        if re.match(".*SearchTag", item['name']):
                            print(file.split(os.path.sep)[-1], end="| ")
                            print(item['name'], end="| ")
                            print(item.get_text())
                            break
    trade_spider()
    

It opens a text file, goes into my set directory (os.chdir(..)), seraches
through all files ending .html, reads the content and if it finds tag with
name attribute "SearchTag" it takes the related HTML text and prints it to my
open text file. After one match there is a break and it will continue with the
next one. So what I read is, that BS4 does this all in memory, which increases
porcessing time significantly.

That's why I wanted to alter my code with using either lxml (prefered) or
html.parser.

Anyone of you being a genius and is able to alter my code to use lxml parser
without changing the initial easy idea I had on this?

Any help appreciated on this as I am totally stucked....

UPDATE:

    
    
    import lxml.etree as et
    import os
    import glob
    
    import contextlib
    
    
    @contextlib.contextmanager
    def stdout2file(fname):
        import sys
        f = open(fname, 'w')
        sys.stdout = f
        yield
        sys.stdout = sys.__stdout__
        f.close()
    
    
    def skip_to(fle, line):
            with open(fle) as f:
                pos = 0
                cur_line = f.readline().strip()
                while not cur_line.startswith(line):
                    pos = f.tell()
                    cur_line = f.readline()
                f.seek(pos)
                return et.parse(f)
    
    
    def trade_spider():
        os.chdir(r"F:\04_Independent Auditors Report")
        with stdout2file("auditfeesexpenses.txt"):
            for file in glob.iglob('**/*.html', recursive=True):
                xml = skip_to(file, "<?xml")
                tree = xml.getroot()
                nsmap = {"ix": tree.nsmap["ix"]}
                fractions = xml.xpath("//ix:nonFraction[contains(@name, 'AuditFeesExpenses')]", namespaces=nsmap)
                for fraction in fractions:
                    print(file.split(os.path.sep)[-1], end="| ")
                    print(fraction.get("name"), end="| ")
                    print(fraction.text, end=" \n")
                    break
    
    trade_spider()
    

I get this error message:

    
    
    Traceback (most recent call last):
      File "C:/Users/6930p/PycharmProjects/untitled/Versuch/lxmlparser.py", line 43, in <module>
        trade_spider()
      File "C:/Users/6930p/PycharmProjects/untitled/Versuch/lxmlparser.py", line 33, in trade_spider
        xml = skip_to(file, "<?xml")
      File "C:/Users/6930p/PycharmProjects/untitled/Versuch/lxmlparser.py", line 26, in skip_to
        return et.parse(f)
      File "lxml.etree.pyx", line 3427, in lxml.etree.parse (src\lxml\lxml.etree.c:79720)
      File "parser.pxi", line 1803, in lxml.etree._parseDocument (src\lxml\lxml.etree.c:116182)
      File "parser.pxi", line 1823, in lxml.etree._parseFilelikeDocument (src\lxml\lxml.etree.c:116474)
      File "parser.pxi", line 1718, in lxml.etree._parseDocFromFilelike (src\lxml\lxml.etree.c:115235)
      File "parser.pxi", line 1139, in lxml.etree._BaseParser._parseDocFromFilelike (src\lxml\lxml.etree.c:110109)
      File "parser.pxi", line 573, in lxml.etree._ParserContext._handleParseResultDoc (src\lxml\lxml.etree.c:103323)
      File "parser.pxi", line 679, in lxml.etree._handleParseResult (src\lxml\lxml.etree.c:104936)
      File "lxml.etree.pyx", line 324, in lxml.etree._ExceptionContext._raise_if_stored (src\lxml\lxml.etree.c:10656)
      File "parser.pxi", line 362, in lxml.etree._FileReaderContext.copyToBuffer (src\lxml\lxml.etree.c:100828)
      File "C:\Users\6930p\Anaconda3\lib\encodings\cp1252.py", line 23, in decode
        return codecs.charmap_decode(input,self.errors,decoding_table)[0]
    UnicodeDecodeError: 'charmap' codec can't decode byte 0x8f in position 1789: character maps to <undefined>
    


Answer: There is a bit of work to tidy up the html as per your html file in
[pastebin](http://pastebin.com/rsyVhThY), the following finds `nonFraction`
tags with name attributes containing `'AuditFeesExpenses'`:

    
    
    import lxml.etree as et
    
    def skip_to(fle, line):
        with open(fle) as f:
            pos = 0
            cur_line = f.readline().strip()
            while not cur_line.startswith(line):
                pos = f.tell()
                cur_line = f.readline()
            f.seek(pos)
            return et.parse(f)
    
    xml = skip_to("/home/padraic/Downloads/sample_html_file.html","<?xml")
    tree = xml.getroot()
    # one mapping is None ->  None: 'http://www.w3.org/1999/xhtml'
    nsmap = {k: v for k, v in tree.nsmap.items() if k}
    
    print(xml.xpath("//ix:nonFraction[contains(@name, 'AuditFeesExpenses')]", namespaces=nsmap))
    

Output:

    
    
    [<Element {http://www.xbrl.org/2008/inlineXBRL}nonFraction at 0x7f5b9e91c560>, <Element {http://www.xbrl.org/2008/inlineXBRL}nonFraction at 0x7f5b9e91c5a8>]
    

To pull the text and name:

    
    
    fractions = xml.xpath("//ix:nonFraction[contains(@name, 'AuditFeesExpenses')]", namespaces=nsmap)
    
    for fraction in fractions:
        print(fraction.get("name"))
        print(fraction.text)
    

Which will give you:

    
    
    ns19:AuditFeesExpenses
    1,850
    ns19:AuditFeesExpenses
    2,400
    

Also if you are just using the ix namespace you can just pull that

    
    
    tree = xml.getroot()
    nsmap = {"ix":tree.nsmap["ix"]}
    
    fractions = xml.xpath("//ix:nonFraction[contains(@name, 'AuditFeesExpenses')]", namespaces=nsmap)
    
    for fraction in fractions:
        print(fraction.get("name"))
        print(fraction.text)
    

So the full woking code:

    
    
    def trade_spider():
        os.chdir(r"C:\Users\Independent Auditors Report")
        with stdout2file("auditfeesexpenses.txt"):
            for file in glob.iglob('**/*.html', recursive=True):
                xml = skip_to(file, "<?xml")
                tree = xml.getroot()
                nsmap = {"ix": tree.nsmap["ix"]}
                fractions = xml.xpath("//ix:nonFraction[contains(@name, 'AuditFeesExpenses')]", namespaces=nsmap)
                for fraction in fractions:
                    print(file.split(os.path.sep)[-1], end="| ")
                    print(fraction.get("name"), end="| ")
                    print(fraction.text, end="|")
    

In place of _os.chdir_ you can also:

    
    
    for file in glob.iglob('C:/Users/Independent Auditors Report/**/*.html', recursive=True):
    

