import logging
import subprocess
import xml.etree.ElementTree as ET

from tqdm import tqdm


logger = logging.getLogger('root')

POST_TYPE_QUESTION = '1'
POST_TYPE_ANSWER = '2'


class SEDataReader(object):
    """
    NOTE: - a typical xml string for question in original data looks like
                    <row Id="4" PostTypeId="1" AcceptedAnswerId="7" CreationDate="2008-07-31T21:42:52.667" Score="543" ViewCount="34799" Body="&lt;p&gt;I want to use a track-bar to change a form's opacity.&lt;/p&gt;&#xA;&#xA;&lt;p&gt;This is my code:&lt;/p&gt;&#xA;&#xA;&lt;pre&gt;&lt;code&gt;decimal trans = trackBar1.Value / 5000;&#xA;this.Opacity = trans;&#xA;&lt;/code&gt;&lt;/pre&gt;&#xA;&#xA;&lt;p&gt;When I build the application, it gives the following error:&lt;/p&gt;&#xA;&#xA;&lt;blockquote&gt;&#xA;  &lt;p&gt;Cannot implicitly convert type &lt;code&gt;'decimal'&lt;/code&gt; to &lt;code&gt;'double'&lt;/code&gt;.&lt;/p&gt;&#xA;&lt;/blockquote&gt;&#xA;&#xA;&lt;p&gt;I tried using &lt;code&gt;trans&lt;/code&gt; and &lt;code&gt;double&lt;/code&gt; but then the control doesn't work. This code worked fine in a past VB.NET project.&lt;/p&gt;&#xA;" OwnerUserId="8" LastEditorUserId="3151675" LastEditorDisplayName="Rich B" LastEditDate="2017-09-27T05:52:59.927" LastActivityDate="2018-02-22T16:40:13.577" Title="While applying opacity to a form, should we use a decimal or a double value?" Tags="&lt;c#&gt;&lt;winforms&gt;&lt;type-conversion&gt;&lt;decimal&gt;&lt;opacity&gt;" AnswerCount="13" CommentCount="1" FavoriteCount="39" CommunityOwnedDate="2012-10-31T16:42:47.213" />

    """

    num_skipped_sample_none = 0
    num_skipped_other_posttype = 0
    num_retrieved = 0

    def __init__(self, data_file_path):
        """
        data_file_path : (string) path to the posts.xml file
        """
        self.data_file_path = data_file_path

    def question_data_filter(self, sample):
        return dict((k, sample.get(k, None)) for k in ['Id', 'Title', 'Body', 'Tags', 'ParentId', 'PostTypeId'])

    def n_items_unfiltered(self):
        out = subprocess.check_output(['wc', '-l', self.data_file_path])
        return int(out.split()[0])

    def read_items(self, allowed_post_types=(POST_TYPE_QUESTION, ), min_score=0, max_year=None):
        questions_dict = dict() if POST_TYPE_ANSWER in allowed_post_types else None

        with open(self.data_file_path, 'r') as f:
            for l in tqdm(f):
                try:
                    sample = ET.fromstring(l.strip()).attrib
                except ET.ParseError as e:
                    print('(Ignoring) ERROR in parsing line (QUESTION READER):\n{}\n'.format(l.strip))
                    sample = None
                if sample:
                    if questions_dict is not None and sample['PostTypeId'] == POST_TYPE_QUESTION:
                        filtered_sample = self.question_data_filter(sample)
                        questions_dict[filtered_sample['Id']] = filtered_sample

                    has_min_score = int(sample['Score']) >= min_score
                    has_max_year = True if max_year is None else int(sample['CreationDate'][:4]) <= max_year

                    if sample['PostTypeId'] in allowed_post_types and has_max_year and has_min_score:
                        SEDataReader.num_retrieved += 1
                        filtered_sample = self.question_data_filter(sample)
                        if sample['PostTypeId'] == POST_TYPE_ANSWER:
                            q = questions_dict.get(filtered_sample['ParentId'], None)
                            if q is None:
                                print('Skipping answer because parent question is unknown')
                                continue
                            else:
                                filtered_sample['Title'] = q.get('Title')
                                filtered_sample['Tags'] = q.get('Tags')

                        yield filtered_sample
                    else:
                        SEDataReader.num_skipped_other_posttype += 1

                else:
                    SEDataReader.num_skipped_sample_none += 1

[(8, 'logging.getLogger', 'logging.getLogger', False, 'import logging\n'), (35, 'subprocess.check_output', 'subprocess.check_output', False, 'import subprocess\n'), (42, 'tqdm.tqdm', 'tqdm', False, 'from tqdm import tqdm\n')]