text: #!/usr/bin/env python
  
  import logging
  import re
  import sys
  
  from typing import Optional, Union, cast, Literal, TypedDict
  
  from lxml.etree import _ElementTree, tounicode
  from lxml.html import HtmlElement, document_fromstring, fragment_fromstring
  
  from .cleaners import clean_attributes, html_cleaner
  from .debug import describe, text_content
  from .htmls import build_doc, get_author, get_body, get_title, shorten_title
  from lxml.etree import tostring
  
  
  log = logging.getLogger("readability.readability")
  
  REGEXES = {
      "unlikelyCandidatesRe": re.compile(
          r"combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter",
          re.I,
      ),
      "okMaybeItsACandidateRe": re.compile(r"and|article|body|column|main|shadow", re.I),
      "positiveRe": re.compile(
          r"article|body|content|entry|hentry|main|page|pagination|post|text|blog|story",
          re.I,
      ),
      "negativeRe": re.compile(
          r"combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget",
          re.I,
      ),
      "divToPElementsRe": re.compile(
          r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", re.I
      ),
      #'replaceBrsRe': re.compile(r'(<br[^>]*>[ \n\r\t]*){2,}',re.I),
      #'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I),
      #'trimRe': re.compile(r'^\s+|\s+$/'),
      #'normalizeRe': re.compile(r'\s{2,}/'),
      #'killBreaksRe': re.compile(r'(<br\s*\/?>(\s|&nbsp;?)*){1,}/'),
      "videoRe": re.compile(r"https?:\/\/(www\.)?(youtube|vimeo)\.com", re.I),
      # skipFootnoteLink:      /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
  }
  
  
  class Unparseable(ValueError):
      pass
  
  
  def tostring_(s):
      return tostring(s, encoding="utf-8")
  
  
  class ScoreInfo(TypedDict):
      content_score: float
      elem: HtmlElement
  
  
  Candidates = dict[HtmlElement, ScoreInfo]
  
  
  def count_commas(s: str) -> int:
      """
      count commas:
        - english comma [,](\u002c)
        - Chinese full-width comma [，](\uff0c)
        - Chinese seperator / Japanese comma [、](\u3001)
      """
      count = 0
      for c in s:
          if c in "\u002c\uff0c\u3001":
              count += 1
      return count
  
  
  def to_int(x: Optional[str]) -> int:
      if not x:
          return 0
      x = x.strip()
      if x.endswith("px"):
          return int(x[:-2])
      if x.endswith("em"):
          return int(x[:-2]) * 12
      return int(x)
  
  
  def clean_spaces(text: str) -> str:
      # Many spaces make the following regexes run forever
      text = re.sub(r"\s{255,}", " " * 255, text)
      text = re.sub(r"\s*\n\s*", "\n", text)
      text = re.sub(r"\t|[ \t]{2,}", " ", text)
      return text.strip()
  
  
  def text_length(i: HtmlElement) -> int:
      return len(clean_spaces(i.text_content() or ""))
  
  
  def compile_pattern(elements: Union[list[str], re.Pattern]) -> Optional[re.Pattern]:
      """
      compile a list of str to `|` sepreated regex pattern
      """
      if not elements:
          return None
      elif isinstance(elements, re.Pattern):
          return elements
      if isinstance(elements, (list, tuple)):
          return re.compile("|".join([re.escape(x.strip()) for x in elements]), re.U)
      else:
          raise Exception("Unknown type for the pattern: {}".format(type(elements)))
  
  
  class Document:
      """Class to build a etree document out of html."""
  
      def __init__(
          self,
          source: str,
          positive_keywords: list[str] = None,
          negative_keywords: list[str] = None,
          url: Optional[str] = None,
          min_text_length: int = 25,
          retry_length: int = 250,
          record_xpath: bool = False,
          handle_failures: Literal["discard", "ignore", None] = "discard",
          # make_links_absolute: bool = False,
      ):
          """Generate the document
  
          :param source: string of the html content.
          :param positive_keywords: regex, list or comma-separated string of patterns in classes and ids
          :param negative_keywords: regex, list or comma-separated string in classes and ids
          :param min_text_length: Tunable. Set to a higher value for more precise detection of longer texts.
          :param retry_length: Tunable. Set to a lower value for better detection of very small texts.
          :param record_xpath: If set to True, adds x="..." attribute to each HTML node,
              containing xpath path pointing to original document path (allows to
              reconstruct selected summary in original document).
          :param handle_failures: Parameter passed to `lxml` for handling failure during exception.
              Support options = ["discard", "ignore", None]
  
          Examples:
              positive_keywords=["news-item", "block"]
              positive_keywords=["news-item, block"]
              positive_keywords=re.compile("news|block")
              negative_keywords=["mysidebar", "related", "ads"]
  
          The Document class is not re-enterable.
          It is designed to create a new Document() for each HTML file to process it.
  
          API methods:
          .title() -- full title
          .short_title() -- cleaned up title
          .content() -- full content
          .summary() -- cleaned up content
          """
          self.source: Union[str, HtmlElement] = source
          self.html: Optional[HtmlElement] = None
          self.encoding: str = ""
          self.positive_keywords = compile_pattern(positive_keywords)
          self.negative_keywords = compile_pattern(negative_keywords)
          self.url: Optional[str] = url
          self.min_text_length = min_text_length
          self.retry_length = retry_length
          self.record_xpath = record_xpath
          self.handle_failures = handle_failures
          # self.make_links_absolute = make_links_absolute
  
      def _html(self, force: bool = False) -> HtmlElement:
          """Returns the lxml document node from self.html.
  
          :param force: forcefully rebuild html tree
          """
          if force or self.html is None:
              self.html = self._parse(self.source)
              if self.record_xpath:
                  root = self.html.getroottree()
                  for i in self.html.getiterator():
                      i.attrib["x"] = root.getpath(i)
          return self.html
  
      def _parse(self, s: Union[str, HtmlElement]) -> HtmlElement:
          """Parse string to a lxml HtmlElement object and cleanup it.
          if s is already a HtmlElement, won't build tree twice.
  
          :param s: the string to parse
          """
          if isinstance(s, (_ElementTree, HtmlElement)):
              doc: HtmlElement = cast(HtmlElement, s)
              self.encoding = "utf-8"
          else:
              doc, self.encoding = build_doc(s)
          doc = cast(HtmlElement, html_cleaner.clean_html(doc))
          doc.resolve_base_href(handle_failures=self.handle_failures)
          return doc
  
      def content(self) -> str:
          """Returns document body"""
          return get_body(self._html(force=True))
  
      def title(self) -> str:
          """Returns document title"""
          return get_title(self._html(force=True))
  
      def author(self) -> str:
          """Returns document author"""
          return get_author(self._html(force=True))
  
      def short_title(self) -> str:
          """Returns cleaned up document title"""
          return shorten_title(self._html(force=True))
  
      def get_clean_html(self) -> str:
          """
          An internal method, which can be overridden in subclasses, for example,
          to disable or to improve DOM-to-text conversion in .summary() method
          """
          return clean_attributes(tounicode(self.html, method="html"))
  
      def summary(self, html_partial: bool = False):
          """
          Given a HTML file, extracts the text of the article.
  
          :param html_partial: return only the div of the document, don't wrap
                               in html and body tags.
  
          Warning: It mutates internal DOM representation of the HTML document,
          so it is better to call other API methods before this one.
          """
          try:
              ruthless = True
              while True:
                  self._html(force=True)
                  html = cast(HtmlElement, self.html)
                  for i in self.tags(html, "script", "style"):
                      i.drop_tree()
                  for i in self.tags(html, "body"):
                      i.set("id", "readability-body")
                  if ruthless:
                      self.remove_unlikely_candidates()
                  self.transform_misused_divs_into_paragraphs()
                  candidates = self.score_paragraphs()
  
                  best_candidate = self.select_best_candidate(candidates)
  
                  if best_candidate:
                      article = self.complement_article_with_siblings(
                          candidates, best_candidate, html_partial=html_partial
                      )
                  else:
                      if ruthless:
                          log.info("ruthless removal did not work. ")
                          ruthless = False
                          log.debug(
                              "ended up stripping too much - going for a safer _parse"
                          )
                          # try again
                          continue
                      else:
                          log.debug(
                              "Ruthless and lenient parsing did not work. Returning raw html"
                          )
                          article = html.find("body")
                          if article is None:
                              article = self.html
  
                  # trying to guard against bad links like <a href="http://[http://...">
                  # such support is added in lxml 3.3.0
                  if self.url:
                      cast(HtmlElement, article).make_links_absolute(
                          self.url,
                          resolve_base_href=True,
                          handle_failures=self.handle_failures,
                      )
  
                  cleaned_article = self.sanitize(cast(HtmlElement, article), candidates)
  
                  article_length = len(cleaned_article or "")
                  retry_length = self.retry_length
                  of_acceptable_length = article_length >= retry_length
                  if ruthless and not of_acceptable_length:
                      ruthless = False
                      # Loop through and try again.
                      continue
                  else:
                      return cleaned_article
          except Exception as e:
              log.exception("error getting summary: ")
              # raise_with_traceback(Unparseable, sys.exc_info()[2], str(e))
              raise Unparseable(str(e)).with_traceback(sys.exc_info()[2])
  
      def complement_article_with_siblings(
          self,
          candidates: Candidates,
          best_candidate: ScoreInfo,
          html_partial: bool = False,
      ):
          # Now that we have the top candidate, look through its siblings for
          # content that might also be related.
          # Things like preambles, content split by ads that we removed, etc.
          sibling_score_threshold = max([10, best_candidate["content_score"] * 0.2])
          # create a new html document with a html->body->div
          if html_partial:
              output = fragment_fromstring("<div/>")
              div = output
          else:
              output = document_fromstring("<div/>")
              # We don't want to append directly to output, but the div in html->body->div
              div = output.getchildren()[0].getchildren()[0]
          best_elem = best_candidate["elem"]
          parent = best_elem.getparent()
          siblings = parent.getchildren() if parent is not None else [best_elem]
          for sibling in siblings:
              # in lxml there no concept of simple text
              # if isinstance(sibling, NavigableString): continue
              append = False
              if sibling is best_elem:
                  append = True
              sibling_key = sibling  # HashableElement(sibling)
              if (
                  sibling_key in candidates
                  and candidates[sibling_key]["content_score"] >= sibling_score_threshold
              ):
                  append = True
  
              if sibling.tag == "p":
                  link_density = self.get_link_density(sibling)
                  node_content = sibling.text or ""
                  node_length = len(node_content)
  
                  if node_length > 80 and link_density < 0.25:
                      append = True
                  elif (
                      node_length <= 80
                      and link_density == 0
                      and re.search(r"\.( |$)", node_content)
                  ):
                      append = True
  
              if append:
                  div.append(sibling)
          # if output is not None:
          #    output.append(best_elem)
          return output
  
      def select_best_candidate(self, candidates: Candidates) -> Optional[ScoreInfo]:
          if not candidates:
              return None
  
          sorted_candidates = sorted(
              candidates.values(), key=lambda x: x["content_score"], reverse=True
          )
          for candidate in sorted_candidates[:5]:
              elem = candidate["elem"]
              log.debug("Top 5 : %6.3f %s", candidate["content_score"], describe(elem))
  
          best_candidate = sorted_candidates[0]
          return best_candidate
  
      def get_link_density(self, elem: HtmlElement) -> float:
          link_length = 0
          for i in self.tags(elem, "a"):
              link_length += text_length(i)
          # if len(elem.findall(".//div") or elem.findall(".//p")):
          #    link_length = link_length
          total_length = text_length(elem)
          return float(link_length) / max(total_length, 1)
  
      def score_paragraphs(self):
          """
          Add scores to paragraphs. Iterate over all good tags and computer score for their
          parents.
  
          The formula is:
  
          text_score = min(sentence_count / 100, 3)
          parent_score = node_score + sum([text_score1, text_score2, ...])
          grand_parent = parent / 2
  
          """
          MIN_LEN = self.min_text_length
          candidates: Candidates = {}  # Python dicts are ordered since 3.7
          for elem in self.tags(self._html(), "p", "pre", "td"):
              parent_node = elem.getparent()
              if parent_node is None:
                  continue
              grand_parent_node = parent_node.getparent()
  
              inner_text = clean_spaces(elem.text_content() or "")
              inner_text_len = len(inner_text)
  
              # If this paragraph is too short(default 25), don't even count it.
              if inner_text_len < MIN_LEN:
                  continue
  
              if parent_node not in candidates:
                  candidates[parent_node] = self.score_node(parent_node)
  
              if grand_parent_node is not None and grand_parent_node not in candidates:
                  candidates[grand_parent_node] = self.score_node(grand_parent_node)
  
              content_score = 1
              content_score += count_commas(inner_text) + 1
              content_score += min((inner_text_len / 100), 3)
              # It seems to be useless to consider current node itself.
              # NOTE: This section is NOT found in the original js implementation
              # if elem not in candidates:
              #     candidates[elem] = self.score_node(elem)
              #     candidates[elem]["content_score"] += content_score
              # End of useless
              candidates[parent_node]["content_score"] += content_score
              if grand_parent_node is not None:
                  candidates[grand_parent_node]["content_score"] += content_score / 2.0
  
          # Scale the final candidates score based on link density. Good content
          # should have a relatively small link density (5% or less) and be
          # mostly unaffected by this operation.
          for elem, candidate in candidates.items():
              ld = self.get_link_density(elem)
              score = candidate["content_score"]
              new_score = score * (1 - ld)
              candidate["content_score"] = new_score
              log.debug(
                  "Scaled %6.3f->%6.3f(link:%.3f) for %s",
                  score,
                  new_score,
                  ld,
                  describe(elem),
              )
  
          return candidates
  
      def class_weight(self, node: HtmlElement) -> int:
          """
          Compute node score by class and id keywords, the score weight is 25
          """
          weight = 0
          for feature in [node.get("class", None), node.get("id", None)]:
              if feature:
                  if REGEXES["negativeRe"].search(feature):
                      weight -= 25
  
                  if REGEXES["positiveRe"].search(feature):
                      weight += 25
  
                  if self.positive_keywords and self.positive_keywords.search(feature):
                      weight += 25
  
                  if self.negative_keywords and self.negative_keywords.search(feature):
                      weight -= 25
  
          if self.positive_keywords and self.positive_keywords.match("tag-" + node.tag):
              weight += 25
  
          if self.negative_keywords and self.negative_keywords.match("tag-" + node.tag):
              weight -= 25
  
          return weight
  
      def score_node(self, elem: HtmlElement) -> ScoreInfo:
          """
          score node by their class_weights and tag names.
          """
          content_score = self.class_weight(elem)
          name = elem.tag.lower()
          if name in ("div", "article"):
              content_score += 5
          elif name in ("pre", "td", "blockquote"):
              content_score += 3
          elif name in ("address", "ol", "ul", "dl", "dd", "dt", "li", "form", "aside"):
              content_score -= 3
          elif name in (
              "h1",
              "h2",
              "h3",
              "h4",
              "h5",
              "h6",
              "th",
              "header",
              "footer",
              "nav",
          ):
              content_score -= 5
          return {"content_score": content_score, "elem": elem}
  
      def remove_unlikely_candidates(self) -> None:
          """
          Iterate over the entire dom tree and remove elements whose id or classes match
          unlikelyCandidatesRe pattern.
          """
          html = cast(HtmlElement, self.html)
          for elem in self.tags(html, "*"):
              s = "%s %s" % (elem.get("class", ""), elem.get("id", ""))
              if len(s) < 2:
                  continue
              if (
                  REGEXES["unlikelyCandidatesRe"].search(s)
                  and (not REGEXES["okMaybeItsACandidateRe"].search(s))
                  and elem.tag not in ("html", "body")
              ):
                  log.debug("Removing unlikely candidate - %s", describe(elem))
                  elem.drop_tree()
  
      def transform_misused_divs_into_paragraphs(self):
          html = cast(HtmlElement, self.html)
          for elem in self.tags(html, "div"):
              # transform <div/>s that do not contain other block elements into <p/>s
              # FIXME: The current implementation ignores all descendants that
              # are not direct children of elem
              # This results in incorrect results in case there is an <img/>
              # buried within an <a/> for example
              if not REGEXES["divToPElementsRe"].search(
                  str(b"".join(map(tostring_, list(elem))))
              ):
                  log.debug("Altering %s to p", describe(elem))
                  elem.tag = "p"
                  log.debug("Fixed element %s", describe(elem))
  
          for elem in self.tags(html, "div"):
              if elem.text and elem.text.strip():
                  p = fragment_fromstring("<p/>")
                  p.text = elem.text
                  elem.text = None
                  elem.insert(0, p)
                  log.debug("Appended %s to %s", tounicode(p), describe(elem))
  
              for pos, child in reversed(list(enumerate(elem))):
                  if child.tail and child.tail.strip():
                      p = fragment_fromstring("<p/>")
                      p.text = child.tail
                      child.tail = None
                      elem.insert(pos + 1, p)
                      # print "Inserted "+tounicode(p)+" to "+describe(elem)
                  if child.tag == "br":
                      # print 'Dropped <br> at '+describe(elem)
                      child.drop_tree()
  
      def tags(self, node: HtmlElement, *tag_names: str, reverse=False):
          for tag_name in tag_names:
              iterator = node.findall(".//%s" % tag_name)
              if reverse:
                  iterator = reversed(iterator)
              # FIXME list -> iterator -> list, which is meaningless
              for e in iterator:
                  yield e
  
      def sanitize(self, node: HtmlElement, candidates: Candidates):
          """
          Drop some non-article elements inside the extracted article
          """
          MIN_LEN = self.min_text_length
  
          # drop bad headers
          for header in self.tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
              if self.class_weight(header) < 0 or self.get_link_density(header) > 0.33:
                  header.drop_tree()
  
          # drop forms
          for elem in self.tags(node, "form", "textarea"):
              elem.drop_tree()
  
          # drop iframes
          for elem in self.tags(node, "iframe"):
              if "src" in elem.attrib and REGEXES["videoRe"].search(elem.attrib["src"]):
                  elem.text = "VIDEO"  # ADD content to iframe text node to force <iframe/> proper output
              else:
                  elem.drop_tree()
  
          allowed = set()
          # Conditionally cleanup <table>s, <ul>s, and <div>s
          for el in self.tags(
              node,
              "table",
              "ul",
              "div",
              "aside",
              "header",
              "footer",
              "section",
              reverse=True,
          ):
              if el in allowed:
                  continue
              weight = self.class_weight(el)
              if el in candidates:
                  content_score = candidates[el]["content_score"]
                  # print '!',el, '-> %6.3f' % content_score
              else:
                  content_score = 0
              tag = el.tag
  
              if weight + content_score < 0:
                  log.debug(
                      "Removed %s with score %6.3f and weight %-3s",
                      describe(el),
                      content_score,
                      weight,
                  )
                  el.drop_tree()
              elif count_commas(el.text_content()) < 10:
                  counts = {}
                  for kind in ["p", "img", "li", "a", "embed", "input"]:
                      counts[kind] = len(list(self.tags(el, kind)))
                  counts["li"] -= 100
                  counts["input"] -= len(el.findall('.//input[@type="hidden"]'))
  
                  # Count the text length excluding any surrounding whitespace
                  content_length = text_length(el)
                  link_density = self.get_link_density(el)
                  parent_node = el.getparent()
                  if parent_node is not None:
                      if parent_node in candidates:
                          content_score = candidates[parent_node]["content_score"]
                      else:
                          content_score = 0
                  # if parent_node is not None:
                  # pweight = self.class_weight(parent_node) + content_score
                  # pname = describe(parent_node)
                  # else:
                  # pweight = 0
                  # pname = "no parent"
                  to_remove = False
                  reason = ""
  
                  # if el.tag == 'div' and counts["img"] >= 1:
                  #    continue
                  if counts["p"] and counts["img"] > 1 + counts["p"] * 1.3:
                      reason = "too many images (%s)" % counts["img"]
                      to_remove = True
                  elif counts["li"] > counts["p"] and tag not in ("ol", "ul"):
                      reason = "more <li>s than <p>s"
                      to_remove = True
                  elif counts["input"] > (counts["p"] / 3):
                      reason = "less than 3x <p>s than <input>s"
                      to_remove = True
                  elif content_length < MIN_LEN and counts["img"] == 0:
                      reason = (
                          "too short content length %s without a single image"
                          % content_length
                      )
                      to_remove = True
                  elif content_length < MIN_LEN and counts["img"] > 2:
                      reason = (
                          "too short content length %s and too many images"
                          % content_length
                      )
                      to_remove = True
                  elif weight < 25 and link_density > 0.2:
                      reason = "too many links %.3f for its weight %s" % (
                          link_density,
                          weight,
                      )
                      to_remove = True
                  elif weight >= 25 and link_density > 0.5:
                      reason = "too many links %.3f for its weight %s" % (
                          link_density,
                          weight,
                      )
                      to_remove = True
                  elif (counts["embed"] == 1 and content_length < 75) or counts[
                      "embed"
                  ] > 1:
                      reason = (
                          "<embed/>s with too short content length, or too many <embed/>s"
                      )
                      to_remove = True
                  elif not content_length:
                      reason = "no content"
                      to_remove = True
                      # if el.tag == 'div' and counts['img'] >= 1 and to_remove:
                      #     imgs = el.findall('.//img')
                      #     valid_img = False
                      #     log.debug(tounicode(el))
                      #     for img in imgs:
                      #         height = img.get('height')
                      #         text_length = img.get('text_length')
                      #         log.debug ("height %s text_length %s" %(repr(height), repr(text_length)))
                      #         if to_int(height) >= 100 or to_int(text_length) >= 100:
                      #             valid_img = True
                      #             log.debug("valid image" + tounicode(img))
                      #             break
                      #     if valid_img:
                      #         to_remove = False
                      #         log.debug("Allowing %s" %el.text_content())
                      #         for desnode in self.tags(el, "table", "ul", "div"):
                      #             allowed.add(desnode)
  
                      # find x(1) non empty preceding and succeeding siblings
                      siblings = 0
                      for sib in el.itersiblings():
                          # log.debug("%s", sib.text_content())
                          sib_content_length = text_length(sib)
                          if sib_content_length:
                              siblings += sib_content_length
                              break
                      for sib in el.itersiblings(preceding=True):
                          # log.debug("%s", sib.text_content())
                          sib_content_length = text_length(sib)
                          if sib_content_length:
                              siblings += sib_content_length
                              break
                      if siblings > 1000:
                          to_remove = False
                          log.debug(
                              "Allowing %s, siblings len %s", describe(el), siblings
                          )
                          for desnode in self.tags(el, "table", "ul", "div", "section"):
                              allowed.add(desnode)
  
                  if to_remove:
                      log.debug(
                          "Removed %6.3f %s with weight %s cause it has %s.",
                          content_score,
                          describe(el),
                          weight,
                          reason,
                      )
                      # log.debug("pname %s pweight %.3f" %(pname, pweight))
                      el.drop_tree()
                  else:
                      log.debug(
                          "Not removing %s of length %s: %s",
                          describe(el),
                          content_length,
                          text_content(el),
                      )
  
          self.html = node
          return self.get_clean_html()
  
  
  def main():
      VERBOSITY = {1: logging.WARNING, 2: logging.INFO, 3: logging.DEBUG}
  
      from argparse import ArgumentParser
  
      parser = ArgumentParser()
      parser.add_argument("-v", "--verbose", action="count", default=0)
      parser.add_argument("-f", "--file", default=None, help="file to parse")
      parser.add_argument(
          "-b", "--browser", default=None, action="store_true", help="open in browser"
      )
      parser.add_argument(
          "-l", "--log", default=None, help="save logs into file (appended)"
      )
      parser.add_argument(
          "-u", "--url", default=None, help="use URL instead of a local file"
      )
      parser.add_argument("-x", "--xpath", default=None, help="add original xpath")
      parser.add_argument(
          "-p",
          "--positive-keywords",
          default=None,
          help="positive keywords (comma-separated)",
          action="store",
      )
      parser.add_argument(
          "-n",
          "--negative-keywords",
          default=None,
          help="negative keywords (comma-separated)",
          action="store",
      )
      args = parser.parse_args()
  
      if args.verbose:
          logging.basicConfig(
              level=VERBOSITY[args.verbose],
              filename=args.log,
              format="%(asctime)s: %(levelname)s: %(message)s (at %(filename)s: %(lineno)d)",
          )
  
      if not args.file and not args.url:
          parser.print_help()
          sys.exit(1)
  
      file = None
      if args.url:
          headers = {"User-Agent": "Mozilla/5.0"}
          import urllib.request
  
          request = urllib.request.Request(args.url, None, headers)
          file = urllib.request.urlopen(request)
      else:
          file = open(args[0], "rt")
      try:
          doc = Document(
              file.read(),
              url=args.url,
              positive_keywords=args.positive_keywords,
              negative_keywords=args.negative_keywords,
          )
          if args.browser:
              from .browser import open_in_browser
  
              result = "<h2>" + doc.short_title() + "</h2><br/>" + doc.summary()
              open_in_browser(result)
          else:
              result = "Title:" + doc.short_title() + "\n" + doc.summary()
              print(result)
      finally:
          file.close()
  
  
  if __name__ == "__main__":
      main()
  