text: # -*- coding: utf-8 -*-
  from bs4 import BeautifulSoup
  import settings
  import model
  import misc
  import time
  import datetime
  import urllib.parse
  import logging
  
  logging.basicConfig(
      format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)
  BASE_URL = u"http://%s.lianjia.com/" % (settings.CITY)
  
  
  def get_house_percommunity(communityname):
      url = BASE_URL + u"ershoufang/rs" + \
          urllib.parse.quote(communityname.encode('utf8')) + "/"
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
  
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
  
      if total_pages == None:
          row = model.Houseinfo.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + \
                  u"ershoufang/d%drs%s/" % (page,
                                            urllib.parse.quote(communityname.encode('utf8')))
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
  
          nameList = soup.findAll("div", {"class": "info"})
          i = 0
          log_progress("GetHouseByCommunitylist",
                       communityname, page + 1, total_pages)
          data_source = []
          hisprice_data_source = []
          for name in nameList:  # per house loop
              i = i + 1
              info_dict = {}
              try:
                  housetitle = name.find("div", {"class": "prop-title"})
                  info_dict.update({u'title': housetitle.a.get('title')})
                  info_dict.update({u'link': housetitle.a.get('href')})
                  info_dict.update({u'houseID': housetitle.a.get('key')})
  
                  houseaddr = name.find("span", {"class": "info-col row1-text"})
                  info = houseaddr.get_text().split('|')
                  info_dict.update({u'housetype': info[0].strip()})
                  info_dict.update({u'square': info[1].strip()})
                  info_dict.update({u'floor': info[2].strip()})
                  try:
                      info_dict.update({u'direction': info[3].strip()})
                  except:
                      info_dict.update({u'direction': ''})
                  info_dict.update({u'decoration': ''})
  
                  housefloor = name.find("span", {"class": "info-col row2-text"})
                  detail = housefloor.get_text().split('|')
                  info_dict.update({u'years': detail[-1].strip()})
  
                  community = name.find("a", {"class": "laisuzhou"})
                  info_dict.update(
                      {u'community': community.span.get_text().strip()})
                  info_dict.update({u'followInfo': ''})
  
                  tax = name.find("div", {"class": "property-tag-container"})
                  info_dict.update({u'taxtype': "".join(tax.get_text().split())})
  
                  totalPrice = name.find(
                      "span", {"class": "total-price strong-num"})
                  info_dict.update(
                      {u'totalPrice': totalPrice.get_text().strip()})
  
                  unitPrice = name.find(
                      "span", {"class": "info-col price-item minor"})
                  info_dict.update({u'unitPrice': unitPrice.get_text().strip()})
              except:
                  continue
              # houseinfo insert into mysql
              data_source.append(info_dict)
              hisprice_data_source.append(
                  {"houseID": info_dict["houseID"], "totalPrice": info_dict["totalPrice"]})
              # model.Houseinfo.insert(**info_dict).upsert().execute()
              #model.Hisprice.insert(houseID=info_dict['houseID'], totalPrice=info_dict['totalPrice']).upsert().execute()
  
          with model.database.atomic():
              model.Houseinfo.insert_many(data_source).upsert().execute()
              model.Hisprice.insert_many(hisprice_data_source).upsert().execute()
          time.sleep(1)
  
  
  def get_sell_percommunity(communityname):
      url = BASE_URL + u"chengjiao/rs" + \
          urllib.parse.quote(communityname.encode('utf8')) + "/"
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
  
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
  
      if total_pages == None:
          row = model.Sellinfo.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + \
                  u"chengjiao/d%drs%s/" % (page,
                                           urllib.parse.quote(communityname.encode('utf8')))
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
          i = 0
          log_progress("GetSellByCommunitylist",
                       communityname, page + 1, total_pages)
          data_source = []
          for name in soup.findAll("div", {"class": "info"}):
              i = i + 1
              info_dict = {}
              try:
                  housetitle = name.findAll("div", {"class": "info-row"})[0]
                  info_dict.update({u'title': housetitle.a.get('title')})
                  info_dict.update({u'link': housetitle.a.get('href')})
                  info_dict.update({u'houseID': housetitle.a.get('key')})
                  houseinfo = housetitle.get_text().strip().split(' ')
                  info_dict.update({u'housetype': houseinfo[1].strip()})
                  info_dict.update(
                      {u'square': houseinfo[2].strip('').split('\n')[0]})
  
                  houseaddr = name.find("div", {"class": "row1-text"})
                  info = houseaddr.get_text().split('|')
                  info_dict.update({u'floor': info[0].strip()})
                  try:
                      info_dict.update({u'direction': info[1].strip()})
                  except:
                      info_dict.update({u'direction': ''})
                  info_dict.update({u'status': info[2].strip()})
  
                  years = name.find("span", {"class": "c-prop-tag2"})
                  info_dict.update({u'years': years.get_text().strip()})
  
                  community = name.find("span", {"class": "cj-text"})
                  info_dict.update({u'community': community.get_text().strip()})
  
                  totalPrice = name.find("span", {"class": "strong-num"})
                  info_dict.update(
                      {u'totalPrice': totalPrice.get_text().strip()})
  
                  unitPrice = name.find(
                      "div", {"class": "info-col price-item minor"})
                  info_dict.update({u'unitPrice': unitPrice.get_text().strip()})
  
                  source = name.find(
                      "div", {"class": "info-col deal-item minor"})
                  info_dict.update({u'source': source.get_text().strip()})
  
                  dealdate = name.find(
                      "div", {"class": "info-col deal-item main strong-num"})
                  info_dict.update(
                      {u'dealdate': dealdate.get_text().strip().replace('.', '-')})
  
              except:
                  continue
              # Sellinfo insert into mysql
              data_source.append(info_dict)
              # model.Sellinfo.insert(**info_dict).upsert().execute()
  
          with model.database.atomic():
              model.Sellinfo.insert_many(data_source).upsert().execute()
          time.sleep(1)
  
  
  def get_community_perregion(regionname=u'pudong'):
      url = BASE_URL + u"xiaoqu/" + regionname + "/"
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
  
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
  
      if total_pages == None:
          row = model.Community.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + u"xiaoqu/" + regionname + "/d%d/" % page
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
  
          nameList = soup.findAll("div", {"class": "info-panel"})
          i = 0
          log_progress("GetCommunityByRegionlist",
                       regionname, page + 1, total_pages)
          data_source = []
          for name in nameList:  # Per house loop
              i = i + 1
              info_dict = {}
              try:
                  communitytitle = name.find("a", {"name": "selectDetail"})
                  title = communitytitle.get_text().strip('\n')
                  link = communitytitle.get('href')
                  id = communitytitle.get('key')
                  info_dict.update({u'title': title})
                  info_dict.update({u'link': link})
                  info_dict.update({u'id': id})
  
                  district = name.find("a", {"class": "ad"})
                  info_dict.update({u'district': district.get_text()})
  
                  cons = name.find("div", {"class": "con"})
                  bizcircle = cons.findAll("a")
                  info_dict.update(
                      {u'bizcircle': bizcircle[1].get_text().strip()})
  
                  try:
                      tagList = name.find("span", {"class": "fang-subway-ex"})
                      info_dict.update({u'tagList': tagList.get_text().strip()})
                  except:
                      info_dict.update({u'tagList': ''})
  
                  onsale = name.find("span", {"class": "num"})
                  info_dict.update({u'onsale': onsale.get_text().strip()})
  
                  price = name.find("div", {"class": "price"})
                  info_dict.update({u'price': price.span.get_text().strip()})
  
                  communityinfo = get_communityinfo_by_url(link)
                  for key, value in communityinfo.iteritems():
                      info_dict.update({key: value})
  
              except:
                  continue
              # communityinfo insert into mysql
              data_source.append(info_dict)
              # model.Community.insert(**info_dict).upsert().execute()
  
          with model.database.atomic():
              model.Community.insert_many(data_source).upsert().execute()
          time.sleep(1)
  
  
  def get_rent_percommunity(communityname):
      url = BASE_URL + u"zufang/rs" + \
          urllib.parse.quote(communityname.encode('utf8')) + "/"
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
  
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
  
      if total_pages == None:
          row = model.Rentinfo.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + \
                  u"rent/d%drs%s/" % (page,
                                      urllib.parse.quote(communityname.encode('utf8')))
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
          i = 0
          log_progress("GetRentByCommunitylist",
                       communityname, page + 1, total_pages)
          data_source = []
          nameList = soup.findAll("div", {"class": "info-panel"})
          for name in nameList:
              i = i + 1
              info_dict = {}
              try:
                  info = name.find("a", {"name": "selectDetail"})
                  info_dict.update({u'title': info.get('title')})
                  info_dict.update({u'link': info.get('href')})
                  info_dict.update({u'houseID': info.get('key')})
  
                  where = name.find("div", {"class": "where"})
                  wheres = where.find_all("span")
                  info_dict.update({u'region': wheres[0].get_text().strip()})
                  info_dict.update({u'zone': wheres[1].get_text().strip()})
                  info_dict.update({u'meters': wheres[2].get_text().strip()})
  
                  other = name.find("div", {"class": "con"})
                  info_dict.update({u'other': "".join(other.get_text().split())})
  
                  info_dict.update({u'subway': ""})
                  info_dict.update({u'decoration': ""})
                  info_dict.update({u'heating': ""})
  
                  price = name.find("div", {"class": "price"})
                  info_dict.update(
                      {u'price': int(price.span.get_text().strip())})
  
                  pricepre = name.find("div", {"class": "price-pre"})
                  info_dict.update(
                      {u'pricepre': "".join(pricepre.get_text().split())})
  
              except:
                  continue
              # Rentinfo insert into mysql
              data_source.append(info_dict)
              # model.Rentinfo.insert(**info_dict).upsert().execute()
  
              with model.database.atomic():
                  model.Rentinfo.insert_many(data_source).upsert().execute()
              time.sleep(1)
  
  
  def get_house_perregion(district):
      url = BASE_URL + u"ershoufang/%s/" % district
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
      if total_pages == None:
          row = model.Houseinfo.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + u"ershoufang/%s/d%d/" % (district, page)
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
          i = 0
          log_progress("GetHouseByRegionlist", district, page + 1, total_pages)
          data_source = []
          hisprice_data_source = []
          nameList = soup.findAll("div", {"class": "info"})
          for name in nameList:  # per house loop
              i = i + 1
              info_dict = {}
              try:
                  housetitle = name.find("div", {"class": "prop-title"})
                  info_dict.update({u'title': housetitle.a.get('title')})
                  info_dict.update({u'link': housetitle.a.get('href')})
                  info_dict.update({u'houseID': housetitle.a.get('key')})
  
                  houseaddr = name.find("span", {"class": "info-col row1-text"})
                  info = houseaddr.get_text().split('|')
                  info_dict.update({u'housetype': info[0].strip()})
                  info_dict.update({u'square': info[1].strip()})
                  info_dict.update({u'floor': info[2].strip()})
                  try:
                      info_dict.update({u'direction': info[3].strip()})
                  except:
                      info_dict.update({u'direction': ''})
                  info_dict.update({u'decoration': ''})
  
                  housefloor = name.find("span", {"class": "info-col row2-text"})
                  detail = housefloor.get_text().split('|')
                  info_dict.update({u'years': detail[-1].strip()})
  
                  community = name.find("a", {"class": "laisuzhou"})
                  info_dict.update(
                      {u'community': community.span.get_text().strip()})
                  info_dict.update({u'followInfo': ''})
  
                  tax = name.find("div", {"class": "property-tag-container"})
                  info_dict.update({u'taxtype': "".join(tax.get_text().split())})
  
                  totalPrice = name.find(
                      "span", {"class": "total-price strong-num"})
                  info_dict.update(
                      {u'totalPrice': totalPrice.get_text().strip()})
  
                  unitPrice = name.find(
                      "span", {"class": "info-col price-item minor"})
                  info_dict.update({u'unitPrice': unitPrice.get_text().strip()})
              except:
                  continue
              # houseinfo insert into mysql
              data_source.append(info_dict)
              hisprice_data_source.append(
                  {"houseID": info_dict["houseID"], "totalPrice": info_dict["totalPrice"]})
              # model.Houseinfo.insert(**info_dict).upsert().execute()
              #model.Hisprice.insert(houseID=info_dict['houseID'], totalPrice=info_dict['totalPrice']).upsert().execute()
  
              with model.database.atomic():
                  model.Houseinfo.insert_many(data_source).upsert().execute()
                  model.Hisprice.insert_many(
                      hisprice_data_source).upsert().execute()
              time.sleep(1)
  
  
  def get_rent_perregion(district):
      url = BASE_URL + u"zufang/%s/" % district
      source_code = misc.get_source_code(url)
      soup = BeautifulSoup(source_code, 'lxml')
      if check_block(soup):
          return
      total_pages = misc.get_sh_total_pages(url)
      if total_pages == None:
          row = model.Rentinfo.select().count()
          raise RuntimeError("Finish at %s because total_pages is None" % row)
  
      for page in range(total_pages):
          if page > 0:
              url_page = BASE_URL + u"zufang/%s/d%d/" % (district, page)
              source_code = misc.get_source_code(url_page)
              soup = BeautifulSoup(source_code, 'lxml')
          i = 0
          log_progress("GetRentByRegionlist", district, page + 1, total_pages)
          data_source = []
          nameList = soup.findAll("div", {"class": "info-panel"})
          for name in nameList:
              i = i + 1
              info_dict = {}
              try:
                  info = name.find("a", {"name": "selectDetail"})
                  info_dict.update({u'title': info.get('title')})
                  info_dict.update({u'link': info.get('href')})
                  info_dict.update({u'houseID': info.get('key')})
  
                  where = name.find("div", {"class": "where"})
                  wheres = where.find_all("span")
                  info_dict.update({u'region': wheres[0].get_text().strip()})
                  info_dict.update({u'zone': wheres[1].get_text().strip()})
                  info_dict.update({u'meters': wheres[2].get_text().strip()})
  
                  other = name.find("div", {"class": "con"})
                  info_dict.update({u'other': "".join(other.get_text().split())})
  
                  info_dict.update({u'subway': ""})
                  info_dict.update({u'decoration': ""})
                  info_dict.update({u'heating': ""})
  
                  price = name.find("div", {"class": "price"})
                  info_dict.update(
                      {u'price': int(price.span.get_text().strip())})
  
                  pricepre = name.find("div", {"class": "price-pre"})
                  info_dict.update(
                      {u'pricepre': "".join(pricepre.get_text().split())})
  
              except:
                  continue
              # Rentinfo insert into mysql
              data_source.append(info_dict)
              # model.Rentinfo.insert(**info_dict).upsert().execute()
  
          with model.database.atomic():
              model.Rentinfo.insert_many(data_source).upsert().execute()
          time.sleep(1)
  
  
  def get_communityinfo_by_url(url):
      source_code = misc.get_source_code(BASE_URL + url)
      soup = BeautifulSoup(source_code, 'lxml')
  
      if check_block(soup):
          return
  
      communityinfos = soup.findAll("div", {"class": "col-2 clearfix"})
      res = {}
      for info in communityinfos:
          try:
              infos = info.findAll("li")
              housetype = infos[0].find("span", {"class": "other"})
              year = infos[1].find("span", {"class": "other"})
              cost = infos[2].find("span", {"class": "other"})
              service = infos[3].span.find(text=True, recursive=False)
              company = infos[4].span.find(text=True, recursive=False)
              res.update({'housetype': housetype.get_text().strip()})
              res.update({'year': year.get_text().strip()})
              res.update({'cost': cost.get_text().strip()})
              res.update({'service': service.strip()})
              res.update({'company': company.strip()})
  
          except:
              continue
      return res
  
  
  def check_block(soup):
      if soup.title.string == "414 Request-URI Too Large":
          logging.error(
              "Lianjia block your ip, please verify captcha manually at lianjia.com")
          return True
      return False
  
  
  def log_progress(function, address, page, total):
      logging.info("Progress: %s %s: current page %d total pages %d" %
                   (function, address, page, total))
  