Scrape google apps page and store application details in database Below is a python script which scrapes specific google apps url for example https://play.google.com/store/apps/details?id=com.mojang.minecraftpe and save the developer email,application name and price in mysql database with database name "app_database" and inside table named "app_table". There are lot of data type conversions and workarounds here and i think that might not come under best practices. How can I improve this.

from lxml import html
import requests
import MySQLdb
conn = MySQLdb.connect("localhost","root","pA$$W0rd","app_database")
cursor=conn.cursor()
class AppCrawler:

    def __init__(self,starting_url,depth):
        self.starting_url=starting_url
        self.depth=depth
        self.current_depth=0
        self.depth_links=[]
        self.apps=[]

    def crawl(self):
        app=self.get_app_from_link(self.starting_url)
        self.apps.append(app)
        self.depth_links.append(app.links)

        while self.current_depth &lt; self.depth:      #0 &lt; 0
            current_links=[]
            for link in self.depth_links[self.current_depth]:
                current_app = self.get_app_from_link(link)
                current_links.extend(current_app.links)
                self.apps.append(current_app)
            self.current_depth += 1
            self.depth_links.append(current_links)

        return

    def get_app_from_link(self,link):
        start_page=requests.get(link)
        #print start_page.text
        tree = html.fromstring(start_page.text)
        # the tree.xpath(...) gives an array 
        name = tree.xpath('//div[@class="id-app-title"]/text()')[0]  

        #developer=tree.xpath('//a[@class="dev-link"]//*/div/@href')
        temp_addr=tree.xpath('//a[contains(@href,"mailto") and @class="dev-link"]/@href')[0]
        developer=temp_addr.split(':')

        price=tree.xpath('//button[@class="price buy id-track-click id-track-impression"]/span[2]/text()')
        temp_price=price[0].split()
        price=str(temp_price[0].lstrip('$'))
        if price=="Install":
            price="0"
        templinks = tree.xpath("//a[@class='title']/@href")

        links=[]
        length=len(templinks)
        domain="https://play.google.com"
        for i in templinks:
            i="https://play.google.com"+i
            links.append(i)
#        print links            
        #print len(testlinks)
        #links = [h for h in tree.xpath("//a[@class='title']/@href")]
        #print links
        #return name,developer[1],price,links
        app = App(name,developer[1],price,links)
        return app

class App:
    def __init__(self, name,developer,price,links):
        self.name=name
        self.developer=developer
        self.price = price
        self.links=links

    def __str__(self):
        return(self.name.encode('UTF-8')+"|"+
        self.developer.encode('UTF-8')+"|"+
        self.price.encode('UTF-8'))


crawler = AppCrawler('https://play.google.com/store/apps/details?id=com.mojang.minecraftpe',0)
crawler.crawl()
#print type(crawler.apps)

query="INSERT INTO app_table (name,developer,price) values('%s','%s','%d ')"
for app in crawler.apps:
    data=str(app).split('|',2)
    price=float(data[2])
    try:
        cursor.execute(query % (data[0],data[1],price))
        print "OK"
        conn.commit()
    #row=x.fetcha
    except:
        print "Failed"
        conn.rollback()
    print data[0],data[1],price
This is being ported over to python 3.x from python 2.x


Docstrings: You should include a docstring at the beginning of every method/class/module you write. This will allow documentation to identify what your code is supposed to do. PEP-8 Docstring Conventions.
Variable / Operator Spacing: When not in default parameters, you should put spaces between variables, the =, and the content it's being assigned. Also, there should be spaces between variables that are being passed to methods. PEP-8 Whitespace Conventions.
Import Ordering: Standard library imports should be put first, then related third party imports, then local libraries/files, all grouped and separated by a blank line. PEP-8 Import Conventions.
String Formatting: Your __str__ in your App class is a bit messy. If you see this behavior, something is afoot. This is a red flag to me. I use f"" to directly include variables into my strings, instead of having to type cast them or, in your case, having to use multiple + to concatenate the variables and the strings.
Unnecessary return statements: Your return in your crawl method is unnecessary. Python automatically returns at the end of functions, so having that return isn't needed.


Updated Code

"""
Module Docstring:
A description of your program goes here
"""

import requests
import MySQLdb

from lxml import html

class AppCrawler:
    """
    Class for crawling a link, and gathering information about an app
    """
    def __init__(self, starting_url, depth):
        """
        AppCrawler Class Init Method

        :param starting_url: The starting url for the crawler
        :param depth: How deep the app crawler should go

        """
        self.starting_url = starting_url
        self.depth = depth
        self.current_dept = 0
        self.depth_links = []
        self.apps = []

    def crawl(self):
        """
        Crawls to find links connected to apps
        """
        app = self.get_app_from_link(self.starting_url)
        self.apps.append(app)
        self.depth_links.append(app.links)

        while self.current_depth &lt; self.depth: #0 &lt; 0
            current_links = []
            for link in self.depth_links[self.current_depth]:
                current_app = self.get_app_from_link(link)
                current_links.extend(current_app.links)
                self.apps.append(current_app)
            self.current_depth += 1
            self.depth_links.append(current_links)

    def get_app_from_link(self, link):
        """
        Gets the app from the passed `link`

        :param link: Link to the app

        """
        start_page = requests.get(link)

        tree = html.fromstring(start_page.text)

        name = tree.xpath('//div[@class="id-app-title"]/text()')[0]  

        temp_addr = tree.xpath('//a[contains(@href,"mailto") and @class="dev-link"]/@href')[0]
        developer = temp_addr.split(':')

        price = tree.xpath('//button[@class="price buy id-track-click id-track-impression"]/span[2]/text()')
        temp_price = price[0].split()
        price = str(temp_price[0].lstrip('$'))
        if price == "Install":
            price = "0"
        templinks = tree.xpath("//a[@class='title']/@href")

        links = []
        length = len(templinks)
        domain = "https://play.google.com"
        for i in templinks:
            i = f"https://play.google.com{i}"
            links.append(i)
        app = App(name, developer[1], price, links)
        return app

class App:
    """
    Class to represent each app getting crawled
    """
    def __init__(self, name, developer, price, links):
        """
        App Class Init Method

        :param name: Name of the app
        :param developer: Developer of the app
        :param price: Price of the app
        :param links: list of links to the app

        """
        self.name = name
        self.developer = developer
        self.price = price
        self.links = links

    def __str__(self):
        """
        Returns a string representation of the app
        """
        return f"{self.name.encode('UTF-8')}|{self.developer.encode('UTF-8')}|{self.price.encode('UTF-8')}"

if __name__ == '__main__':

    conn = MySQLdb.connect("localhost","root","pA$$W0rd","app_database")
    cursor = conn.cursor()

    crawler = AppCrawler('https://play.google.com/store/apps/details?id=com.mojang.minecraftpe', 0)
    crawler.crawl()

    query = "INSERT INTO app_table (name,developer,price) values('%s','%s','%d ')"
    for app in crawler.apps:
        data = str(app).split('|', 2)
        price = float(data[2])
        try:
            cursor.execute(query % (data[0], data[1], price))
            print("OK")
            conn.commit()
        except:
            print("Failed")
            conn.rollback()
        print(data[0], data[1], price)