dockers/scraper/schedular.py
def save(e, file):
    '''
    Parse the data scraped and insert it MYSQL
    '''

    # Since pickles don't work, i needed to convert it to json.txt
    with open('streetlocations.txt') as json_file:
        streetdict = json.load(json_file)

    # Connect to database
    conn = connect()
    mycursor = conn.cursor()

    # This is the weird part, because it first loaded files in the beta version
    # convert file name to dates
    now = file[4:14].split('-')
    now = [int(i) for i in now]
    scrape_date = file[4:14]

    new = 0
    for single in e:
        try:
            # Get url
            single, url = single.split(';')

            # Split the rest
            l = single.split('-')
            if 'Kamer' in l[2]:
                l.pop(0)

            # somethings numbers join the text ~ stupid kamernet
            loc = l[0].strip()
            if loc[1].isdigit():
                loc = loc[2:]

            # Extracted all the data
            price = (int(''.join([i for i in l[1] if i.isdigit()])))
            incl = (('incl' in l[2]))

            ind = (l[2].index('m2'))
            size = (int(l[2][ind-3:ind-1].strip()))

            if not 'Onbepaalde tijd' in single:
                onbep = 0 ##
            else:
                onbep = -1

            then = '20' + l[4][1:3] +'-' + l[3] + "-" +l[2][-2:]
            street = loc[:-9]+ " Amsterdam Netherlands"

            # Machine learning street dictonairy here! :P
            try:
                lat = streetdict[street][0]
                lon = streetdict[street][1]
            except:
                # Self learning dictonairy
                lat, lon = latlon_by_term(street)
                streetdict[street] = (lat, lon)

            # Insert in my sql
            call = make_insert(mycursor,loc,
                            scrape_date+' '+file[15:-2].replace('_', ':'),
                            then,
                            size,
                            url,
                            price,
                            lat,
                            lon,
                            onbep)
            new += 1

        except Exception as x:
            print(x)

    # Save count and send text
    save_log(mycursor, new)
    send_text(mycursor, new)

    # Commit queue
    conn.commit()

    Parse the data scraped and insert it MYSQL
    
Parse the data scraped and insert it MYSQL
Parse the data scraped and insert it MYSQL
def save(e, file):
    with open('streetlocations.txt') as json_file:
        streetdict = json.load(json_file)
    conn = connect()
    mycursor = conn.cursor()
    now = file[4:14].split('-')
    now = [int(i) for i in now]
    scrape_date = file[4:14]
    new = 0
    for single in e:
        try:
            single, url = single.split(';')
            l = single.split('-')
            if 'Kamer' in l[2]:
                l.pop(0)
            loc = l[0].strip()
            if loc[1].isdigit():
                loc = loc[2:]
            price = (int(''.join([i for i in l[1] if i.isdigit()])))
            incl = (('incl' in l[2]))
            ind = (l[2].index('m2'))
            size = (int(l[2][ind-3:ind-1].strip()))
            if not 'Onbepaalde tijd' in single:
                onbep = 0 
            else:
                onbep = -1
            then = '20' + l[4][1:3] +'-' + l[3] + "-" +l[2][-2:]
            street = loc[:-9]+ " Amsterdam Netherlands"
            try:
                lat = streetdict[street][0]
                lon = streetdict[street][1]
            except:
                lat, lon = latlon_by_term(street)
                streetdict[street] = (lat, lon)
            call = make_insert(mycursor,loc,
                            scrape_date+' '+file[15:-2].replace('_', ':'),
                            then,
                            size,
                            url,
                            price,
                            lat,
                            lon,
                            onbep)
            new += 1
        except Exception as x:
            print(x)
    save_log(mycursor, new)
    send_text(mycursor, new)
    conn.commit()
def save ( e , file ) : with open ( 'streetlocations.txt' ) as json_file : streetdict = json . load ( json_file ) conn = connect ( ) mycursor = conn . cursor ( ) now = file [ 4 : 14 ] . split ( '-' ) now = [ int ( i ) for i in now ] scrape_date = file [ 4 : 14 ] new = 0 for single in e : try : single , url = single . split ( ';' ) l = single . split ( '-' ) if 'Kamer' in l [ 2 ] : l . pop ( 0 ) loc = l [ 0 ] . strip ( ) if loc [ 1 ] . isdigit ( ) : loc = loc [ 2 : ] price = ( int ( '' . join ( [ i for i in l [ 1 ] if i . isdigit ( ) ] ) ) ) incl = ( ( 'incl' in l [ 2 ] ) ) ind = ( l [ 2 ] . index ( 'm2' ) ) size = ( int ( l [ 2 ] [ ind - 3 : ind - 1 ] . strip ( ) ) ) if not 'Onbepaalde tijd' in single : onbep = 0 else : onbep = - 1 then = '20' + l [ 4 ] [ 1 : 3 ] + '-' + l [ 3 ] + "-" + l [ 2 ] [ - 2 : ] street = loc [ : - 9 ] + " Amsterdam Netherlands" try : lat = streetdict [ street ] [ 0 ] lon = streetdict [ street ] [ 1 ] except : lat , lon = latlon_by_term ( street ) streetdict [ street ] = ( lat , lon ) call = make_insert ( mycursor , loc , scrape_date + ' ' + file [ 15 : - 2 ] . replace ( '_' , ':' ) , then , size , url , price , lat , lon , onbep ) new += 1 except Exception as x : print ( x ) save_log ( mycursor , new ) send_text ( mycursor , new ) conn . commit ( )
Parse the data scraped and insert it MYSQL
Parse the data scraped and insert it MYSQL
'''
    Parse the data scraped and insert it MYSQL
    ''' # Since pickles don't work, i needed to convert it to json.txt # Connect to database # This is the weird part, because it first loaded files in the beta version # convert file name to dates # Get url # Split the rest # somethings numbers join the text ~ stupid kamernet # Extracted all the data ## # Machine learning street dictonairy here! :P # Self learning dictonairy # Insert in my sql # Save count and send text # Commit queue
e
file