original_data/process_data.py
def process_data():
    """Convert original data to json file
    """
    # Bitcoin price in USD
    price = read_datafile('price.txt')
    # Difficulty index for bitcoin mining
    difficulty = read_datafile('difficulty.txt')
    # Google trend index
    gtread = read_datafile('gtrend.txt')
    # Number of active address 
    sentaddr = read_datafile('sentaddr.txt')
    # Number of transaction per day
    transaction = read_datafile('transaction.txt')
    # Number of tweets per day
    # Tweets file lacks of the data before the date '2014/04/09', 
    # so we manually add missing data from 2010/07/17.
    tweets = add_missing_data(
        start_date='2010/07/17',
        end_date='2014/04/09',
        init_value=300,
        scale_factor=0.002);
    tweets += read_datafile('tweets.txt')

    json_data = {}
    json_data['date'] = []
    json_data['price'] = []
    json_data['growth_60_day'] = []
    json_data['hot'] = []
    json_data['bubble'] = []

    # Get Date, Price, and 60_day_growth
    last_price = 0.0
    accumulate_60 = 0.0
    my_q = queue = collections.deque()
    for data in price:
        key, value = data.split(':')
        json_data['date'].append(key)
        json_data['price'].append('%0.2f' % float(value))
        if last_price == 0.0:
            last_price = float(value)
        if len(my_q) >= 60:
            accumulate_60 -= my_q.popleft()
        day_growth = (float(value) - last_price) / last_price
        accumulate_60 += day_growth
        my_q.append(day_growth)
        last_price = float(value)
        json_data['growth_60_day'].append(int(accumulate_60 * 100))

    # Get keywords Hot value
    for i in range(len(gtread)):
        gtrend_key, gtread_value = gtread[i].split(':')
        assert gtrend_key == json_data['date'][i]
        tweets_key, tweets_value = tweets[i].split(':')
        assert tweets_key == json_data['date'][i]
        gtread_value = 0.0 if gtread_value == 'null' else float(gtread_value)
        tweets_value = 0.0 if tweets_value == 'null' else float(tweets_value)
        hot_value = get_hot_value(gtread_value, tweets_value)
        json_data['hot'].append(int(hot_value))
    while len(json_data['hot']) < len(json_data['price']):
        json_data['hot'].append(json_data['hot'][-1])
    assert len(json_data['hot']) == len(json_data['price'])

    # Get bubble index
    for i in range(len(price)):
        while len(difficulty) < len(price):
            difficulty_value = difficulty[len(difficulty) - 1].split(':')[1]
            difficulty.append(key + ':' + difficulty_value)
        difficulty_key, difficulty_value = difficulty[i].split(':')
        assert difficulty_key == json_data['date'][i]
        while len(sentaddr) < len(price):
            sentaddr_value = difficulty[len(sentaddr) - 1].split(':')[1]
            sentaddr.append(key + ':' + sentaddr_value)
        sentaddr_key, sentaddr_value = sentaddr[i].split(':')
        assert sentaddr_key == json_data['date'][i]
        while len(transaction) < len(price):
            transaction_value = transaction[len(transaction) - 1].split(':')[1]
            transaction.append(key + ':' + transaction_value)
        transaction_key, transaction_value = transaction[i].split(':')
        assert transaction_key == json_data['date'][i]

        difficulty_value = 0.0 if difficulty_value == 'null' else float(difficulty_value)
        sentaddr_value = 0.0 if sentaddr_value == 'null' else float(sentaddr_value)
        transaction_value = 0.0 if transaction_value == 'null' else float(transaction_value)

        bubble_index = get_bubble_index(
            float(json_data['price'][i]),
            float(json_data['growth_60_day'][i]),
            float(json_data['hot'][i]),
            float(difficulty_value),
            float(sentaddr_value),
            float(transaction_value))

        json_data['bubble'].append(int(bubble_index))

    assert len(json_data['bubble']) == len(json_data['price'])

    # Write json data to file
    file = open('../data.json', "w")
    file.write("data = '")
    file.write(json.dumps(json_data))
    file.write("'")
    file.close()
Convert original data to json file
    
Convert original data to json file
Convert original data to json file
def process_data():
    price = read_datafile('price.txt')
    difficulty = read_datafile('difficulty.txt')
    gtread = read_datafile('gtrend.txt')
    sentaddr = read_datafile('sentaddr.txt')
    transaction = read_datafile('transaction.txt')
    tweets = add_missing_data(
        start_date='2010/07/17',
        end_date='2014/04/09',
        init_value=300,
        scale_factor=0.002);
    tweets += read_datafile('tweets.txt')
    json_data = {}
    json_data['date'] = []
    json_data['price'] = []
    json_data['growth_60_day'] = []
    json_data['hot'] = []
    json_data['bubble'] = []
    last_price = 0.0
    accumulate_60 = 0.0
    my_q = queue = collections.deque()
    for data in price:
        key, value = data.split(':')
        json_data['date'].append(key)
        json_data['price'].append('%0.2f' % float(value))
        if last_price == 0.0:
            last_price = float(value)
        if len(my_q) >= 60:
            accumulate_60 -= my_q.popleft()
        day_growth = (float(value) - last_price) / last_price
        accumulate_60 += day_growth
        my_q.append(day_growth)
        last_price = float(value)
        json_data['growth_60_day'].append(int(accumulate_60 * 100))
    for i in range(len(gtread)):
        gtrend_key, gtread_value = gtread[i].split(':')
        assert gtrend_key == json_data['date'][i]
        tweets_key, tweets_value = tweets[i].split(':')
        assert tweets_key == json_data['date'][i]
        gtread_value = 0.0 if gtread_value == 'null' else float(gtread_value)
        tweets_value = 0.0 if tweets_value == 'null' else float(tweets_value)
        hot_value = get_hot_value(gtread_value, tweets_value)
        json_data['hot'].append(int(hot_value))
    while len(json_data['hot']) < len(json_data['price']):
        json_data['hot'].append(json_data['hot'][-1])
    assert len(json_data['hot']) == len(json_data['price'])
    for i in range(len(price)):
        while len(difficulty) < len(price):
            difficulty_value = difficulty[len(difficulty) - 1].split(':')[1]
            difficulty.append(key + ':' + difficulty_value)
        difficulty_key, difficulty_value = difficulty[i].split(':')
        assert difficulty_key == json_data['date'][i]
        while len(sentaddr) < len(price):
            sentaddr_value = difficulty[len(sentaddr) - 1].split(':')[1]
            sentaddr.append(key + ':' + sentaddr_value)
        sentaddr_key, sentaddr_value = sentaddr[i].split(':')
        assert sentaddr_key == json_data['date'][i]
        while len(transaction) < len(price):
            transaction_value = transaction[len(transaction) - 1].split(':')[1]
            transaction.append(key + ':' + transaction_value)
        transaction_key, transaction_value = transaction[i].split(':')
        assert transaction_key == json_data['date'][i]
        difficulty_value = 0.0 if difficulty_value == 'null' else float(difficulty_value)
        sentaddr_value = 0.0 if sentaddr_value == 'null' else float(sentaddr_value)
        transaction_value = 0.0 if transaction_value == 'null' else float(transaction_value)
        bubble_index = get_bubble_index(
            float(json_data['price'][i]),
            float(json_data['growth_60_day'][i]),
            float(json_data['hot'][i]),
            float(difficulty_value),
            float(sentaddr_value),
            float(transaction_value))
        json_data['bubble'].append(int(bubble_index))
    assert len(json_data['bubble']) == len(json_data['price'])
    file = open('../data.json', "w")
    file.write("data = '")
    file.write(json.dumps(json_data))
    file.write("'")
    file.close()
def process_data ( ) : price = read_datafile ( 'price.txt' ) difficulty = read_datafile ( 'difficulty.txt' ) gtread = read_datafile ( 'gtrend.txt' ) sentaddr = read_datafile ( 'sentaddr.txt' ) transaction = read_datafile ( 'transaction.txt' ) tweets = add_missing_data ( start_date = '2010/07/17' , end_date = '2014/04/09' , init_value = 300 , scale_factor = 0.002 ) ; tweets += read_datafile ( 'tweets.txt' ) json_data = { } json_data [ 'date' ] = [ ] json_data [ 'price' ] = [ ] json_data [ 'growth_60_day' ] = [ ] json_data [ 'hot' ] = [ ] json_data [ 'bubble' ] = [ ] last_price = 0.0 accumulate_60 = 0.0 my_q = queue = collections . deque ( ) for data in price : key , value = data . split ( ':' ) json_data [ 'date' ] . append ( key ) json_data [ 'price' ] . append ( '%0.2f' % float ( value ) ) if last_price == 0.0 : last_price = float ( value ) if len ( my_q ) >= 60 : accumulate_60 -= my_q . popleft ( ) day_growth = ( float ( value ) - last_price ) / last_price accumulate_60 += day_growth my_q . append ( day_growth ) last_price = float ( value ) json_data [ 'growth_60_day' ] . append ( int ( accumulate_60 * 100 ) ) for i in range ( len ( gtread ) ) : gtrend_key , gtread_value = gtread [ i ] . split ( ':' ) assert gtrend_key == json_data [ 'date' ] [ i ] tweets_key , tweets_value = tweets [ i ] . split ( ':' ) assert tweets_key == json_data [ 'date' ] [ i ] gtread_value = 0.0 if gtread_value == 'null' else float ( gtread_value ) tweets_value = 0.0 if tweets_value == 'null' else float ( tweets_value ) hot_value = get_hot_value ( gtread_value , tweets_value ) json_data [ 'hot' ] . append ( int ( hot_value ) ) while len ( json_data [ 'hot' ] ) < len ( json_data [ 'price' ] ) : json_data [ 'hot' ] . append ( json_data [ 'hot' ] [ - 1 ] ) assert len ( json_data [ 'hot' ] ) == len ( json_data [ 'price' ] ) for i in range ( len ( price ) ) : while len ( difficulty ) < len ( price ) : difficulty_value = difficulty [ len ( difficulty ) - 1 ] . split ( ':' ) [ 1 ] difficulty . append ( key + ':' + difficulty_value ) difficulty_key , difficulty_value = difficulty [ i ] . split ( ':' ) assert difficulty_key == json_data [ 'date' ] [ i ] while len ( sentaddr ) < len ( price ) : sentaddr_value = difficulty [ len ( sentaddr ) - 1 ] . split ( ':' ) [ 1 ] sentaddr . append ( key + ':' + sentaddr_value ) sentaddr_key , sentaddr_value = sentaddr [ i ] . split ( ':' ) assert sentaddr_key == json_data [ 'date' ] [ i ] while len ( transaction ) < len ( price ) : transaction_value = transaction [ len ( transaction ) - 1 ] . split ( ':' ) [ 1 ] transaction . append ( key + ':' + transaction_value ) transaction_key , transaction_value = transaction [ i ] . split ( ':' ) assert transaction_key == json_data [ 'date' ] [ i ] difficulty_value = 0.0 if difficulty_value == 'null' else float ( difficulty_value ) sentaddr_value = 0.0 if sentaddr_value == 'null' else float ( sentaddr_value ) transaction_value = 0.0 if transaction_value == 'null' else float ( transaction_value ) bubble_index = get_bubble_index ( float ( json_data [ 'price' ] [ i ] ) , float ( json_data [ 'growth_60_day' ] [ i ] ) , float ( json_data [ 'hot' ] [ i ] ) , float ( difficulty_value ) , float ( sentaddr_value ) , float ( transaction_value ) ) json_data [ 'bubble' ] . append ( int ( bubble_index ) ) assert len ( json_data [ 'bubble' ] ) == len ( json_data [ 'price' ] ) file = open ( '../data.json' , "w" ) file . write ( "data = '" ) file . write ( json . dumps ( json_data ) ) file . write ( "'" ) file . close ( )
Convert original data to json file
Convert original data to json file
"""Convert original data to json file
    """ # Bitcoin price in USD # Difficulty index for bitcoin mining # Google trend index # Number of active address  # Number of transaction per day # Number of tweets per day # Tweets file lacks of the data before the date '2014/04/09',  # so we manually add missing data from 2010/07/17. # Get Date, Price, and 60_day_growth # Get keywords Hot value # Get bubble index # Write json data to file