python 2.7 : appending log return to csv
Question: With the following code I'm trying to grab data from a website every 5 mins,
timestamp it, calculate its logn return and append all that to a csv file.

Grabbing the data, time stamping it and appending to csv works, but when I try
to figure out how to include the log return I'm kind of stuck.

    
    
    import time
    from time import strftime, gmtime
    import numpy as np
    import urllib2
    from urllib2 import urlopen
    from math import log
    
    coiAr = []
    dateAr = []
    logReAr = []
    
    def mcapGrab():
        while True:
            try:
                sourceCode = urllib2.urlopen('http://coinmarketcap.com').read() 
    
                mcapUSD = sourceCode.split('<strong>Total Market Cap: <span id="total-marketcap" data-usd="')[1].split('"')[0]
                coiAr.append(float(mcapUSD.replace(',','')))
    
                date = strftime('%d %b %Y %H:%M:%S', gmtime())
                dateAr.append(date)         
    #           if len(coiAr) > 0:
    #               indexLog = 1
    #               logRe = log(coiAr[indexLog]/coiAr[indexLog-1])
    #               logReAr.append(logRe)
    #               indexLog += 1
    #           else:
    #               logReAr.append(0)
    
                for eachMcap in coiAr:
                    saveLine = date+','+str(eachMcap)+'\n'
                    saveFile = open('mcapdata.csv', 'a')
                    saveFile.write(saveLine)
                    saveFile.close()
    
                    s = 0
                    print dateAr[s]+','+str(coiAr[s])
                    time.sleep(300)
                    s+= 1
    
    
            except Exception, e: 
                print 'Failed to grab market cap', str(e)
    
    mcapGrab()
    

I've commented out the section where I attempt to calc and append log return
but doesn't work.

Any help would be appreciated!


Answer: Don't use global lists; just write each entry to the file _as you find it_.
Using the `csv` module would make this all a bit easier still:

    
    
    import csv
    
    sourceCode = urllib2.urlopen('http://coinmarketcap.com').read() 
    mcapUSD = sourceCode.split('<strong>Total Market Cap: <span id="total-marketcap" data-usd="')[1].split('"')[0]
    mcap = float(mcapUSD.replace(',','')
    
    # read previous value from the CSV first
    with open('mcapdata.csv', 'rb') as infh:
        last = None
        for row in csv.reader(infh):
            last = row[1]  # second column
    
    # Now calculate the log value based on the last value
    logRe = log(mcap/float(last))
    
    # write new values
    with open(mcapdata.csv', 'ab') as outfh:
        date = strftime('%d %b %Y %H:%M:%S', gmtime())
        csv.writer(outfh).writerow([date, mcap, logRe])
    

This code will read the `mcapdata.csv` file, picking out just the last value
written to it. You could instead also keep all of the rows in memory and just
picking out the last entry in a list of lists.

