#!/usr/bin/python
#
# So this script is in a bit of a hack state right now. 
# This script reads 
#
#
#


# Graciously copied and modified from:
# http://graphics.cs.cmu.edu/projects/im2gps/flickr_code.html
#Image querying script written by Tamara Berg,
#and extended heavily James Hays

#9/26/2007 added dynamic timeslices to query more efficiently.
#8/18/2008 added new fields and set maximum time slice.
#8/19/2008 this is a much simpler function which gets ALL geotagged photos of
# sufficient accuracy.  No queries, no negative constraints.
# divides up the query results into multiple files
# 1/5/2009
# now uses date_taken instead of date_upload to get more diverse blocks of images
# 1/13/2009 - uses the original im2gps keywords, not as negative constraints though

import sys, string, math, time, socket
from flickrapi2 import FlickrAPI
from datetime import datetime
import pycurl
import os
import shutil
socket.setdefaulttimeout(30)  #30 second time out on sockets before they throw
#an exception.  I've been having trouble with urllib.urlopen hanging in the 
#flickr API.  This will show up as exceptions.IOError.

#the time out needs to be pretty long, it seems, because the flickr servers can be slow
#to respond to our big searches.

#returns a query and the search times to attempt to get a desired number of photos
#this needs serious refactoring -KAS
def DoSearch(fapi,query_string,desired_photos):
    # number of seconds to skip per query  
    #timeskip = 62899200 #two years
    #timeskip = 604800  #one week
    timeskip = 172800  #two days
    #timeskip = 86400 #one day
    #timeskip = 3600 #one hour
    #timeskip = 2257 #for resuming previous query

    #mintime = 1121832000 #from im2gps
    #mintime = 1167407788 # resume crash england
    #mintime = 1177828976 #resume crash japan
    #mintime = 1187753798 #resume crash greece
    mintime = 1171416400 #resume crash WashingtonDC
    maxtime = mintime+timeskip
    endtime = 1192165200  #10/12/2007, at the end of im2gps queries



    print datetime.fromtimestamp(mintime)
    print datetime.fromtimestamp(endtime)

    while (maxtime < endtime):

        #new approach - adjust maxtime until we get the desired number of images 
        #within a block. We'll need to keep upper bounds and lower
        #lower bound is well defined (mintime), but upper bound is not. We can't 
        #search all the way from endtime.
    
        lower_bound = mintime + 900 #lower bound OF the upper time limit. must be at least 15 minutes or zero results
        upper_bound = mintime + timeskip * 20 #upper bound of the upper time limit
        maxtime     = .95 * lower_bound + .05 * upper_bound
    
        print '\nBinary search on time range upper bound' 
        print 'Lower bound is ' + str(datetime.fromtimestamp(lower_bound))
        print 'Upper bound is ' + str(datetime.fromtimestamp(upper_bound))
    
        keep_going = 6 #search stops after a fixed number of iterations
        while( keep_going > 0 and maxtime < endtime):
            try:
                rsp = fapi.photos_search(api_key=flickrAPIKey,
                                        ispublic="1",
                                        media="photos",
                                        per_page="250", 
                                        page="1",
                                        has_geo = "0", #bbox="-180, -90, 180, 90",
                                        text=query_string,
                                        accuracy="6", #6 is region level. 
                                        min_upload_date=str(mintime),
                                        max_upload_date=str(maxtime))
                          
                #we want to catch these failures somehow and keep going.
                time.sleep(1)
                fapi.testFailure(rsp)
                total_images = rsp.photos[0]['total'];
                null_test = int(total_images); #want to make sure this won't crash later on for some reason
                null_test = float(total_images);
        
                print '\nnumimgs: ' + total_images
                print 'mintime: ' + str(mintime) + ' maxtime: ' + str(maxtime) + ' timeskip:  ' + str(maxtime - mintime)
            
                if( int(total_images) > desired_photos ):
                    print 'too many photos in block, reducing maxtime'
                    upper_bound = maxtime
                    maxtime = (lower_bound + maxtime) / 2 #midpoint between current value and lower bound.
                
                if( int(total_images) < desired_photos):
                    print 'too few photos in block, increasing maxtime'
                    lower_bound = maxtime
                    maxtime = (upper_bound + maxtime) / 2
                
                print 'Lower bound is ' + str(datetime.fromtimestamp(lower_bound))
                print 'Upper bound is ' + str(datetime.fromtimestamp(upper_bound))
            
                if( int(total_images) > 0): #only if we're not in a degenerate case
                    keep_going = keep_going - 1
                else:
                    upper_bound = upper_bound + timeskip;    
            
            except KeyboardInterrupt:
                print('Keyboard exception while querying for images, exiting\n')
                raise
            except:
                print sys.exc_info()[0]
                #print type(inst)     # the exception instance
                #print inst.args      # arguments stored in .args
                #print inst           # __str__ allows args to printed directly
                print ('Exception encountered while querying for images\n')

        #end of while binary search    
        print 'finished binary search'
        return([mintime,maxtime,total_images,rsp])



###########################################################################
# Modify this section to reflect your data and specific search 
###########################################################################
# flickr auth information:
# change these to your flickr api keys and secret
flickrAPIKey = "fa33550d413b36b3fddc473a931a3b3b"  # API key
flickrSecret = "7fd481bff0916055"                  # shared "secret"
rootpath = "../data/" #where do you want the data
desired_photos = 1000 #how many photos do you want to try and get
query_file_name = 'query.dat' #The file to get the queries from



#query_file_name = 'place_rec_queries_fall08.txt'
query_file = open(query_file_name, 'r')

#aggregate all of the positive and negative queries together.
pos_queries = []  #an empty list
neg_queries = ''  #a string
num_queries = 0

for line in query_file:
    if line[0] != '#' and len(line) > 1:  #line end character is 2 long?
      print line[0:len(line)-1]
      if line[0] != '-':
        pos_queries = pos_queries + [line[0:len(line)-1]]
        num_queries = num_queries + 1
      if line[0] == '-':
        neg_queries = neg_queries + ' ' + line[0:len(line)-1]
        
query_file.close()
print 'positive queries:  '
print pos_queries
print 'negative queries:  ' + neg_queries
print 'num_queries = ' + str(num_queries)
#this is the desired number of photos in each block

    
# make a new FlickrAPI instance
fapi = FlickrAPI(flickrAPIKey, flickrSecret)

for current_tag in range(0, num_queries):
  
    print('TOP OF LOOP')
    # change this to the location where you want to put your output file
    try:
        stats = os.stat(rootpath)
    except OSError:
        os.mkdir(rootpath)
        
    outpath = rootpath+pos_queries[current_tag]+'/'
    try:
        os.mkdir(outpath)
    except OSError:
        shutil.rmtree(outpath,True)
        os.mkdir(outpath)
       
    out_file = open(rootpath + pos_queries[current_tag] + '.txt','w')
    ###########################################################################
    
    #form the query string.
    query_string = pos_queries[current_tag] + ' ' + neg_queries
    print '\n\nquery_string is ' + query_string
    
    total_images_queried = 0;
    [mintime,maxtime,total_images,rsp] = DoSearch(fapi,query_string,desired_photos)

    print('GETTING TOTATL IMAGES:'+str(total_images))
    s = '\nmintime: ' + str(mintime) + ' maxtime: ' + str(maxtime)
    print s
    out_file.write(s + '\n') 
    i = getattr(rsp,'photos',None)
    if i:
            
        s = 'numimgs: ' + total_images
        print s
        out_file.write(s + '\n')
        
        current_image_num = 1;
        
        num = 4 # CHANGE THIS BACK int(rsp.photos[0]['pages'])
        s =  'total pages: ' + str(num)
        print s
        out_file.write(s + '\n')
        
        #only visit 16 pages max, to try and avoid the dreaded duplicate bug
        #16 pages = 4000 images, should be duplicate safe.  Most interesting pictures will be taken.
        
        num_visit_pages = min(16,num)
        
        s = 'visiting only ' + str(num_visit_pages) + ' pages ( up to ' + str(num_visit_pages * 250) + ' images)'
        print s
        out_file.write(s + '\n')
        
        total_images_queried = total_images_queried + min((num_visit_pages * 250), int(total_images))

        #print 'stopping before page ' + str(int(math.ceil(num/3) + 1)) + '\n'
    
        pagenum = 1;

        counter = -1
        while( pagenum <= num_visit_pages ):
        #for pagenum in range(1, num_visit_pages + 1):  #page one is searched twice
            
            print '  page number ' + str(pagenum)
            
            try:
                print("PAGE")
                print(pagenum)
                # WARNING THIS QUERY HAS TO MATCH THE SEARCH QUERY!!!!
                rsp = fapi.photos_search(api_key=flickrAPIKey,
                                        ispublic="1",
                                        media="photos",
                                        per_page="250", 
                                        page=str(pagenum),
                                        has_geo = "0",
                                        text=query_string,
                                        #extras = "tags, original_format, license, geo, date_taken, date_upload, o_dims, views",
                                        #accuracy="6", #6 is region level. 
                                        min_upload_date=str(1121832000),#mintime),
                                        max_upload_date=str(1192165200))#maxtime))
                
                #rsp = fapi.photos_search(api_key=flickrAPIKey,
                 #                   ispublic="1",
                  #                  media="photos",
                   #                 per_page="250", 
                    #                page='0', #str(pagenum),
                     #               sort="interestingness-desc",
                      #              has_geo = "0", #bbox="-180, -90, 180, 90",
                       #             text=query_string,
                        #            #accuracy="6", #6 is region level.  most things seem 10 or better.
                         #           extras = "tags, original_format, license, geo, date_taken, date_upload, o_dims, views",
                          #          min_upload_date=str(mintime),
                           #         max_upload_date=str(maxtime))
                                    ##min_taken_date=str(datetime.fromtimestamp(mintime)),
                                    ##max_taken_date=str(datetime.fromtimestamp(maxtime)))
                time.sleep(1)
                fapi.testFailure(rsp)
            except KeyboardInterrupt:
                print('Keyboard exception while querying for images, exiting\n')
                raise
            except:
                print sys.exc_info()[0]
                #print type(inst)     # the exception instance
                #print inst.args      # arguments stored in .args
                #print inst           # __str__ allows args to printed directly
                print ('Exception encountered while querying for images\n')
            else:
                print('got a response')
                # and print them
                k = getattr(rsp,'photos',None)
                if k:
                    print('In K')
                    m = getattr(rsp.photos[0],'photo',None)
                    if m:
                        print('In    M')
                        for b in rsp.photos[0].photo:
                            print('In b')
                            if b!=None:
                                counter = counter + 1
                                ##print(http://farm{farm-id}.static.flickr.com/{server-id}/{id}_{secret}.jpg)
                                
                                myurl = 'http://farm'+b['farm']+".static.flickr.com/"+b['server']+"/"+b['id']+"_"+b['secret']+'.jpg'
                                fname = outpath+pos_queries[current_tag]+str(counter)+'.jpg' #b['id']+"_"+b['secret']+'.jpg'
                                print(myurl)
                                print(fname)
                                mycurl = pycurl.Curl()
                                mycurl.setopt(pycurl.URL, str(myurl))
                                myfile = open(fname,"wb")
                                mycurl.setopt(pycurl.WRITEDATA, myfile)
                                mycurl.setopt(pycurl.FOLLOWLOCATION, 1)
                                mycurl.setopt(pycurl.MAXREDIRS, 5)
                                mycurl.setopt(pycurl.NOSIGNAL, 1)
                                mycurl.perform()
                                mycurl.close()
                                myfile.close()
                                out_file.write('URL: '+myurl+'\n')
                                out_file.write('File: '+ fname+'\n')
                                out_file.write('photo: ' + b['id'] + ' ' + b['secret'] + ' ' + b['server'] + '\n')
                                out_file.write('owner: ' + b['owner'] + '\n') 
                                out_file.write('title: ' + b['title'].encode("ascii","replace") + '\n')
                                
                                out_file.write('originalsecret: ' + b['originalsecret'] + '\n')
                                out_file.write('originalformat: ' + b['originalformat'] + '\n')
                                out_file.write('o_height: ' + b['o_height'] + '\n')
                                out_file.write('o_width: ' + b['o_width'] + '\n')
                                out_file.write('datetaken: ' + b['datetaken'].encode("ascii","replace") + '\n')
                                out_file.write('dateupload: ' + b['dateupload'].encode("ascii","replace") + '\n')
                                
                                out_file.write('tags: ' + b['tags'].encode("ascii","replace") + '\n')
                                
                                out_file.write('license: ' + b['license'].encode("ascii","replace") + '\n')
                                out_file.write('latitude: '  + b['latitude'].encode("ascii","replace") + '\n')
                                out_file.write('longitude: ' + b['longitude'].encode("ascii","replace") + '\n')
                                out_file.write('accuracy: '  + b['accuracy'].encode("ascii","replace") + '\n')
                                
                                out_file.write('views: ' + b['views'] + '\n')
                                out_file.write('interestingness: ' + str(current_image_num) + ' out of ' + str(total_images) + '\n');
                                out_file.write('\n')
                                current_image_num = current_image_num + 1;
               
            print('')
            pagenum = pagenum + 1;  #this is in the else exception block.  Itwon't increment for a failure.
            #this block is indented such that it will only run if there are no exceptions
            #in the original query.  That means if there are exceptions, mintime won't be incremented
            #and it will try again
            timeskip = maxtime - mintime #used for initializing next binary search
            mintime  = maxtime

    out_file.write('Total images queried: ' + str(total_images_queried) + '\n')
    out_file.close

[]