import datetime
import pymongo
import os
import sys
import time
from courtreader import readers

# Connect to database
client = pymongo.MongoClient(os.environ['DISTRICT_DB'])
db = client.va_district_court_cases

# Connect to District Court Reader
reader = readers.DistrictCourtReader()
reader.connect()

# Fill in cases
while True:
    case = db.cases.find_one({
        'FIPSCode': sys.argv[1],
        'date_collected': {'$exists': False}
    })
    if case is None: break
    print case['CaseNumber']
    case_details = reader.get_case_details_by_number( \
        case['FIPSCode'], case['CaseNumber'])
    case_details['date_collected'] = datetime.datetime.utcnow()
    updated_case = dict(case.items() + case_details.items())
    db.cases.replace_one({'_id': case['_id']}, updated_case)
    time.sleep(2)
print 'Finished'

import boto.utils
import datetime
import pymongo
import os
import sys
import time
import uuid
from courtreader import readers

# Connect to database
client = pymongo.MongoClient(os.environ['DISTRICT_DB'])
db = client.va_district_court_cases

# Connect to District Court Reader
reader = readers.DistrictCourtReader()
reader.connect()

# get some info about this process
process_id = str(uuid.uuid4())
cwd = os.getcwd()
ec2_id = None
try:
    ec2_id = boto.utils.get_instance_metadata(timeout=1, num_retries=1)['instance-id']
except:
    pass

# create db record for this process
db.scrapers.insert_one({
    'process_id': process_id,
    'cwd': cwd,
    'ec2_id': ec2_id
})

fips_code = sys.argv[1]

# Fill in cases
while True:
    case = db.cases.find_one({
        'FIPSCode': fips_code,
        'date_collected': {'$exists': False}
    })
    if case is None: break
    print case['CaseNumber']
    case_details = reader.get_case_details_by_number( \
        case['FIPSCode'], case['CaseNumber'])
    case_details['date_collected'] = datetime.datetime.utcnow()
    updated_case = dict(case.items() + case_details.items())
    db.cases.replace_one({'_id': case['_id']}, updated_case)
    db.scrapers.update_one({'process_id': process_id}, {
        '$set': {
            'fips_code':  fips_code,
            'last_update':  datetime.datetime.utcnow()
        }
    })
    time.sleep(2)
db.scrapers.remove({'process_id': process_id})
db['completed_courts'].replace_one({'fips_code': fips_code}, {
    'fips_code': fips_code,
    'completed_time': datetime.datetime.utcnow()
}, upsert=True)
print 'Finished'

Save scraper settings to database
Save scraper settings to database

This is the first step in allowing multiple processes to run on
different servers. Coming in the next commit!
