#!/bin/bash
# Crawl all pages from year 2000 to 2012

zone_type=$1
startyear=2000
if [ $zone_type = "region" ]; then
    startyear=2008
fi

if [ $zone_type = "department" ]; then
    startyear=2008
fi

if [ $zone_type = "epci" ]; then
    startyear=2007
fi

for year in $(seq $startyear 2013);
do
    scrapy crawl localfinance -o scraped_data/${zone_type}_$year.json -t jsonlines -a year=$year -a zone_type=$zone_type
done


#!/bin/bash
# Crawl all pages from year 2000 to 2012

zone_type=$1
startyear=2000
if [ $zone_type = "region" ]; then
    startyear=2008
fi

if [ $zone_type = "department" ]; then
    startyear=2008
fi

if [ $zone_type = "epci" ]; then
    startyear=2007
fi

for year in $(seq $startyear 2014);
do
    scrapy crawl localfinance -o scraped_data/${zone_type}_$year.json -t jsonlines -a year=$year -a zone_type=$zone_type
done


Add 2014 in crawl all years bash script
Add 2014 in crawl all years bash script
