python urlopen 403 inspite of URL being accessible via browsers
Question: ..Hi folks ..the following URL
[https://www.nseindia.com/products/dynaContent/equities/indices/historicalindices.jsp?toDate=30-06-2016&fromDate=29-06-2016&indexType=NIFTY%2050](https://www.nseindia.com/products/dynaContent/equities/indices/historicalindices.jsp?toDate=30-06-2016&fromDate=29-06-2016&indexType=NIFTY%2050)
works perfectly fine while being accessed via browser but my python code below
keeps throwing a 403. I trapped the error message which says "access denied"
but the funny thing is the explanation which reads - You don't have permission
to access
"http://www.nseindia.com/products/dynaContent/equities/indices/historicalindices.jsp"
on this server

Any pointers will be greatly appreciated !

Code inserted below (contents of INDEX file are just 2 lines "NIFTY 50" and
"NIFTY MIDCAP 50")

    
    
        from urllib import urlencode
        import urllib2
        from bs4 import BeautifulSoup
        from datetime import datetime
        import csv
        import time
        import datetime
    
        arr = [1,3,5,10]
        url = "http://www.nseindia.com/products/dynaContent/equities/indices/historicalindices.jsp"
        fo = open("options/multiyearreturn/INDEX_DATA.txt", "wb")
    
        def is_number(s):
            try:
                float(s)
                return True
            except ValueError:
                return False
    
        with open('options/multiyearreturn/INDEX', 'rb') as csvfile:
            spamreader = csv.reader(csvfile, delimiter=',')
            for row in spamreader:
                print row
                # cant take today since it will load with a days lag
                ToDt = datetime.datetime.now() - datetime.timedelta(days=1)
                if datetime.datetime.now().weekday() == 5:
                    ToDt = ToDt - datetime.timedelta(days=1)
                elif datetime.datetime.now().weekday() == 6:
                    ToDt = ToDt - datetime.timedelta(days=2)
    
                for x in range(len(arr)):
                    frmDt1 = ToDt - datetime.timedelta(days=1)
                    if frmDt1.weekday() == 5:
                         frmDt1 = frmDt1 -  datetime.timedelta(days=1)
                    elif frmDt1.weekday() == 6:
                         frmDt1 = frmDt1 -  datetime.timedelta(days=2)
    
                    values = {'indexType' : row[0], 'fromDate' : frmDt1.strftime("%d-%m-%Y"), 'toDate' : ToDt.strftime("%d-%m-%Y"), "User-Agent" : "Magic Browser" }
                    data = urlencode(values).replace('+','%20')
                    req = urllib2.Request(url, data)
                    print data
                    try:
                        response = urllib2.urlopen(req)
                    except urllib2.HTTPError, e:
                        print e.fp.read()
                    the_page = response.read()
                    soup = BeautifulSoup( the_page )
                    MFDTLS = soup.findAll('td', {'class': 'number'})
    


Answer: just added this additional line in the headers

    
    
                    hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
    

the "Accept" header did the trick for me. Appreciate help!

