This is the curl command I used -->

```
 curl "https://api.coursera.org/api/courses.v1?start=1&limit=11?includes=instructorIds,partnerIds,specializations,s12nlds,v1Details,v2Details&fields=instructorIds,partnerIds,specializations,s12nlds,description" 

```

I played around with the query parameters -start and limit, but it just repeats the same 100 courses out of 2150 courses.Here is the link to the course catalog API -->

<https://docs.google.com/document/d/15gwppUMLp0s1OhbzFZvFSeTbvFkRfSFIkiIKrEP6cUA/edit>

Python code:

```
 import requests
 import json
 from bs4 import BeautifulSoup
 import csv
 import sys
 reload(sys)
 sys.setdefaultencoding('utf-8')

if __name__ == "__main__":
headers = ({
    "x-user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 
    (KHTML, like Gecko) Chrome/53.0.2785.92 Safari/537.36 
    FKUA/website/41/website/Desktop"})
d = open('result.json', 'r')
data = json.load(d)
print(data)
d.close()

with open("coursera.csv", 'a') as f:

    # Wrote the header once and toggle comment

    header = f.write('instructorIds' + ',' + 'courseType' + ',' + 'name' + ',' + 'partnerIds' + ',' +
                     'slug' + ',' + 'specializations' + ',' + 'course_id' + ',' + 'description' + "\n")

    for i in range(len(data['elements'])):

                instructorIds = data['elements'][i]['instructorIds']

                instructorIds = str(instructorIds)
                if instructorIds:
                    instructorIds = instructorIds.rstrip().replace(',', '')
                    instructorIds = instructorIds.rstrip().replace('\n', '')
                    instructorIds = instructorIds.rstrip().replace('u', '')
                    instructorIds = instructorIds.rstrip().replace('[', '')
                    instructorIds = instructorIds.rstrip().replace(']', '')
                else:
                    instructorIds = ' '
                print(instructorIds)
                courseType = data['elements'][i]['courseType']
                courseType = str(courseType)
                print(courseType)
                name = data['elements'][i]['name']
                name = str(name)
                print(name)
                partnerIds = data['elements'][i]['partnerIds']
                partnerIds = str(partnerIds)
                if partnerIds:
                    partnerIds = partnerIds.rstrip().replace(',', '')
                    partnerIds = partnerIds.rstrip().replace('\n', '')
                    partnerIds = partnerIds.rstrip().replace('u', '')
                    partnerIds = partnerIds.rstrip().replace('[', '')
                    partnerIds = partnerIds.rstrip().replace(']', '')
                else:
                    partnerIds = ' '
                print(partnerIds)
                slug = data['elements'][i]['slug']
                slug = str(slug)
                print(slug)
                specializations = data['elements'][i]['specializations']
                specializations = str(specializations)
                if specializations:
                    specializations = specializations.rstrip().replace(',', '')
                    specializations = specializations.rstrip().replace('\n', '')
                    specializations = specializations.rstrip().replace('u', '')
                    specializations = specializations.rstrip().replace('[', '')
                    specializations = specializations.rstrip().replace(']', '')
                else:
                    specializations = ' '
                print(specializations)
                course_id = data['elements'][i]['id']
                course_id = str(course_id)
                print(course_id)
                description = data['elements'][i]['description']
                description = str(description)
                print(description)

                if description:
                          description = description.rstrip().replace(',', '')
                          description = description.rstrip().replace('\n', '')
                else:
                     description = ' '

                                ####################################################################
                    ### writing the attributes in a csv file

                f.write(instructorIds + ',' + courseType + ',' + name + ',' + partnerIds + ',' + slug + ',' + specializations + ',' + course_id + ',' + description + "\n")

```

Please suggest a way I can scrape all the courses.