app/scrapers/sections_scraper.rb
def parse_sections(url, semester)
  # Parse with Nokogiri
  page = Nokogiri::HTML(open(url))
  course_divs = page.search("div.course-sections")
  section_array = []

  # for each of the courses on the page
  course_divs.each do |course_div|
    course_id = course_div.attr('id')
    # for each section of the course
    course_div.search("div.section").each do |section|
      # add section to array to add
      instructors = section.search('span.section-instructors').text.gsub(/\t|\r\n/,'').encode('UTF-8', :invalid => :replace).split(',').map(&:strip)
      dept = course_id.match(/^([A-Z]{4})\d{3}[A-Z]?$/)[1]

      # add course and department to professor object for each instructor
      profs = []
      instructors.each do |x|
        if x != 'Instructor: TBA'
          professor_name = x.squeeze(' ')
          profs << professor_name
        end
      end

      meetings = []
      section.search('div.class-days-container div.row').each do |meeting|
        start_time = meeting.search('span.class-start-time').text
        end_time = meeting.search('span.class-end-time').text

        meetings << {
          days: meeting.search('span.section-days').text,
          start_time: start_time,
          end_time: end_time,
          start_seconds: time_to_int(start_time),
          end_seconds: time_to_int(end_time),
          building: meeting.search('span.building-code').text,
          room: meeting.search('span.class-room').text,
          classtype: meeting.search('span.class-type').text || "Lecture"
        }
      end
      number = section.search('span.section-id').text.gsub(/\s/, '')
      open_seats = section.search('span.open-seats-count').text
      waitlist = section.search('span.waitlist-count').text
      section_array << {
        section_id: "#{course_id}-#{number}",
        course_id: course_id,
        number: number,
        instructors: profs,
        seats: section.search('span.total-seats-count').text,
        semester: semester,
        meetings: meetings,
        open_seats: open_seats,
        waitlist: waitlist
      }
    end
  end

  return section_array
end
# Parses a given section page
# Returns [sections, professors]
# TODO: Remove semester param, infer from url
Parses a given section page
Returns [sections, professors]
TODO: Remove semester param, infer from url
Parses a given section page Returns [ sections professors ] TODO : Remove semester param infer from url
def parse_sections(url, semester)
  page = Nokogiri::HTML(open(url))
  course_divs = page.search("div.course-sections")
  section_array = []
  course_divs.each do |course_div|
    course_id = course_div.attr('id')
    course_div.search("div.section").each do |section|
      instructors = section.search('span.section-instructors').text.gsub(/\t|\r\n/,'').encode('UTF-8', :invalid => :replace).split(',').map(&:strip)
      dept = course_id.match(/^([A-Z]{4})\d{3}[A-Z]?$/)[1]
      profs = []
      instructors.each do |x|
        if x != 'Instructor: TBA'
          professor_name = x.squeeze(' ')
          profs << professor_name
        end
      end
      meetings = []
      section.search('div.class-days-container div.row').each do |meeting|
        start_time = meeting.search('span.class-start-time').text
        end_time = meeting.search('span.class-end-time').text
        meetings << {
          days: meeting.search('span.section-days').text,
          start_time: start_time,
          end_time: end_time,
          start_seconds: time_to_int(start_time),
          end_seconds: time_to_int(end_time),
          building: meeting.search('span.building-code').text,
          room: meeting.search('span.class-room').text,
          classtype: meeting.search('span.class-type').text || "Lecture"
        }
      end
      number = section.search('span.section-id').text.gsub(/\s/, '')
      open_seats = section.search('span.open-seats-count').text
      waitlist = section.search('span.waitlist-count').text
      section_array << {
        section_id: "#{course_id}-#{number}",
        course_id: course_id,
        number: number,
        instructors: profs,
        seats: section.search('span.total-seats-count').text,
        semester: semester,
        meetings: meetings,
        open_seats: open_seats,
        waitlist: waitlist
      }
    end
  end
  return section_array
end
def parse_sections ( url , semester ) page = Nokogiri :: HTML ( open ( url ) ) course_divs = page . search ( "div.course-sections" ) section_array = [ ] course_divs . each do | course_div | course_id = course_div . attr ( 'id' ) course_div . search ( "div.section" ) . each do | section | instructors = section . search ( 'span.section-instructors' ) . text . gsub ( / \t | \r \n / , '' ) . encode ( 'UTF-8' , :invalid => :replace ) . split ( ',' ) . map ( & :strip ) dept = course_id . match ( / ^([A-Z]{4}) \d {3}[A-Z]?$ / ) [ 1 ] profs = [ ] instructors . each do | x | if x != 'Instructor: TBA' professor_name = x . squeeze ( ' ' ) profs << professor_name end end meetings = [ ] section . search ( 'div.class-days-container div.row' ) . each do | meeting | start_time = meeting . search ( 'span.class-start-time' ) . text end_time = meeting . search ( 'span.class-end-time' ) . text meetings << { days : meeting . search ( 'span.section-days' ) . text , start_time : start_time , end_time : end_time , start_seconds : time_to_int ( start_time ) , end_seconds : time_to_int ( end_time ) , building : meeting . search ( 'span.building-code' ) . text , room : meeting . search ( 'span.class-room' ) . text , classtype : meeting . search ( 'span.class-type' ) . text || "Lecture" } end number = section . search ( 'span.section-id' ) . text . gsub ( / \s / , '' ) open_seats = section . search ( 'span.open-seats-count' ) . text waitlist = section . search ( 'span.waitlist-count' ) . text section_array << { section_id : "#{course_id}-#{number}" , course_id : course_id , number : number , instructors : profs , seats : section . search ( 'span.total-seats-count' ) . text , semester : semester , meetings : meetings , open_seats : open_seats , waitlist : waitlist } end end return section_array end
Parses a given section page
Returns [sections, professors]
TODO: Remove semester param, infer from url
Parses a given section page Returns [ sections professors ] TODO : Remove semester param infer from url
# Parse with Nokogiri # for each of the courses on the page # for each section of the course # add section to array to add # add course and department to professor object for each instructor
url
semester