Parsing json element
Question: I am using Scrapy and a Regex to parse some none standard web source code. I
then wish to parse the first element of the dictionary returned:

    
    
    from scrapy.contrib.spiders import CrawlSpider, Rule
    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
    from scrapy.selector import Selector
    from scrapy.item import Item
    from scrapy.spider import BaseSpider
    from scrapy import log
    from scrapy.cmdline import execute
    from scrapy.utils.markup import remove_tags
    import time
    import re
    import json
    import requests
    
    
    class ExampleSpider(CrawlSpider):
        name = "goal2"
        allowed_domains = ["whoscored.com"]
        start_urls = ["http://www.whoscored.com"]
        download_delay = 5
    
        rules = [Rule(SgmlLinkExtractor(allow=('\Teams'),deny=(),), follow=False, callback='parse_item')]
    
        def parse_item(self, response):
    
            sel = Selector(response)
            titles = sel.xpath("normalize-space(//title)")
            print '-' * 170
            myheader = titles.extract()[0]
            print '********** Page Title:', myheader.encode('utf-8'), '**********'
            print '-' * 170
    
            match1 = re.search(re.escape("DataStore.prime('stage-player-stat', defaultTeamPlayerStatsConfigParams.defaultParams , ") \
                         + '(\[.*\])' + re.escape(");"), response.body)
    
    
            if match1 is not None:
                playerdata1 = match1.group(1)
    
                teamid = json.loads(playerdata1[0])
                print teamid
    

The key for the first element of 'playerdata1' is called 'TeamId'. I assumed
the above method would work, however I am getting the following error:

    
    
        teamid = json.loads(playerdata1[0])
      File "C:\Python27\lib\json\__init__.py", line 338, in loads
        return _default_decoder.decode(s)
      File "C:\Python27\lib\json\decoder.py", line 366, in decode
        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
      File "C:\Python27\lib\json\decoder.py", line 382, in raw_decode
        obj, end = self.scan_once(s, idx)
    exceptions.ValueError: Expecting object: line 1 column 1 (char 0)
    

Can anyone see what the issue is here?


Answer: `match1.group(1)` returns _one string_. You then index that string:

    
    
    teamid = json.loads(playerdata1[0])
    

Here, `[0]` will give you the _just the first character_ of that string.
Remove the indexing expression there to use the _whole_ string:

    
    
    teamid = json.loads(playerdata1)
    

Now `teamid` is a list with player objects:

    
    
    >>> len(teamid)
    22
    >>> teamid[0].keys()
    [u'FirstName', u'LastName', u'KnownName', u'Field', u'GameStarted', u'AerialWon', u'TeamRegionCode', u'SecondYellow', u'ShotsBlocked', u'TotalShots', u'Assists', u'Red', u'Name', u'PositionText', u'Ranking', u'PositionLong', u'PlayerId', u'SubOff', u'Dispossesed', u'TeamId', u'TotalTackles', u'TotalLongBalls', u'Goals', u'SubOn', u'WasDribbled', u'AerialLost', u'Turnovers', u'ShotsOnTarget', u'WSName', u'Fouls', u'ManOfTheMatch', u'Height', u'TeamName', u'RegionCode', u'TotalPasses', u'TotalThroughBalls', u'Dribbles', u'DateOfBirth', u'OwnGoals', u'WasFouled', u'TotalClearances', u'Rating', u'PlayedPositionsRaw', u'Weight', u'AccurateLongBalls', u'OffsidesWon', u'AccuratePasses', u'Yellow', u'KeyPasses', u'TotalCrosses', u'AccurateCrosses', u'IsCurrentPlayer', u'Age', u'PositionShort', u'AccurateThroughBalls', u'Interceptions', u'Offsides']
    

