Default initialization of items using Scrapy I am writing a scraper, using Scrapy, in which I need to populate items with default values at first.

Could my method be improved?

 def sp_offer_page(self, response):
            item = MallCrawlerItem()
            for key, value in self.get_default_item_dict().iteritems():
                item[key] = value
            item['is_coupon'] = 1
            item['mall'] = 'abc'
            # some other logic 
            yield item

    def get_default_item_dict(self):     # populate item with Default Values
        return {'mall': 'null', 'store': 'null', 'bonus': 'null',
                'per_action': 0, 'more_than': 0,
                'up_to': 0, 'deal_url': 'null', 'category': 'null', 'expiration': 'null', 'discount': 'null',
                'is_coupon': 0, 'code': 'null'}
You can use Field() default values as suggested.

class MalCrawlerItem(Item):
    mall = Field(default='null')
    store = Field(default='null')
    bonus= Field(default='null')
    per_action = Field(default='null')
    more_than = Field(default='null')
    up_to = Field(default='null')
    deal_url = Field(default='null')
    category = Field(default='null')
    expiration = Field(default='null')
    discount = Field(default='null')
    is_coupon = Field(default='null')
    code = Field(default='null')


You can also use Item copy in the constructor.

#initialize default item
default = MallCrawlerItem()
default['mall'] ='null'
default['store'] = 'null'
default['bonus'] = 'null'
default['per_action'] = 0
default['more_than'] = 0
default['up_to'] = 0
default['deal_url'] = 'null'
default['category'] = 'null'
default['expiration'] = 'null'
default['discount'] = 'null'
default['is_coupon'] =  0
default['code'] = 'null'

#copy to new instance
item = MallCrawlerItem(default)


The benefits of this approach are:


constructor copy is the fastest method outside of class definition
you can create specific default item for your current function. e.g. initializing fixed fields of item before entering a for loop