fetch_data.py
def parse_html(self,dataset_type=Myntra,url_root='https://myntra.com'):
        '''
            From each html, parse the data
        '''
        self.products = {}      
        for efile in self.files:
            
            file_path = os.path.join(config.HOME,config.PAGES,efile)
            if not os.path.exists(file_path):
                self.logger.error(f'File not found {efile}')

            try:
                f = open(file_path,'r')
                raw_product_info = json.load(f)
                
                fields = [f.name for f in dataclasses.fields(dataset_type)]
                
                for _,products in raw_product_info.items():
                    for product in products:                  
                        myntra = Myntra(**{k:v for k,v in product.items() if k in fields})                        
                        myntra.searchImage = utils.decode_unicode_url(myntra.searchImage)                        
                        myntra.landingPageUrl = utils.decode_unicode_url(os.path.join(url_root,
                            myntra.landingPageUrl))
                        product_id = next(self._count)
                        self.products[product_id] = myntra
                           
            except ValueError as Err:
                self.logger.error(f'Exception thrown at reading the file {efile}')

        return self.products

            From each html, parse the data
        
From each html, parse the data
From each html parse the data
def parse_html(self,dataset_type=Myntra,url_root='https://myntra.com'):
        self.products = {}      
        for efile in self.files:
            file_path = os.path.join(config.HOME,config.PAGES,efile)
            if not os.path.exists(file_path):
                self.logger.error(f'File not found {efile}')
            try:
                f = open(file_path,'r')
                raw_product_info = json.load(f)
                fields = [f.name for f in dataclasses.fields(dataset_type)]
                for _,products in raw_product_info.items():
                    for product in products:                  
                        myntra = Myntra(**{k:v for k,v in product.items() if k in fields})                        
                        myntra.searchImage = utils.decode_unicode_url(myntra.searchImage)                        
                        myntra.landingPageUrl = utils.decode_unicode_url(os.path.join(url_root,
                            myntra.landingPageUrl))
                        product_id = next(self._count)
                        self.products[product_id] = myntra
            except ValueError as Err:
                self.logger.error(f'Exception thrown at reading the file {efile}')
        return self.products
def parse_html ( self , dataset_type = Myntra , url_root = 'https://myntra.com' ) : self . products = { } for efile in self . files : file_path = os . path . join ( config . HOME , config . PAGES , efile ) if not os . path . exists ( file_path ) : self . logger . error ( f'File not found {efile}' ) try : f = open ( file_path , 'r' ) raw_product_info = json . load ( f ) fields = [ f . name for f in dataclasses . fields ( dataset_type ) ] for _ , products in raw_product_info . items ( ) : for product in products : myntra = Myntra ( ** { k : v for k , v in product . items ( ) if k in fields } ) myntra . searchImage = utils . decode_unicode_url ( myntra . searchImage ) myntra . landingPageUrl = utils . decode_unicode_url ( os . path . join ( url_root , myntra . landingPageUrl ) ) product_id = next ( self . _count ) self . products [ product_id ] = myntra except ValueError as Err : self . logger . error ( f'Exception thrown at reading the file {efile}' ) return self . products
From each html, parse the data
From each html parse the data
'''
            From each html, parse the data
        '''
self
dataset_type
url_root