def read_data(self):
        """
        Read parameter data from CSV files, if needed. Data that may be
        defined in CSV files is read before generate_model() so that it
        only has to be read from disk once, even if generate_model() is
        repeatedly called.

        Note on indexing: if subset_t is set, a subset of the data is
        selected, and all data including their indices are subsetted.
        d._dt.index maps a simple [0, data length] index to the actual
        t index used.

        Data is stored in the `self.data`  for each
        `param` and technology `y`: ``self.data[param][y]``

        """
        @utils.memoize
        def _get_option_from_csv(filename):
            """Read CSV time series"""
            d_path = os.path.join(self.config_model.data_path, filename)
            # [self.slice] to do time subset if needed
            # Results in e.g. d.r_eff['csp'] being a dataframe
            # of efficiencies for each time step t at location x
            df = pd.read_csv(d_path, index_col=0)[self.slice]
            # Fill columns that weren't defined with NaN
            # missing_cols = list(set(self.data._x) - set(df.columns))
            # for c in missing_cols:
            #     df[c] = np.nan

            # Ensure that the read file's index matches the data's timesteps
            mismatch = df.index.difference(d._dt.index)
            if len(mismatch) > 0:
                e = exceptions.ModelError
                entries = mismatch.tolist()
                raise e('File has invalid index. Ensure that it has the same '
                        'date range and resolution as set_t.csv: {}.\n\n'
                        'Invalid entries: {}'.format(filename, entries))
            return df

        d = self.data
        self.debug.data_sources = utils.AttrDict()
        # Data for storage initialization parameter
        d.s_init = pd.DataFrame(index=d._x, columns=d._y_pc)
        for y in d.s_init.columns:
            for x in d.s_init.index:
                d.s_init.at[x, y] = self.get_option(y + '.constraints.s_init',
                                                    x=x)
        # Parameters that may be defined over (x, t) for a given technology y
        d.params = ['r', 'e_eff']
        d._y_def_r = set()
        d._y_def_e_eff = set()
        # TODO could allow params in d.params to be defined only over
        # x instead of either static or over (x, t) via CSV!
        for param in d.params:
            d[param] = utils.AttrDict()
            for y in d._y:
                d[param][y] = pd.DataFrame(np.nan, index=d._dt.index,
                                           columns=d._x)
                # TODO this whole process could be refactored for efficiency
                # to read files only once,
                # create a dict of files: {'f1.csv': ['x1', 'x2'],
                #                          'f2.csv': ['x3'],
                #                          'model_config': ['x4, x5']}
                for x in d._x:
                    # If this y is actually not defined at this x,
                    # and is also not a transmission tech,
                    # continue (but set the param to 0 first)
                    # TODO this is a bit of a hack -- e.g. the extra check
                    # for transmission tech is necessary because we set
                    # e_eff to 0 for all transmission (as transmission techs
                    # don't show up in the config_model.locations[x].techs)
                    # Keep an eye out in case this causes other problems
                    if (y not in self.config_model.locations[x].techs
                            and y not in d._y_transmission):
                        d[param][y].loc[:, x] = 0
                        continue
                    option = self.get_option(y + '.constraints.' + param, x=x)
                    if (isinstance(option, str)
                            and not option.startswith('file')):
                        e = exceptions.ModelError
                        raise e('Invalid value for `{}.{}.{}`:'
                                ' `{}`'.format(param, y, x, option))
                    k = param + '.' + y + '.' + x
                    if (isinstance(option, str)
                            and option.startswith('file')):
                        if param == 'r':
                            d._y_def_r.add(y)
                        elif param == 'e_eff':
                            d._y_def_e_eff.add(y)
                        try:
                            # Parse 'file=filename' option
                            f = option.split('=')[1]
                        except IndexError:
                            # If set to just 'file', set filename with y and
                            # param, e.g. 'csp_r_eff.csv'
                            f = y + '_' + param + '.csv'
                        df = _get_option_from_csv(f)
                        # Set up x_map if that option has been set
                        try:
                            x_map = self.get_option(y + '.x_map', x=x)
                        except exceptions.OptionNotSetError:
                            x_map = None
                        # If x_map is available, remap the current col
                        if x_map:
                            # TODO this is a hack and will take up a lot
                            # of memory due to data duplication in case
                            # of a lot of mappings pointing to the same
                            # column in the data
                            # Format is <name in model config>:<name in data>
                            x_map_dict = {i.split(':')[0].strip():
                                          i.split(':')[1].strip()
                                          for i in x_map.split(',')}
                            x_map_str = 'x_map: \'{}\''.format(x_map)
                            # Get the mapping for this x from x_map
                            # NB not removing old columns in case
                            # those are also used somewhere!
                            try:
                                x_m = x_map_dict[x]
                            except KeyError:
                                e = exceptions.ModelError
                                raise e('x_map defined but does not map '
                                        'location defined in model config: '
                                        '{}, with {}'.format(x, x_map_str))
                            if x_m not in df.columns:
                                e = exceptions.ModelError
                                raise e('Trying to map to to a column not '
                                        'contained in data: {}, for region '
                                        '{}, with {}'
                                        .format(x_m, x, x_map_str))
                            df[x] = df[x_m]
                        try:
                            d[param][y].loc[:, x] = df[x]
                            self.debug.data_sources.set_key(k, 'file:' + f)
                        except __HOLE__:
                            # If could not be read from file, set it to zero
                            d[param][y].loc[:, x] = 0
                            # Depending on whether or not the tech is allowed
                            # at this location, set _NA_ for the data source,
                            # or raise an error
                            if self.data.locations.at[x, y] == 0:
                                self.debug.data_sources.set_key(k, '_NA_')
                            else:
                                w = exceptions.ModelWarning
                                message = ('Could not load data for {}, '
                                           'with given option: '
                                           '{}'.format(k, option))
                                warnings.warn(message, w)
                                v = 'file:_NOT_FOUND_'
                                self.debug.data_sources.set_key(k, v)
                    else:
                        d[param][y].loc[:, x] = option
                        self.debug.data_sources.set_key(k, 'model_config')
                        if (param == 'r' and option != float('inf')):
                            d._y_def_r.add(y)
                    # Convert power to energy for r, if necessary
                    if param == 'r':
                        r_unit = self.get_option(y + '.constraints.r_unit', x=x)
                        if r_unit == 'power':
                            r_scale = d.time_res_data
                            d[param][y].loc[:, x] = (d[param][y].loc[:, x]
                                                     * r_scale)
                    # Scale r to a given maximum if necessary
                    scale = self.get_option(y + '.constraints.r_scale_to_peak',
                                            x=x)
                    if param == 'r' and scale:
                        scaled = self.scale_to_peak(d[param][y][x], scale)
                        d[param][y].loc[:, x] = scaled
        ds = self.debug.data_sources
        missing_data = [kk for kk in ds.keys_nested()
                        if ds.get_key(kk) == 'file:_NOT_FOUND_']
        if len(missing_data) > 0:
            message = ('The following parameter values could not be read '
                       'from file. They were automatically set to `0`: '
                       + ', '.join(missing_data))
            warnings.warn(message, exceptions.ModelWarning)
        # Finally, check data consistency. For now, demand must be <= 0,
        # and supply >=0, at all times.
        # FIXME update these checks on implementing conditional param updates.
        for y in d._y_def_r:
            base_tech = self.get_parent(y)
            # Check each column:
            for c in d.r[y].columns:
                series = d.r[y][c]
                err_suffix = 'for tech: {}, at location: {}'.format(y, c)
                if base_tech == 'demand':
                    err = 'Demand resource must be <=0, ' + err_suffix
                    assert (series <= 0).all(), err
                elif base_tech == 'supply':
                    err = 'Supply resource must be >=0, ' + err_suffix
                    assert (series >= 0).all(), err
dataset/ETHPy150Open calliope-project/calliope/calliope/core.py/Model.read_data