text: // Copyright(c) 2017-2020, Intel Corporation
  //
  // Redistribution  and  use  in source  and  binary  forms,  with  or  without
  // modification, are permitted provided that the following conditions are met:
  //
  // * Redistributions of  source code  must retain the  above copyright notice,
  //   this list of conditions and the following disclaimer.
  // * Redistributions in binary form must reproduce the above copyright notice,
  //   this list of conditions and the following disclaimer in the documentation
  //   and/or other materials provided with the distribution.
  // * Neither the name  of Intel Corporation  nor the names of its contributors
  //   may be used to  endorse or promote  products derived  from this  software
  //   without specific prior written permission.
  //
  // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,  BUT NOT LIMITED TO,  THE
  // IMPLIED WARRANTIES OF  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  // ARE DISCLAIMED.  IN NO EVENT  SHALL THE COPYRIGHT OWNER  OR CONTRIBUTORS BE
  // LIABLE  FOR  ANY  DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL,  EXEMPLARY,  OR
  // CONSEQUENTIAL  DAMAGES  (INCLUDING,  BUT  NOT LIMITED  TO,  PROCUREMENT  OF
  // SUBSTITUTE GOODS OR SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS
  // INTERRUPTION)  HOWEVER CAUSED  AND ON ANY THEORY  OF LIABILITY,  WHETHER IN
  // CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)
  // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE
  // POSSIBILITY OF SUCH DAMAGE.
  
  #ifdef HAVE_CONFIG_H
  #include <config.h>
  #endif // HAVE_CONFIG_H
  
  #define _GNU_SOURCE
  #include <pthread.h>
  #include <glob.h>
  #include <dirent.h>
  #include <unistd.h>
  #include <string.h>
  #include <stdlib.h>
  #include <stdio.h>
  #include <fcntl.h>
  #include <errno.h>
  #include <sys/stat.h>
  #include <regex.h>
  #undef _GNU_SOURCE
  
  #include <opae/types.h>
  #include <opae/log.h>
  #include <opae/types_enum.h>
  
  #include "types_int.h"
  #include "sysfs_int.h"
  #include "common_int.h"
  
  // substring that identifies a sysfs directory as the FME device.
  #define FPGA_SYSFS_FME "fme"
  #define FPGA_SYSFS_FME_LEN 3
  // substring that identifies a sysfs directory as the AFU device.
  #define FPGA_SYSFS_PORT "port"
  #define FPGA_SYSFS_PORT_LEN 4
  #define OPAE_KERNEL_DRIVERS 2
  
  
  typedef struct _sysfs_formats {
  	const char *sysfs_class_path;
  	const char *sysfs_pcidrv_fpga;
  	const char *sysfs_device_fmt;
  	const char *sysfs_region_fmt;
  	const char *sysfs_device_glob;
  	const char *sysfs_fme_glob;
  	const char *sysfs_port_glob;
  	const char *sysfs_compat_id;
  	const char *sysfs_fme_pwr_glob;
  	const char *sysfs_fme_temp_glob;
  	const char *sysfs_fme_perf_glob;
  	const char *sysfs_port_err;
  	const char *sysfs_port_err_clear;
  	const char *sysfs_bmc_glob;
  	const char *sysfs_max10_glob;
  } sysfs_formats;
  
  static sysfs_formats sysfs_path_table[OPAE_KERNEL_DRIVERS] = {
  	// upstream driver sysfs formats
  	{.sysfs_class_path = "/sys/class/fpga_region",
  	 .sysfs_pcidrv_fpga = "fpga_region",
  	 .sysfs_device_fmt = "(region)([0-9])+",
  	 .sysfs_region_fmt = "dfl-(fme|port)\\.([0-9]+)",
  	 .sysfs_device_glob = "region*",
  	 .sysfs_fme_glob = "dfl-fme.*",
  	 .sysfs_port_glob = "dfl-port.*",
  	 .sysfs_compat_id = "/dfl-fme-region.*/fpga_region/region*/compat_id",
  	 .sysfs_fme_temp_glob = "hwmon/hwmon*/temp*_*",
  	 .sysfs_fme_pwr_glob = "hwmon/hwmon*/power*_*",
  	 .sysfs_fme_perf_glob = "*perf",
  	 .sysfs_port_err = "errors/errors",
  	 .sysfs_port_err_clear = "errors/errors",
  	 .sysfs_bmc_glob = "avmmi-bmc.*/bmc_info",
  	 .sysfs_max10_glob = "dfl*/*spi*/*spi*/*spi*"
  	},
  	// intel driver sysfs formats
  	{.sysfs_class_path = "/sys/class/fpga",
  	 .sysfs_pcidrv_fpga = "fpga",
  	 .sysfs_device_fmt = "(intel-fpga-dev\\.)([0-9]+)",
  	 .sysfs_region_fmt = "intel-fpga-(fme|port)\\.([0-9]+)",
  	 .sysfs_device_glob = "intel-fpga-dev.*",
  	 .sysfs_fme_glob = "intel-fpga-fme.*",
  	 .sysfs_port_glob = "intel-fpga-port.*",
  	 .sysfs_compat_id = "pr/interface_id",
  	 .sysfs_fme_temp_glob = "thermal_mgmt/*",
  	 .sysfs_fme_pwr_glob = "power_mgmt/*",
  	 .sysfs_fme_perf_glob = "*perf",
  	 .sysfs_port_err = "errors/errors",
  	 .sysfs_port_err_clear = "errors/clear",
  	 .sysfs_bmc_glob = "avmmi-bmc.*/bmc_info",
  	 .sysfs_max10_glob = "spi-*/spi_master/spi*/spi*.*"
  	} };
  
  // RE_MATCH_STRING is index 0 in a regex match array
  #define RE_MATCH_STRING 0
  // RE_DEVICE_GROUPS is the matching groups for the device regex in the
  // sysfs_path_table above.
  // Currently this only has three groups:
  // * The matching string itself - group 0
  // * The prefix (either 'region' or 'intel-fpga-dev.') - group 1
  // * The number - group 2
  // These indices are used when indexing a regex match object
  #define RE_DEVICE_GROUPS 3
  #define RE_DEVICE_GROUP_PREFIX 1
  #define RE_DEVICE_GROUP_NUM 2
  
  // RE_REGION_GROUPS is the matching groups for the region regex in the
  // sysfs_path_table above.
  // Currently this only has three groups:
  // * The matching string itself - group 0
  // * The type ('fme' or 'port') - group 1
  // * The number - group 2
  // These indices are used when indexing a regex match object
  #define RE_REGION_GROUPS 3
  #define RE_REGION_GROUP_TYPE 1
  #define RE_REGION_GROUP_NUM 2
  
  static sysfs_formats *_sysfs_format_ptr;
  static uint32_t _sysfs_device_count;
  /* mutex to protect sysfs device data structures */
  pthread_mutex_t _sysfs_device_lock = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
  
  #define SYSFS_FORMAT(s) (_sysfs_format_ptr ? _sysfs_format_ptr->s : NULL)
  
  #define SYSFS_MAX_DEVICES 128
  static sysfs_fpga_device _devices[SYSFS_MAX_DEVICES];
  
  #define PCIE_PATH_PATTERN "([0-9a-fA-F]{4}):([0-9a-fA-F]{2}):([0-9a-fA-F]{2})\\.([0-9])/fpga"
  #define PCIE_PATH_PATTERN_GROUPS 5
  
  #define PARSE_MATCH_INT(_p, _m, _v, _b, _l)                                    \
  	do {                                                                   \
  		errno = 0;                                                     \
  		_v = strtoul(_p + _m.rm_so, NULL, _b);                         \
  		if (errno) {                                                   \
  			OPAE_MSG("error parsing int");                         \
  			goto _l;                                               \
  		}                                                              \
  	} while (0)
  
  #define FREE_IF(var)                                                           \
  	do {                                                                   \
  		if (var) {                                                     \
  			free(var);                                             \
  			var = NULL;                                            \
  		}                                                              \
  	} while (0)
  
  STATIC int parse_pcie_info(sysfs_fpga_device *device, char *buffer)
  {
  	char err[128] = {0};
  	regex_t re;
  	regmatch_t matches[PCIE_PATH_PATTERN_GROUPS] = { {0} };
  	int res = FPGA_EXCEPTION;
  
  	int reg_res = regcomp(&re, PCIE_PATH_PATTERN, REG_EXTENDED | REG_ICASE);
  	if (reg_res) {
  		OPAE_ERR("Error compling regex");
  		return FPGA_EXCEPTION;
  	}
  	reg_res = regexec(&re, buffer, PCIE_PATH_PATTERN_GROUPS, matches, 0);
  	if (reg_res) {
  		regerror(reg_res, &re, err, 128);
  		OPAE_ERR("Error executing regex: %s", err);
  		res = FPGA_EXCEPTION;
  		goto out;
  	} else {
  		PARSE_MATCH_INT(buffer, matches[1], device->segment, 16, out);
  		PARSE_MATCH_INT(buffer, matches[2], device->bus, 16, out);
  		PARSE_MATCH_INT(buffer, matches[3], device->device, 16, out);
  		PARSE_MATCH_INT(buffer, matches[4], device->function, 10, out);
  	}
  	res = FPGA_OK;
  
  out:
  	regfree(&re);
  	return res;
  }
  
  int sysfs_parse_attribute64(const char *root, const char *attr_path, uint64_t *value)
  {
  	uint64_t pg_size = (uint64_t)sysconf(_SC_PAGE_SIZE);
  	char path[SYSFS_PATH_MAX] = { 0, };
  	char buffer[pg_size];
  	int fd = -1;
  	ssize_t bytes_read = 0;
  
  	snprintf(path, sizeof(path),
  		 "%s/%s", root, attr_path);
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("Error opening %s: %s", path, strerror(errno));
  		return FPGA_EXCEPTION;
  	}
  	bytes_read = eintr_read(fd, buffer, pg_size);
  	if (bytes_read < 0) {
  		OPAE_ERR("Error reading from %s: %s", path,
  			 strerror(errno));
  		close(fd);
  		return FPGA_EXCEPTION;
  	}
  
  	*value = strtoull(buffer, NULL, 0);
  
  	close(fd);
  	return FPGA_OK;
  }
  
  STATIC int parse_device_vendor_id(sysfs_fpga_device *device)
  {
  	uint64_t value = 0;
  	int res = sysfs_parse_attribute64(device->sysfs_path, "device/device", &value);
  	if (res) {
  		OPAE_MSG("Error parsing device_id for device: %s",
  			 device->sysfs_path);
  		return res;
  	}
  	device->device_id = value;
  
  	res = sysfs_parse_attribute64(device->sysfs_path, "device/vendor", &value);
  
  	if (res) {
  		OPAE_ERR("Error parsing vendor_id for device: %s",
  			 device->sysfs_path);
  		return res;
  	}
  	device->vendor_id = value;
  
  	return FPGA_OK;
  }
  
  STATIC sysfs_fpga_region *make_region(sysfs_fpga_device *device, char *name,
  				      int num, fpga_objtype type)
  {
  	size_t len;
  
  	sysfs_fpga_region *region = malloc(sizeof(sysfs_fpga_region));
  	if (region == NULL) {
  		OPAE_ERR("error creating region");
  		return NULL;
  	}
  	region->device = device;
  	region->type = type;
  	region->number = num;
  
  	// sysfs path of region is sysfs path of device + / + name
  	if (snprintf(region->sysfs_path, SYSFS_PATH_MAX,
  		     "%s/%s", device->sysfs_path, name) < 0) {
  		free(region);
  		OPAE_ERR("snprintf buffer overflow");
  		return NULL;
  	}
  
  	len = strnlen(name, SYSFS_PATH_MAX - 1);
  	memcpy(region->sysfs_name, name, len);
  	region->sysfs_name[len] = '\0';
  
  	return region;
  }
  
  /**
   * @brief Match a device node given a format pattern
   *
   * @param fmt A regex pattern for the device node
   * @param inpstr A sysfs path to a potential device node
   * @param(out) prefix[] A prefix string for the device node
   * @param prefix_len capacity of prefix (max length)
   * @param(out) num The sysfs number encoded in the name
   *
   * @note fmt is expected to be a regex pattern in our sysfs_format_table
   *       Matching input strings could could look like:
   *       * region0 where 'region' is the prefix and 0 is the num
   *       * intel-fpga-dev.0 where 'intel-fpga-dev.' is the prefix and 0 is the
   *       num
   *
   *
   * @return FPGA_OK if a match is found, FPGA_NOT_FOUND it no match is found,
   *         FPGA_EXCEPTION if an error is encountered
   */
  STATIC fpga_result re_match_device(const char *fmt, char *inpstr, char prefix[],
  				   size_t prefix_len, int *num)
  {
  	int reg_res = 0;
  	fpga_result res = FPGA_EXCEPTION;
  	regmatch_t matches[RE_DEVICE_GROUPS];
  	char err[128];
  	char *ptr = NULL;
  	char *end = NULL;
  	regex_t re;
  
  	ASSERT_NOT_NULL(fmt);
  	ASSERT_NOT_NULL(inpstr);
  	ASSERT_NOT_NULL(prefix);
  	ASSERT_NOT_NULL(num);
  	reg_res = regcomp(&re, fmt, REG_EXTENDED);
  	if (reg_res) {
  		regerror(reg_res, &re, err, sizeof(err));
  		OPAE_ERR("Error compiling regex: %s", err);
  		return FPGA_EXCEPTION;
  	}
  	reg_res = regexec(&re, inpstr, RE_DEVICE_GROUPS, matches, 0);
  	if (reg_res) {
  		return FPGA_NOT_FOUND;
  	}
  
  	ptr = inpstr + matches[RE_DEVICE_GROUP_PREFIX].rm_so;
  	end = inpstr + matches[RE_DEVICE_GROUP_PREFIX].rm_eo;
  
  	if ((size_t)(end - ptr) >= prefix_len) {
  		OPAE_ERR("Regex result too long");
  		res = FPGA_EXCEPTION;
  		goto out_free;
  	}
  
  	strncpy(prefix, ptr, end - ptr);
  	*(prefix + (end - ptr)) = '\0';
  
  	ptr = inpstr + matches[RE_DEVICE_GROUP_NUM].rm_so;
  	errno = 0;
  	*num = strtoul(ptr, NULL, 10);
  	if (errno) {
  		OPAE_ERR("Error parsing number: %s", inpstr);
  		goto out_free;
  	}
  	res = FPGA_OK;
  out_free:
  	regfree(&re);
  	return res;
  }
  
  /**
   * @brief Match a device node given a format pattern
   *
   * @param fmt A regex pattern for the device node
   * @param inpstr A sysfs path to a potential device node
   * @param(out) type[] A type string for the device node
   * @param type_len capacity of type (max length)
   * @param(out) num The sysfs number encoded in the name
   *
   * @note fmt is expected to be a regex pattern in our sysfs_format_table
   *       Matching input strings could could look like:
   *       * dfl-fme.0 where 'fme' is the type and 0 is the num
   *       * dfl-port.1 where 'port' is the type and 1 is the num
   *       * intel-fpga-fme.0 where 'fme' is the type and 0 is the num
   *       * intel-fpga-port.1 where 'port' is the type and 1 is the num
   *
   *
   * @return FPGA_OK if a match is found, FPGA_NOT_FOUND it no match is found,
   *         FPGA_EXCEPTION if an error is encountered
   */
  STATIC fpga_result re_match_region(const char *fmt, char *inpstr, char type[],
  				   size_t type_len, int *num)
  {
  	int reg_res = 0;
  	fpga_result res = FPGA_EXCEPTION;
  	regmatch_t matches[RE_REGION_GROUPS];
  	char err[128];
  	char *ptr = NULL;
  	char *end = NULL;
  	regex_t re;
  
  	ASSERT_NOT_NULL(fmt);
  	ASSERT_NOT_NULL(inpstr);
  	ASSERT_NOT_NULL(type);
  	ASSERT_NOT_NULL(num);
  	reg_res = regcomp(&re, fmt, REG_EXTENDED);
  	if (reg_res) {
  		regerror(reg_res, &re, err, sizeof(err));
  		OPAE_ERR("Error compiling regex: %s", err);
  		return FPGA_EXCEPTION;
  	}
  	reg_res = regexec(&re, inpstr, RE_REGION_GROUPS, matches, 0);
  	if (reg_res) {
  		res = FPGA_NOT_FOUND;
  		goto out_free;
  	}
  
  	ptr = inpstr + matches[RE_REGION_GROUP_TYPE].rm_so;
  	end = inpstr + matches[RE_REGION_GROUP_TYPE].rm_eo;
  
  	if ((size_t)(end - ptr) >= type_len) {
  		OPAE_ERR("Error copying type from string: %s", inpstr);
  		goto out_free;
  	}
  
  	strncpy(type, ptr, end - ptr);
  	*(type + (end - ptr)) = '\0';
  
  	ptr = inpstr + matches[RE_REGION_GROUP_NUM].rm_so;
  	errno = 0;
  	*num = strtoul(ptr, NULL, 10);
  	if (errno) {
  		OPAE_ERR("Error parsing number: %s", inpstr);
  		goto out_free;
  	}
  	res = FPGA_OK;
  out_free:
  	regfree(&re);
  	return res;
  }
  
  
  STATIC int find_regions(sysfs_fpga_device *device)
  {
  	int num = -1;
  	char type[8];
  	fpga_result res = FPGA_OK;
  	fpga_result match_res = FPGA_NOT_FOUND;
  	fpga_objtype region_type = FPGA_DEVICE;
  	sysfs_fpga_region **region_ptr = NULL;
  	struct dirent *dirent = NULL;
  	DIR *dir = opendir(device->sysfs_path);
  	if (!dir) {
  		OPAE_ERR("failed to open device path: %s", device->sysfs_path);
  		return FPGA_EXCEPTION;
  	}
  
  	while ((dirent = readdir(dir)) != NULL) {
  		res = FPGA_OK;
  		if (!strcmp(dirent->d_name, "."))
  			continue;
  		if (!strcmp(dirent->d_name, ".."))
  			continue;
  
  		match_res = re_match_region(SYSFS_FORMAT(sysfs_region_fmt),
  					    dirent->d_name, type, sizeof(type),
  					    &num);
  		if (match_res == FPGA_OK) {
  			if (!strncmp(FPGA_SYSFS_FME, type,
  				     FPGA_SYSFS_FME_LEN)) {
  				region_type = FPGA_DEVICE;
  				region_ptr = &device->fme;
  			} else if (!strncmp(FPGA_SYSFS_PORT, type,
  					    FPGA_SYSFS_PORT_LEN)) {
  				region_type = FPGA_ACCELERATOR;
  				region_ptr = &device->port;
  			}
  
  			if (region_ptr)
  				*region_ptr = make_region(device,
  				dirent->d_name, num, region_type);
  
  			region_ptr = NULL;
  
  		} else if (match_res != FPGA_NOT_FOUND) {
  			res = match_res;
  			break;
  		}
  	}
  
  	if (dir)
  		closedir(dir);
  	if (!device->fme && !device->port) {
  		OPAE_DBG("did not find fme/port in device: %s", device->sysfs_path);
  		return FPGA_NOT_FOUND;
  	}
  
  	return res;
  }
  
  
  STATIC int make_device(sysfs_fpga_device *device, const char *sysfs_class_fpga,
  		       char *dir_name, int num)
  {
  	int res = FPGA_OK;
  	char buffer[SYSFS_PATH_MAX] = { 0, };
  	ssize_t sym_link_len = 0;
  	size_t len;
  
  	if (snprintf(device->sysfs_path, SYSFS_PATH_MAX,
  		     "%s/%s", sysfs_class_fpga, dir_name) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		return FPGA_EXCEPTION;
  	}
  
  	len = strnlen(dir_name, SYSFS_PATH_MAX - 1);
  	memcpy(device->sysfs_name, dir_name, len);
  	device->sysfs_name[len] = '\0';
  
  	sym_link_len = readlink(device->sysfs_path, buffer, SYSFS_PATH_MAX);
  	if (sym_link_len < 0) {
  		OPAE_ERR("Error reading sysfs link: %s", device->sysfs_path);
  		return FPGA_EXCEPTION;
  	}
  
  	device->number = num;
  	res = parse_pcie_info(device, buffer);
  
  	if (res) {
  		OPAE_ERR("Could not parse symlink");
  		return res;
  	}
  
  	res = parse_device_vendor_id(device);
  	if (res) {
  		OPAE_MSG("Could not parse vendor/device id");
  		return res;
  	}
  
  	return find_regions(device);
  }
  
  
  
  STATIC int sysfs_device_destroy(sysfs_fpga_device *device)
  {
  	ASSERT_NOT_NULL(device);
  	if (device->fme) {
  		free(device->fme);
  		device->fme = NULL;
  	}
  	if (device->port) {
  		free(device->port);
  		device->port = NULL;
  	}
  	return FPGA_OK;
  }
  
  int sysfs_device_count(void)
  {
  	int res = 0, count = 0;
  	if (!opae_mutex_lock(res, &_sysfs_device_lock)) {
  		count = _sysfs_device_count;
  	}
  
  	if (opae_mutex_unlock(res, &_sysfs_device_lock)) {
  		count = 0;
  	}
  
  	return count;
  }
  
  fpga_result sysfs_foreach_device(device_cb cb, void *context)
  {
  	uint32_t i = 0;
  	int res = 0;
  	fpga_result result = FPGA_OK;
  	if (opae_mutex_lock(res, &_sysfs_device_lock)) {
  		return FPGA_EXCEPTION;
  	}
  
  	result = sysfs_finalize();
  	if (result) {
  		goto out_unlock;
  	}
  	result = sysfs_initialize();
  	if (result) {
  		goto out_unlock;
  	}
  	for (; i < _sysfs_device_count; ++i) {
  		result = cb(&_devices[i], context);
  		if (result) {
  			goto out_unlock;
  		}
  	}
  
  out_unlock:
  	opae_mutex_unlock(res, &_sysfs_device_lock);
  
  	return result;
  }
  
  int sysfs_initialize(void)
  {
  	int stat_res = -1;
  	int res = FPGA_OK;
  	uint32_t i = 0;
  	struct stat st;
  	DIR *dir = NULL;
  	struct dirent *dirent = NULL;
  	int num = -1;
  	char prefix[64] = {0};
  
  	for (i = 0; i < OPAE_KERNEL_DRIVERS; ++i) {
  		errno = 0;
  		stat_res = stat(sysfs_path_table[i].sysfs_class_path, &st);
  		if (!stat_res) {
  			_sysfs_format_ptr = &sysfs_path_table[i];
  			break;
  		}
  		if (errno != ENOENT) {
  			OPAE_ERR("Error while inspecting sysfs: %s",
  				 strerror(errno));
  			return FPGA_EXCEPTION;
  		}
  	}
  	if (i == OPAE_KERNEL_DRIVERS) {
  		OPAE_DBG(
  			"No valid sysfs class files found - a suitable driver may not be loaded");
  		return FPGA_NO_DRIVER;
  	}
  
  	_sysfs_device_count = 0;
  
  	const char *sysfs_class_fpga = SYSFS_FORMAT(sysfs_class_path);
  	if (!sysfs_class_fpga) {
  		OPAE_ERR("Invalid fpga class path: %s", sysfs_class_fpga);
  		res = FPGA_EXCEPTION;
  		goto out_free;
  	}
  
  	// open the root sysfs class directory
  	// look in the directory and get device objects
  	dir = opendir(sysfs_class_fpga);
  	if (!dir) {
  		OPAE_MSG("failed to open device path: %s", sysfs_class_fpga);
  		res = FPGA_EXCEPTION;
  		goto out_free;
  	}
  
  	while ((dirent = readdir(dir))) {
  		if (!strcmp(dirent->d_name, "."))
  			continue;
  		if (!strcmp(dirent->d_name, ".."))
  			continue;
  		res = re_match_device(SYSFS_FORMAT(sysfs_device_fmt),
  				      dirent->d_name, prefix, sizeof(prefix),
  				      &num);
  		if (res == FPGA_OK) {
  			// increment our device count after filling out details
  			// of the discovered device in our _devices array
  			if (opae_mutex_lock(res, &_sysfs_device_lock)) {
  				goto out_free;
  			}
  			if (make_device(&_devices[_sysfs_device_count++],
  					sysfs_class_fpga, dirent->d_name,
  					num)) {
  				OPAE_MSG("Error processing device: %s",
  					 dirent->d_name);
  				_sysfs_device_count--;
  			}
  			if (opae_mutex_unlock(res, &_sysfs_device_lock)) {
  				goto out_free;
  			}
  		}
  	}
  
  	if (!_sysfs_device_count) {
  		OPAE_DBG("Error discovering fpga devices");
  		res = FPGA_NO_DRIVER;
  	}
  out_free:
  	if (dir)
  		closedir(dir);
  	return res;
  }
  
  int sysfs_finalize(void)
  {
  	uint32_t i = 0;
  	int res = 0;
  	if (opae_mutex_lock(res, &_sysfs_device_lock)) {
  		OPAE_ERR("Error locking mutex");
  		return FPGA_EXCEPTION;
  	}
  	for (; i < _sysfs_device_count; ++i) {
  		sysfs_device_destroy(&_devices[i]);
  	}
  	_sysfs_device_count = 0;
  	_sysfs_format_ptr = NULL;
  	if (opae_mutex_unlock(res, &_sysfs_device_lock)) {
  		OPAE_ERR("Error unlocking mutex");
  		return FPGA_EXCEPTION;
  	}
  	return FPGA_OK;
  }
  
  const sysfs_fpga_device *sysfs_get_device(size_t num)
  {
  	const sysfs_fpga_device *ptr = NULL;
  	int res = 0;
  	if (!opae_mutex_lock(res, &_sysfs_device_lock)) {
  		if (num >= _sysfs_device_count) {
  			OPAE_ERR("No such device with index: %d", num);
  		} else {
  			ptr = &_devices[num];
  		}
  		if (opae_mutex_unlock(res, &_sysfs_device_lock)) {
  			ptr = NULL;
  		}
  	}
  
  	return ptr;
  }
  
  fpga_result sysfs_get_interface_id(fpga_token token, fpga_guid guid)
  {
  	fpga_result res = FPGA_OK;
  	char path[SYSFS_PATH_MAX];
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  	res = cat_token_sysfs_path(path, token, SYSFS_FORMAT(sysfs_compat_id));
  	if (res) {
  		return res;
  	}
  	res = opae_glob_path(path, SYSFS_PATH_MAX - 1);
  	if (res) {
  		return res;
  	}
  	return sysfs_read_guid(path, guid);
  }
  
  
  
  fpga_result sysfs_get_fme_pwr_path(fpga_token token, char *sysfs_pwr)
  {
  	fpga_result res = FPGA_OK;
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  
  	if (sysfs_pwr == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  	res = cat_token_sysfs_path(sysfs_pwr, token, SYSFS_FORMAT(sysfs_fme_pwr_glob));
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	// check for path is valid
  	res = check_sysfs_path_is_valid(sysfs_pwr);
  	if (res != FPGA_OK) {
  		OPAE_MSG("Invalid path %s", sysfs_pwr);
  		return res;
  	}
  
  	return res;
  }
  
  fpga_result sysfs_get_fme_temp_path(fpga_token token, char *sysfs_temp)
  {
  	fpga_result res = FPGA_OK;
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  
  	if (sysfs_temp == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	res = cat_token_sysfs_path(sysfs_temp, token, SYSFS_FORMAT(sysfs_fme_temp_glob));
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	// check for path is valid
  	res = check_sysfs_path_is_valid(sysfs_temp);
  	if (res != FPGA_OK) {
  		OPAE_MSG("Invalid path %s", sysfs_temp);
  		return res;
  	}
  
  	return res;
  }
  
  fpga_result sysfs_get_fme_perf_path(fpga_token token, char *sysfs_perf)
  {
  	fpga_result res = FPGA_OK;
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  
  	if (sysfs_perf == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	res = cat_token_sysfs_path(sysfs_perf, token, SYSFS_FORMAT(sysfs_fme_perf_glob));
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	// check for path is valid
  	res = check_sysfs_path_is_valid(sysfs_perf);
  	if (res != FPGA_OK) {
  		OPAE_MSG("Invalid path %s", sysfs_perf);
  		return res;
  	}
  
  	return res;
  }
  
  fpga_result sysfs_get_port_error_path(fpga_handle handle, char *sysfs_port_error)
  {
  	fpga_result result = FPGA_OK;
  	char sysfs_path[SYSFS_PATH_MAX] = { 0, };
  
  	if (sysfs_port_error == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	result = get_port_sysfs(handle, sysfs_path);
  	if (result != FPGA_OK) {
  		OPAE_ERR("Failed to get port syfs path");
  		return result;
  	}
  
  	if (!SYSFS_FORMAT(sysfs_port_err)) {
  		OPAE_ERR("_sysfs_format_ptr is not set.");
  		return FPGA_EXCEPTION;
  	}
  
  	if (snprintf(sysfs_port_error, SYSFS_PATH_MAX,
  		     "%s/%s", sysfs_path, _sysfs_format_ptr->sysfs_port_err) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		return FPGA_EXCEPTION;
  	}
  
  	return result;
  }
  
  fpga_result sysfs_get_port_error_clear_path(fpga_handle handle, char *sysfs_port_error_clear)
  {
  	fpga_result result = FPGA_OK;
  	char sysfs_path[SYSFS_PATH_MAX] = { 0, };
  
  	if (sysfs_port_error_clear == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	result = get_port_sysfs(handle, sysfs_path);
  	if (result != FPGA_OK) {
  		OPAE_ERR("Failed to get port syfs path");
  		return result;
  	}
  
  	if (!SYSFS_FORMAT(sysfs_port_err_clear)) {
  		OPAE_ERR("_sysfs_format_ptr is not set.");
  		return FPGA_EXCEPTION;
  	}
  
  	if (snprintf(sysfs_port_error_clear, SYSFS_PATH_MAX,
  		     "%s/%s", sysfs_path,
  		     _sysfs_format_ptr->sysfs_port_err_clear) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		return FPGA_EXCEPTION;
  	}
  
  	return result;
  }
  
  fpga_result sysfs_get_bmc_path(fpga_token token, char *sysfs_bmc)
  {
  	fpga_result res = FPGA_OK;
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  
  	if (sysfs_bmc == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	res = cat_token_sysfs_path(sysfs_bmc, token, SYSFS_FORMAT(sysfs_bmc_glob));
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	return opae_glob_path(sysfs_bmc, SYSFS_PATH_MAX - 1);
  }
  
  fpga_result sysfs_get_max10_path(fpga_token token, char *sysfs_max10)
  {
  	fpga_result res = FPGA_OK;
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  	ASSERT_NOT_NULL(_token);
  
  	if (sysfs_max10 == NULL) {
  		OPAE_ERR("Invalid input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	res = cat_token_sysfs_path(sysfs_max10, token, SYSFS_FORMAT(sysfs_max10_glob));
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	return opae_glob_path(sysfs_max10, SYSFS_PATH_MAX - 1);
  }
  
  fpga_result sysfs_get_fme_pr_interface_id(const char *sysfs_sysfs_path, fpga_guid guid)
  {
  	fpga_result res = FPGA_OK;
  	char sysfs_path[SYSFS_PATH_MAX] = { 0, };
  
  	if (!SYSFS_FORMAT(sysfs_compat_id)) {
  		OPAE_ERR("_sysfs_format_ptr is not set.");
  		return FPGA_EXCEPTION;
  	}
  
  	snprintf(sysfs_path, sizeof(sysfs_path),
  		 "%s/%s",
  		 sysfs_sysfs_path,
  		 _sysfs_format_ptr->sysfs_compat_id);
  
  	res = opae_glob_path(sysfs_path, SYSFS_PATH_MAX - 1);
  	if (res)
  		return res;
  
  	return sysfs_read_guid(sysfs_path, guid);
  }
  
  fpga_result sysfs_get_guid(fpga_token token, const char *sysfspath, fpga_guid guid)
  {
  	fpga_result res = FPGA_OK;
  	char sysfs_path[SYSFS_PATH_MAX] = { 0, };
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  
  	if (_token == NULL || sysfspath == NULL)
  		return FPGA_EXCEPTION;
  
  	if (snprintf(sysfs_path, sizeof(sysfs_path),
  		     "%s/%s", _token->sysfspath, sysfspath) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		return FPGA_EXCEPTION;
  	}
  
  	res = opae_glob_path(sysfs_path, SYSFS_PATH_MAX - 1);
  	if (res)
  		return res;
  
  	return sysfs_read_guid(sysfs_path, guid);
  }
  
  int sysfs_filter(const struct dirent *de)
  {
  	return de->d_name[0] != '.';
  }
  
  
  /**
   * @brief Get a path to an fme node given a path to a port node
   *
   * @param sysfs_port sysfs path to a port node
   * @param(out) sysfs_fme realpath to an fme node in sysfs
   *
   * @return FPGA_OK if able to find the path to the fme
   *         FPGA_EXCEPTION if errors encountered during copying,
   *         formatting strings
   *         FPGA_NOT_FOUND if unable to find fme path or any relevant paths
   */
  fpga_result sysfs_get_fme_path(const char *sysfs_port, char *sysfs_fme)
  {
  	fpga_result result = FPGA_EXCEPTION;
  	char sysfs_path[SYSFS_PATH_MAX]   = { 0, };
  	char fpga_path[SYSFS_PATH_MAX]    = { 0, };
  	// subdir candidates to look for when locating "fpga*" node in sysfs
  	// order is important here because a physfn node is the exception
  	// (will only exist when a port is on a VF) and will be used to point
  	// to the PF that the FME is on
  	const char *fpga_globs[] = {"device/physfn/fpga*", "device/fpga*", NULL};
  	int i = 0;
  	size_t len;
  
  	// now try globbing fme resource sysfs path + a candidate
  	// sysfs_port is expected to be the sysfs path to a port
  	for (; fpga_globs[i]; ++i) {
  
  		snprintf(sysfs_path, SYSFS_PATH_MAX,
  			 "%s/../%s", sysfs_port, fpga_globs[i]);
  
  		result = opae_glob_path(sysfs_path, SYSFS_PATH_MAX - 1);
  		if (result == FPGA_OK) {
  			// we've found a path to the "fpga*" node
  			break;
  		} else if (result != FPGA_NOT_FOUND) {
  			return result;
  		}
  	}
  
  	if (!fpga_globs[i]) {
  		OPAE_ERR("Could not find path to port device/fpga*");
  		return FPGA_NOT_FOUND;
  	}
  
  
  	// format a string to look for in the subdirectory of the "fpga*" node
  	// this subdirectory should include glob patterns for the current
  	// driver
  	// -- intel-fpga-dev.*/intel-fpga-fme.*
  	// -- region*/dfl-fme.*
  
  	if (!SYSFS_FORMAT(sysfs_device_glob)) {
  		OPAE_ERR("_sysfs_format_ptr is not set.");
  		return FPGA_EXCEPTION;
  	}
  
  	snprintf(fpga_path, sizeof(fpga_path),
  		 "/%s/%s",
  		 _sysfs_format_ptr->sysfs_device_glob,
  		 _sysfs_format_ptr->sysfs_fme_glob);
  
  
  	len = strnlen(sysfs_path, SYSFS_PATH_MAX - 1);
  	strncat(sysfs_path, fpga_path, SYSFS_PATH_MAX - len);
  
  	result = opae_glob_path(sysfs_path, SYSFS_PATH_MAX - 1);
  	if (result)
  		return result;
  
  	// copy the assembled and verified path to the output param
  	if (!realpath(sysfs_path, sysfs_fme))
  		return FPGA_EXCEPTION;
  
  	return FPGA_OK;
  }
  
  //
  // sysfs access (read/write) functions
  //
  
  fpga_result sysfs_read_int(const char *path, int *i)
  {
  	int fd;
  	int res;
  	char buf[SYSFS_PATH_MAX];
  	int b;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed", path);
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek failed");
  		goto out_close;
  	}
  
  	b = 0;
  
  	do {
  		res = read(fd, buf + b, sizeof(buf) - b);
  		if (res <= 0) {
  			OPAE_MSG("Read from %s failed", path);
  			goto out_close;
  		}
  		b += res;
  		if (((unsigned)b > sizeof(buf)) || (b <= 0)) {
  			OPAE_MSG("Unexpected size reading from %s", path);
  			goto out_close;
  		}
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && (unsigned)b < sizeof(buf));
  
  	// erase \n
  	buf[b - 1] = 0;
  
  	*i = atoi(buf);
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  fpga_result sysfs_read_u32(const char *path, uint32_t *u)
  {
  	int fd;
  	int res;
  	char buf[SYSFS_PATH_MAX];
  	int b;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed", path);
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek failed");
  		goto out_close;
  	}
  
  	b = 0;
  
  	do {
  		res = read(fd, buf + b, sizeof(buf) - b);
  		if (res <= 0) {
  			OPAE_MSG("Read from %s failed", path);
  			goto out_close;
  		}
  		b += res;
  		if (((unsigned)b > sizeof(buf)) || (b <= 0)) {
  			OPAE_MSG("Unexpected size reading from %s", path);
  			goto out_close;
  		}
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && (unsigned)b < sizeof(buf));
  
  	// erase \n
  	buf[b - 1] = 0;
  
  	*u = strtoul(buf, NULL, 0);
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  // read tuple separated by 'sep' character
  fpga_result sysfs_read_u32_pair(const char *path, uint32_t *u1, uint32_t *u2,
  				char sep)
  {
  	int fd;
  	int res;
  	char buf[SYSFS_PATH_MAX];
  	int b;
  	char *c;
  	uint32_t x1, x2;
  
  	if (sep == '\0') {
  		OPAE_MSG("invalid separation character");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed", path);
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek failed");
  		goto out_close;
  	}
  
  	b = 0;
  
  	do {
  		res = read(fd, buf + b, sizeof(buf) - b);
  		if (res <= 0) {
  			OPAE_MSG("Read from %s failed", path);
  			goto out_close;
  		}
  		b += res;
  		if (((unsigned)b > sizeof(buf)) || (b <= 0)) {
  			OPAE_MSG("Unexpected size reading from %s", path);
  			goto out_close;
  		}
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && (unsigned)b < sizeof(buf));
  
  	// erase \n
  	buf[b - 1] = 0;
  
  	// read first value
  	x1 = strtoul(buf, &c, 0);
  	if (*c != sep) {
  		OPAE_MSG("couldn't find separation character '%c' in '%s'", sep,
  			 path);
  		goto out_close;
  	}
  	// read second value
  	x2 = strtoul(c + 1, &c, 0);
  	if (*c != '\0') {
  		OPAE_MSG("unexpected character '%c' in '%s'", *c, path);
  		goto out_close;
  	}
  
  	*u1 = x1;
  	*u2 = x2;
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  fpga_result sysfs_read_u64(const char *path, uint64_t *u)
  {
  	int fd = -1;
  	int res = 0;
  	char buf[SYSFS_PATH_MAX] = {0};
  	int b = 0;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed", path);
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek failed");
  		goto out_close;
  	}
  
  	do {
  		res = read(fd, buf + b, sizeof(buf) - b);
  		if (res <= 0) {
  			OPAE_MSG("Read from %s failed", path);
  			goto out_close;
  		}
  		b += res;
  		if (((unsigned)b > sizeof(buf)) || (b <= 0)) {
  			OPAE_MSG("Unexpected size reading from %s", path);
  			goto out_close;
  		}
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && (unsigned)b < sizeof(buf));
  
  	// erase \n
  	buf[b - 1] = 0;
  
  	*u = strtoull(buf, NULL, 0);
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  fpga_result sysfs_write_u64(const char *path, uint64_t u)
  {
  	int fd = -1;
  	int res = 0;
  	char buf[SYSFS_PATH_MAX] = {0};
  	int b = 0;
  	int len;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_WRONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed: %s", path, strerror(errno));
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek: %s", strerror(errno));
  		goto out_close;
  	}
  
  	len = snprintf(buf, sizeof(buf), "0x%lx\n", u);
  
  	do {
  		res = write(fd, buf + b, len - b);
  		if (res <= 0) {
  			OPAE_ERR("Failed to write");
  			goto out_close;
  		}
  		b += res;
  
  		if (b > len || b <= 0) {
  			OPAE_MSG("Unexpected size writing to %s", path);
  			goto out_close;
  		}
  
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && b < len);
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  
  fpga_result sysfs_write_u64_decimal(const char *path, uint64_t u)
  {
  	int fd = -1;
  	int res = 0;
  	char buf[SYSFS_PATH_MAX] = {0};
  	int b = 0;
  	int len;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_WRONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed: %s", path, strerror(errno));
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek: %s", strerror(errno));
  		goto out_close;
  	}
  
  	len = snprintf(buf, sizeof(buf), "%ld\n", u);
  
  	do {
  		res = write(fd, buf + b, len - b);
  		if (res <= 0) {
  			OPAE_ERR("Failed to write");
  			goto out_close;
  		}
  		b += res;
  
  		if (b > len || b <= 0) {
  			OPAE_MSG("Unexpected size writing to %s", path);
  			goto out_close;
  		}
  
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && b < len);
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  fpga_result sysfs_read_guid(const char *path, fpga_guid guid)
  {
  	int fd;
  	int res;
  	char buf[SYSFS_PATH_MAX] = { 0, };
  	int b;
  
  	int i;
  	char tmp;
  	unsigned octet;
  
  	if (path == NULL) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	fd = open(path, O_RDONLY);
  	if (fd < 0) {
  		OPAE_MSG("open(%s) failed", path);
  		return FPGA_NOT_FOUND;
  	}
  
  	if ((off_t)-1 == lseek(fd, 0, SEEK_SET)) {
  		OPAE_MSG("seek failed");
  		goto out_close;
  	}
  
  	b = 0;
  
  	do {
  		res = read(fd, buf + b, sizeof(buf) - b);
  		if (res <= 0) {
  			OPAE_MSG("Read from %s failed", path);
  			goto out_close;
  		}
  		b += res;
  		if (((unsigned)b > sizeof(buf)) || (b <= 0)) {
  			OPAE_MSG("Unexpected size reading from %s", path);
  			goto out_close;
  		}
  	} while (buf[b - 1] != '\n' && buf[b - 1] != '\0'
  		 && (unsigned)b < sizeof(buf));
  
  	// erase \n
  	buf[b - 1] = 0;
  
  	for (i = 0; i < 32; i += 2) {
  		tmp = buf[i + 2];
  		buf[i + 2] = 0;
  
  		octet = 0;
  		sscanf(&buf[i], "%x", &octet);
  		guid[i / 2] = (uint8_t)octet;
  
  		buf[i + 2] = tmp;
  	}
  
  	close(fd);
  	return FPGA_OK;
  
  out_close:
  	close(fd);
  	return FPGA_NOT_FOUND;
  }
  
  fpga_result check_sysfs_path_is_valid(const char *sysfs_path)
  {
  	fpga_result result = FPGA_OK;
  	char path[SYSFS_PATH_MAX] = { 0, };
  	struct stat stats;
  	size_t len;
  
  	if (!sysfs_path) {
  		OPAE_ERR("Invalid input path");
  		return FPGA_INVALID_PARAM;
  	}
  
  	len = strnlen(sysfs_path, SYSFS_PATH_MAX - 1);
  	memcpy(path, sysfs_path, len);
  	path[len] = '\0';
  
  	result = opae_glob_path(path, SYSFS_PATH_MAX - 1);
  	if (result) {
  		return result;
  	}
  
  	if (stat(path, &stats) != 0) {
  		OPAE_ERR("stat failed: %s", strerror(errno));
  		return FPGA_NOT_FOUND;
  	}
  
  	if (S_ISDIR(stats.st_mode) || S_ISREG(stats.st_mode)) {
  		return FPGA_OK;
  	}
  
  	return FPGA_EXCEPTION;
  }
  
  
  fpga_result sysfs_path_is_valid(const char *root, const char *attr_path)
  {
  	char path[SYSFS_PATH_MAX]    = { 0, };
  	fpga_result result          = FPGA_OK;
  	struct stat stats;
  
  	if (!root || !attr_path) {
  		OPAE_ERR("input path is NULL");
  		return FPGA_INVALID_PARAM;
  	}
  
  	snprintf(path, sizeof(path),
  		 "%s/%s", root, attr_path);
  
  	result = opae_glob_path(path, SYSFS_PATH_MAX - 1);
  	if (result) {
  		return result;
  	}
  
  	if (stat(path, &stats) != 0) {
  		OPAE_ERR("stat failed: %s", strerror(errno));
  		return FPGA_NOT_FOUND;
  	}
  
  	if (S_ISDIR(stats.st_mode) || S_ISREG(stats.st_mode)) {
  		return FPGA_OK;
  	}
  
  	return FPGA_EXCEPTION;
  }
  
  //
  // sysfs convenience functions to access device components by device number
  //
  
  fpga_result sysfs_get_socket_id(int dev, int subdev, uint8_t *socket_id)
  {
  	fpga_result result;
  	char spath[SYSFS_PATH_MAX] = { 0, };
  	int i;
  
  	snprintf(spath, SYSFS_PATH_MAX,
  		 SYSFS_FPGA_CLASS_PATH SYSFS_FME_PATH_FMT
  		 "/" FPGA_SYSFS_SOCKET_ID,
  		 dev, subdev);
  
  	i = 0;
  	result = sysfs_read_int(spath, &i);
  	if (FPGA_OK != result)
  		return result;
  
  	*socket_id = (uint8_t)i;
  
  	return FPGA_OK;
  }
  
  fpga_result sysfs_get_afu_id(int dev, int subdev, fpga_guid guid)
  {
  	char spath[SYSFS_PATH_MAX] = { 0, };
  
  	snprintf(spath, SYSFS_PATH_MAX,
  		 SYSFS_FPGA_CLASS_PATH SYSFS_AFU_PATH_FMT
  		 "/" FPGA_SYSFS_AFU_GUID,
  		 dev, subdev);
  
  	return sysfs_read_guid(spath, guid);
  }
  
  fpga_result sysfs_get_pr_id(int dev, int subdev, fpga_guid guid)
  {
  	char spath[SYSFS_PATH_MAX] = { 0, };
  
  	snprintf(spath, SYSFS_PATH_MAX,
  		 SYSFS_FPGA_CLASS_PATH SYSFS_FME_PATH_FMT
  		 "/" FPGA_SYSFS_FME_INTERFACE_ID,
  		 dev, subdev);
  
  	return sysfs_read_guid(spath, guid);
  }
  
  fpga_result sysfs_get_slots(int dev, int subdev, uint32_t *slots)
  {
  	char spath[SYSFS_PATH_MAX] = { 0, };
  
  	snprintf(spath, SYSFS_PATH_MAX,
  		 SYSFS_FPGA_CLASS_PATH SYSFS_FME_PATH_FMT
  		 "/" FPGA_SYSFS_NUM_SLOTS,
  		 dev, subdev);
  
  	return sysfs_read_u32(spath, slots);
  }
  
  fpga_result sysfs_get_bitstream_id(int dev, int subdev, uint64_t *id)
  {
  	char spath[SYSFS_PATH_MAX] = { 0, };
  
  	snprintf(spath, SYSFS_PATH_MAX,
  		 SYSFS_FPGA_CLASS_PATH SYSFS_FME_PATH_FMT
  		 "/" FPGA_SYSFS_BITSTREAM_ID,
  		 dev, subdev);
  
  	return sysfs_read_u64(spath, id);
  }
  
  /**
   * @brief Get a path to a port node given a handle to an resource
   *
   * @param handle Open handle to an fme resource (FPGA_DEVICE)
   * @param(out) sysfs_port realpath to a port node in sysfs
   *
   * @return FPGA_OK if able to find the path to the port
   *         FPGA_EXCEPTION if errors encountered during copying,
   *         formatting strings
   *         FPGA_NOT_FOUND if unable to find fme path or any relevant paths
   */
  fpga_result get_port_sysfs(fpga_handle handle, char *sysfs_port)
  {
  
  	struct _fpga_token *_token;
  	struct _fpga_handle *_handle      = (struct _fpga_handle *)handle;
  	char sysfs_path[SYSFS_PATH_MAX]   = { 0, };
  	char fpga_path[SYSFS_PATH_MAX]    = { 0, };
  	fpga_result result                = FPGA_OK;
  	int i = 0;
  	size_t len;
  
  	// subdir candidates to look for when locating "fpga*" node in sysfs
  	// order is important here because a virtfn* node is the exception
  	// (will only exist when a port is on a VF) and will be used to point
  	// to the VF that the port is on
  	const char *fpga_globs[] = {"device/virtfn*/fpga*", "device/fpga*", NULL};
  	if (sysfs_port == NULL) {
  		OPAE_ERR("Invalid output pointer");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (_handle == NULL) {
  		OPAE_ERR("Invalid handle");
  		return FPGA_INVALID_PARAM;
  	}
  
  	_token = (struct _fpga_token *)_handle->token;
  	if (_token == NULL) {
  		OPAE_ERR("Token not found");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (!strstr(_token->sysfspath, FPGA_SYSFS_FME)) {
  		OPAE_ERR("Invalid sysfspath in token");
  		return FPGA_INVALID_PARAM;
  	}
  
  	// now try globbing fme token's sysfs path + a candidate
  	for (; fpga_globs[i]; ++i) {
  
  		if (snprintf(sysfs_path, SYSFS_PATH_MAX,
  			 "%s/../%s", _token->sysfspath, fpga_globs[i]) < 0) {
  			OPAE_ERR("snprintf buffer overflow");
  			return FPGA_EXCEPTION;
  		}
  
  		result = opae_glob_path(sysfs_path, SYSFS_PATH_MAX - 1);
  		if (result == FPGA_OK) {
  			// we've found a path to the "fpga*" node
  			break;
  		} else if (result != FPGA_NOT_FOUND) {
  			return result;
  		}
  	}
  
  	if (!fpga_globs[i]) {
  		OPAE_ERR("Could not find path to port device/fpga");
  		return FPGA_EXCEPTION;
  	}
  
  	if (!SYSFS_FORMAT(sysfs_device_glob) ||
  	    !SYSFS_FORMAT(sysfs_port_glob)) {
  		OPAE_ERR("NULL glob pattern");
  		return FPGA_EXCEPTION;
  	}
  
  	// format a string to look for in the subdirectory of the "fpga*" node
  	// this subdirectory should include glob patterns for the current
  	// driver
  	// -- intel-fgga-dev.*/intel-fpga-port.*
  	// -- region*/dfl-port.*
  	snprintf(fpga_path, SYSFS_PATH_MAX, "/%s/%s",
  		 SYSFS_FORMAT(sysfs_device_glob),
  		 SYSFS_FORMAT(sysfs_port_glob));
  
  	// now concatenate the subdirectory to the "fpga*" node
  	len = strnlen(fpga_path, SYSFS_PATH_MAX - 1);
  	strncat(sysfs_path, fpga_path, len + 1);
  
  	result = opae_glob_path(sysfs_path, sizeof(sysfs_path) - 1);
  	if (result) {
  		return result;
  	}
  
  
  	// copy the assembled and verified path to the output param
  	if (!realpath(sysfs_path, sysfs_port)) {
  		return FPGA_EXCEPTION;
  	}
  
  	return FPGA_OK;
  }
  
  enum fpga_hw_type opae_id_to_hw_type(uint16_t vendor_id, uint16_t device_id)
  {
  	enum fpga_hw_type hw_type = FPGA_HW_UNKNOWN;
  
  	if (vendor_id == 0x8086) {
  
  		switch (device_id) {
  		case 0xbcbc: /* FALLTHROUGH */
  		case 0xbcbd: /* FALLTHROUGH */
  		case 0xbcbe: /* FALLTHROUGH */
  		case 0xbcbf: /* FALLTHROUGH */
  		case 0xbcc0: /* FALLTHROUGH */
  		case 0xbcc1: /* FALLTHROUGH */
  		case 0x09cb:
  			hw_type = FPGA_HW_MCP;
  		break;
  
  		case 0x09c4: /* FALLTHROUGH */
  		case 0x09c5:
  			hw_type = FPGA_HW_DCP_RC;
  		break;
  
  		case 0x0b2b: /* FALLTHROUGH */
  		case 0x0b2c:
  		case 0xaf00:
  		case 0xbcce:
  			hw_type = FPGA_HW_DCP_D5005;
  		break;
  
  		case 0x0b30: /* FALLTHROUGH */
  		case 0x0b31:
  			hw_type = FPGA_HW_DCP_N3000;
  		break;
  
  		default:
  			OPAE_ERR("unknown device id: 0x%04x", device_id);
  		}
  
  	} else if (vendor_id == 0x1c2c) { /* Silicom Denmark */
  
  		switch (device_id) {
  		case 0x1000: /* Lightning Creek */
  			hw_type = FPGA_HW_DCP_N5010;
  		break;
  		case 0x1001: /* Lightning Creek featuring E810 Adapter */
  			hw_type = FPGA_HW_DCP_N5011;
  		break;
  		default:
  			OPAE_ERR("unknown Silicom device id: 0x%04x", device_id);
  		}
  
  	} else {
  		OPAE_ERR("unknown vendor id: 0x%04x", vendor_id);
  	}
  
  	return hw_type;
  }
  
  // get fpga hardware type from handle
  fpga_result get_fpga_hw_type(fpga_handle handle, enum fpga_hw_type *hw_type)
  {
  	struct _fpga_token *_token = NULL;
  	struct _fpga_handle *_handle = (struct _fpga_handle *)handle;
  	char sysfs_path[SYSFS_PATH_MAX] = {0};
  	fpga_result result = FPGA_OK;
  	int err = 0;
  	uint64_t vendor_id = 0;
  	uint64_t device_id = 0;
  
  	if (_handle == NULL) {
  		OPAE_ERR("Invalid handle");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (hw_type == NULL) {
  		OPAE_ERR("Invalid input Parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (pthread_mutex_lock(&_handle->lock)) {
  		OPAE_MSG("Failed to lock handle mutex");
  		return FPGA_EXCEPTION;
  	}
  
  	_token = (struct _fpga_token *)_handle->token;
  	if (_token == NULL) {
  		OPAE_ERR("Token not found");
  		result = FPGA_INVALID_PARAM;
  		goto out_unlock;
  	}
  
  	if (snprintf(sysfs_path, SYSFS_PATH_MAX,
  		     "%s/../device/vendor", _token->sysfspath) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		result = FPGA_EXCEPTION;
  		goto out_unlock;
  	}
  
  	result = sysfs_read_u64(sysfs_path, &vendor_id);
  	if (result != 0) {
  		OPAE_ERR("Failed to read vendor ID");
  		goto out_unlock;
  	}
  
  	if (snprintf(sysfs_path, SYSFS_PATH_MAX,
  		     "%s/../device/device", _token->sysfspath) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		result = FPGA_EXCEPTION;
  		goto out_unlock;
  	}
  
  	result = sysfs_read_u64(sysfs_path, &device_id);
  	if (result != 0) {
  		OPAE_ERR("Failed to read device ID");
  		goto out_unlock;
  	}
  
  	*hw_type = opae_id_to_hw_type((uint16_t)vendor_id,
  				      (uint16_t)device_id);
  
  out_unlock:
  	err = pthread_mutex_unlock(&_handle->lock);
  	if (err)
  		OPAE_ERR("pthread_mutex_unlock() failed: %s", strerror(err));
  	return result;
  }
  
  /*
   * The rlpath path is assumed to be of the form:
   * ../../devices/pci0000:5e/0000:5e:00.0/fpga/intel-fpga-dev.0
   */
  fpga_result sysfs_sbdf_from_path(const char *sysfspath, int *s, int *b, int *d,
  				 int *f)
  {
  	int res;
  	char rlpath[SYSFS_PATH_MAX];
  	char *p;
  
  	res = readlink(sysfspath, rlpath, sizeof(rlpath)-1);
  	if (-1 == res) {
  		OPAE_MSG("Can't read link %s (no driver?)", sysfspath);
  		return FPGA_NO_DRIVER;
  	}
  
  	// Find the BDF from the link path.
  	rlpath[res] = 0;
  	p = strrchr(rlpath, '/');
  	if (!p) {
  		OPAE_MSG("Invalid link %s (no driver?)", rlpath);
  		return FPGA_NO_DRIVER;
  	}
  	*p = 0;
  	p = strrchr(rlpath, '/');
  	if (!p) {
  		OPAE_MSG("Invalid link %s (no driver?)", rlpath);
  		return FPGA_NO_DRIVER;
  	}
  	*p = 0;
  	p = strrchr(rlpath, '/');
  	if (!p) {
  		OPAE_MSG("Invalid link %s (no driver?)", rlpath);
  		return FPGA_NO_DRIVER;
  	}
  	++p;
  
  	//           11
  	// 012345678901
  	// ssss:bb:dd.f
  	*f = (int)strtoul(p + 11, NULL, 16);
  	*(p + 10) = 0;
  
  	*d = (int)strtoul(p + 8, NULL, 16);
  	*(p + 7) = 0;
  
  	*b = (int)strtoul(p + 5, NULL, 16);
  	*(p + 4) = 0;
  
  	*s = (int)strtoul(p, NULL, 16);
  
  	return FPGA_OK;
  }
  
  fpga_result sysfs_objectid_from_path(const char *sysfspath, uint64_t *object_id)
  {
  	char sdevpath[SYSFS_PATH_MAX] = { 0, };
  	uint32_t major = 0;
  	uint32_t minor = 0;
  	fpga_result result;
  
  	snprintf(sdevpath, SYSFS_PATH_MAX,
  		 "%s/dev", sysfspath);
  
  	result = sysfs_read_u32_pair(sdevpath, &major, &minor, ':');
  	if (FPGA_OK != result)
  		return result;
  
  	*object_id = ((major & 0xFFF) << 20) | (minor & 0xFFFFF);
  
  	return FPGA_OK;
  }
  
  ssize_t eintr_read(int fd, void *buf, size_t count)
  {
  	ssize_t bytes_read = 0, total_read = 0;
  	char *ptr = buf;
  	while (total_read < (ssize_t)count) {
  		bytes_read = read(fd, ptr + total_read, count - total_read);
  
  		if (bytes_read < 0) {
  			if (errno == EINTR) {
  				continue;
  			}
  			return bytes_read;
  		} else if (bytes_read == 0) {
  			return lseek(fd, 0, SEEK_CUR);
  		} else {
  			total_read += bytes_read;
  		}
  	}
  	return total_read;
  }
  
  ssize_t eintr_write(int fd, void *buf, size_t count)
  {
  	ssize_t bytes_written = 0, total_written = 0;
  	char *ptr = buf;
  
  	if (!buf) {
  		return -1;
  	}
  
  	while (total_written < (ssize_t)count) {
  		bytes_written =
  			write(fd, ptr + total_written, count - total_written);
  		if (bytes_written < 0) {
  			if (errno == EINTR) {
  				continue;
  			}
  			return bytes_written;
  		}
  		total_written += bytes_written;
  	}
  	return total_written;
  }
  
  fpga_result cat_token_sysfs_path(char *dest, fpga_token token, const char *path)
  {
  	struct _fpga_token *_token = (struct _fpga_token *)token;
  
  	if (!dest) {
  		OPAE_ERR("destination str is NULL");
  		return FPGA_EXCEPTION;
  	}
  
  	if (!path) {
  		OPAE_ERR("path str is NULL");
  		return FPGA_EXCEPTION;
  	}
  
  	if (snprintf(dest, SYSFS_PATH_MAX,
  		     "%s/%s", _token->sysfspath, path) < 0) {
  		OPAE_ERR("snprintf buffer overflow");
  		return FPGA_EXCEPTION;
  	}
  
  	return FPGA_OK;
  }
  
  
  fpga_result cat_sysfs_path(char *dest, const char *path)
  {
  	size_t len_dest;
  	size_t len_path;
  
  	if (!dest || !path) {
  		OPAE_ERR("NULL pointer in name");
  		return FPGA_INVALID_PARAM;
  	}
  
  	len_dest = strnlen(dest, SYSFS_PATH_MAX);
  	len_path = strnlen(path, SYSFS_PATH_MAX);
  
  	if (len_dest + len_path > SYSFS_PATH_MAX) {
  		OPAE_ERR("concat strings too long");
  		return FPGA_EXCEPTION;
  	}
  
  	strncat(dest, path, SYSFS_PATH_MAX);
  
  	return FPGA_OK;
  }
  
  fpga_result cat_handle_sysfs_path(char *dest, fpga_handle handle,
  				  const char *path)
  {
  	struct _fpga_handle *_handle = (struct _fpga_handle *)(handle);
  	return cat_token_sysfs_path(dest, _handle->token, path);
  }
  
  STATIC char *cstr_dup(const char *str)
  {
  	size_t s;
  	char *p;
  
  	if (!str) {
  		OPAE_ERR("NULL param to cstr_dup");
  		return NULL;
  	}
  
  	s = strnlen(str, PATH_MAX - 1);
  	p = malloc(s+1);
  	if (!p) {
  		OPAE_ERR("malloc failed");
  		return NULL;
  	}
  
  	strncpy(p, str, s + 1);
  	p[s] = '\0';
  
  	return p;
  }
  
  struct _fpga_object *alloc_fpga_object(const char *sysfspath, const char *name)
  {
  	struct _fpga_object *obj = calloc(1, sizeof(struct _fpga_object));
  	if (obj) {
  		pthread_mutexattr_t mattr;
  		if (pthread_mutexattr_init(&mattr)) {
  			OPAE_ERR("pthread_mutexattr_init() failed");
  			goto out_err;
  		}
  		if (pthread_mutexattr_settype(&mattr,
  					      PTHREAD_MUTEX_RECURSIVE)) {
  			OPAE_ERR("pthread_mutexattr_settype() failed");
  			pthread_mutexattr_destroy(&mattr);
  			goto out_err;
  		}
  		if (pthread_mutex_init(&obj->lock, &mattr)) {
  			OPAE_ERR("pthread_mutex_init() failed");
  			pthread_mutexattr_destroy(&mattr);
  			goto out_err;
  		}
  
  		pthread_mutexattr_destroy(&mattr);
  		obj->handle = NULL;
  		obj->path = cstr_dup(sysfspath);
  		obj->name = cstr_dup(name);
  		obj->perm = 0;
  		obj->size = 0;
  		obj->max_size = 0;
  		obj->buffer = NULL;
  		obj->objects = NULL;
  	}
  	return obj;
  out_err:
  	if (obj) {
  		free(obj);
  		obj = NULL;
  	}
  	return obj;
  }
  
  fpga_result destroy_fpga_object(struct _fpga_object *obj)
  {
  	fpga_result res = FPGA_OK;
  	FREE_IF(obj->path);
  	FREE_IF(obj->name);
  	FREE_IF(obj->buffer);
  	while (obj->size && obj->objects) {
  		res = destroy_fpga_object(
  			(struct _fpga_object *)obj->objects[--obj->size]);
  		if (res) {
  			OPAE_ERR("Error freeing subobject");
  			return res;
  		}
  	}
  	FREE_IF(obj->objects);
  
  	if (pthread_mutex_unlock(&obj->lock)) {
  		OPAE_MSG("pthread_mutex_unlock() failed");
  	}
  
  	if (pthread_mutex_destroy(&obj->lock)) {
  		OPAE_ERR("Error destroying mutex");
  		res = FPGA_EXCEPTION;
  	}
  	free(obj);
  	return res;
  }
  
  fpga_result opae_glob_path(char *path, size_t len)
  {
  	fpga_result res = FPGA_OK;
  	glob_t pglob;
  	pglob.gl_pathc = 0;
  	pglob.gl_pathv = NULL;
  	int globres = glob(path, 0, NULL, &pglob);
  	size_t glob_len;
  	if (!globres) {
  		if (pglob.gl_pathc > 1) {
  			OPAE_MSG("Ambiguous object key - using first one");
  		}
  		glob_len = strnlen(pglob.gl_pathv[0], len-1);
  		memcpy(path, pglob.gl_pathv[0], glob_len);
  		path[glob_len] = '\0';
  		globfree(&pglob);
  	} else {
  		switch (globres) {
  		case GLOB_NOSPACE:
  			res = FPGA_NO_MEMORY;
  			break;
  		case GLOB_NOMATCH:
  			res = FPGA_NOT_FOUND;
  			break;
  		default:
  			res = FPGA_EXCEPTION;
  		}
  		if (pglob.gl_pathv) {
  			globfree(&pglob);
  		}
  	}
  	return res;
  }
  
  
  fpga_result opae_glob_paths(const char *path, size_t found_max, char *found[],
  			    size_t *num_found)
  {
  	fpga_result res = FPGA_OK;
  	glob_t pglob;
  	pglob.gl_pathc = 0;
  	pglob.gl_pathv = NULL;
  	int globres = glob(path, 0, NULL, &pglob);
  	size_t i = 0;
  	size_t to_copy = 0;
  
  	if (!globres) {
  		*num_found = pglob.gl_pathc;
  		to_copy = *num_found < found_max ? *num_found : found_max;
  		while (found && i < to_copy) {
  			found[i] = cstr_dup(pglob.gl_pathv[i]);
  			if (!found[i]) {
  				// we had an error duplicating the string
  				// undo what we've duplicated so far
  				while (i) {
  					free(found[--i]);
  					found[i] = NULL;
  				}
  				OPAE_ERR("Could not copy globbed path");
  				res = FPGA_EXCEPTION;
  				goto out_free;
  			}
  			i++;
  		}
  
  	} else {
  		switch (globres) {
  		case GLOB_NOSPACE:
  			res = FPGA_NO_MEMORY;
  			break;
  		case GLOB_NOMATCH:
  			res = FPGA_NOT_FOUND;
  			break;
  		default:
  			res = FPGA_EXCEPTION;
  		}
  	}
  out_free:
  	if (pglob.gl_pathv) {
  		globfree(&pglob);
  	}
  	return res;
  }
  
  #define MAX_SYSOBJECT_FILESIZE 0x40000
  static ssize_t find_eof(int fd)
  {
  	uint64_t pg_size = (uint64_t)sysconf(_SC_PAGE_SIZE);
  	char buffer[pg_size];
  	ssize_t bytes_read = 0, total_read = 0;
  	while (total_read <= MAX_SYSOBJECT_FILESIZE) {
  		bytes_read = read(fd, buffer, pg_size);
  		if (bytes_read < 0) {
  			if (errno == EINTR) {
  				continue;
  			}
  			lseek(fd, 0, SEEK_SET);
  			return bytes_read;
  		} else if (bytes_read == 0) {
  			break;
  		} else {
  			total_read += bytes_read;
  		}
  	}
  	lseek(fd, 0, SEEK_SET);
  	return total_read;
  }
  
  static fpga_result sync_object_size(struct _fpga_object *_obj, int fd)
  {
  	off_t size;
  	uint8_t *buffer;
  	size = find_eof(fd);
  	if (size > 0) {
  		buffer = realloc(_obj->buffer, size);
  		if (!buffer) {
  			return FPGA_NO_MEMORY;
  		}
  		_obj->buffer = buffer;
  		_obj->max_size = size;
  	}
  	return FPGA_OK;
  }
  
  fpga_result sync_object(fpga_object obj)
  {
  	struct _fpga_object *_obj;
  	int fd = -1;
  	fpga_result res = FPGA_OK;
  	ssize_t bytes_read = 0;
  	ASSERT_NOT_NULL(obj);
  	_obj = (struct _fpga_object *)obj;
  	fd = open(_obj->path, _obj->perm);
  	if (fd < 0) {
  		OPAE_ERR("Error opening %s: %s", _obj->path, strerror(errno));
  		return FPGA_EXCEPTION;
  	}
  
  	if (_obj->max_size == 0) {
  		res = sync_object_size(_obj, fd);
  		if (res != FPGA_OK) {
  			close(fd);
  			return res;
  		}
  	}
  
  	bytes_read = eintr_read(fd, _obj->buffer, _obj->max_size);
  	if (bytes_read < 0) {
  		close(fd);
  		return FPGA_EXCEPTION;
  	}
  	_obj->size = bytes_read;
  	close(fd);
  	return FPGA_OK;
  }
  
  fpga_result make_sysfs_group(char *sysfspath, const char *name,
  			     fpga_object *object, int flags, fpga_handle handle)
  {
  	struct dirent **namelist;
  	int n;
  	size_t pathlen = strlen(sysfspath);
  	char *ptr = NULL;
  	fpga_object subobj;
  	fpga_result res = FPGA_OK;
  	struct _fpga_object *group;
  
  	if (flags & FPGA_OBJECT_GLOB) {
  		res = opae_glob_path(sysfspath, SYSFS_PATH_MAX - 1);
  	}
  	if (res != FPGA_OK) {
  		return res;
  	}
  
  	n = scandir(sysfspath, &namelist, sysfs_filter, alphasort);
  	if (n < 0) {
  		OPAE_ERR("Error calling scandir: %s", strerror(errno));
  		switch (errno) {
  		case ENOMEM:
  			return FPGA_NO_MEMORY;
  		case ENOENT:
  			return FPGA_NOT_FOUND;
  		}
  		return FPGA_EXCEPTION;
  	}
  
  	if (n == 0) {
  		OPAE_ERR("Group is empty");
  		return FPGA_EXCEPTION;
  	}
  
  	group = alloc_fpga_object(sysfspath, name);
  	if (!group) {
  		res = FPGA_NO_MEMORY;
  		goto out_free_namelist;
  	}
  
  	group->handle = handle;
  	group->type = FPGA_SYSFS_DIR;
  	if (flags & FPGA_OBJECT_RECURSE_ONE
  	    || flags & FPGA_OBJECT_RECURSE_ALL) {
  		ptr = sysfspath + pathlen;
  		*ptr++ = '/';
  		group->objects = calloc(n, sizeof(fpga_object));
  		if (!group->objects) {
  			res = FPGA_NO_MEMORY;
  			goto out_free_group;
  		}
  		group->size = 0;
  		while (n--) {
  			strncpy(ptr, namelist[n]->d_name,
  				SYSFS_PATH_MAX - pathlen + 1);
  			if (flags & FPGA_OBJECT_RECURSE_ONE) {
  				flags &= ~FPGA_OBJECT_RECURSE_ONE;
  			}
  			if (!make_sysfs_object(
  				    sysfspath, namelist[n]->d_name,
  				    &subobj, flags, handle)) {
  				group->objects[group->size++] = subobj;
  			}
  			free(namelist[n]);
  		}
  		free(namelist);
  	} else {
  		while (n--) {
  			free(namelist[n]);
  		}
  		free(namelist);
  	}
  
  	*object = (fpga_object)group;
  	return FPGA_OK;
  
  out_free_group:
  	if (destroy_fpga_object(group)) {
  		OPAE_ERR("Error destroying object");
  	}
  
  out_free_namelist:
  	while (n--)
  		free(namelist[n]);
  	free(namelist);
  
  	return res;
  }
  
  
  fpga_result make_sysfs_array(char *sysfspath, const char *name,
  			     fpga_object *object, int flags, fpga_handle handle,
  			     char *objects[], size_t num_objects)
  {
  	fpga_result res = FPGA_OK;
  	size_t i = 0;
  	struct _fpga_object *array = alloc_fpga_object(sysfspath, name);
  	char *oname = NULL;
  	if (!array) {
  		OPAE_ERR(
  			"Error allocating memory for container of fpga_objects");
  		return FPGA_NO_MEMORY;
  	}
  	array->objects = calloc(num_objects, sizeof(fpga_object));
  	if (!array->objects) {
  		OPAE_ERR("Error allocating memory for array of fpga_objects");
  		destroy_fpga_object(array);
  		return FPGA_NO_MEMORY;
  	}
  
  	array->handle = handle;
  	array->type = FPGA_SYSFS_LIST;
  	array->size = num_objects;
  	for (i = 0; i < num_objects; ++i) {
  		oname = strrchr(objects[i], '/');
  		if (!oname) {
  			OPAE_ERR("Error with sysfs path: %s", objects[i]);
  			res = FPGA_EXCEPTION;
  			goto out_err;
  		}
  		res = make_sysfs_object(objects[i], oname+1, &array->objects[i],
  					flags & ~FPGA_OBJECT_GLOB, handle);
  		if (res) {
  			goto out_err;
  		}
  	}
  	*object = (fpga_object)array;
  	return res;
  out_err:
  	if (destroy_fpga_object(array)) {
  		OPAE_ERR("Error destroying object");
  	}
  	return res;
  }
  
  #define MAX_SYSOBJECT_GLOB 128
  #define MAX_SYSOBJECT_GLOB_RESURSIVE_DEPTH 5
  fpga_result make_sysfs_object(char *sysfspath, const char *name,
  			      fpga_object *object, int flags,
  			      fpga_handle handle)
  {
  	struct _fpga_object *obj = NULL;
  	struct stat objstat;
  	int statres;
  	fpga_result res = FPGA_OK;
  	char *object_paths[MAX_SYSOBJECT_GLOB] = { NULL };
  	size_t found = 0;
  	size_t len;
  	char full_path[SYSFS_PATH_MAX] = { 0 };
  
  	if (flags & FPGA_OBJECT_GLOB) {
  
  		// Check "**"recursive pattern in sysfs path
  		if (strstr(sysfspath, "/**/")) {
  			res = find_glob_path(sysfspath, full_path);
  			if (res != FPGA_OK) {
  				OPAE_ERR("Invalid recursive input path");
  				return res;
  			}
  			len = strnlen(full_path, SYSFS_PATH_MAX - 1);
  			memcpy(sysfspath, full_path, len);
  			sysfspath[len] = '\0';
  		} // end
  
  		res = opae_glob_paths(sysfspath, MAX_SYSOBJECT_GLOB,
  				      object_paths, &found);
  		if (res) {
  			return res;
  		}
  		if (found == 1) {
  			len = strnlen(object_paths[0], SYSFS_PATH_MAX - 1);
  			memcpy(sysfspath, object_paths[0], len);
  			sysfspath[len] = '\0';
  			res = make_sysfs_object(sysfspath, name, object,
  						flags & ~FPGA_OBJECT_GLOB,
  						handle);
  		} else {
  			res = make_sysfs_array(sysfspath, name, object, flags,
  					       handle, object_paths, found);
  		}
  		// opae_glob_paths allocates memory for each path found
  		// let's free it here since we don't need it any longer
  		while (found) {
  			free(object_paths[--found]);
  		}
  		return res;
  	}
  
  	statres = stat(sysfspath, &objstat);
  	if (statres < 0) {
  		OPAE_MSG("Error accessing %s: %s", sysfspath, strerror(errno));
  		switch (errno) {
  		case ENOENT:
  			res = FPGA_NOT_FOUND;
  			goto out_free;
  		case ENOMEM:
  			res = FPGA_NO_MEMORY;
  			goto out_free;
  		case EACCES:
  			res = FPGA_NO_ACCESS;
  			goto out_free;
  		}
  		res = FPGA_EXCEPTION;
  		goto out_free;
  	}
  
  	if (S_ISDIR(objstat.st_mode)) {
  		return make_sysfs_group(sysfspath, name, object, flags, handle);
  	}
  	obj = alloc_fpga_object(sysfspath, name);
  	if (!obj) {
  		return FPGA_NO_MEMORY;
  	}
  	obj->handle = handle;
  	obj->type = FPGA_SYSFS_FILE;
  	obj->buffer = calloc(objstat.st_size, sizeof(uint8_t));
  	obj->max_size = objstat.st_size;
  	if (handle && (objstat.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH))) {
  		if ((objstat.st_mode & (S_IRUSR | S_IRGRP | S_IROTH))) {
  			obj->perm = O_RDWR;
  		} else {
  			obj->perm = O_WRONLY;
  		}
  	} else {
  		obj->perm = O_RDONLY;
  	}
  	*object = (fpga_object)obj;
  	if (obj->perm == O_RDONLY || obj->perm == O_RDWR) {
  		return sync_object((fpga_object)obj);
  	}
  
  	return FPGA_OK;
  out_free:
  
  
  	free(obj);
  	return res;
  }
  
  
  fpga_result find_glob_path(const char *sysfspath, char *path)
  {
  	int resurse_depth = MAX_SYSOBJECT_GLOB_RESURSIVE_DEPTH;
  	char pattern[SYSFS_PATH_MAX] = { 0 };
  	char full_path[SYSFS_PATH_MAX] = { 0 };
  	char prefix_path[SYSFS_PATH_MAX] = { 0 };
  	char *object_paths[MAX_SYSOBJECT_GLOB] = { NULL };
  	fpga_result res = FPGA_OK;
  	size_t found = 0;
  	size_t len = 0;
  
  	if (sysfspath == NULL ||
  		path == NULL) {
  		OPAE_ERR("Invalid Input parameters");
  		return FPGA_INVALID_PARAM;
  	}
  
  	if (strstr(sysfspath, "/**/")) {
  		char *p = strstr(sysfspath, "/**/");
  		if (p == NULL)
  			return FPGA_INVALID_PARAM;
  
  		// search for multipule pattern "/**/"
  		char *ptr = p;
  		while (ptr != NULL) {
  			ptr++;
  			found++;
  			if (found > 1) {
  				return FPGA_INVALID_PARAM;
  			}
  			ptr = strstr(ptr, "/**/");
  		}
  		found = 0;
  
  		// Prefix substring
  		memcpy(prefix_path, sysfspath, p - sysfspath);
  		*(prefix_path + (p - sysfspath)) = '\0';
  
  		// while loop depth 5
  		while (resurse_depth) {
  			memset(full_path, 0, sizeof(full_path));
  			len = strnlen(pattern, SYSFS_PATH_MAX - 1);
  			strncat(pattern, "*/", SYSFS_PATH_MAX - len);
  
  			if (snprintf(full_path, SYSFS_PATH_MAX,
  				"%s%s%s", prefix_path, pattern, p + 4) < 0) {
  				OPAE_ERR("snprintf buffer overflow");
  				return FPGA_EXCEPTION;
  			}
  
  			res = opae_glob_paths(full_path, MAX_SYSOBJECT_GLOB,
  				object_paths, &found);
  
  			resurse_depth--;
  
  			if (res) {
  				continue;
  			}
  
  			if (found > 0) {
  				len = strnlen(object_paths[0], SYSFS_PATH_MAX - 1);
  				memcpy(path, object_paths[0], len);
  				path[len] = '\0';
  				break;
  			}
  
  		} // end
  
  		while (found) {
  			free(object_paths[--found]);
  		}
  	} else {
  		return FPGA_NOT_FOUND;
  	}
  
  	return res;
  }
  