text: // SPDX-License-Identifier: GPL-2.0-or-later
  /*
   * Copyright (C) 2017 Free Electrons
   * Copyright (C) 2017 NextThing Co
   *
   * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
   */
  
  #include <linux/slab.h>
  
  #include "internals.h"
  
  /*
   * Special Micron status bit 3 indicates that the block has been
   * corrected by on-die ECC and should be rewritten.
   */
  #define NAND_ECC_STATUS_WRITE_RECOMMENDED	BIT(3)
  
  /*
   * On chips with 8-bit ECC and additional bit can be used to distinguish
   * cases where a errors were corrected without needing a rewrite
   *
   * Bit 4 Bit 3 Bit 0 Description
   * ----- ----- ----- -----------
   * 0     0     0     No Errors
   * 0     0     1     Multiple uncorrected errors
   * 0     1     0     4 - 6 errors corrected, recommend rewrite
   * 0     1     1     Reserved
   * 1     0     0     1 - 3 errors corrected
   * 1     0     1     Reserved
   * 1     1     0     7 - 8 errors corrected, recommend rewrite
   */
  #define NAND_ECC_STATUS_MASK		(BIT(4) | BIT(3) | BIT(0))
  #define NAND_ECC_STATUS_UNCORRECTABLE	BIT(0)
  #define NAND_ECC_STATUS_4_6_CORRECTED	BIT(3)
  #define NAND_ECC_STATUS_1_3_CORRECTED	BIT(4)
  #define NAND_ECC_STATUS_7_8_CORRECTED	(BIT(4) | BIT(3))
  
  struct nand_onfi_vendor_micron {
  	u8 two_plane_read;
  	u8 read_cache;
  	u8 read_unique_id;
  	u8 dq_imped;
  	u8 dq_imped_num_settings;
  	u8 dq_imped_feat_addr;
  	u8 rb_pulldown_strength;
  	u8 rb_pulldown_strength_feat_addr;
  	u8 rb_pulldown_strength_num_settings;
  	u8 otp_mode;
  	u8 otp_page_start;
  	u8 otp_data_prot_addr;
  	u8 otp_num_pages;
  	u8 otp_feat_addr;
  	u8 read_retry_options;
  	u8 reserved[72];
  	u8 param_revision;
  } __packed;
  
  struct micron_on_die_ecc {
  	bool forced;
  	bool enabled;
  	void *rawbuf;
  };
  
  struct micron_nand {
  	struct micron_on_die_ecc ecc;
  };
  
  static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode)
  {
  	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode};
  
  	return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature);
  }
  
  /*
   * Configure chip properties from Micron vendor-specific ONFI table
   */
  static int micron_nand_onfi_init(struct nand_chip *chip)
  {
  	struct nand_parameters *p = &chip->parameters;
  
  	if (p->onfi) {
  		struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor;
  
  		chip->read_retries = micron->read_retry_options;
  		chip->setup_read_retry = micron_nand_setup_read_retry;
  	}
  
  	if (p->supports_set_get_features) {
  		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list);
  		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list);
  		set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list);
  		set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list);
  	}
  
  	return 0;
  }
  
  static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd,
  					      int section,
  					      struct mtd_oob_region *oobregion)
  {
  	if (section >= 4)
  		return -ERANGE;
  
  	oobregion->offset = (section * 16) + 8;
  	oobregion->length = 8;
  
  	return 0;
  }
  
  static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd,
  					       int section,
  					       struct mtd_oob_region *oobregion)
  {
  	if (section >= 4)
  		return -ERANGE;
  
  	oobregion->offset = (section * 16) + 2;
  	oobregion->length = 6;
  
  	return 0;
  }
  
  static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = {
  	.ecc = micron_nand_on_die_4_ooblayout_ecc,
  	.free = micron_nand_on_die_4_ooblayout_free,
  };
  
  static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd,
  					      int section,
  					      struct mtd_oob_region *oobregion)
  {
  	struct nand_chip *chip = mtd_to_nand(mtd);
  
  	if (section)
  		return -ERANGE;
  
  	oobregion->offset = mtd->oobsize - chip->ecc.total;
  	oobregion->length = chip->ecc.total;
  
  	return 0;
  }
  
  static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd,
  					       int section,
  					       struct mtd_oob_region *oobregion)
  {
  	struct nand_chip *chip = mtd_to_nand(mtd);
  
  	if (section)
  		return -ERANGE;
  
  	oobregion->offset = 2;
  	oobregion->length = mtd->oobsize - chip->ecc.total - 2;
  
  	return 0;
  }
  
  static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = {
  	.ecc = micron_nand_on_die_8_ooblayout_ecc,
  	.free = micron_nand_on_die_8_ooblayout_free,
  };
  
  static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable)
  {
  	struct micron_nand *micron = nand_get_manufacturer_data(chip);
  	u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, };
  	int ret;
  
  	if (micron->ecc.forced)
  		return 0;
  
  	if (micron->ecc.enabled == enable)
  		return 0;
  
  	if (enable)
  		feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN;
  
  	ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature);
  	if (!ret)
  		micron->ecc.enabled = enable;
  
  	return ret;
  }
  
  static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
  					   void *buf, int page,
  					   int oob_required)
  {
  	struct micron_nand *micron = nand_get_manufacturer_data(chip);
  	struct mtd_info *mtd = nand_to_mtd(chip);
  	unsigned int step, max_bitflips = 0;
  	bool use_datain = false;
  	int ret;
  
  	if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
  		if (status & NAND_STATUS_FAIL)
  			mtd->ecc_stats.failed++;
  
  		return 0;
  	}
  
  	/*
  	 * The internal ECC doesn't tell us the number of bitflips that have
  	 * been corrected, but tells us if it recommends to rewrite the block.
  	 * If it's the case, we need to read the page in raw mode and compare
  	 * its content to the corrected version to extract the actual number of
  	 * bitflips.
  	 * But before we do that, we must make sure we have all OOB bytes read
  	 * in non-raw mode, even if the user did not request those bytes.
  	 */
  	if (!oob_required) {
  		/*
  		 * We first check which operation is supported by the controller
  		 * before running it. This trick makes it possible to support
  		 * all controllers, even the most constraints, without almost
  		 * any performance hit.
  		 *
  		 * TODO: could be enhanced to avoid repeating the same check
  		 * over and over in the fast path.
  		 */
  		if (!nand_has_exec_op(chip) ||
  		    !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
  				       true))
  			use_datain = true;
  
  		if (use_datain)
  			ret = nand_read_data_op(chip, chip->oob_poi,
  						mtd->oobsize, false, false);
  		else
  			ret = nand_change_read_column_op(chip, mtd->writesize,
  							 chip->oob_poi,
  							 mtd->oobsize, false);
  		if (ret)
  			return ret;
  	}
  
  	micron_nand_on_die_ecc_setup(chip, false);
  
  	ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf,
  				mtd->writesize + mtd->oobsize);
  	if (ret)
  		return ret;
  
  	for (step = 0; step < chip->ecc.steps; step++) {
  		unsigned int offs, i, nbitflips = 0;
  		u8 *rawbuf, *corrbuf;
  
  		offs = step * chip->ecc.size;
  		rawbuf = micron->ecc.rawbuf + offs;
  		corrbuf = buf + offs;
  
  		for (i = 0; i < chip->ecc.size; i++)
  			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
  
  		offs = (step * 16) + 4;
  		rawbuf = micron->ecc.rawbuf + mtd->writesize + offs;
  		corrbuf = chip->oob_poi + offs;
  
  		for (i = 0; i < chip->ecc.bytes + 4; i++)
  			nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]);
  
  		if (WARN_ON(nbitflips > chip->ecc.strength))
  			return -EINVAL;
  
  		max_bitflips = max(nbitflips, max_bitflips);
  		mtd->ecc_stats.corrected += nbitflips;
  	}
  
  	return max_bitflips;
  }
  
  static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status)
  {
  	struct mtd_info *mtd = nand_to_mtd(chip);
  
  	/*
  	 * With 8/512 we have more information but still don't know precisely
  	 * how many bit-flips were seen.
  	 */
  	switch (status & NAND_ECC_STATUS_MASK) {
  	case NAND_ECC_STATUS_UNCORRECTABLE:
  		mtd->ecc_stats.failed++;
  		return 0;
  	case NAND_ECC_STATUS_1_3_CORRECTED:
  		mtd->ecc_stats.corrected += 3;
  		return 3;
  	case NAND_ECC_STATUS_4_6_CORRECTED:
  		mtd->ecc_stats.corrected += 6;
  		/* rewrite recommended */
  		return 6;
  	case NAND_ECC_STATUS_7_8_CORRECTED:
  		mtd->ecc_stats.corrected += 8;
  		/* rewrite recommended */
  		return 8;
  	default:
  		return 0;
  	}
  }
  
  static int
  micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
  				 int oob_required, int page)
  {
  	struct mtd_info *mtd = nand_to_mtd(chip);
  	bool use_datain = false;
  	u8 status;
  	int ret, max_bitflips = 0;
  
  	ret = micron_nand_on_die_ecc_setup(chip, true);
  	if (ret)
  		return ret;
  
  	ret = nand_read_page_op(chip, page, 0, NULL, 0);
  	if (ret)
  		goto out;
  
  	ret = nand_status_op(chip, &status);
  	if (ret)
  		goto out;
  
  	/*
  	 * We first check which operation is supported by the controller before
  	 * running it. This trick makes it possible to support all controllers,
  	 * even the most constraints, without almost any performance hit.
  	 *
  	 * TODO: could be enhanced to avoid repeating the same check over and
  	 * over in the fast path.
  	 */
  	if (!nand_has_exec_op(chip) ||
  	    !nand_read_data_op(chip, buf, mtd->writesize, false, true))
  		use_datain = true;
  
  	if (use_datain) {
  		ret = nand_exit_status_op(chip);
  		if (ret)
  			goto out;
  
  		ret = nand_read_data_op(chip, buf, mtd->writesize, false,
  					false);
  		if (!ret && oob_required)
  			ret = nand_read_data_op(chip, chip->oob_poi,
  						mtd->oobsize, false, false);
  	} else {
  		ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize,
  						 false);
  		if (!ret && oob_required)
  			ret = nand_change_read_column_op(chip, mtd->writesize,
  							 chip->oob_poi,
  							 mtd->oobsize, false);
  	}
  
  	if (chip->ecc.strength == 4)
  		max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
  							       buf, page,
  							       oob_required);
  	else
  		max_bitflips = micron_nand_on_die_ecc_status_8(chip, status);
  
  out:
  	micron_nand_on_die_ecc_setup(chip, false);
  
  	return ret ? ret : max_bitflips;
  }
  
  static int
  micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf,
  				  int oob_required, int page)
  {
  	int ret;
  
  	ret = micron_nand_on_die_ecc_setup(chip, true);
  	if (ret)
  		return ret;
  
  	ret = nand_write_page_raw(chip, buf, oob_required, page);
  	micron_nand_on_die_ecc_setup(chip, false);
  
  	return ret;
  }
  
  enum {
  	/* The NAND flash doesn't support on-die ECC */
  	MICRON_ON_DIE_UNSUPPORTED,
  
  	/*
  	 * The NAND flash supports on-die ECC and it can be
  	 * enabled/disabled by a set features command.
  	 */
  	MICRON_ON_DIE_SUPPORTED,
  
  	/*
  	 * The NAND flash supports on-die ECC, and it cannot be
  	 * disabled.
  	 */
  	MICRON_ON_DIE_MANDATORY,
  };
  
  #define MICRON_ID_INTERNAL_ECC_MASK	GENMASK(1, 0)
  #define MICRON_ID_ECC_ENABLED		BIT(7)
  
  /*
   * Try to detect if the NAND support on-die ECC. To do this, we enable
   * the feature, and read back if it has been enabled as expected. We
   * also check if it can be disabled, because some Micron NANDs do not
   * allow disabling the on-die ECC and we don't support such NANDs for
   * now.
   *
   * This function also has the side effect of disabling on-die ECC if
   * it had been left enabled by the firmware/bootloader.
   */
  static int micron_supports_on_die_ecc(struct nand_chip *chip)
  {
  	u8 id[5];
  	int ret;
  
  	if (!chip->parameters.onfi)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	if (nanddev_bits_per_cell(&chip->base) != 1)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	/*
  	 * We only support on-die ECC of 4/512 or 8/512
  	 */
  	if  (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	/* 0x2 means on-die ECC is available. */
  	if (chip->id.len != 5 ||
  	    (chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	/*
  	 * It seems that there are devices which do not support ECC officially.
  	 * At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports
  	 * enabling the ECC feature but don't reflect that to the READ_ID table.
  	 * So we have to guarantee that we disable the ECC feature directly
  	 * after we did the READ_ID table command. Later we can evaluate the
  	 * ECC_ENABLE support.
  	 */
  	ret = micron_nand_on_die_ecc_setup(chip, true);
  	if (ret)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	ret = nand_readid_op(chip, 0, id, sizeof(id));
  	if (ret)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	ret = micron_nand_on_die_ecc_setup(chip, false);
  	if (ret)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	if (!(id[4] & MICRON_ID_ECC_ENABLED))
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	ret = nand_readid_op(chip, 0, id, sizeof(id));
  	if (ret)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	if (id[4] & MICRON_ID_ECC_ENABLED)
  		return MICRON_ON_DIE_MANDATORY;
  
  	/*
  	 * We only support on-die ECC of 4/512 or 8/512
  	 */
  	if  (chip->base.eccreq.strength != 4 && chip->base.eccreq.strength != 8)
  		return MICRON_ON_DIE_UNSUPPORTED;
  
  	return MICRON_ON_DIE_SUPPORTED;
  }
  
  static int micron_nand_init(struct nand_chip *chip)
  {
  	struct mtd_info *mtd = nand_to_mtd(chip);
  	struct micron_nand *micron;
  	int ondie;
  	int ret;
  
  	micron = kzalloc(sizeof(*micron), GFP_KERNEL);
  	if (!micron)
  		return -ENOMEM;
  
  	nand_set_manufacturer_data(chip, micron);
  
  	ret = micron_nand_onfi_init(chip);
  	if (ret)
  		goto err_free_manuf_data;
  
  	chip->options |= NAND_BBM_FIRSTPAGE;
  
  	if (mtd->writesize == 2048)
  		chip->options |= NAND_BBM_SECONDPAGE;
  
  	ondie = micron_supports_on_die_ecc(chip);
  
  	if (ondie == MICRON_ON_DIE_MANDATORY &&
  	    chip->ecc.mode != NAND_ECC_ON_DIE) {
  		pr_err("On-die ECC forcefully enabled, not supported\n");
  		ret = -EINVAL;
  		goto err_free_manuf_data;
  	}
  
  	if (chip->ecc.mode == NAND_ECC_ON_DIE) {
  		if (ondie == MICRON_ON_DIE_UNSUPPORTED) {
  			pr_err("On-die ECC selected but not supported\n");
  			ret = -EINVAL;
  			goto err_free_manuf_data;
  		}
  
  		if (ondie == MICRON_ON_DIE_MANDATORY) {
  			micron->ecc.forced = true;
  			micron->ecc.enabled = true;
  		}
  
  		/*
  		 * In case of 4bit on-die ECC, we need a buffer to store a
  		 * page dumped in raw mode so that we can compare its content
  		 * to the same page after ECC correction happened and extract
  		 * the real number of bitflips from this comparison.
  		 * That's not needed for 8-bit ECC, because the status expose
  		 * a better approximation of the number of bitflips in a page.
  		 */
  		if (chip->base.eccreq.strength == 4) {
  			micron->ecc.rawbuf = kmalloc(mtd->writesize +
  						     mtd->oobsize,
  						     GFP_KERNEL);
  			if (!micron->ecc.rawbuf) {
  				ret = -ENOMEM;
  				goto err_free_manuf_data;
  			}
  		}
  
  		if (chip->base.eccreq.strength == 4)
  			mtd_set_ooblayout(mtd,
  					  &micron_nand_on_die_4_ooblayout_ops);
  		else
  			mtd_set_ooblayout(mtd,
  					  &micron_nand_on_die_8_ooblayout_ops);
  
  		chip->ecc.bytes = chip->base.eccreq.strength * 2;
  		chip->ecc.size = 512;
  		chip->ecc.strength = chip->base.eccreq.strength;
  		chip->ecc.algo = NAND_ECC_BCH;
  		chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
  		chip->ecc.write_page = micron_nand_write_page_on_die_ecc;
  
  		if (ondie == MICRON_ON_DIE_MANDATORY) {
  			chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
  			chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
  		} else {
  			if (!chip->ecc.read_page_raw)
  				chip->ecc.read_page_raw = nand_read_page_raw;
  			if (!chip->ecc.write_page_raw)
  				chip->ecc.write_page_raw = nand_write_page_raw;
  		}
  	}
  
  	return 0;
  
  err_free_manuf_data:
  	kfree(micron->ecc.rawbuf);
  	kfree(micron);
  
  	return ret;
  }
  
  static void micron_nand_cleanup(struct nand_chip *chip)
  {
  	struct micron_nand *micron = nand_get_manufacturer_data(chip);
  
  	kfree(micron->ecc.rawbuf);
  	kfree(micron);
  }
  
  static void micron_fixup_onfi_param_page(struct nand_chip *chip,
  					 struct nand_onfi_params *p)
  {
  	/*
  	 * MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the
  	 * revision number field of the ONFI parameter page. Assume ONFI
  	 * version 1.0 if the revision number is 00 00.
  	 */
  	if (le16_to_cpu(p->revision) == 0)
  		p->revision = cpu_to_le16(ONFI_VERSION_1_0);
  }
  
  const struct nand_manufacturer_ops micron_nand_manuf_ops = {
  	.init = micron_nand_init,
  	.cleanup = micron_nand_cleanup,
  	.fixup_onfi_param_page = micron_fixup_onfi_param_page,
  };
  