// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved. */

#include <devlink.h>

#include "fw_reset.h"
#include "diag/fw_tracer.h"
#include "lib/tout.h"

enum {
	MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
	MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
	MLX5_FW_RESET_FLAGS_PENDING_COMP,
	MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
	MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED
};

struct mlx5_fw_reset {
	struct mlx5_core_dev *dev;
	struct mlx5_nb nb;
	struct workqueue_struct *wq;
	struct work_struct fw_live_patch_work;
	struct work_struct reset_request_work;
	struct work_struct reset_unload_work;
	struct work_struct reset_reload_work;
	struct work_struct reset_now_work;
	struct work_struct reset_abort_work;
	struct delayed_work reset_timeout_work;
	unsigned long reset_flags;
	struct timer_list timer;
#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
	struct completion done;
	int ret;
#endif
};

enum {
	MLX5_FW_RST_STATE_IDLE = 0,
	MLX5_FW_RST_STATE_TOGGLE_REQ = 4,
};

enum {
	MLX5_RST_STATE_BIT_NUM = 12,
	MLX5_RST_ACK_BIT_NUM = 22,
};

static u8 mlx5_get_fw_rst_state(struct mlx5_core_dev *dev)
{
	return (ioread32be(&dev->iseg->initializing) >> MLX5_RST_STATE_BIT_NUM) & 0xF;
}

static void mlx5_set_fw_rst_ack(struct mlx5_core_dev *dev)
{
	iowrite32be(BIT(MLX5_RST_ACK_BIT_NUM), &dev->iseg->initializing);
}

#if defined(HAVE_DEVLINK_PARAM) && (defined(HAVE_DEVLINK_PARAMS_PUBLISHED) || defined(HAVE_DEVLINK_REGISTER_GET_1_PARAMS))
static int mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink *devlink, u32 id,
						     struct devlink_param_gset_ctx *ctx)
{
	struct mlx5_core_dev *dev = devlink_priv(devlink);
	struct mlx5_fw_reset *fw_reset;

	#if !defined(HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET) && defined(HAVE_DEVLINK_PARAM_REGISTER)
        	if (mlx5_dev_is_lightweight(dev))
                	return -EOPNOTSUPP;
	#endif
	fw_reset = dev->priv.fw_reset;

	if (ctx->val.vbool)
		clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	else
		set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	return 0;
}

static int mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink *devlink, u32 id,
						     struct devlink_param_gset_ctx *ctx)
{
	struct mlx5_core_dev *dev = devlink_priv(devlink);
	struct mlx5_fw_reset *fw_reset;

	#if !defined(HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET) && defined(HAVE_DEVLINK_PARAM_REGISTER)
        	if (mlx5_dev_is_lightweight(dev))
                	return -EOPNOTSUPP;
	#endif
	fw_reset = dev->priv.fw_reset;

	ctx->val.vbool = !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
				   &fw_reset->reset_flags);
	return 0;
}
#endif

static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
			     u8 reset_type_sel, u8 sync_resp, bool sync_start)
{
	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};

	MLX5_SET(mfrl_reg, in, reset_level, reset_level);
	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);

	return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
}

static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level,
			       u8 *reset_type, u8 *reset_state)
{
	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	int err;

	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
	if (err)
		return err;

	if (reset_level)
		*reset_level = MLX5_GET(mfrl_reg, out, reset_level);
	if (reset_type)
		*reset_type = MLX5_GET(mfrl_reg, out, reset_type);
	if (reset_state)
		*reset_state = MLX5_GET(mfrl_reg, out, reset_state);

	return 0;
}

int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
{
	return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL);
}

#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev,
					     struct netlink_ext_ack *extack)
{
	u8 reset_state;

	if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state))
		goto out;

	switch (reset_state) {
	case MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION:
	case MLX5_MFRL_REG_RESET_STATE_RESET_IN_PROGRESS:
		NL_SET_ERR_MSG_MOD(extack, "Sync reset was already triggered");
		return -EBUSY;
	case MLX5_MFRL_REG_RESET_STATE_TIMEOUT:
		NL_SET_ERR_MSG_MOD(extack, "Sync reset got timeout");
		return -ETIMEDOUT;
	case MLX5_MFRL_REG_RESET_STATE_NACK:
		NL_SET_ERR_MSG_MOD(extack, "One of the hosts disabled reset");
		return -EPERM;
	}

out:
	NL_SET_ERR_MSG_MOD(extack, "Sync reset failed");
	return -EIO;
}

int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
				 struct netlink_ext_ack *extack)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	int err;

	set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);

	MLX5_SET(mfrl_reg, in, reset_level, MLX5_MFRL_REG_RESET_LEVEL3);
	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, 1);
	err = mlx5_access_reg(dev, in, sizeof(in), out, sizeof(out),
			      MLX5_REG_MFRL, 0, 1, false);
	if (!err)
		return 0;

	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	if (err == -EREMOTEIO && MLX5_CAP_MCAM_FEATURE(dev, reset_state))
		return mlx5_fw_reset_get_reset_state_err(dev, extack);

	NL_SET_ERR_MSG_MOD(extack, "Sync reset command failed");
	return mlx5_cmd_check(dev, err, in, out);
}
#endif /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */

int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
}

#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unloaded)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	/* if this is the driver that initiated the fw reset, devlink completed the reload */
	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
		complete(&fw_reset->done);
	} else {
#ifdef HAVE_DEVL_TRAP_GROUPS_REGISTER
		if (!unloaded)
			mlx5_unload_one(dev, false);
		if (mlx5_health_wait_pci_up(dev))
			mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
		else
#endif
			mlx5_load_one(dev, true);
		devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0,
							BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
	}
}
#endif

static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	del_timer_sync(&fw_reset->timer);
}

static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
		mlx5_core_warn(dev, "Reset request was already cleared\n");
		return -EALREADY;
	}

	mlx5_stop_sync_reset_poll(dev);
	if (poll_health)
		mlx5_start_health_poll(dev);
	return 0;
}

static void mlx5_sync_reset_reload_work(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_reload_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
#ifndef HAVE_DEVL_TRAP_GROUPS_REGISTER
	int err;
#endif

	cancel_delayed_work(&fw_reset->reset_timeout_work);
	mlx5_sync_reset_clear_reset_requested(dev, false);
	mlx5_enter_error_state(dev, true);
#ifndef HAVE_DEVL_TRAP_GROUPS_REGISTER
	mlx5_unload_one(dev, false);
	err = mlx5_health_wait_pci_up(dev);
	if (err) {
		mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
#ifndef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
		return;
#endif
	}
#endif /* HAVE_DEVL_TRAP_GROUPS_REGISTER */

#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
#ifndef HAVE_DEVL_TRAP_GROUPS_REGISTER
	fw_reset->ret = err;
#endif
	mlx5_fw_reset_complete_reload(dev, false);
#else /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */
	mlx5_load_one(dev, true);
#endif /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */
}

#define MLX5_RESET_POLL_INTERVAL	(HZ / 10)
#ifdef HAVE_TIMER_SETUP
static void poll_sync_reset(struct timer_list *t)
#else
static void poll_sync_reset(unsigned long data)
#endif
{
#ifdef HAVE_TIMER_SETUP
	struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer);
#else
	struct mlx5_fw_reset *fw_reset = (struct mlx5_fw_reset *)data;
#endif
	struct mlx5_core_dev *dev = fw_reset->dev;
	u32 fatal_error;

	if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
		return;

	fatal_error = mlx5_health_check_fatal_sensors(dev);

	if (fatal_error) {
		mlx5_core_warn(dev, "Got Device Reset\n");
		if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
			queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
		else
			mlx5_core_err(dev, "Device is being removed, Drop new reset work\n");
		return;
	}

	mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
}

static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

#ifdef HAVE_TIMER_SETUP
	timer_setup(&fw_reset->timer, poll_sync_reset, 0);
#else
	init_timer(&fw_reset->timer);
	fw_reset->timer.data = (unsigned long)fw_reset;
	fw_reset->timer.function = poll_sync_reset;
#endif
	fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
	add_timer(&fw_reset->timer);
}

static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
}

static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
}

static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
		mlx5_core_warn(dev, "Reset request was already set\n");
		return -EALREADY;
	}
	mlx5_stop_health_poll(dev, true);
	mlx5_start_sync_reset_poll(dev);
	return 0;
}

static void mlx5_fw_live_patch_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      fw_live_patch_work);
	struct mlx5_core_dev *dev = fw_reset->dev;

	mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
		       fw_rev_min(dev), fw_rev_sub(dev));

	if (mlx5_fw_tracer_reload(dev->tracer))
		mlx5_core_err(dev, "Failed to reload FW tracer\n");
}

static const struct pci_device_id rshim_device_ids[] = {
	{ PCI_VDEVICE(MELLANOX, 0xc2d2) }, /* BlueField1 RShim device ID */
	{ PCI_VDEVICE(MELLANOX, 0xc2d3) }, /* BlueField2 RShim device ID */
	{ PCI_VDEVICE(MELLANOX, 0xc2d6) }, /* BlueField2 RShim device ID2 */
	{ PCI_VDEVICE(MELLANOX, 0xc2d5) }  /* BlueField3 RShim device ID */
};

static bool mlx5_is_rshim_pci_device(struct mlx5_core_dev *dev, u16 dev_id)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(rshim_device_ids); ++i) {
		if (rshim_device_ids[i].device == dev_id)
			return true;
	}

	return false;
}

#if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
static int mlx5_check_hotplug_interrupt(struct mlx5_core_dev *dev)
{
	struct pci_dev *bridge = dev->pdev->bus->self;
	u16 reg16;
	int err;

	if (!bridge)
		return -EOPNOTSUPP;

	err = pcie_capability_read_word(bridge, PCI_EXP_SLTCTL, &reg16);
	if (err)
		return err;

	if ((reg16 & PCI_EXP_SLTCTL_HPIE) && (reg16 & PCI_EXP_SLTCTL_DLLSCE)) {
		mlx5_core_warn(dev, "FW reset is not supported as HotPlug is enabled\n");
		return -EOPNOTSUPP;
	}

	return 0;
}
#endif

static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
{
	struct pci_bus *bridge_bus = dev->pdev->bus;
	struct pci_dev *sdev;
	u16 sdev_id;
	int err;

	/* Check that all functions under the pci bridge are PFs of
	 * this device otherwise fail this function.
	 */
	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
		if (err)
			return pcibios_err_to_errno(err);
		if (sdev_id == dev_id)
			continue;

		if (mlx5_is_rshim_pci_device(dev, sdev_id))
			continue;

		mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
		return -EPERM;
	}
	return 0;
}

static bool mlx5_is_reset_now_capable(struct mlx5_core_dev *dev)
{
	u16 dev_id;
	int err;

	if (!MLX5_CAP_GEN(dev, fast_teardown)) {
		mlx5_core_warn(dev, "fast teardown is not supported by firmware\n");
		return false;
	}

#if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
	err = mlx5_check_hotplug_interrupt(dev);
	if (err)
		return false;
#endif

	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
	if (err)
		return false;
	return (!mlx5_check_dev_ids(dev, dev_id));
}

static void mlx5_sync_reset_request_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_request_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
	unsigned long sync_reset_timeout;
	int err;

	if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) ||
	    !mlx5_is_reset_now_capable(dev)) {
		err = mlx5_fw_reset_set_reset_sync_nack(dev);
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
			       err ? "Failed" : "Sent");
		return;
	}
	if (mlx5_sync_reset_set_reset_requested(dev))
		return;

	sync_reset_timeout = msecs_to_jiffies(mlx5_tout_ms(dev, PCI_SYNC_UPDATE));
	schedule_delayed_work(&fw_reset->reset_timeout_work, sync_reset_timeout);
	err = mlx5_fw_reset_set_reset_sync_ack(dev);
	if (err)
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
	else
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
}

#ifndef PCI_EXP_RTCAP_CRSVIS
#define  PCI_EXP_RTCAP_CRSVIS	0x0001	/* CRS Software Visibility capability */
#endif

static int mlx5_pci_config_hw_control(struct pci_dev *root_port,
				      bool new_val, bool *prev_val)
{
	u16 root_ctl, root_cap;
	bool curr_val;
	int ret;

	pcie_capability_read_word(root_port, PCI_EXP_RTCAP, &root_cap);
	if (!(root_cap & PCI_EXP_RTCAP_CRSVIS))
		return 0;

	pcie_capability_read_word(root_port, PCI_EXP_RTCTL, &root_ctl);
	curr_val = !(root_ctl & PCI_EXP_RTCTL_CRSSVE);

	if (prev_val)
		*prev_val = curr_val;

	if (curr_val == new_val)
		return 0;

	if (new_val)
		ret = pcie_capability_clear_word(root_port, PCI_EXP_RTCTL,
						 PCI_EXP_RTCTL_CRSSVE);
	else
		ret = pcie_capability_set_word(root_port, PCI_EXP_RTCTL,
					       PCI_EXP_RTCTL_CRSSVE);

	if (ret) {
		dev_err(&root_port->dev, "Failed to config CRSSVE bit, err(%d)\n", ret);
		return ret;
	}

	return 0;
}

static void mlx5_pci_restore(struct pci_dev *dev)
{
	struct pci_dev *pdev;

	pci_restore_state(dev);
	if (!dev->subordinate)
		return;
	list_for_each_entry(pdev, &dev->subordinate->devices, bus_list)
		mlx5_pci_restore(pdev);
}

static void mlx5_pci_unlock(struct pci_dev *dev)
{
	struct pci_dev *pdev;

	pci_cfg_access_unlock(dev);
	if (!dev->subordinate)
		return;
	list_for_each_entry(pdev, &dev->subordinate->devices, bus_list)
		mlx5_pci_unlock(pdev);
}

static bool mlx5_pci_is_dev_same_id(struct pci_dev *pdev, u16 dev_id)
{
	u16 pdev_id;

	if (pci_read_config_word(pdev, PCI_DEVICE_ID, &pdev_id))
		return false;
	if (pdev_id != dev_id)
		return false;
	return true;
}

static bool mlx5_pci_are_all_devs_same_id(struct pci_dev *dev, u16 nic_dev_id)
{
	struct pci_dev *pdev;
	int type;

	if (!dev->subordinate) {
		type = pci_pcie_type(dev);
		if (type != PCI_EXP_TYPE_ENDPOINT &&
		    type != PCI_EXP_TYPE_LEG_END)
			return true;
		return mlx5_pci_is_dev_same_id(dev, nic_dev_id);
	}

	list_for_each_entry(pdev, &dev->subordinate->devices, bus_list)
		if (!mlx5_pci_are_all_devs_same_id(pdev, nic_dev_id))
			return false;

	return true;
}

static void mlx5_pci_save_locked(struct pci_dev *dev)
{
	struct pci_dev *pdev;

	pci_cfg_access_lock(dev);
	pci_save_state(dev);
	if (!dev->subordinate)
		return;
	list_for_each_entry(pdev, &dev->subordinate->devices, bus_list)
		mlx5_pci_save_locked(pdev);
}

static int mlx5_check_reset_criteria(struct mlx5_core_dev *dev,
				     struct pci_dev *root_port)
{
	u16 dev_id;
	int err;

	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
	if (err)
		return err;

	if (!mlx5_pci_are_all_devs_same_id(root_port, dev_id))
		return -EPERM;

	return 0;
}

static int mlx5_reset_pci_topology(struct mlx5_core_dev *dev,
				   struct pci_dev *root_port)
{
	struct pci_dev *bridge;
	unsigned long timeout;
	int err;

	bridge = dev->pdev->bus->self;

	/* Disable PCI link */
	err = pcie_capability_set_word(bridge, PCI_EXP_LNKCTL,
				       PCI_EXP_LNKCTL_LD);
	if (err) {
		mlx5_core_err(dev, "Failed to disable pci link, err (%d)\n", err);
		return err;
	}
	msleep(500);

	/* Wait for PCI link comes back with timeout */
	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE));
	do {
		if (pci_device_is_present(root_port))
			break;
		msleep(20);
	} while (!time_after(jiffies, timeout));

	if (pci_device_is_present(root_port)) {
		mlx5_core_info(dev, "PCIe topology is ready\n");
	} else {
		mlx5_core_err(dev, "PCIe topology is not ready after %lu ms\n",
				msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE)));
		return -ETIMEDOUT;
	}
	return 0;
}

#ifndef HAVE_PCIE_FIND_ROOT_PORT
static inline struct pci_dev *pcie_find_root_port(struct pci_dev *dev)
{
	while (dev) {
		if (pci_is_pcie(dev) &&
		    pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
			return dev;
		dev = pci_upstream_bridge(dev);
	}

	return NULL;
}
#endif

static int mlx5_pci_link_toggle_ecpf(struct mlx5_core_dev *dev)
{
	struct pci_dev *root_port;
	bool prev;
	int err;

	root_port = pcie_find_root_port(dev->pdev);
	if (!root_port) {
		mlx5_core_err(dev, "Failed to find root port\n");
		return -ENODEV;
	}

	err = mlx5_check_reset_criteria(dev, root_port);
	if (err) {
		mlx5_core_err(dev, "Device does not match reset criteria, err(%d)\n",
			      err);
		return err;
	}

	err = mlx5_pci_config_hw_control(root_port, true, &prev);
	if (err) {
		mlx5_core_err(dev, "Failed to enable hardware retry\n");
		return -EIO;
	}

	mlx5_pci_save_locked(root_port);

	err = mlx5_reset_pci_topology(dev, root_port);
	if (err)
		goto err_reset;

	mlx5_pci_restore(root_port);
err_reset:
	mlx5_pci_unlock(root_port);
	mlx5_pci_config_hw_control(root_port, prev, NULL);
	return err;
}

static int mlx5_pci_link_toggle_pf(struct mlx5_core_dev *dev)
{
	struct pci_bus *bridge_bus = dev->pdev->bus;
	struct pci_dev *bridge = bridge_bus->self;
	unsigned long timeout;
	struct pci_dev *sdev;
	u16 reg16, dev_id;
	int cap, err;
	u32 reg32;

	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
	if (err)
		return pcibios_err_to_errno(err);

	err = mlx5_check_dev_ids(dev, dev_id);
	if (err)
		return err;

	if (!bridge)
		return -EOPNOTSUPP;

	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
	if (!cap)
		return -EOPNOTSUPP;

	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		pci_save_state(sdev);
		pci_cfg_access_lock(sdev);
	}
	/* PCI link toggle */
	err = pcie_capability_set_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
	if (err)
		return pcibios_err_to_errno(err);
	msleep(2500);
	err = pcie_capability_clear_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
	if (err)
		return err;

	/* Check link */
	err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, &reg32);
	if (err)
		return pcibios_err_to_errno(err);
	if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) {
		mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32);
		msleep(1000);
		goto restore;
	}

	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE));
	do {
		err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, &reg16);
		if (err)
			return pcibios_err_to_errno(err);
		if (reg16 & PCI_EXP_LNKSTA_DLLLA)
			break;
		msleep(20);
	} while (!time_after(jiffies, timeout));

	if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
		mlx5_core_info(dev, "PCI Link up\n");
	} else {
		mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n",
			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
		err = -ETIMEDOUT;
		goto restore;
	}

	do {
		err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &reg16);
		if (err)
			return pcibios_err_to_errno(err);
		if (reg16 == dev_id)
			break;
		msleep(20);
	} while (!time_after(jiffies, timeout));

	if (reg16 == dev_id) {
		mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n");
	} else {
		mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n",
			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
		err = -ETIMEDOUT;
	}

restore:
	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		pci_cfg_access_unlock(sdev);
		pci_restore_state(sdev);
	}

	return err;
}

static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev)
{
	if (mlx5_core_is_ecpf(dev))
		return mlx5_pci_link_toggle_ecpf(dev);
	else
		return mlx5_pci_link_toggle_pf(dev);
}

static void mlx5_sync_reset_now_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_now_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
	int err;

	cancel_delayed_work(&fw_reset->reset_timeout_work);
	if (mlx5_sync_reset_clear_reset_requested(dev, false))
		return;

	mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");

	err = mlx5_cmd_fast_teardown_hca(dev);
	if (err) {
		mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
		goto done;
	}

	err = mlx5_pci_link_toggle(dev);
	if (err) {
		mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err);
		set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags);
	}

	mlx5_enter_error_state(dev, true);
#ifndef HAVE_DEVL_TRAP_GROUPS_REGISTER
	mlx5_unload_one(dev, false);
#endif
done:
#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
	fw_reset->ret = err;
	mlx5_fw_reset_complete_reload(dev, false);
#else
	mlx5_load_one(dev, true);
#endif
}

static void mlx5_sync_reset_unload_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset;
	struct mlx5_core_dev *dev;
	unsigned long timeout;
	bool reset_action;
	u8 rst_state;
	int err;

	fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work);
	dev = fw_reset->dev;

	cancel_delayed_work(&fw_reset->reset_timeout_work);
	if (mlx5_sync_reset_clear_reset_requested(dev, false))
		return;

	mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n");

	err = mlx5_cmd_fast_teardown_hca(dev);
	if (err)
		mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err);
	else
		mlx5_enter_error_state(dev, true);

#ifdef HAVE_DEVL_TRAP_GROUPS_REGISTER
	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags))
		mlx5_unload_one_devl_locked(dev, false);
	else
#endif
		mlx5_unload_one(dev, false);

	mlx5_set_fw_rst_ack(dev);
	mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");

	reset_action = false;
	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
	do {
		rst_state = mlx5_get_fw_rst_state(dev);
		if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
		    rst_state == MLX5_FW_RST_STATE_IDLE) {
			reset_action = true;
			break;
		}
		msleep(20);
	} while (!time_after(jiffies, timeout));

	if (!reset_action) {
		mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
			      rst_state);
#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
		fw_reset->ret = -ETIMEDOUT;
#endif
		goto done;
	}

	mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n", rst_state);
	if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
		err = mlx5_pci_link_toggle(dev);
		if (err) {
			mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, err %d\n", err);
#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
			fw_reset->ret = err;
#endif
		}
	}

done:
#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
	mlx5_fw_reset_complete_reload(dev, true);
#else /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */
	mlx5_load_one(dev, true);
#endif /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */
}

static void mlx5_sync_reset_abort_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_abort_work);
	struct mlx5_core_dev *dev = fw_reset->dev;

	cancel_delayed_work(&fw_reset->reset_timeout_work);
	if (mlx5_sync_reset_clear_reset_requested(dev, true))
		return;
	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
}

static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
{
	struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
	u8 sync_event_rst_type;

	sync_fw_update_eqe = &eqe->data.sync_fw_update;
	sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
	switch (sync_event_rst_type) {
	case MLX5_SYNC_RST_STATE_RESET_REQUEST:
		queue_work(fw_reset->wq, &fw_reset->reset_request_work);
		break;
	case MLX5_SYNC_RST_STATE_RESET_UNLOAD:
		queue_work(fw_reset->wq, &fw_reset->reset_unload_work);
		break;
	case MLX5_SYNC_RST_STATE_RESET_NOW:
		queue_work(fw_reset->wq, &fw_reset->reset_now_work);
		break;
	case MLX5_SYNC_RST_STATE_RESET_ABORT:
		queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
		break;
	}
}

static void mlx5_sync_reset_timeout_work(struct work_struct *work)
{
	struct delayed_work *dwork = container_of(work, struct delayed_work,
						  work);
	struct mlx5_fw_reset *fw_reset = container_of(dwork, struct mlx5_fw_reset,
						      reset_timeout_work);
	struct mlx5_core_dev *dev = fw_reset->dev;

	if (mlx5_sync_reset_clear_reset_requested(dev, true))
		return;
	mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n");
}

static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
{
	struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
	struct mlx5_eqe *eqe = data;

	if (test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
		return NOTIFY_DONE;

	switch (eqe->sub_type) {
	case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
		queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
		break;
	case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
		mlx5_sync_reset_events_handle(fw_reset, eqe);
		break;
	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
}

#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
{
	unsigned long pci_sync_update_timeout = mlx5_tout_ms(dev, PCI_SYNC_UPDATE);
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	unsigned long timeout;
	int err;

	if (MLX5_CAP_GEN(dev, pci_sync_for_fw_update_with_driver_unload))
		pci_sync_update_timeout += mlx5_tout_ms(dev, RESET_UNLOAD);
	timeout = msecs_to_jiffies(pci_sync_update_timeout);
	if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
		mlx5_core_warn(dev, "FW sync reset timeout after %lu seconds\n",
			       pci_sync_update_timeout / 1000);
		err = -ETIMEDOUT;
		goto out;
	}
	err = fw_reset->ret;
	if (test_and_clear_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags)) {
#ifdef HAVE_DEVL_TRAP_GROUPS_REGISTER
		mlx5_unload_one_devl_locked(dev, false);
		mlx5_load_one_devl_locked(dev, true);
#else
		mlx5_load_one(dev, true);
#endif
	}
out:
	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	return err;
}
#endif /* HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION */

void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
	mlx5_eq_notifier_register(dev, &fw_reset->nb);
}

void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
{
	mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb);
}

void mlx5_drain_fw_reset(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	set_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags);
	cancel_work_sync(&fw_reset->fw_live_patch_work);
	cancel_work_sync(&fw_reset->reset_request_work);
	cancel_work_sync(&fw_reset->reset_unload_work);
	cancel_work_sync(&fw_reset->reset_reload_work);
	cancel_work_sync(&fw_reset->reset_now_work);
	cancel_work_sync(&fw_reset->reset_abort_work);
	cancel_delayed_work(&fw_reset->reset_timeout_work);
}


#if defined(HAVE_DEVLINK_PARAM) && (defined(HAVE_DEVLINK_PARAMS_PUBLISHED) || defined(HAVE_DEVLINK_REGISTER_GET_1_PARAMS))
static const struct devlink_param mlx5_fw_reset_devlink_params[] = {
#ifdef HAVE_DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET
	DEVLINK_PARAM_GENERIC(ENABLE_REMOTE_DEV_RESET, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
			      mlx5_fw_reset_enable_remote_dev_reset_get,
			      mlx5_fw_reset_enable_remote_dev_reset_set, NULL),
#else
	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ENABLE_REMOTE_DEV_RESET,
			     "enable_remote_dev_reset", DEVLINK_PARAM_TYPE_BOOL,
			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
			     mlx5_fw_reset_enable_remote_dev_reset_get,
			     mlx5_fw_reset_enable_remote_dev_reset_set, NULL),
#endif
};

#ifndef HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET
int mlx5_register_devlink_fw_params(struct mlx5_core_dev *dev)
{
	return devlink_params_register(priv_to_devlink(dev), mlx5_fw_reset_devlink_params,
				       ARRAY_SIZE(mlx5_fw_reset_devlink_params));
}

void mlx5_unregister_devlink_fw_params(struct mlx5_core_dev *dev)
{
	devlink_params_unregister(priv_to_devlink(dev), mlx5_fw_reset_devlink_params,
				  ARRAY_SIZE(mlx5_fw_reset_devlink_params));
}
#endif /* !HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET */
#endif /* HAVE_DEVLINK_PARAM && (HAVE_DEVLINK_PARAMS_PUBLISHED || HAVE_DEVLINK_REGISTER_GET_1_PARAMS) */

int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);
#ifdef HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET
	int err;
#endif

	if (!fw_reset)
		return -ENOMEM;
	fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
	if (!fw_reset->wq) {
		kfree(fw_reset);
		return -ENOMEM;
	}

	fw_reset->dev = dev;
	dev->priv.fw_reset = fw_reset;

#ifdef HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET
	err = devl_params_register(priv_to_devlink(dev),
				   mlx5_fw_reset_devlink_params,
				   ARRAY_SIZE(mlx5_fw_reset_devlink_params));
	if (err) {
		destroy_workqueue(fw_reset->wq);
		kfree(fw_reset);
		return err;
	}
#endif

	INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
	INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
	INIT_WORK(&fw_reset->reset_unload_work, mlx5_sync_reset_unload_event);
	INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
	INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
	INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
	INIT_DELAYED_WORK(&fw_reset->reset_timeout_work, mlx5_sync_reset_timeout_work);

#ifdef HAVE_DEVLINK_RELOAD_DOWN_SUPPORT_RELOAD_ACTION
	init_completion(&fw_reset->done);
#endif
	return 0;
}

void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

#ifdef HAVE_DEVL_PARAM_DRIVERINIT_VALUE_GET
	devl_params_unregister(priv_to_devlink(dev),
			       mlx5_fw_reset_devlink_params,
			       ARRAY_SIZE(mlx5_fw_reset_devlink_params));
#endif
	destroy_workqueue(fw_reset->wq);
	kfree(dev->priv.fw_reset);
}
