mobility.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Support for Partition Mobility/Migration
 *
 * Copyright (C) 2010 Nathan Fontenot
 * Copyright (C) 2010 IBM Corporation
 */


#define pr_fmt(fmt) "mobility: " fmt

#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/kobject.h>
#include <linux/nmi.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/stat.h>
#include <linux/stop_machine.h>
#include <linux/completion.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/stringify.h>

#include <asm/machdep.h>
#include <asm/nmi.h>
#include <asm/rtas.h>
#include "pseries.h"
#include "vas.h"	/* vas_migration_handler() */
#include "papr-hvpipe.h"	/* hvpipe_migration_handler() */
#include "../../kernel/cacheinfo.h"

static struct kobject *mobility_kobj;

struct update_props_workarea {
	__be32 phandle;
	__be32 state;
	__be64 reserved;
	__be32 nprops;
} __packed;

#define NODE_ACTION_MASK	0xff000000
#define NODE_COUNT_MASK		0x00ffffff

#define DELETE_DT_NODE	0x01000000
#define UPDATE_DT_NODE	0x02000000
#define ADD_DT_NODE	0x03000000

#define MIGRATION_SCOPE	(1)
#define PRRN_SCOPE -2

#ifdef CONFIG_PPC_WATCHDOG
static unsigned int nmi_wd_lpm_factor = 200;

#ifdef CONFIG_SYSCTL
static const struct ctl_table nmi_wd_lpm_factor_ctl_table[] = {
	{
		.procname	= "nmi_wd_lpm_factor",
		.data		= &nmi_wd_lpm_factor,
		.maxlen		= sizeof(int),
		.mode		= 0644,
		.proc_handler	= proc_douintvec_minmax,
	},
};

static int __init register_nmi_wd_lpm_factor_sysctl(void)
{
	register_sysctl("kernel", nmi_wd_lpm_factor_ctl_table);

	return 0;
}
device_initcall(register_nmi_wd_lpm_factor_sysctl);
#endif /* CONFIG_SYSCTL */
#endif /* CONFIG_PPC_WATCHDOG */

static int mobility_rtas_call(int token, char *buf, s32 scope)
{
	int rc;

	spin_lock(&rtas_data_buf_lock);

	memcpy(rtas_data_buf, buf, RTAS_DATA_BUF_SIZE);
	rc = rtas_call(token, 2, 1, NULL, rtas_data_buf, scope);
	memcpy(buf, rtas_data_buf, RTAS_DATA_BUF_SIZE);

	spin_unlock(&rtas_data_buf_lock);
	return rc;
}

static int delete_dt_node(struct device_node *dn)
{
	struct device_node *pdn;
	bool is_platfac;

	pdn = of_get_parent(dn);
	is_platfac = of_node_is_type(dn, "ibm,platform-facilities") ||
		     of_node_is_type(pdn, "ibm,platform-facilities");
	of_node_put(pdn);

	/*
	 * The drivers that bind to nodes in the platform-facilities
	 * hierarchy don't support node removal, and the removal directive
	 * from firmware is always followed by an add of an equivalent
	 * node. The capability (e.g. RNG, encryption, compression)
	 * represented by the node is never interrupted by the migration.
	 * So ignore changes to this part of the tree.
	 */
	if (is_platfac) {
		pr_notice("ignoring remove operation for %pOFfp\n", dn);
		return 0;
	}

	pr_debug("removing node %pOFfp\n", dn);
	dlpar_detach_node(dn);
	return 0;
}

static int update_dt_property(struct device_node *dn, struct property **prop,
			      const char *name, u32 vd, char *value)
{
	struct property *new_prop = *prop;
	int more = 0;

	/* A negative 'vd' value indicates that only part of the new property
	 * value is contained in the buffer and we need to call
	 * ibm,update-properties again to get the rest of the value.
	 *
	 * A negative value is also the two's compliment of the actual value.
	 */
	if (vd & 0x80000000) {
		vd = ~vd + 1;
		more = 1;
	}

	if (new_prop) {
		/* partial property fixup */
		char *new_data = kzalloc(new_prop->length + vd, GFP_KERNEL);
		if (!new_data)
			return -ENOMEM;

		memcpy(new_data, new_prop->value, new_prop->length);
		memcpy(new_data + new_prop->length, value, vd);

		kfree(new_prop->value);
		new_prop->value = new_data;
		new_prop->length += vd;
	} else {
		new_prop = kzalloc(sizeof(*new_prop), GFP_KERNEL);
		if (!new_prop)
			return -ENOMEM;

		new_prop->name = kstrdup(name, GFP_KERNEL);
		if (!new_prop->name) {
			kfree(new_prop);
			return -ENOMEM;
		}

		new_prop->length = vd;
		new_prop->value = kzalloc(new_prop->length, GFP_KERNEL);
		if (!new_prop->value) {
			kfree(new_prop->name);
			kfree(new_prop);
			return -ENOMEM;
		}

		memcpy(new_prop->value, value, vd);
		*prop = new_prop;
	}

	if (!more) {
		pr_debug("updating node %pOF property %s\n", dn, name);
		of_update_property(dn, new_prop);
		*prop = NULL;
	}

	return 0;
}

static int update_dt_node(struct device_node *dn, s32 scope)
{
	struct update_props_workarea *upwa;
	struct property *prop = NULL;
	int i, rc, rtas_rc;
	char *prop_data;
	char *rtas_buf;
	int update_properties_token;
	u32 nprops;
	u32 vd;

	update_properties_token = rtas_function_token(RTAS_FN_IBM_UPDATE_PROPERTIES);
	if (update_properties_token == RTAS_UNKNOWN_SERVICE)
		return -EINVAL;

	rtas_buf = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
	if (!rtas_buf)
		return -ENOMEM;

	upwa = (struct update_props_workarea *)&rtas_buf[0];
	upwa->phandle = cpu_to_be32(dn->phandle);

	do {
		rtas_rc = mobility_rtas_call(update_properties_token, rtas_buf,
					scope);
		if (rtas_rc < 0)
			break;

		prop_data = rtas_buf + sizeof(*upwa);
		nprops = be32_to_cpu(upwa->nprops);

		/* On the first call to ibm,update-properties for a node the
		 * first property value descriptor contains an empty
		 * property name, the property value length encoded as u32,
		 * and the property value is the node path being updated.
		 */
		if (*prop_data == 0) {
			prop_data++;
			vd = be32_to_cpu(*(__be32 *)prop_data);
			prop_data += vd + sizeof(vd);
			nprops--;
		}

		for (i = 0; i < nprops; i++) {
			char *prop_name;

			prop_name = prop_data;
			prop_data += strlen(prop_name) + 1;
			vd = be32_to_cpu(*(__be32 *)prop_data);
			prop_data += sizeof(vd);

			switch (vd) {
			case 0x00000000:
				/* name only property, nothing to do */
				break;

			case 0x80000000:
				of_remove_property(dn, of_find_property(dn,
							prop_name, NULL));
				prop = NULL;
				break;

			default:
				rc = update_dt_property(dn, &prop, prop_name,
							vd, prop_data);
				if (rc) {
					pr_err("updating %s property failed: %d\n",
					       prop_name, rc);
				}

				prop_data += vd;
				break;
			}

			cond_resched();
		}

		cond_resched();
	} while (rtas_rc == 1);

	kfree(rtas_buf);
	return 0;
}

static int add_dt_node(struct device_node *parent_dn, __be32 drc_index)
{
	struct device_node *dn;
	int rc;

	dn = dlpar_configure_connector(drc_index, parent_dn);
	if (!dn)
		return -ENOENT;

	/*
	 * Since delete_dt_node() ignores this node type, this is the
	 * necessary counterpart. We also know that a platform-facilities
	 * node returned from dlpar_configure_connector() has children
	 * attached, and dlpar_attach_node() only adds the parent, leaking
	 * the children. So ignore these on the add side for now.
	 */
	if (of_node_is_type(dn, "ibm,platform-facilities")) {
		pr_notice("ignoring add operation for %pOF\n", dn);
		dlpar_free_cc_nodes(dn);
		return 0;
	}

	rc = dlpar_attach_node(dn, parent_dn);
	if (rc)
		dlpar_free_cc_nodes(dn);

	pr_debug("added node %pOFfp\n", dn);

	return rc;
}

static int pseries_devicetree_update(s32 scope)
{
	char *rtas_buf;
	__be32 *data;
	int update_nodes_token;
	int rc;

	update_nodes_token = rtas_function_token(RTAS_FN_IBM_UPDATE_NODES);
	if (update_nodes_token == RTAS_UNKNOWN_SERVICE)
		return 0;

	rtas_buf = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
	if (!rtas_buf)
		return -ENOMEM;

	do {
		rc = mobility_rtas_call(update_nodes_token, rtas_buf, scope);
		if (rc && rc != 1)
			break;

		data = (__be32 *)rtas_buf + 4;
		while (be32_to_cpu(*data) & NODE_ACTION_MASK) {
			int i;
			u32 action = be32_to_cpu(*data) & NODE_ACTION_MASK;
			u32 node_count = be32_to_cpu(*data) & NODE_COUNT_MASK;

			data++;

			for (i = 0; i < node_count; i++) {
				struct device_node *np;
				__be32 phandle = *data++;
				__be32 drc_index;

				np = of_find_node_by_phandle(be32_to_cpu(phandle));
				if (!np) {
					pr_warn("Failed lookup: phandle 0x%x for action 0x%x\n",
						be32_to_cpu(phandle), action);
					continue;
				}

				switch (action) {
				case DELETE_DT_NODE:
					delete_dt_node(np);
					break;
				case UPDATE_DT_NODE:
					update_dt_node(np, scope);
					break;
				case ADD_DT_NODE:
					drc_index = *data++;
					add_dt_node(np, drc_index);
					break;
				}

				of_node_put(np);
				cond_resched();
			}
		}

		cond_resched();
	} while (rc == 1);

	kfree(rtas_buf);
	return rc;
}

void post_mobility_fixup(void)
{
	int rc;

	rtas_activate_firmware();

	/*
	 * We don't want CPUs to go online/offline while the device
	 * tree is being updated.
	 */
	cpus_read_lock();

	/*
	 * It's common for the destination firmware to replace cache
	 * nodes.  Release all of the cacheinfo hierarchy's references
	 * before updating the device tree.
	 */
	cacheinfo_teardown();

	rc = pseries_devicetree_update(MIGRATION_SCOPE);
	if (rc)
		pr_err("device tree update failed: %d\n", rc);

	cacheinfo_rebuild();

	cpus_read_unlock();

	/* Possibly switch to a new L1 flush type */
	pseries_setup_security_mitigations();

	/* Reinitialise system information for hv-24x7 */
	read_24x7_sys_info();

	return;
}

static int poll_vasi_state(u64 handle, unsigned long *res)
{
	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
	long hvrc;
	int ret;

	hvrc = plpar_hcall(H_VASI_STATE, retbuf, handle);
	switch (hvrc) {
	case H_SUCCESS:
		ret = 0;
		*res = retbuf[0];
		break;
	case H_PARAMETER:
		ret = -EINVAL;
		break;
	case H_FUNCTION:
		ret = -EOPNOTSUPP;
		break;
	case H_HARDWARE:
	default:
		pr_err("unexpected H_VASI_STATE result %ld\n", hvrc);
		ret = -EIO;
		break;
	}
	return ret;
}

static int wait_for_vasi_session_suspending(u64 handle)
{
	unsigned long state;
	int ret;

	/*
	 * Wait for transition from H_VASI_ENABLED to
	 * H_VASI_SUSPENDING. Treat anything else as an error.
	 */
	while (true) {
		ret = poll_vasi_state(handle, &state);

		if (ret != 0 || state == H_VASI_SUSPENDING) {
			break;
		} else if (state == H_VASI_ENABLED) {
			ssleep(1);
		} else {
			pr_err("unexpected H_VASI_STATE result %lu\n", state);
			ret = -EIO;
			break;
		}
	}

	/*
	 * Proceed even if H_VASI_STATE is unavailable. If H_JOIN or
	 * ibm,suspend-me are also unimplemented, we'll recover then.
	 */
	if (ret == -EOPNOTSUPP)
		ret = 0;

	return ret;
}

static void wait_for_vasi_session_completed(u64 handle)
{
	unsigned long state = 0;
	int ret;

	pr_info("waiting for memory transfer to complete...\n");

	/*
	 * Wait for transition from H_VASI_RESUMED to H_VASI_COMPLETED.
	 */
	while (true) {
		ret = poll_vasi_state(handle, &state);

		/*
		 * If the memory transfer is already complete and the migration
		 * has been cleaned up by the hypervisor, H_PARAMETER is return,
		 * which is translate in EINVAL by poll_vasi_state().
		 */
		if (ret == -EINVAL || (!ret && state == H_VASI_COMPLETED)) {
			pr_info("memory transfer completed.\n");
			break;
		}

		if (ret) {
			pr_err("H_VASI_STATE return error (%d)\n", ret);
			break;
		}

		if (state != H_VASI_RESUMED) {
			pr_err("unexpected H_VASI_STATE result %lu\n", state);
			break;
		}

		msleep(500);
	}
}

static void prod_single(unsigned int target_cpu)
{
	long hvrc;
	int hwid;

	hwid = get_hard_smp_processor_id(target_cpu);
	hvrc = plpar_hcall_norets(H_PROD, hwid);
	if (hvrc == H_SUCCESS)
		return;
	pr_err_ratelimited("H_PROD of CPU %u (hwid %d) error: %ld\n",
			   target_cpu, hwid, hvrc);
}

static void prod_others(void)
{
	unsigned int cpu;

	for_each_online_cpu(cpu) {
		if (cpu != smp_processor_id())
			prod_single(cpu);
	}
}

static u16 clamp_slb_size(void)
{
#ifdef CONFIG_PPC_64S_HASH_MMU
	u16 prev = mmu_slb_size;

	slb_set_size(SLB_MIN_SIZE);

	return prev;
#else
	return 0;
#endif
}

static int do_suspend(void)
{
	u16 saved_slb_size;
	int status;
	int ret;

	pr_info("calling ibm,suspend-me on CPU %i\n", smp_processor_id());

	/*
	 * The destination processor model may have fewer SLB entries
	 * than the source. We reduce mmu_slb_size to a safe minimum
	 * before suspending in order to minimize the possibility of
	 * programming non-existent entries on the destination. If
	 * suspend fails, we restore it before returning. On success
	 * the OF reconfig path will update it from the new device
	 * tree after resuming on the destination.
	 */
	saved_slb_size = clamp_slb_size();

	ret = rtas_ibm_suspend_me(&status);
	if (ret != 0) {
		pr_err("ibm,suspend-me error: %d\n", status);
		slb_set_size(saved_slb_size);
	}

	return ret;
}

/**
 * struct pseries_suspend_info - State shared between CPUs for join/suspend.
 * @counter: Threads are to increment this upon resuming from suspend
 *           or if an error is received from H_JOIN. The thread which performs
 *           the first increment (i.e. sets it to 1) is responsible for
 *           waking the other threads.
 * @done: False if join/suspend is in progress. True if the operation is
 *        complete (successful or not).
 */
struct pseries_suspend_info {
	atomic_t counter;
	bool done;
};

static int do_join(void *arg)
{
	struct pseries_suspend_info *info = arg;
	atomic_t *counter = &info->counter;
	long hvrc;
	int ret;

retry:
	/* Must ensure MSR.EE off for H_JOIN. */
	hard_irq_disable();
	hvrc = plpar_hcall_norets(H_JOIN);

	switch (hvrc) {
	case H_CONTINUE:
		/*
		 * All other CPUs are offline or in H_JOIN. This CPU
		 * attempts the suspend.
		 */
		ret = do_suspend();
		break;
	case H_SUCCESS:
		/*
		 * The suspend is complete and this cpu has received a
		 * prod, or we've received a stray prod from unrelated
		 * code (e.g. paravirt spinlocks) and we need to join
		 * again.
		 *
		 * This barrier orders the return from H_JOIN above vs
		 * the load of info->done. It pairs with the barrier
		 * in the wakeup/prod path below.
		 */
		smp_mb();
		if (READ_ONCE(info->done) == false) {
			pr_info_ratelimited("premature return from H_JOIN on CPU %i, retrying",
					    smp_processor_id());
			goto retry;
		}
		ret = 0;
		break;
	case H_BAD_MODE:
	case H_HARDWARE:
	default:
		ret = -EIO;
		pr_err_ratelimited("H_JOIN error %ld on CPU %i\n",
				   hvrc, smp_processor_id());
		break;
	}

	if (atomic_inc_return(counter) == 1) {
		pr_info("CPU %u waking all threads\n", smp_processor_id());
		WRITE_ONCE(info->done, true);
		/*
		 * This barrier orders the store to info->done vs subsequent
		 * H_PRODs to wake the other CPUs. It pairs with the barrier
		 * in the H_SUCCESS case above.
		 */
		smp_mb();
		prod_others();
	}
	/*
	 * Execution may have been suspended for several seconds, so reset
	 * the watchdogs. touch_nmi_watchdog() also touches the soft lockup
	 * watchdog.
	 */
	rcu_cpu_stall_reset();
	touch_nmi_watchdog();

	return ret;
}

/*
 * Abort reason code byte 0. We use only the 'Migrating partition' value.
 */
enum vasi_aborting_entity {
	ORCHESTRATOR        = 1,
	VSP_SOURCE          = 2,
	PARTITION_FIRMWARE  = 3,
	PLATFORM_FIRMWARE   = 4,
	VSP_TARGET          = 5,
	MIGRATING_PARTITION = 6,
};

static void pseries_cancel_migration(u64 handle, int err)
{
	u32 reason_code;
	u32 detail;
	u8 entity;
	long hvrc;

	entity = MIGRATING_PARTITION;
	detail = abs(err) & 0xffffff;
	reason_code = (entity << 24) | detail;

	hvrc = plpar_hcall_norets(H_VASI_SIGNAL, handle,
				  H_VASI_SIGNAL_CANCEL, reason_code);
	if (hvrc)
		pr_err("H_VASI_SIGNAL error: %ld\n", hvrc);
}

static int pseries_suspend(u64 handle)
{
	const unsigned int max_attempts = 5;
	unsigned int retry_interval_ms = 1;
	unsigned int attempt = 1;
	int ret;

	while (true) {
		struct pseries_suspend_info info;
		unsigned long vasi_state;
		int vasi_err;

		info = (struct pseries_suspend_info) {
			.counter = ATOMIC_INIT(0),
			.done = false,
		};

		ret = stop_machine(do_join, &info, cpu_online_mask);
		if (ret == 0)
			break;
		/*
		 * Encountered an error. If the VASI stream is still
		 * in Suspending state, it's likely a transient
		 * condition related to some device in the partition
		 * and we can retry in the hope that the cause has
		 * cleared after some delay.
		 *
		 * A better design would allow drivers etc to prepare
		 * for the suspend and avoid conditions which prevent
		 * the suspend from succeeding. For now, we have this
		 * mitigation.
		 */
		pr_notice("Partition suspend attempt %u of %u error: %d\n",
			  attempt, max_attempts, ret);

		if (attempt == max_attempts)
			break;

		vasi_err = poll_vasi_state(handle, &vasi_state);
		if (vasi_err == 0) {
			if (vasi_state != H_VASI_SUSPENDING) {
				pr_notice("VASI state %lu after failed suspend\n",
					  vasi_state);
				break;
			}
		} else if (vasi_err != -EOPNOTSUPP) {
			pr_err("VASI state poll error: %d", vasi_err);
			break;
		}

		pr_notice("Will retry partition suspend after %u ms\n",
			  retry_interval_ms);

		msleep(retry_interval_ms);
		retry_interval_ms *= 10;
		attempt++;
	}

	return ret;
}

static int pseries_migrate_partition(u64 handle)
{
	int ret;
	unsigned int factor = 0;

#ifdef CONFIG_PPC_WATCHDOG
	factor = nmi_wd_lpm_factor;
#endif
	/*
	 * When the migration is initiated, the hypervisor changes VAS
	 * mappings to prepare before OS gets the notification and
	 * closes all VAS windows. NX generates continuous faults during
	 * this time and the user space can not differentiate these
	 * faults from the migration event. So reduce this time window
	 * by closing VAS windows at the beginning of this function.
	 */
	vas_migration_handler(VAS_SUSPEND);
	hvpipe_migration_handler(HVPIPE_SUSPEND);

	ret = wait_for_vasi_session_suspending(handle);
	if (ret)
		goto out;

	if (factor)
		watchdog_hardlockup_set_timeout_pct(factor);

	ret = pseries_suspend(handle);
	if (ret == 0) {
		post_mobility_fixup();
		/*
		 * Wait until the memory transfer is complete, so that the user
		 * space process returns from the syscall after the transfer is
		 * complete. This allows the user hooks to be executed at the
		 * right time.
		 */
		wait_for_vasi_session_completed(handle);
	} else
		pseries_cancel_migration(handle, ret);

	if (factor)
		watchdog_hardlockup_set_timeout_pct(0);

out:
	vas_migration_handler(VAS_RESUME);
	hvpipe_migration_handler(HVPIPE_RESUME);

	return ret;
}

int rtas_syscall_dispatch_ibm_suspend_me(u64 handle)
{
	return pseries_migrate_partition(handle);
}

static ssize_t migration_store(const struct class *class,
			       const struct class_attribute *attr, const char *buf,
			       size_t count)
{
	u64 streamid;
	int rc;

	rc = kstrtou64(buf, 0, &streamid);
	if (rc)
		return rc;

	rc = pseries_migrate_partition(streamid);
	if (rc)
		return rc;

	return count;
}

/*
 * Used by drmgr to determine the kernel behavior of the migration interface.
 *
 * Version 1: Performs all PAPR requirements for migration including
 *	firmware activation and device tree update.
 */
#define MIGRATION_API_VERSION	1

static CLASS_ATTR_WO(migration);
static CLASS_ATTR_STRING(api_version, 0444, __stringify(MIGRATION_API_VERSION));

static int __init mobility_sysfs_init(void)
{
	int rc;

	mobility_kobj = kobject_create_and_add("mobility", kernel_kobj);
	if (!mobility_kobj)
		return -ENOMEM;

	rc = sysfs_create_file(mobility_kobj, &class_attr_migration.attr);
	if (rc)
		pr_err("unable to create migration sysfs file (%d)\n", rc);

	rc = sysfs_create_file(mobility_kobj, &class_attr_api_version.attr.attr);
	if (rc)
		pr_err("unable to create api_version sysfs file (%d)\n", rc);

	return 0;
}
machine_device_initcall(pseries, mobility_sysfs_init);