#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/memory.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/memory-tiers.h>
#include <linux/memory_hotplug.h>
#include <linux/string_helpers.h>
#include "dax-private.h"
#include "bus.h"
#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5)
static const char *kmem_name;
static bool any_hotremove_failed;
static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r)
{
struct dev_dax_range *dax_range = &dev_dax->ranges[i];
struct range *range = &dax_range->range;
*r = memory_block_aligned_range(range);
if (r->start >= r->end) {
r->start = range->start;
r->end = range->end;
return -ENOSPC;
}
return 0;
}
struct dax_kmem_data {
const char *res_name;
int mgid;
int state;
struct mutex lock;
struct resource *res[];
};
static DEFINE_MUTEX(kmem_memory_type_lock);
static LIST_HEAD(kmem_memory_types);
static struct memory_dev_type *kmem_find_alloc_memory_type(int adist)
{
guard(mutex)(&kmem_memory_type_lock);
return mt_find_alloc_memory_type(adist, &kmem_memory_types);
}
static void kmem_put_memory_types(void)
{
guard(mutex)(&kmem_memory_type_lock);
mt_put_memory_types(&kmem_memory_types);
}
static bool dax_kmem_state_is_online(int state)
{
return state == MMOP_ONLINE ||
state == MMOP_ONLINE_KERNEL ||
state == MMOP_ONLINE_MOVABLE;
}
static int dax_kmem_do_hotplug(struct dev_dax *dev_dax,
struct dax_kmem_data *data,
int online_type)
{
struct device *dev = &dev_dax->dev;
int i, rc, added = 0;
mhp_t mhp_flags;
if (dax_kmem_state_is_online(data->state))
return -EINVAL;
if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE)
return -EINVAL;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range range;
rc = dax_kmem_range(dev_dax, i, &range);
if (rc)
continue;
if (!data->res[i])
continue;
mhp_flags = MHP_NID_IS_MGID;
if (dev_dax->memmap_on_memory)
mhp_flags |= MHP_MEMMAP_ON_MEMORY;
rc = __add_memory_driver_managed(data->mgid, range.start,
range_len(&range), kmem_name, mhp_flags,
online_type);
if (rc) {
dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
i, range.start, range.end);
if (data->res[i]) {
remove_resource(data->res[i]);
kfree(data->res[i]);
data->res[i] = NULL;
}
if (added)
continue;
return rc;
}
added++;
}
return added;
}
static int dax_kmem_init_resources(struct dev_dax *dev_dax,
struct dax_kmem_data *data)
{
struct device *dev = &dev_dax->dev;
int i, rc, mapped = 0;
for (i = 0; i < dev_dax->nr_range; i++) {
struct resource *res;
struct range range;
rc = dax_kmem_range(dev_dax, i, &range);
if (rc)
continue;
if (data->res[i])
continue;
res = request_mem_region(range.start, range_len(&range),
data->res_name);
if (!res) {
dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n",
i, range.start, range.end);
if (mapped)
continue;
return -EBUSY;
}
data->res[i] = res;
res->flags = IORESOURCE_SYSTEM_RAM;
mapped++;
}
return mapped;
}
#ifdef CONFIG_MEMORY_HOTREMOVE
static int dax_kmem_do_hotremove(struct dev_dax *dev_dax,
struct dax_kmem_data *data)
{
struct device *dev = &dev_dax->dev;
struct range *ranges;
int i, nr_ranges = 0, rc;
ranges = kmalloc_objs(*ranges, dev_dax->nr_range);
if (!ranges)
return -ENOMEM;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range range;
if (!data->res[i])
continue;
if (dax_kmem_range(dev_dax, i, &range))
continue;
ranges[nr_ranges++] = range;
}
if (!nr_ranges) {
kfree(ranges);
return 0;
}
rc = offline_and_remove_memory_ranges(ranges, nr_ranges);
kfree(ranges);
if (rc) {
dev_err(dev, "hotremove failed, device left online: %d\n", rc);
return rc;
}
for (i = 0; i < dev_dax->nr_range; i++) {
if (!data->res[i])
continue;
remove_resource(data->res[i]);
kfree(data->res[i]);
data->res[i] = NULL;
}
return 0;
}
#else
static int dax_kmem_do_hotremove(struct dev_dax *dev_dax,
struct dax_kmem_data *data)
{
return -EBUSY;
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax,
struct dax_kmem_data *data)
{
int i;
if (WARN(((data->state != DAX_KMEM_UNPLUGGED) &&
(data->state != MMOP_OFFLINE)),
"Hotplug memory regions stuck online until reboot"))
return;
for (i = 0; i < dev_dax->nr_range; i++) {
if (!data->res[i])
continue;
remove_resource(data->res[i]);
kfree(data->res[i]);
data->res[i] = NULL;
}
}
static int dax_kmem_parse_state(const char *buf)
{
int online_type;
if (sysfs_streq(buf, "unplugged"))
return DAX_KMEM_UNPLUGGED;
online_type = mhp_online_type_from_str(buf);
if (online_type == MMOP_OFFLINE)
return -EINVAL;
return online_type;
}
static ssize_t state_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct dax_kmem_data *data = dev_get_drvdata(dev);
const char *state_str;
if (data->state == DAX_KMEM_UNPLUGGED)
state_str = "unplugged";
else
state_str = mhp_online_type_to_str(data->state);
return sysfs_emit(buf, "%s\n", state_str ?: "unknown");
}
static ssize_t state_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t len)
{
struct dev_dax *dev_dax = to_dev_dax(dev);
struct dax_kmem_data *data = dev_get_drvdata(dev);
int online_type;
int rc;
online_type = dax_kmem_parse_state(buf);
if (online_type < DAX_KMEM_UNPLUGGED)
return online_type;
guard(mutex)(&data->lock);
if (data->state == online_type)
return len;
if (online_type == DAX_KMEM_UNPLUGGED) {
rc = dax_kmem_do_hotremove(dev_dax, data);
if (rc)
return rc;
data->state = DAX_KMEM_UNPLUGGED;
return len;
}
if (data->state != DAX_KMEM_UNPLUGGED)
return -EBUSY;
rc = dax_kmem_init_resources(dev_dax, data);
if (rc < 0)
return rc;
rc = dax_kmem_do_hotplug(dev_dax, data, online_type);
if (rc < 0) {
dax_kmem_cleanup_resources(dev_dax, data);
return rc;
}
data->state = online_type;
return len;
}
static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
{
struct device *dev = &dev_dax->dev;
unsigned long total_len = 0, orig_len = 0;
struct dax_kmem_data *data;
struct memory_dev_type *mtype;
int i, rc;
int numa_node;
int adist = MEMTIER_DEFAULT_DAX_ADISTANCE;
int online_type = mhp_get_default_online_type();
numa_node = dev_dax->target_node;
if (numa_node < 0) {
dev_warn(dev, "rejecting DAX region with invalid node: %d\n",
numa_node);
return -EINVAL;
}
mt_calc_adistance(numa_node, &adist);
mtype = kmem_find_alloc_memory_type(adist);
if (IS_ERR(mtype))
return PTR_ERR(mtype);
for (i = 0; i < dev_dax->nr_range; i++) {
struct range range;
orig_len += range_len(&dev_dax->ranges[i].range);
rc = dax_kmem_range(dev_dax, i, &range);
if (rc) {
dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n",
i, range.start, range.end);
continue;
}
total_len += range_len(&range);
}
if (!total_len) {
dev_warn(dev, "rejecting DAX region without any memory after alignment\n");
return -EINVAL;
} else if (total_len != orig_len) {
char buf[16];
string_get_size(orig_len - total_len, 1, STRING_UNITS_2,
buf, sizeof(buf));
dev_warn(dev, "DAX region truncated by %s due to alignment\n", buf);
}
init_node_memory_type(numa_node, mtype);
rc = -ENOMEM;
data = kzalloc_flex(*data, res, dev_dax->nr_range);
if (!data)
goto err_dax_kmem_data;
data->res_name = kstrdup(dev_name(dev), GFP_KERNEL);
if (!data->res_name)
goto err_res_name;
rc = memory_group_register_static(numa_node, PFN_UP(total_len));
if (rc < 0)
goto err_reg_mgid;
data->mgid = rc;
data->state = DAX_KMEM_UNPLUGGED;
mutex_init(&data->lock);
dev_set_drvdata(dev, data);
rc = dax_kmem_init_resources(dev_dax, data);
if (rc < 0)
goto err_resources;
rc = dax_kmem_do_hotplug(dev_dax, data, online_type);
if (rc < 0)
goto err_hotplug;
data->state = online_type;
return 0;
err_hotplug:
dax_kmem_cleanup_resources(dev_dax, data);
err_resources:
dev_set_drvdata(dev, NULL);
memory_group_unregister(data->mgid);
err_reg_mgid:
kfree(data->res_name);
err_res_name:
kfree(data);
err_dax_kmem_data:
clear_node_memory_type(numa_node, mtype);
return rc;
}
#ifdef CONFIG_MEMORY_HOTREMOVE
static int dax_kmem_remove_ranges(struct dev_dax *dev_dax,
struct dax_kmem_data *data)
{
struct device *dev = &dev_dax->dev;
int i, rc = 0;
for (i = 0; i < dev_dax->nr_range; i++) {
struct range range;
if (!data->res[i] || dax_kmem_range(dev_dax, i, &range))
continue;
if (remove_memory(range.start, range_len(&range))) {
dev_warn(dev, "mapping%d: %#llx-%#llx stuck online until reboot\n",
i, range.start, range.end);
rc = -EBUSY;
continue;
}
remove_resource(data->res[i]);
kfree(data->res[i]);
data->res[i] = NULL;
}
return rc;
}
static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
{
int node = dev_dax->target_node;
struct device *dev = &dev_dax->dev;
struct dax_kmem_data *data = dev_get_drvdata(dev);
if (dax_kmem_remove_ranges(dev_dax, data)) {
dev_err(dev, "Hotplug regions stuck online until reboot\n");
any_hotremove_failed = true;
return;
}
memory_group_unregister(data->mgid);
kfree(data->res_name);
kfree(data);
dev_set_drvdata(dev, NULL);
clear_node_memory_type(node, NULL);
}
#else
static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
{
any_hotremove_failed = true;
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
static DEVICE_ATTR_RW(state);
static struct attribute *dev_dax_kmem_attrs[] = {
&dev_attr_state.attr,
NULL,
};
ATTRIBUTE_GROUPS(dev_dax_kmem);
static struct dax_device_driver device_dax_kmem_driver = {
.probe = dev_dax_kmem_probe,
.remove = dev_dax_kmem_remove,
.type = DAXDRV_KMEM_TYPE,
.drv = {
.dev_groups = dev_dax_kmem_groups,
},
};
static int __init dax_kmem_init(void)
{
int rc;
kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL);
if (!kmem_name)
return -ENOMEM;
rc = dax_driver_register(&device_dax_kmem_driver);
if (rc)
goto error_dax_driver;
return rc;
error_dax_driver:
kmem_put_memory_types();
kfree_const(kmem_name);
return rc;
}
static void __exit dax_kmem_exit(void)
{
dax_driver_unregister(&device_dax_kmem_driver);
if (!any_hotremove_failed)
kfree_const(kmem_name);
kmem_put_memory_types();
}
MODULE_AUTHOR("Intel Corporation");
MODULE_DESCRIPTION("KMEM DAX: map dax-devices as System-RAM");
MODULE_LICENSE("GPL v2");
module_init(dax_kmem_init);
module_exit(dax_kmem_exit);
MODULE_ALIAS_DAX_DEVICE(0