diff --git a/Documentation/admin-guide/cgroup-v1/rdma.rst b/Documentation/admin-guide/cgroup-v1/rdma.rst index e69369b7252e..8d0c3a796ee3 100644 --- a/Documentation/admin-guide/cgroup-v1/rdma.rst +++ b/Documentation/admin-guide/cgroup-v1/rdma.rst @@ -90,6 +90,13 @@ Following resources can be accounted by rdma controller. hca_object Maximum number of HCA Objects ========== ============================= +RDMA devices from all network namespaces are listed. Each line starts with +the device name. If more than one device has the same name, ``index=N`` +follows the name, where ``N`` is the system-wide RDMA device index, unique +among registered devices. When configuring a limit, the index is optional +for a globally unique name and required for a duplicate name. A write without +the required index fails with ``-ENOTUNIQ``. + 2. Usage Examples ================= @@ -97,6 +104,7 @@ Following resources can be accounted by rdma controller. echo mlx4_0 hca_handle=2 hca_object=2000 > /sys/fs/cgroup/rdma/1/rdma.max echo ocrdma1 hca_handle=3 > /sys/fs/cgroup/rdma/2/rdma.max + echo "rxe0 index=5 hca_handle=2" > /sys/fs/cgroup/rdma/3/rdma.max (b) Query resource limit:: diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 993446ab66d0..df74d554d2cd 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2752,6 +2752,11 @@ RDMA The "rdma" controller regulates the distribution and accounting of RDMA resources. +RDMA devices from all network namespaces are listed. Each line starts with +the device name. If more than one device has the same name, ``index=N`` +follows the name, where ``N`` is the system-wide RDMA device index, unique +among registered devices. + RDMA Interface Files ~~~~~~~~~~~~~~~~~~~~ @@ -2760,7 +2765,11 @@ RDMA Interface Files except root that describes current configured resource limit for a RDMA/IB device. - Lines are keyed by device name and are not ordered. + Lines are keyed by device name and are not ordered. A write may + include ``index=N`` after the device name. The index is optional + when the name is globally unique. If multiple devices have that + name, the index is required and a write without it fails with + ``-ENOTUNIQ``. Each line contains space separated resource name and its configured limit that can be distributed. @@ -2776,6 +2785,10 @@ RDMA Interface Files mlx4_0 hca_handle=2 hca_object=2000 ocrdma1 hca_handle=3 hca_object=max + For devices with duplicate names, select the device by index:: + + echo "rxe0 index=5 hca_handle=2" > rdma.max + rdma.current A read-only file that describes current resource usage. It exists for all the cgroup except root. diff --git a/drivers/infiniband/core/cgroup.c b/drivers/infiniband/core/cgroup.c index 1f037fe01450..8611b4e32cfb 100644 --- a/drivers/infiniband/core/cgroup.c +++ b/drivers/infiniband/core/cgroup.c @@ -17,6 +17,7 @@ void ib_device_register_rdmacg(struct ib_device *device) { device->cg_device.name = device->name; + device->cg_device.index = device->index; rdmacg_register_device(&device->cg_device); } diff --git a/include/linux/cgroup_rdma.h b/include/linux/cgroup_rdma.h index 404e746552ca..9a5c9ee728e7 100644 --- a/include/linux/cgroup_rdma.h +++ b/include/linux/cgroup_rdma.h @@ -34,6 +34,7 @@ struct rdmacg_device { struct list_head dev_node; struct list_head rpools; char *name; + u32 index; }; /* diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 5e82a03b3270..9489f3df0bf3 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -19,6 +19,7 @@ #define RDMACG_MAX_STR "max" enum rdmacg_limit_tokens { + RDMACG_DEVICE_INDEX, RDMACG_HCA_HANDLE_VAL, RDMACG_HCA_HANDLE_MAX, RDMACG_HCA_OBJECT_VAL, @@ -27,6 +28,7 @@ enum rdmacg_limit_tokens { }; static const match_table_t rdmacg_limit_tokens = { + { RDMACG_DEVICE_INDEX, "index=%u" }, { RDMACG_HCA_HANDLE_VAL, "hca_handle=%d" }, { RDMACG_HCA_HANDLE_MAX, "hca_handle=max" }, { RDMACG_HCA_OBJECT_VAL, "hca_object=%d" }, @@ -464,17 +466,53 @@ void rdmacg_unregister_device(struct rdmacg_device *device) } EXPORT_SYMBOL(rdmacg_unregister_device); -static struct rdmacg_device *rdmacg_get_device_locked(const char *name) +static struct rdmacg_device * +rdmacg_get_device_locked(const char *name, bool has_index, u32 index) { + struct rdmacg_device *match = NULL; struct rdmacg_device *device; lockdep_assert_held(&rdmacg_mutex); - list_for_each_entry(device, &rdmacg_devices, dev_node) - if (!strcmp(name, device->name)) - return device; + list_for_each_entry(device, &rdmacg_devices, dev_node) { + if (strcmp(name, device->name)) + continue; - return NULL; + if (has_index) { + if (device->index == index) + return device; + continue; + } + + if (match) + return ERR_PTR(-ENOTUNIQ); + match = device; + } + + return match ?: ERR_PTR(-ENODEV); +} + +static bool +rdmacg_device_name_unique_locked(const struct rdmacg_device *device) +{ + struct rdmacg_device *other; + + lockdep_assert_held(&rdmacg_mutex); + + list_for_each_entry(other, &rdmacg_devices, dev_node) + if (other != device && !strcmp(other->name, device->name)) + return false; + + return true; +} + +static void rdmacg_print_device_key(struct seq_file *sf, + const struct rdmacg_device *device) +{ + seq_puts(sf, device->name); + if (!rdmacg_device_name_unique_locked(device)) + seq_printf(sf, " index=%u", device->index); + seq_putc(sf, ' '); } static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, @@ -488,6 +526,8 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, char *p; int *new_limits; unsigned long enables = 0; + u32 dev_index = 0; + bool has_index = false; int i = 0, ret = 0; /* extract the device name first */ @@ -503,7 +543,7 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, goto err; } - /* parse resource limit tokens */ + /* parse the optional device index and resource limit tokens */ while ((p = strsep(&options, " \t\n"))) { substring_t args[MAX_OPT_ARGS]; int tok, intval; @@ -513,6 +553,13 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, tok = match_token(p, rdmacg_limit_tokens, args); switch (tok) { + case RDMACG_DEVICE_INDEX: + if (has_index || match_uint(&args[0], &dev_index)) { + ret = -EINVAL; + goto parse_err; + } + has_index = true; + break; case RDMACG_HCA_HANDLE_VAL: if (match_int(&args[0], &intval) || intval < 0) { ret = -EINVAL; @@ -546,9 +593,9 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of, /* acquire lock to synchronize with hot plug devices */ mutex_lock(&rdmacg_mutex); - device = rdmacg_get_device_locked(dev_name); - if (!device) { - ret = -ENODEV; + device = rdmacg_get_device_locked(dev_name, has_index, dev_index); + if (IS_ERR(device)) { + ret = PTR_ERR(device); goto dev_err; } @@ -626,7 +673,7 @@ static int rdmacg_resource_read(struct seq_file *sf, void *v) mutex_lock(&rdmacg_mutex); list_for_each_entry(device, &rdmacg_devices, dev_node) { - seq_printf(sf, "%s ", device->name); + rdmacg_print_device_key(sf, device); rpool = find_cg_rpool_locked(cg, device); print_rpool_values(sf, rpool); @@ -650,7 +697,7 @@ static int rdmacg_events_show(struct seq_file *sf, void *v) list_for_each_entry(device, &rdmacg_devices, dev_node) { rpool = find_cg_rpool_locked(cg, device); - seq_printf(sf, "%s ", device->name); + rdmacg_print_device_key(sf, device); for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i], @@ -679,7 +726,7 @@ static int rdmacg_events_local_show(struct seq_file *sf, void *v) list_for_each_entry(device, &rdmacg_devices, dev_node) { rpool = find_cg_rpool_locked(cg, device); - seq_printf(sf, "%s ", device->name); + rdmacg_print_device_key(sf, device); for (i = 0; i < RDMACG_RESOURCE_MAX; i++) { seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu", rdmacg_resource_names[i],