diff --git a/arch/arm64/configs/vendor/pineapple_GKI.config b/arch/arm64/configs/vendor/pineapple_GKI.config index e9abb6d3e9b6..05f782220c7f 100644 --- a/arch/arm64/configs/vendor/pineapple_GKI.config +++ b/arch/arm64/configs/vendor/pineapple_GKI.config @@ -116,6 +116,7 @@ CONFIG_QCOM_MEM_BUF_DEV=m CONFIG_QCOM_MEM_BUF_DEV_GH=y CONFIG_QCOM_MEM_BUF_GH=y CONFIG_QCOM_MEM_BUF_MSGQ=m +CONFIG_QCOM_MEM_OFFLINE=m # CONFIG_QCOM_MINIDUMP is not set CONFIG_QCOM_PANIC_ON_NOTIF_TIMEOUT=y CONFIG_QCOM_PANIC_ON_PDR_NOTIF_TIMEOUT=y diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 8aa1439bb6b5..747b90342c8d 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -631,6 +631,17 @@ config QCOM_CPU_VENDOR_HOOKS having scope to upstream. If unsure, say N. +config QCOM_MEM_OFFLINE + tristate "Dynamic Memory Region Offline driver" + depends on MEMORY_HOTPLUG + help + Add support for DDR Self-Refresh power management through the dynamic + memory offline framework. This driver interfaces between the memory + hotplug subsystem and AOP which hot adds or removes memory blocks and + controls the start/stop of self-refresh of these DDR regions. This + helps reduce power consumption during idle mode of the system. + If unsure, say N + config QCOM_HUNG_TASK_ENH tristate "QTI Hung Task Enhancement" depends on ARCH_QCOM diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 0a7ea3833436..bd994f4bfcf1 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -63,6 +63,7 @@ minidump-$(CONFIG_QCOM_MINIDUMP_PANIC_DUMP) += minidump_memory.o obj-$(CONFIG_QCOM_VA_MINIDUMP) += qcom_va_minidump.o obj-$(CONFIG_QCOM_LOGBUF_VENDOR_HOOKS) += qcom_logbuf_vh.o obj-$(CONFIG_QCOM_HUNG_TASK_ENH) += hung_task_enh.o +obj-$(CONFIG_QCOM_MEM_OFFLINE) += mem-offline.o obj-$(CONFIG_GH_TLMM_VM_MEM_ACCESS) += gh_tlmm_vm_mem_access.o obj-$(CONFIG_QCOM_RAMDUMP) += qcom_ramdump.o obj-$(CONFIG_QCOM_ICC_BWMON) += icc-bwmon.o diff --git a/drivers/soc/qcom/mem-offline.c b/drivers/soc/qcom/mem-offline.c new file mode 100644 index 000000000000..1ba98e9e53fb --- /dev/null +++ b/drivers/soc/qcom/mem-offline.c @@ -0,0 +1,1542 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MSM_RPM_SMD +#include +#endif + +#define RPM_DDR_REQ 0x726464 +#define AOP_MSG_ADDR_MASK 0xffffffff +#define AOP_MSG_ADDR_HIGH_SHIFT 32 +#define MAX_LEN 96 + +/** + * bypass_send_msg - skip mem offline/online mesg sent to rpm/aop + */ +static bool bypass_send_msg; +module_param(bypass_send_msg, bool, 0644); +MODULE_PARM_DESC(bypass_send_msg, + "skip mem offline/online mesg sent to rpm/aop."); + +static unsigned long start_section_nr, end_section_nr; +static struct kobject *kobj; +static unsigned int sections_per_block; +static u32 offline_granule; +static bool is_rpm_controller; +static DECLARE_BITMAP(movable_bitmap, 1024); +#define MODULE_CLASS_NAME "mem-offline" +#define MEMBLOCK_NAME "memory%lu" +#define SEGMENT_NAME "segment%lu" +#define BUF_LEN 100 + +struct section_stat { + unsigned long success_count; + unsigned long fail_count; + unsigned long avg_time; + unsigned long best_time; + unsigned long worst_time; + unsigned long total_time; + unsigned long last_recorded_time; + ktime_t resident_time; + ktime_t resident_since; +}; + +enum memory_states { + MEMORY_ONLINE, + MEMORY_OFFLINE, + MAX_STATE, +}; + +struct segment_info { + signed long start_addr; + unsigned long seg_size; + unsigned long num_kernel_blks; + unsigned int bitmask_kernel_blk; + enum memory_states state; +}; + +#define MAX_NUM_SEGMENTS 16 +#define MAX_NUM_DDR_REGIONS 10 + +struct ddr_region { + /* region physical address */ + unsigned long start_address; + + /* size of region in bytes */ + unsigned long length; + + /* size of segments in MB (1024 * 1024 bytes) */ + unsigned long granule_size; + + /* index of first full segment in a region */ + unsigned int segments_start_idx; + + /* offset in bytes to first full segment */ + unsigned long segments_start_offset; + +}; + +static struct segment_info *segment_infos; + +static struct ddr_region *ddr_regions; + +static int differing_segment_sizes; + +static int num_ddr_regions, num_segments; + +/* + * start_addr_HIGH, start_addr_LOW, + * length_HIGH, length_LOW, + * segment_start_offset_HIGH, segment_start_offset_LOW, + * segment_start_idx_HIGH, segment_start_idx_LOW, + * granule_size_HIGH, granule_size_LOW + */ +#define DDR_REGIONS_NUM_CELLS 10 + +static enum memory_states *mem_sec_state; + +static phys_addr_t bootmem_dram_end_addr; + +static phys_addr_t offlinable_region_start_addr; + +static struct mem_offline_mailbox { + struct mbox_client cl; + struct mbox_chan *mbox; +} mailbox; + +struct memory_refresh_request { + u64 start; /* Lower bit signifies action + * 0 - disable self-refresh + * 1 - enable self-refresh + * upper bits are for base address + */ + u32 size; /* size of memory region */ +}; + +static struct section_stat *mem_info; + +static int nopasr; +module_param_named(nopasr, nopasr, uint, 0644); + +static void record_stat(unsigned long sec, ktime_t delay, int mode) +{ + unsigned int total_sec = end_section_nr - start_section_nr + 1; + unsigned int blk_nr = (sec - start_section_nr + mode * total_sec) / + sections_per_block; + ktime_t now, delta; + + if (sec > end_section_nr) + return; + + if (delay < mem_info[blk_nr].best_time || !mem_info[blk_nr].best_time) + mem_info[blk_nr].best_time = delay; + + if (delay > mem_info[blk_nr].worst_time) + mem_info[blk_nr].worst_time = delay; + + ++mem_info[blk_nr].success_count; + if (mem_info[blk_nr].fail_count) + --mem_info[blk_nr].fail_count; + + mem_info[blk_nr].total_time += delay; + + mem_info[blk_nr].avg_time = + mem_info[blk_nr].total_time / mem_info[blk_nr].success_count; + + mem_info[blk_nr].last_recorded_time = delay; + + now = ktime_get(); + mem_info[blk_nr].resident_since = now; + + /* since other state has gone inactive, update the stats */ + mode = mode ? MEMORY_ONLINE : MEMORY_OFFLINE; + blk_nr = (sec - start_section_nr + mode * total_sec) / + sections_per_block; + delta = ktime_sub(now, mem_info[blk_nr].resident_since); + mem_info[blk_nr].resident_time = + ktime_add(mem_info[blk_nr].resident_time, delta); + mem_info[blk_nr].resident_since = 0; +} + +static int mem_region_refresh_control(unsigned long pfn, + unsigned long nr_pages, + bool enable) +{ +#ifdef CONFIG_MSM_RPM_SMD + struct memory_refresh_request mem_req; + struct msm_rpm_kvp rpm_kvp; + + mem_req.start = enable; + mem_req.start |= pfn << PAGE_SHIFT; + mem_req.size = nr_pages * PAGE_SIZE; + + rpm_kvp.key = RPM_DDR_REQ; + rpm_kvp.data = (void *)&mem_req; + rpm_kvp.length = sizeof(mem_req); + + return msm_rpm_send_message(MSM_RPM_CTX_ACTIVE_SET, RPM_DDR_REQ, 0, + &rpm_kvp, 1); +#else + return -EINVAL; +#endif +} + +static int aop_send_msg(unsigned long addr, bool online) +{ + struct qmp_pkt pkt; + char mbox_msg[MAX_LEN]; + unsigned long addr_low, addr_high; + + addr_low = addr & AOP_MSG_ADDR_MASK; + addr_high = (addr >> AOP_MSG_ADDR_HIGH_SHIFT) & AOP_MSG_ADDR_MASK; + + snprintf(mbox_msg, MAX_LEN, + "{class: ddr, event: pasr, addr_hi: 0x%08lx, addr_lo: 0x%08lx, refresh: %s}", + addr_high, addr_low, online ? "on" : "off"); + + pkt.size = MAX_LEN; + pkt.data = mbox_msg; + return (mbox_send_message(mailbox.mbox, &pkt) < 0); +} + +static long get_memblk_bits(unsigned int seg_idx, unsigned long memblk_addr) +{ + if (memblk_addr > segment_infos[seg_idx].start_addr + + segment_infos[seg_idx].seg_size) + return -EINVAL; + + return (1 << ((memblk_addr - segment_infos[seg_idx].start_addr) / + memory_block_size_bytes())); +} + +static long get_segment_addr_to_idx(unsigned long addr) +{ + int i; + + for (i = 0; i < num_segments; i++) { + if (addr >= segment_infos[i].start_addr && + addr < segment_infos[i].start_addr + segment_infos[i].seg_size) + return i; + } + + return -EINVAL; +} + +static int send_msg(struct memory_notify *mn, bool online, int count) +{ + unsigned long segment_size, start, addr, base_addr; + int ret, i, seg_idx; + + if (bypass_send_msg) + return 0; + + addr = __pfn_to_phys(SECTION_ALIGN_DOWN(mn->start_pfn)); + seg_idx = get_segment_addr_to_idx(addr); + base_addr = segment_infos[seg_idx].start_addr; + + for (i = 0; i < count; ++i) { + + seg_idx = get_segment_addr_to_idx(addr); + segment_size = segment_infos[seg_idx].seg_size; + start = __phys_to_pfn(segment_infos[seg_idx].start_addr); + addr = segment_infos[seg_idx].start_addr; + + if (is_rpm_controller) + ret = mem_region_refresh_control(start, + segment_size >> PAGE_SHIFT, + online); + else + ret = aop_send_msg(__pfn_to_phys(start), online); + + if (ret) { + pr_err("PASR: %s %s request addr:0x%llx failed\n", + is_rpm_controller ? "RPM" : "AOP", + online ? "online" : "offline", + __pfn_to_phys(start)); + goto undo; + } + + pr_info("mem-offline: sent msg successfully to %s segment at phys addr 0x%lx\n", + online ? "online" : "offline", __pfn_to_phys(start)); + addr += segment_size; + } + + return 0; +undo: + addr = base_addr; + seg_idx = get_segment_addr_to_idx(addr); + start = __phys_to_pfn(base_addr); + + while (i-- > 0) { + int ret; + + if (is_rpm_controller) + ret = mem_region_refresh_control(start, + segment_size >> PAGE_SHIFT, + !online); + else + ret = aop_send_msg(__pfn_to_phys(start), !online); + + if (ret) + panic("Failed to completely online/offline a hotpluggable segment. A quasi state of memblock can cause randomn system failures."); + segment_size = segment_infos[seg_idx].seg_size; + addr += segment_size; + seg_idx = get_segment_addr_to_idx(addr); + start = __phys_to_pfn(__pfn_to_phys(start) + segment_size); + } + + return ret; +} + +static void set_memblk_bitmap_online(unsigned long addr) +{ + unsigned long seg_idx; + long cur_blk_bit; + + seg_idx = get_segment_addr_to_idx(addr); + cur_blk_bit = get_memblk_bits(seg_idx, addr); + + if (cur_blk_bit < 0) { + pr_err("mem-offline: couldn't get current block bitmap\n"); + return; + } + + if (segment_infos[seg_idx].bitmask_kernel_blk & cur_blk_bit) { + pr_warn("mem-offline: memblk 0x%lx in bitmap already onlined\n", addr); + return; + } + + segment_infos[seg_idx].bitmask_kernel_blk |= cur_blk_bit; +} + +static void set_memblk_bitmap_offline(unsigned long addr) +{ + unsigned long seg_idx; + long cur_blk_bit; + + seg_idx = get_segment_addr_to_idx(addr); + cur_blk_bit = get_memblk_bits(seg_idx, addr); + + if (cur_blk_bit < 0) { + pr_err("mem-offline: couldn't get current block bitmap\n"); + return; + } + + if (!(segment_infos[seg_idx].bitmask_kernel_blk & cur_blk_bit)) { + pr_warn("mem-offline: memblk 0x%lx in bitmap already offlined\n", addr); + return; + } + + segment_infos[seg_idx].bitmask_kernel_blk &= ~cur_blk_bit; +} + +static bool need_to_send_remote_request(struct memory_notify *mn, + enum memory_states request) +{ + int seg_idx; + unsigned long addr; + long cur_blk_bit, mask; + + addr = SECTION_ALIGN_DOWN(mn->start_pfn) << PAGE_SHIFT; + seg_idx = get_segment_addr_to_idx(addr); + + cur_blk_bit = get_memblk_bits(seg_idx, addr); + + if (cur_blk_bit < 0) { + pr_err("mem-offline: couldn't get current block bitmap\n"); + return false; + } + /* + * For MEM_OFFLINE, don't send the request if there are other online + * blocks in the segment. + * For MEM_ONLINE, don't send the request if there is already one + * online block in the segment. + */ + + + /* check if other memblocks are ONLINE, if so then return false */ + mask = segment_infos[seg_idx].bitmask_kernel_blk & (~cur_blk_bit); + if (mask) + return false; + + return true; +} + +/* + * This returns the number of hotpluggable segments in a memory block. + */ +static int get_num_memblock_hotplug_segments(unsigned long addr) +{ + unsigned long segment_size; + unsigned long block_size = memory_block_size_bytes(); + unsigned long end_addr = addr + block_size; + int seg_idx, count = 0; + + seg_idx = get_segment_addr_to_idx(addr); + segment_size = segment_infos[seg_idx].seg_size; + + if (segment_size >= block_size) + return 1; + + while ((addr < end_addr) && (addr + segment_size < end_addr)) { + if (block_size % segment_size) { + pr_warn("PASR is unusable. Offline granule size should be in multiples for memory_block_size_bytes.\n"); + return 0; + } + addr += segment_size; + seg_idx = get_segment_addr_to_idx(addr); + segment_size = segment_infos[seg_idx].seg_size; + count++; + } + + return count; +} + +static int mem_change_refresh_state(struct memory_notify *mn, + enum memory_states state) +{ + int start = SECTION_ALIGN_DOWN(mn->start_pfn); + unsigned long sec_nr = pfn_to_section_nr(start); + bool online = (state == MEMORY_ONLINE) ? true : false; + unsigned long idx = (sec_nr - start_section_nr) / sections_per_block; + int ret, count; + unsigned long addr; + int seg_idx; + + if (mem_sec_state[idx] == state) { + /* we shouldn't be getting this request */ + pr_warn("mem-offline: state of mem%d block already in %s state. Ignoring refresh state change request\n", + sec_nr, online ? "online" : "offline"); + return 0; + } + + addr = __pfn_to_phys(SECTION_ALIGN_DOWN(mn->start_pfn)); + seg_idx = get_segment_addr_to_idx(addr); + + count = get_num_memblock_hotplug_segments(addr); + if (!count) + return -EINVAL; + + if (!need_to_send_remote_request(mn, state)) + goto out; + + ret = send_msg(mn, online, count); + if (ret) { + /* online failures are critical failures */ + if (online) + BUG_ON(IS_ENABLED(CONFIG_BUG_ON_HW_MEM_ONLINE_FAIL)); + return -EINVAL; + } + segment_infos[seg_idx].state = state; +out: + mem_sec_state[idx] = state; + return 0; +} + +static unsigned long get_section_allocated_memory(unsigned long sec_nr) +{ + unsigned long block_sz = memory_block_size_bytes(); + unsigned long pages_per_blk = block_sz / PAGE_SIZE; + unsigned long tot_free_pages = 0, pfn, end_pfn, flags; + unsigned long used; + struct zone *movable_zone = &NODE_DATA(numa_node_id())->node_zones[ZONE_MOVABLE]; + struct page *page; + + if (!populated_zone(movable_zone)) + return 0; + + pfn = section_nr_to_pfn(sec_nr); + end_pfn = pfn + pages_per_blk; + + if (!zone_intersects(movable_zone, pfn, pages_per_blk)) + return 0; + + spin_lock_irqsave(&movable_zone->lock, flags); + while (pfn < end_pfn) { + if (!pfn_valid(pfn) || !PageBuddy(pfn_to_page(pfn))) { + pfn++; + continue; + } + page = pfn_to_page(pfn); + tot_free_pages += 1 << page_private(page); + pfn += 1 << page_private(page); + } + spin_unlock_irqrestore(&movable_zone->lock, flags); + + used = block_sz - (tot_free_pages * PAGE_SIZE); + + return used; +} + +static int mem_event_callback(struct notifier_block *self, + unsigned long action, void *arg) +{ + struct memory_notify *mn = arg; + unsigned long start, end, sec_nr; + static ktime_t cur; + ktime_t delay = 0; + phys_addr_t start_addr, end_addr; + unsigned int idx = end_section_nr - start_section_nr + 1; + unsigned long seg_idx; + + start = SECTION_ALIGN_DOWN(mn->start_pfn); + end = SECTION_ALIGN_UP(mn->start_pfn + mn->nr_pages); + + if ((start != mn->start_pfn) || (end != mn->start_pfn + mn->nr_pages)) { + WARN("mem-offline: %s pfn not aligned to section\n", __func__); + pr_err("mem-offline: start pfn = %lu end pfn = %lu\n", + mn->start_pfn, mn->start_pfn + mn->nr_pages); + return -EINVAL; + } + + start_addr = __pfn_to_phys(start); + end_addr = __pfn_to_phys(end); + sec_nr = pfn_to_section_nr(start); + + if (sec_nr > end_section_nr || sec_nr < start_section_nr) { + if (action == MEM_ONLINE || action == MEM_OFFLINE) + pr_info("mem-offline: %s mem%ld, but not our block. Not performing any action\n", + action == MEM_ONLINE ? "Onlined" : "Offlined", + sec_nr); + return NOTIFY_OK; + } + + switch (action) { + case MEM_GOING_ONLINE: + pr_debug("mem-offline: MEM_GOING_ONLINE : start = 0x%llx end = 0x%llx\n", + start_addr, end_addr); + ++mem_info[(sec_nr - start_section_nr + MEMORY_ONLINE * + idx) / sections_per_block].fail_count; + cur = ktime_get(); + + if (mem_change_refresh_state(mn, MEMORY_ONLINE)) + return NOTIFY_BAD; + + break; + case MEM_ONLINE: + delay = ktime_us_delta(ktime_get(), cur); + record_stat(sec_nr, delay, MEMORY_ONLINE); + cur = 0; + set_memblk_bitmap_online(start_addr); + pr_info("mem-offline: Onlined memory block mem%pK\n", + (void *)sec_nr); + seg_idx = get_segment_addr_to_idx(start_addr); + pr_debug("mem-offline: Segment %d memblk_bitmap 0x%lx\n", + seg_idx, segment_infos[seg_idx].bitmask_kernel_blk); + totalram_pages_add(-(memory_block_size_bytes()/PAGE_SIZE)); + break; + case MEM_GOING_OFFLINE: + pr_debug("mem-offline: MEM_GOING_OFFLINE : start = 0x%llx end = 0x%llx\n", + start_addr, end_addr); + ++mem_info[(sec_nr - start_section_nr + MEMORY_OFFLINE * + idx) / sections_per_block].fail_count; + cur = ktime_get(); + break; + case MEM_OFFLINE: + mem_change_refresh_state(mn, MEMORY_OFFLINE); + /* + * Notifying that something went bad at this stage won't + * help since this is the last stage of memory hotplug. + */ + + delay = ktime_us_delta(ktime_get(), cur); + record_stat(sec_nr, delay, MEMORY_OFFLINE); + cur = 0; + set_memblk_bitmap_offline(start_addr); + pr_info("mem-offline: Offlined memory block mem%pK\n", + (void *)sec_nr); + seg_idx = get_segment_addr_to_idx(start_addr); + pr_debug("mem-offline: Segment %d memblk_bitmap 0x%lx\n", + seg_idx, segment_infos[seg_idx].bitmask_kernel_blk); + totalram_pages_add(memory_block_size_bytes()/PAGE_SIZE); + break; + case MEM_CANCEL_ONLINE: + pr_info("mem-offline: MEM_CANCEL_ONLINE: start = 0x%llx end = 0x%llx\n", + start_addr, end_addr); + mem_change_refresh_state(mn, MEMORY_OFFLINE); + break; + default: + break; + } + return NOTIFY_OK; +} + +static int mem_online_remaining_blocks(void) +{ + unsigned long memblock_end_pfn = __phys_to_pfn(memblock_end_of_DRAM()); + unsigned long ram_end_pfn = __phys_to_pfn(bootmem_dram_end_addr - 1); + unsigned long block_size, memblock, pfn; + unsigned int nid, delta; + phys_addr_t phys_addr; + int fail = 0; + + pr_debug("mem-offline: memblock_end_of_DRAM 0x%lx\n", memblock_end_of_DRAM()); + + block_size = memory_block_size_bytes(); + sections_per_block = block_size / MIN_MEMORY_BLOCK_SIZE; + + start_section_nr = pfn_to_section_nr(memblock_end_pfn); + end_section_nr = pfn_to_section_nr(ram_end_pfn); + + if (memblock_end_of_DRAM() >= bootmem_dram_end_addr) { + pr_info("mem-offline: System booted with no zone movable memory blocks. Cannot perform memory offlining\n"); + return -EINVAL; + } + + if (memblock_end_of_DRAM() % block_size) { + delta = block_size - (memblock_end_of_DRAM() % block_size); + pr_err("mem-offline: !!ERROR!! memblock end of dram address is not aligned to memory block size!\n"); + pr_err("mem-offline: memory%lu could be partially available. %lukB of memory will be missing from RAM!\n", + start_section_nr, delta / SZ_1K); + + /* + * since this section is partially added during boot, we cannot + * add the remaining part of section using add_memory since it + * won't be size aligned to block size. We have to start the + * offlinable region from the next section onwards. + */ + start_section_nr += 1; + + } + + if (bootmem_dram_end_addr % block_size) { + delta = bootmem_dram_end_addr % block_size; + pr_err("mem-offline: !!ERROR!! bootmem end of dram address is not aligned to memory block size!\n"); + pr_err("mem-offline: memory%lu will not be added. %lukB of memory will be missing from RAM!\n", + end_section_nr, delta / SZ_1K); + + /* + * since this section cannot be added, the last section of offlinable + * region will be the previous section. + */ + end_section_nr -= 1; + } + + offlinable_region_start_addr = + section_nr_to_pfn(__pfn_to_phys(start_section_nr)); + + /* + * below check holds true if there were only one offlinable section + * and that was partially added during boot. In such case, bail out. + */ + if (start_section_nr > end_section_nr) + return 1; + + pr_debug("mem-offline: offlinable_region_start_addr 0X%lx\n", + offlinable_region_start_addr); + + for (memblock = start_section_nr; memblock <= end_section_nr; + memblock += sections_per_block) { + if (!test_bit(memblock - start_section_nr, movable_bitmap)) + continue; + + pfn = section_nr_to_pfn(memblock); + phys_addr = __pfn_to_phys(pfn); + + if (phys_addr & (((PAGES_PER_SECTION * sections_per_block) + << PAGE_SHIFT) - 1)) { + fail = 1; + pr_warn("mem-offline: PFN of mem%lu block not aligned to section start. Not adding this memory block\n", + memblock); + continue; + } + nid = memory_add_physaddr_to_nid(phys_addr); + if (add_memory(nid, phys_addr, + MIN_MEMORY_BLOCK_SIZE * sections_per_block, MHP_NONE)) { + pr_warn("mem-offline: Adding memory block mem%lu failed\n", + memblock); + fail = 1; + } + } + + return fail; +} + +static ssize_t show_block_allocated_bytes(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned long allocd_mem = 0; + unsigned long sec_nr; + int ret; + + ret = sscanf(kobject_name(kobj), MEMBLOCK_NAME, &sec_nr); + if (ret != 1) { + pr_err("mem-offline: couldn't get memory block number! ret %d\n", ret); + return 0; + } + + allocd_mem = get_section_allocated_memory(sec_nr); + + return scnprintf(buf, BUF_LEN, "%lu\n", allocd_mem); +} + +static ssize_t show_seg_memblk_start(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned long memblk_start; + unsigned long seg_nr; + int ret; + + ret = sscanf(kobject_name(kobj), SEGMENT_NAME, &seg_nr); + if (ret != 1) { + pr_err("mem-offline: couldn't get segment number! ret %d\n", ret); + return 0; + } + + memblk_start = + pfn_to_section_nr(PFN_DOWN(segment_infos[seg_nr].start_addr)); + + return scnprintf(buf, BUF_LEN, "%lu\n", memblk_start); +} + +static ssize_t show_num_memblks(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned long num_memblks; + unsigned long seg_nr; + int ret; + + ret = sscanf(kobject_name(kobj), SEGMENT_NAME, &seg_nr); + if (ret != 1) { + pr_err("mem-offline: couldn't get num_memblks! ret %d\n", ret); + return 0; + } + num_memblks = segment_infos[seg_nr].num_kernel_blks; + + return scnprintf(buf, BUF_LEN, "%lu\n", num_memblks); +} + +static ssize_t show_seg_size(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned long seg_size; + unsigned long seg_nr; + int ret; + + ret = sscanf(kobject_name(kobj), SEGMENT_NAME, &seg_nr); + if (ret != 1) { + pr_err("mem-offline: couldn't get segment size! ret %d\n", ret); + return 0; + } + seg_size = segment_infos[seg_nr].seg_size; + + return scnprintf(buf, BUF_LEN, "%lu\n", seg_size); +} + +static ssize_t show_mem_offline_granule(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", + (unsigned long)offline_granule * SZ_1M); +} + +static ssize_t show_differing_seg_sizes(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", + (unsigned int)differing_segment_sizes); +} + +static ssize_t show_num_segments(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%lu\n", + (unsigned long)num_segments); +} + +static unsigned int print_blk_residency_percentage(char *buf, size_t sz, + unsigned int tot_blks, ktime_t *total_time, + enum memory_states mode) +{ + unsigned int i; + unsigned int c = 0; + int percent; + unsigned int idx = tot_blks + 1; + + for (i = 0; i <= tot_blks; i++) { + percent = (int)ktime_divns(total_time[i + mode * idx] * 100, + ktime_add(total_time[i + MEMORY_ONLINE * idx], + total_time[i + MEMORY_OFFLINE * idx])); + + c += scnprintf(buf + c, sz - c, "%d%%\t\t", percent); + } + return c; +} +static unsigned int print_blk_residency_times(char *buf, size_t sz, + unsigned int tot_blks, ktime_t *total_time, + enum memory_states mode) +{ + unsigned int i; + unsigned int c = 0; + ktime_t now, delta; + unsigned int idx = tot_blks + 1; + + now = ktime_get(); + for (i = 0; i <= tot_blks; i++) { + if (mem_sec_state[i] == mode) + delta = ktime_sub(now, + mem_info[i + mode * idx].resident_since); + else + delta = 0; + delta = ktime_add(delta, + mem_info[i + mode * idx].resident_time); + c += scnprintf(buf + c, sz - c, "%lus\t\t", + ktime_to_us(delta) / USEC_PER_SEC); + total_time[i + mode * idx] = delta; + } + return c; +} + +static ssize_t show_mem_stats(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + + unsigned int blk_start = start_section_nr / sections_per_block; + unsigned int blk_end = end_section_nr / sections_per_block; + unsigned int tot_blks = blk_end - blk_start; + ktime_t *total_time; + unsigned int idx = tot_blks + 1; + unsigned int c = 0; + unsigned int i, j; + + size_t sz = PAGE_SIZE; + ktime_t total = 0, total_online = 0, total_offline = 0; + + total_time = kcalloc(idx * MAX_STATE, sizeof(*total_time), GFP_KERNEL); + + if (!total_time) + return -ENOMEM; + + for (j = 0; j < MAX_STATE; j++) { + c += scnprintf(buf + c, sz - c, + "\n\t%s\n\t\t\t", j == 0 ? "ONLINE" : "OFFLINE"); + for (i = blk_start; i <= blk_end; i++) + c += scnprintf(buf + c, sz - c, + "%s%d\t\t", "mem", i); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tLast recd time:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, "%luus\t\t", + mem_info[i + j * idx].last_recorded_time); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tAvg time:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, + "%luus\t\t", mem_info[i + j * idx].avg_time); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tBest time:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, + "%luus\t\t", mem_info[i + j * idx].best_time); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tWorst time:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, + "%luus\t\t", mem_info[i + j * idx].worst_time); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tSuccess count:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, + "%lu\t\t", mem_info[i + j * idx].success_count); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, + "\tFail count:\t"); + for (i = 0; i <= tot_blks; i++) + c += scnprintf(buf + c, sz - c, + "%lu\t\t", mem_info[i + j * idx].fail_count); + c += scnprintf(buf + c, sz - c, "\n"); + } + + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\tState:\t\t"); + for (i = 0; i <= tot_blks; i++) { + c += scnprintf(buf + c, sz - c, "%s\t\t", + mem_sec_state[i] == MEMORY_ONLINE ? + "Online" : "Offline"); + } + c += scnprintf(buf + c, sz - c, "\n"); + + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\tOnline time:\t"); + c += print_blk_residency_times(buf + c, sz - c, + tot_blks, total_time, MEMORY_ONLINE); + + + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\tOffline time:\t"); + c += print_blk_residency_times(buf + c, sz - c, + tot_blks, total_time, MEMORY_OFFLINE); + + c += scnprintf(buf + c, sz - c, "\n"); + + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\tOnline %%:\t"); + c += print_blk_residency_percentage(buf + c, sz - c, + tot_blks, total_time, MEMORY_ONLINE); + + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\tOffline %%:\t"); + c += print_blk_residency_percentage(buf + c, sz - c, + tot_blks, total_time, MEMORY_OFFLINE); + c += scnprintf(buf + c, sz - c, "\n"); + c += scnprintf(buf + c, sz - c, "\n"); + + for (i = 0; i <= tot_blks; i++) + total = ktime_add(total, + ktime_add(total_time[i + MEMORY_ONLINE * idx], + total_time[i + MEMORY_OFFLINE * idx])); + + for (i = 0; i <= tot_blks; i++) + total_online = ktime_add(total_online, + total_time[i + MEMORY_ONLINE * idx]); + + total_offline = ktime_sub(total, total_online); + + c += scnprintf(buf + c, sz - c, + "\tAvg Online %%:\t%d%%\n", + ((int)total_online * 100) / total); + c += scnprintf(buf + c, sz - c, + "\tAvg Offline %%:\t%d%%\n", + ((int)total_offline * 100) / total); + + c += scnprintf(buf + c, sz - c, "\n"); + kfree(total_time); + return c; +} + +static struct kobj_attribute stats_attr = + __ATTR(stats, 0444, show_mem_stats, NULL); + +static struct kobj_attribute offline_granule_attr = + __ATTR(offline_granule, 0444, show_mem_offline_granule, NULL); + +static struct attribute *mem_root_attrs[] = { + &stats_attr.attr, + &offline_granule_attr.attr, + NULL, +}; + +static struct attribute_group mem_attr_group = { + .attrs = mem_root_attrs, +}; + +/* memblock allocated bytes attribute group */ +static struct kobj_attribute block_allocated_bytes_attr = + __ATTR(allocated_bytes, 0444, show_block_allocated_bytes, NULL); + +static struct attribute *mem_block_attrs[] = { + &block_allocated_bytes_attr.attr, + NULL, +}; + +static struct attribute_group mem_block_attr_group = { + .attrs = mem_block_attrs, +}; + +/* differing segment attribute group */ +static struct kobj_attribute differing_seg_sizes_attr = + __ATTR(differing_seg_sizes, 0444, show_differing_seg_sizes, NULL); + +static struct kobj_attribute num_segments_attr = + __ATTR(num_segment, 0444, show_num_segments, NULL); + +static struct attribute *differing_segments_attrs[] = { + &differing_seg_sizes_attr.attr, + &num_segments_attr.attr, + NULL, +}; + +static struct attribute_group differing_segments_attr_group = { + .attrs = differing_segments_attrs, +}; + +/* segment info attribute group */ +static struct kobj_attribute seg_memblk_start_attr = + __ATTR(memblk_start, 0444, show_seg_memblk_start, NULL); + +static struct kobj_attribute seg_num_memblks_attr = + __ATTR(num_memblks, 0444, show_num_memblks, NULL); + +static struct kobj_attribute seg_size_attr = + __ATTR(seg_size, 0444, show_seg_size, NULL); + +static struct attribute *seg_info_attrs[] = { + &seg_memblk_start_attr.attr, + &seg_num_memblks_attr.attr, + &seg_size_attr.attr, + NULL, +}; + +static struct attribute_group seg_info_attr_group = { + .attrs = seg_info_attrs, +}; + +static int mem_sysfs_create_seginfo(struct kobject *parent_kobj) +{ + struct kobject *segment_kobj, *seg_info_kobj; + char segmentstr[BUF_LEN]; + unsigned long segnum; + int ret; + + if (sysfs_create_group(parent_kobj, &differing_segments_attr_group)) { + kobject_put(kobj); + return -EINVAL; + } + + ret = scnprintf(segmentstr, sizeof(segmentstr), "seg_info"); + seg_info_kobj = kobject_create_and_add(segmentstr, parent_kobj); + if (!seg_info_kobj) + return -ENOMEM; + + for (segnum = 0; segnum < num_segments; segnum++) { + ret = scnprintf(segmentstr, sizeof(segmentstr), SEGMENT_NAME, segnum); + segment_kobj = kobject_create_and_add(segmentstr, seg_info_kobj); + if (!segment_kobj) + return -ENOMEM; + + if (sysfs_create_group(segment_kobj, &seg_info_attr_group)) + kobject_put(segment_kobj); + } + + return 0; +} +static int mem_sysfs_create_memblocks(struct kobject *parent_kobj) +{ + struct kobject *memblk_kobj; + char memblkstr[BUF_LEN]; + unsigned long memblock; + int ret; + + for (memblock = start_section_nr; memblock <= end_section_nr; + memblock += sections_per_block) { + ret = scnprintf(memblkstr, sizeof(memblkstr), MEMBLOCK_NAME, memblock); + if (ret <= 0) + return -EINVAL; + memblk_kobj = kobject_create_and_add(memblkstr, parent_kobj); + if (!memblk_kobj) + return -ENOMEM; + if (sysfs_create_group(memblk_kobj, &mem_block_attr_group)) + kobject_put(memblk_kobj); + } + + return 0; +} + +static int mem_sysfs_init(void) +{ + if (start_section_nr == end_section_nr) + return -EINVAL; + + kobj = kobject_create_and_add(MODULE_CLASS_NAME, kernel_kobj); + if (!kobj) + return -ENOMEM; + + if (sysfs_create_group(kobj, &mem_attr_group)) + kobject_put(kobj); + + if (mem_sysfs_create_memblocks(kobj)) { + pr_err("mem-offline: failed to create memblock sysfs nodes\n"); + return -EINVAL; + } + + /* create sysfs nodes for segment info if ddr has differing segment sizes */ + if (differing_segment_sizes && mem_sysfs_create_seginfo(kobj)) { + pr_err("mem-offline: failed to create seginfo sysfs nodes\n"); + return -EINVAL; + } + + return 0; +} + +static int mem_parse_dt(struct platform_device *pdev) +{ + const __be32 *val; + struct device_node *node = pdev->dev.of_node; + + val = of_get_property(node, "granule", NULL); + if (!val) { + pr_err("mem-offine: granule property not found in DT\n"); + return -EINVAL; + } + if (!*val) { + pr_err("mem-offine: invalid granule property\n"); + return -EINVAL; + } + offline_granule = be32_to_cpup(val); + if (!offline_granule || (offline_granule & (offline_granule - 1)) || + ((offline_granule * SZ_1M < MIN_MEMORY_BLOCK_SIZE) && + (MIN_MEMORY_BLOCK_SIZE % (offline_granule * SZ_1M)))) { + pr_err("mem-offine: invalid granule property\n"); + return -EINVAL; + } + + if (!of_find_property(node, "mboxes", NULL)) { + is_rpm_controller = true; + return 0; + } + + mailbox.cl.dev = &pdev->dev; + mailbox.cl.tx_block = true; + mailbox.cl.tx_tout = 1000; + mailbox.cl.knows_txdone = false; + + mailbox.mbox = mbox_request_channel(&mailbox.cl, 0); + if (IS_ERR(mailbox.mbox)) { + if (PTR_ERR(mailbox.mbox) != -EPROBE_DEFER) + pr_err("mem-offline: failed to get mailbox channel %pK %ld\n", + mailbox.mbox, PTR_ERR(mailbox.mbox)); + return PTR_ERR(mailbox.mbox); + } + + return 0; +} + +static struct notifier_block hotplug_memory_callback_nb = { + .notifier_call = mem_event_callback, + .priority = 0, +}; + +static unsigned int get_num_offlinable_segments(void) +{ + uint8_t r = 0; // region index + unsigned long region_end, segment_start, segment_size, addr; + unsigned int count = 0; + + /* iterate through regions */ + for (r = 0; r < num_ddr_regions; r++) { + + region_end = ddr_regions[r].start_address + ddr_regions[r].length; + + /* Calculate segment starting address */ + segment_start = ddr_regions[r].start_address + + ddr_regions[r].segments_start_offset; + + /* If DDR region granule_size is 0, this region cannot be offlined */ + if (!ddr_regions[r].granule_size) + continue; + + /* Calculate size of segments in bytes */ + segment_size = ddr_regions[r].granule_size << 20; + + /* now iterate through segments within the region */ + for (addr = segment_start; addr < region_end; addr += segment_size) { + + /* Check if segment extends beyond region */ + if ((addr + segment_size) > region_end) + break; + + /* populate segment info only for ones in offlinable region */ + if (addr < offlinable_region_start_addr) + continue; + + count++; + } + } + + return count; +} + +static int get_segment_region_info(void) +{ + uint8_t r = 0; // region index + unsigned long region_end, segment_start, segment_size, r0_segment_size; + unsigned long num_kernel_blks, seg_idx = 0, addr; + int i; + + num_segments = get_num_offlinable_segments(); + + segment_infos = kcalloc(num_segments, sizeof(*segment_infos), GFP_KERNEL); + if (!segment_infos) + return -ENOMEM; + + for (i = 0; i < num_segments; i++) + segment_infos[i].start_addr = -1; + + r0_segment_size = ddr_regions[0].granule_size << 20; + + /* iterate through regions */ + for (r = 0; r < num_ddr_regions; r++) { + + region_end = ddr_regions[r].start_address + ddr_regions[r].length; + + /* Calculate segment starting address */ + segment_start = ddr_regions[r].start_address + + ddr_regions[r].segments_start_offset; + + /* Calculate size of segments in bytes */ + segment_size = ddr_regions[r].granule_size << 20; + + /* If DDR region granule_size is 0, this region cannot be offlined */ + if (!segment_size) + continue; + + /* Check if we have diferring segment sizes */ + if (r0_segment_size != segment_size) + differing_segment_sizes = 1; + + /* now iterate through segments within the region */ + for (addr = segment_start; addr < region_end; addr += segment_size) { + + /* Check if segment extends beyond region */ + if ((addr + segment_size) > region_end) + break; + + /* populate segment info only for ones in offlinable region */ + if (addr < offlinable_region_start_addr) + continue; + + if (segment_size > memory_block_size_bytes()) + num_kernel_blks = segment_size / memory_block_size_bytes(); + else + num_kernel_blks = 1; + + segment_infos[seg_idx].start_addr = addr; + segment_infos[seg_idx].seg_size = segment_size; + segment_infos[seg_idx].num_kernel_blks = num_kernel_blks; + + segment_infos[seg_idx].bitmask_kernel_blk = + GENMASK_ULL(num_kernel_blks - 1, 0); + + seg_idx++; + } + } + + if (differing_segment_sizes) + pr_info("mem-offline: system has DDR type of differing segment sizes\n"); + + return 0; +} + +static unsigned int get_num_ddr_regions(struct device_node *node) +{ + int i, len; + char str[20]; + + for (i = 0; i < MAX_NUM_DDR_REGIONS; i++) { + + snprintf(str, sizeof(str), "region%d", i); + if (!of_find_property(node, str, &len)) + break; + } + + return i; +} + +static int get_ddr_regions_info(void) +{ + struct device_node *node; + struct property *prop; + int len, num_cells; + u64 val; + int nr_address_cells; + const __be32 *pos; + char str[20]; + int i; + + node = of_find_node_by_name(of_root, "ddr-regions"); + if (!node) { + pr_err("mem-offine: ddr-regions node not found in DT\n"); + return -EINVAL; + } + + num_ddr_regions = get_num_ddr_regions(node); + + ddr_regions = kcalloc(num_ddr_regions, sizeof(*ddr_regions), GFP_KERNEL); + if (!ddr_regions) + return -ENOMEM; + + nr_address_cells = of_n_addr_cells(of_root); + + for (i = 0; i < num_ddr_regions; i++) { + + snprintf(str, sizeof(str), "region%d", i); + prop = of_find_property(node, str, &len); + + if (!prop) + return -EINVAL; + + num_cells = len / sizeof(__be32); + if (num_cells != DDR_REGIONS_NUM_CELLS) + return -EINVAL; + + pos = prop->value; + + val = of_read_number(pos, nr_address_cells); + pos += nr_address_cells; + ddr_regions[i].start_address = val; + + val = of_read_number(pos, nr_address_cells); + pos += nr_address_cells; + ddr_regions[i].length = val; + + val = of_read_number(pos, nr_address_cells); + pos += nr_address_cells; + ddr_regions[i].segments_start_offset = val; + + val = of_read_number(pos, nr_address_cells); + pos += nr_address_cells; + ddr_regions[i].segments_start_idx = val; + + val = of_read_number(pos, nr_address_cells); + pos += nr_address_cells; + ddr_regions[i].granule_size = val; + } + + for (i = 0; i < num_ddr_regions; i++) { + + pr_info("region%d: seg_start 0x%lx len 0x%lx granule 0x%lx seg_start_offset 0x%lx seg_start_idx 0x%lx\n", + i, ddr_regions[i].start_address, ddr_regions[i].length, + ddr_regions[i].granule_size, + ddr_regions[i].segments_start_offset, + ddr_regions[i].segments_start_idx); + } + + return 0; +} + +static int check_segment_granule_alignment(void) +{ + int seg_idx; + unsigned long granule_size; + + for (seg_idx = 0; seg_idx < num_segments; seg_idx++) { + granule_size = segment_infos[seg_idx].seg_size; + + if (granule_size & (granule_size - 1)) { + pr_err("mem-offline: invalid granule property for segment %d granule_size 0x%lx\n", + seg_idx, granule_size); + return -EINVAL; + } + + /* check for granule size alignment */ + if (((granule_size < MIN_MEMORY_BLOCK_SIZE) && + (MIN_MEMORY_BLOCK_SIZE % granule_size)) || + ((granule_size > MIN_MEMORY_BLOCK_SIZE) && + (granule_size % MIN_MEMORY_BLOCK_SIZE))) { + pr_err("mem-offline: granule size for segment %d granule_size 0x%lx is not aligned to memblock size\n", + seg_idx, granule_size); + return -EINVAL; + } + } + + return 0; +} + +static int update_dram_end_address_and_movable_bitmap(phys_addr_t *bootmem_dram_end_addr) +{ + struct device_node *node; + struct property *prop; + int len, num_cells, num_entries; + u64 addr = 0, max_base = 0; + u64 size, base, end, section_size; + u64 movable_start; + int nr_address_cells, nr_size_cells; + const __be32 *pos; + + node = of_find_node_by_name(of_root, "memory"); + if (!node) { + pr_err("mem-offine: memory node not found in DT\n"); + return -EINVAL; + } + + nr_address_cells = of_n_addr_cells(of_root); + nr_size_cells = of_n_size_cells(of_root); + + prop = of_find_property(node, "reg", &len); + if (!prop) { + pr_err("mem-offine: reg node not found in DT\n"); + return -EINVAL; + } + + num_cells = len / sizeof(__be32); + num_entries = num_cells / (nr_address_cells + nr_size_cells); + + pos = prop->value; + + section_size = MIN_MEMORY_BLOCK_SIZE; + movable_start = memblock_end_of_DRAM(); + + while (num_entries--) { + base = of_read_number(pos, nr_address_cells); + size = of_read_number(pos + nr_address_cells, nr_size_cells); + pos += nr_address_cells + nr_size_cells; + + if (base > max_base) { + max_base = base; + addr = base + size; + } + } + + *bootmem_dram_end_addr = addr; + pr_debug("mem-offline: bootmem_dram_end_addr 0x%lx\n", *bootmem_dram_end_addr); + + num_entries = num_cells / (nr_address_cells + nr_size_cells); + pos = prop->value; + while (num_entries--) { + u64 new_base, new_end; + u64 new_start_bitmap, bitmap_size; + + base = of_read_number(pos, nr_address_cells); + size = of_read_number(pos + nr_address_cells, nr_size_cells); + pos += nr_address_cells + nr_size_cells; + end = base + size; + + if (end <= movable_start) + continue; + + if (base < movable_start) + new_base = movable_start; + else + new_base = base; + new_end = end; + + new_start_bitmap = (new_base - movable_start) / section_size; + bitmap_size = (new_end - new_base) / section_size; + bitmap_set(movable_bitmap, new_start_bitmap, bitmap_size); + } + + pr_debug("mem-offline: movable_bitmap is %lx\n", *movable_bitmap); + return 0; +} + +static int mem_offline_driver_probe(struct platform_device *pdev) +{ + unsigned int total_blks; + int ret, i; + ktime_t now; + + if (nopasr) { + pr_info("mem-offline: nopasr mode enabled. Skipping probe\n"); + return 0; + } + + ret = mem_parse_dt(pdev); + if (ret) + return ret; + + ret = update_dram_end_address_and_movable_bitmap(&bootmem_dram_end_addr); + if (ret) + return ret; + + ret = mem_online_remaining_blocks(); + if (ret < 0) + return -ENODEV; + + if (ret > 0) + pr_err("mem-offline: !!ERROR!! Auto onlining some memory blocks failed. System could run with less RAM\n"); + + ret = get_ddr_regions_info(); + if (ret) + return ret; + + ret = get_segment_region_info(); + if (ret) + return ret; + + ret = check_segment_granule_alignment(); + if (ret) + return ret; + + pr_info("mem-offline: num_ddr_regions %d num_segments %d\n", + num_ddr_regions, num_segments); + + total_blks = (end_section_nr - start_section_nr + 1) / + sections_per_block; + mem_info = kcalloc(total_blks * MAX_STATE, sizeof(*mem_info), + GFP_KERNEL); + if (!mem_info) + return -ENOMEM; + + /* record time of online for all blocks */ + now = ktime_get(); + for (i = 0; i < total_blks; i++) + mem_info[i].resident_since = now; + + mem_sec_state = kcalloc(total_blks, sizeof(*mem_sec_state), GFP_KERNEL); + if (!mem_sec_state) { + ret = -ENOMEM; + goto err_free_mem_info; + } + + /* we assume that hardware state of mem blocks are online after boot */ + for (i = 0; i < total_blks; i++) + mem_sec_state[i] = MEMORY_ONLINE; + + if (mem_sysfs_init()) { + ret = -ENODEV; + goto err_free_mem_sec_state; + } + + if (register_hotmemory_notifier(&hotplug_memory_callback_nb)) { + pr_err("mem-offline: Registering memory hotplug notifier failed\n"); + ret = -ENODEV; + goto err_sysfs_remove_group; + } + pr_info("mem-offline: Added memory blocks ranging from mem%lu - mem%lu\n", + start_section_nr, end_section_nr); + + if (bypass_send_msg) + pr_info("mem-offline: bypass mode\n"); + + return 0; + +err_sysfs_remove_group: + sysfs_remove_group(kobj, &mem_attr_group); + kobject_put(kobj); +err_free_mem_sec_state: + kfree(mem_sec_state); +err_free_mem_info: + kfree(mem_info); + return ret; +} + +static const struct of_device_id mem_offline_match_table[] = { + {.compatible = "qcom,mem-offline"}, + {} +}; + +MODULE_DEVICE_TABLE(of, mem_offline_match_table); + +static struct platform_driver mem_offline_driver = { + .probe = mem_offline_driver_probe, + .driver = { + .name = "mem_offline", + .of_match_table = mem_offline_match_table, + }, +}; + +static int __init mem_module_init(void) +{ + return platform_driver_register(&mem_offline_driver); +} +subsys_initcall(mem_module_init); + +static void __exit mem_module_exit(void) +{ + platform_driver_unregister(&mem_offline_driver); +} +module_exit(mem_module_exit); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Memory Offlining Driver"); +MODULE_LICENSE("GPL");