From b5a8d233a588b3acf2a7a3a8da30f8f68f376626 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:50 +0100 Subject: [PATCH 001/633] bus: mhi: core: Add device hardware reset support The MHI specification allows to perform a hard reset of the device when writing to the SOC_RESET register. It can be used to completely restart the device (e.g. in case of unrecoverable MHI error). This is up to the MHI controller driver to determine when this hard reset should be used, and in case of MHI errors, should be used as a reset of last resort (after standard MHI stack reset). This function is a stateless function, the MHI layer do nothing except triggering the reset by writing into the right register(s), this is up to the caller to ensure right mhi_controller state (e.g. unregister the controller if necessary). Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/main.c | 13 +++++++++++++ include/linux/mhi.h | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index d34d7e90e38d..9b425402767c 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -135,6 +135,19 @@ enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) } EXPORT_SYMBOL_GPL(mhi_get_mhi_state); +void mhi_soc_reset(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->reset) { + mhi_cntrl->reset(mhi_cntrl); + return; + } + + /* Generic MHI SoC reset */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + MHI_SOC_RESET_REQ); +} +EXPORT_SYMBOL_GPL(mhi_soc_reset); + int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, struct mhi_buf_info *buf_info) { diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 562862ff819c..54afcae1709a 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -347,6 +347,7 @@ struct mhi_controller_config { * @unmap_single: CB function to destroy TRE buffer * @read_reg: Read a MHI register via the physical link (required) * @write_reg: Write a MHI register via the physical link (required) + * @reset: Controller specific reset function (optional) * @buffer_len: Bounce buffer length * @index: Index of the MHI controller instance * @bounce_buf: Use of bounce buffer @@ -437,6 +438,7 @@ struct mhi_controller { u32 *out); void (*write_reg)(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 val); + void (*reset)(struct mhi_controller *mhi_cntrl); size_t buffer_len; int index; @@ -672,6 +674,13 @@ enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl); */ enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl); +/** + * mhi_soc_reset - Trigger a device reset. This can be used as a last resort + * to reset and recover a device. + * @mhi_cntrl: MHI controller + */ +void mhi_soc_reset(struct mhi_controller *mhi_cntrl); + /** * mhi_device_get - Disable device low power mode * @mhi_dev: Device associated with the channel From d9f23ea69d41d9749873381affe3c00bb1857019 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:51 +0100 Subject: [PATCH 002/633] mhi: pci-generic: Increase number of hardware events If the IPA (IP hardware accelerator) is starved of event ring elements, the modem is crashing (SDX55). That can be prevented by setting a larger number of events (i.e 2 x number of channel ring elements). Tested with FN980m module. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index f5bee76ea061..d3896ef13eee 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -91,7 +91,7 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ - .num_elements = 128, \ + .num_elements = 256, \ .irq_moderation_ms = 5, \ .irq = (ev_ring) + 1, \ .priority = 1, \ From eb96787a5da8e481e53bf2d87a575283d57130a2 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:52 +0100 Subject: [PATCH 003/633] mhi: pci_generic: Enable burst mode for hardware channels Hardware channels have a feature called burst mode that allows to queue transfer ring element(s) (TRE) to a channel without ringing the device doorbell. In that mode, the device is polling the channel context for new elements. This reduces the frequency of host initiated doorbells and increase throughput. Create a new dedicated macro for hardware channels with burst enabled. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index d3896ef13eee..778d13f0db2a 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -76,6 +76,36 @@ struct mhi_pci_dev_info { .offload_channel = false, \ } +#define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_TO_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } \ + +#define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_FROM_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } + #define MHI_EVENT_CONFIG_DATA(ev_ring) \ { \ .num_elements = 128, \ @@ -110,8 +140,8 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_UL(100, "IP_HW0", 128, 1), - MHI_CHANNEL_CONFIG_DL(101, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1), + MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2), }; static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { From 8ccc3279fcad9736b58873b1ad597287ee52757a Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:53 +0100 Subject: [PATCH 004/633] mhi: pci_generic: Add support for reset Add support for resetting the device, reset can be triggered in case of error or manually via sysfs (/sys/bus/pci/devices/*/reset). Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 121 ++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 13 deletions(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 778d13f0db2a..f0dfa625db16 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -8,6 +8,7 @@ * Copyright (C) 2020 Linaro Ltd */ +#include #include #include #include @@ -15,6 +16,7 @@ #define MHI_PCI_DEFAULT_BAR_NUM 0 +#define MHI_POST_RESET_DELAY_MS 500 /** * struct mhi_pci_dev_info - MHI PCI device specific information * @config: MHI controller configuration @@ -177,6 +179,16 @@ static const struct pci_device_id mhi_pci_id_table[] = { }; MODULE_DEVICE_TABLE(pci, mhi_pci_id_table); +enum mhi_pci_device_status { + MHI_PCI_DEV_STARTED, +}; + +struct mhi_pci_device { + struct mhi_controller mhi_cntrl; + struct pci_saved_state *pci_state; + unsigned long status; +}; + static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 *out) { @@ -196,6 +208,20 @@ static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, /* Nothing to do for now */ } +static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) +{ + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + u16 vendor = 0; + + if (pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor)) + return false; + + if (vendor == (u16) ~0 || vendor == 0) + return false; + + return true; +} + static int mhi_pci_claim(struct mhi_controller *mhi_cntrl, unsigned int bar_num, u64 dma_mask) { @@ -291,16 +317,20 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; const struct mhi_controller_config *mhi_cntrl_config; + struct mhi_pci_device *mhi_pdev; struct mhi_controller *mhi_cntrl; int err; dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name); - mhi_cntrl = mhi_alloc_controller(); - if (!mhi_cntrl) + /* mhi_pdev.mhi_cntrl must be zero-initialized */ + mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL); + if (!mhi_pdev) return -ENOMEM; mhi_cntrl_config = info->config; + mhi_cntrl = &mhi_pdev->mhi_cntrl; + mhi_cntrl->cntrl_dev = &pdev->dev; mhi_cntrl->iova_start = 0; mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width); @@ -315,17 +345,21 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width)); if (err) - goto err_release; + return err; err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; - pci_set_drvdata(pdev, mhi_cntrl); + pci_set_drvdata(pdev, mhi_pdev); + + /* Have stored pci confspace at hand for restore in sudden PCI error */ + pci_save_state(pdev); + mhi_pdev->pci_state = pci_store_saved_state(pdev); err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; /* MHI bus does not power up the controller by default */ err = mhi_prepare_for_power_up(mhi_cntrl); @@ -340,33 +374,94 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unprepare; } + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + return 0; err_unprepare: mhi_unprepare_after_power_down(mhi_cntrl); err_unregister: mhi_unregister_controller(mhi_cntrl); -err_release: - mhi_free_controller(mhi_cntrl); return err; } static void mhi_pci_remove(struct pci_dev *pdev) { - struct mhi_controller *mhi_cntrl = pci_get_drvdata(pdev); + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, true); + mhi_unprepare_after_power_down(mhi_cntrl); + } - mhi_power_down(mhi_cntrl, true); - mhi_unprepare_after_power_down(mhi_cntrl); mhi_unregister_controller(mhi_cntrl); - mhi_free_controller(mhi_cntrl); } +static void mhi_pci_reset_prepare(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_info(&pdev->dev, "reset\n"); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* cause internal device reset */ + mhi_soc_reset(mhi_cntrl); + + /* Be sure device reset has been executed */ + msleep(MHI_POST_RESET_DELAY_MS); +} + +static void mhi_pci_reset_done(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + /* Restore initial known working PCI state */ + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + /* Is device status available ? */ + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(&pdev->dev, "reset failed\n"); + return; + } + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to prepare MHI controller\n"); + return; + } + + err = mhi_sync_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to power up MHI controller\n"); + mhi_unprepare_after_power_down(mhi_cntrl); + return; + } + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); +} + +static const struct pci_error_handlers mhi_pci_err_handler = { + .reset_prepare = mhi_pci_reset_prepare, + .reset_done = mhi_pci_reset_done, +}; + static struct pci_driver mhi_pci_driver = { .name = "mhi-pci-generic", .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, - .remove = mhi_pci_remove + .remove = mhi_pci_remove, + .err_handler = &mhi_pci_err_handler, }; module_pci_driver(mhi_pci_driver); From 7389337f0a78ea099c47f0af08f64f20c52ab4ba Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:54 +0100 Subject: [PATCH 005/633] mhi: pci_generic: Add suspend/resume/recovery procedure Add support for system wide suspend/resume. During suspend, MHI device controller must be put in M3 state and PCI bus in D3 state. Add a recovery procedure allowing to reinitialize the device in case of error during resume steps, which can happen if device loses power (and so its context) while system suspend. Reviewed-by Hemant Kumar Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index f0dfa625db16..2552c2ea75e8 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -13,6 +13,7 @@ #include #include #include +#include #define MHI_PCI_DEFAULT_BAR_NUM 0 @@ -186,6 +187,7 @@ enum mhi_pci_device_status { struct mhi_pci_device { struct mhi_controller mhi_cntrl; struct pci_saved_state *pci_state; + struct work_struct recovery_work; unsigned long status; }; @@ -313,6 +315,50 @@ static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl) /* no PM for now */ } +static void mhi_pci_recovery_work(struct work_struct *work) +{ + struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device, + recovery_work); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + int err; + + dev_warn(&pdev->dev, "device recovery started\n"); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* Check if we can recover without full reset */ + pci_set_power_state(pdev, PCI_D0); + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + if (!mhi_pci_is_alive(mhi_cntrl)) + goto err_try_reset; + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) + goto err_try_reset; + + err = mhi_sync_power_up(mhi_cntrl); + if (err) + goto err_unprepare; + + dev_dbg(&pdev->dev, "Recovery completed\n"); + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + return; + +err_unprepare: + mhi_unprepare_after_power_down(mhi_cntrl); +err_try_reset: + if (pci_reset_function(pdev)) + dev_err(&pdev->dev, "Recovery failed\n"); +} + static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; @@ -328,6 +374,8 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!mhi_pdev) return -ENOMEM; + INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work); + mhi_cntrl_config = info->config; mhi_cntrl = &mhi_pdev->mhi_cntrl; @@ -391,6 +439,8 @@ static void mhi_pci_remove(struct pci_dev *pdev) struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + cancel_work_sync(&mhi_pdev->recovery_work); + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { mhi_power_down(mhi_cntrl, true); mhi_unprepare_after_power_down(mhi_cntrl); @@ -456,12 +506,67 @@ static const struct pci_error_handlers mhi_pci_err_handler = { .reset_done = mhi_pci_reset_done, }; +static int __maybe_unused mhi_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + cancel_work_sync(&mhi_pdev->recovery_work); + + /* Transition to M3 state */ + mhi_pm_suspend(mhi_cntrl); + + pci_save_state(pdev); + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); + + return 0; +} + +static int __maybe_unused mhi_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + pci_set_master(pdev); + + err = pci_enable_device(pdev); + if (err) + goto err_recovery; + + /* Exit M3, transition to M0 state */ + err = mhi_pm_resume(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to resume device: %d\n", err); + goto err_recovery; + } + + return 0; + +err_recovery: + /* The device may have loose power or crashed, try recovering it */ + queue_work(system_long_wq, &mhi_pdev->recovery_work); + + return err; +} + +static const struct dev_pm_ops mhi_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mhi_pci_suspend, mhi_pci_resume) +}; + static struct pci_driver mhi_pci_driver = { .name = "mhi-pci-generic", .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, .remove = mhi_pci_remove, .err_handler = &mhi_pci_err_handler, + .driver.pm = &mhi_pci_pm_ops }; module_pci_driver(mhi_pci_driver); From b012ee6bfe2ac99bdf7d70b7776187f416c1cab8 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:55 +0100 Subject: [PATCH 006/633] mhi: pci_generic: Add PCI error handlers In AER capable root complex, errors are reported to the host which can then act accordingly and perform PCI recovering procedure. This patch enables error reporting and implements error_detected, slot_reset and resume callbacks. Reviewed-by Hemant Kumar Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 2552c2ea75e8..2cc44100f0dc 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -8,6 +8,7 @@ * Copyright (C) 2020 Linaro Ltd */ +#include #include #include #include @@ -405,6 +406,8 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_save_state(pdev); mhi_pdev->pci_state = pci_store_saved_state(pdev); + pci_enable_pcie_error_reporting(pdev); + err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config); if (err) return err; @@ -501,7 +504,54 @@ static void mhi_pci_reset_done(struct pci_dev *pdev) set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); } +static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_err(&pdev->dev, "PCI error detected, state = %u\n", state); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } else { + /* Nothing to do */ + return PCI_ERS_RESULT_RECOVERED; + } + + pci_disable_device(pdev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev) +{ + if (pci_enable_device(pdev)) { + dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_RECOVERED; +} + +static void mhi_pci_io_resume(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCI slot reset done\n"); + + queue_work(system_long_wq, &mhi_pdev->recovery_work); +} + static const struct pci_error_handlers mhi_pci_err_handler = { + .error_detected = mhi_pci_error_detected, + .slot_reset = mhi_pci_slot_reset, + .resume = mhi_pci_io_resume, .reset_prepare = mhi_pci_reset_prepare, .reset_done = mhi_pci_reset_done, }; From 8562d4fe34a3ef52da077f77985994bb9ad1f83e Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:56 +0100 Subject: [PATCH 007/633] mhi: pci_generic: Add health-check If the modem crashes for any reason, we may not be able to detect it at MHI level (MHI registers not reachable anymore). This patch implements a health-check mechanism to check regularly that device is alive (MHI layer can communicate with). If device is not alive (because a crash or unexpected reset), the recovery procedure is triggered. Tested successfully with Telit FN980m module. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Reviewed-by: Hemant Kumar Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 2cc44100f0dc..3eac41072ad2 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -14,11 +14,15 @@ #include #include #include +#include #include #define MHI_PCI_DEFAULT_BAR_NUM 0 #define MHI_POST_RESET_DELAY_MS 500 + +#define HEALTH_CHECK_PERIOD (HZ * 2) + /** * struct mhi_pci_dev_info - MHI PCI device specific information * @config: MHI controller configuration @@ -189,6 +193,7 @@ struct mhi_pci_device { struct mhi_controller mhi_cntrl; struct pci_saved_state *pci_state; struct work_struct recovery_work; + struct timer_list health_check_timer; unsigned long status; }; @@ -326,6 +331,8 @@ static void mhi_pci_recovery_work(struct work_struct *work) dev_warn(&pdev->dev, "device recovery started\n"); + del_timer(&mhi_pdev->health_check_timer); + /* Clean up MHI state */ if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { mhi_power_down(mhi_cntrl, false); @@ -351,6 +358,7 @@ static void mhi_pci_recovery_work(struct work_struct *work) dev_dbg(&pdev->dev, "Recovery completed\n"); set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); return; err_unprepare: @@ -360,6 +368,21 @@ static void mhi_pci_recovery_work(struct work_struct *work) dev_err(&pdev->dev, "Recovery failed\n"); } +static void health_check(struct timer_list *t) +{ + struct mhi_pci_device *mhi_pdev = from_timer(mhi_pdev, t, health_check_timer); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(mhi_cntrl->cntrl_dev, "Device died\n"); + queue_work(system_long_wq, &mhi_pdev->recovery_work); + return; + } + + /* reschedule in two seconds */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; @@ -376,6 +399,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work); + timer_setup(&mhi_pdev->health_check_timer, health_check, 0); mhi_cntrl_config = info->config; mhi_cntrl = &mhi_pdev->mhi_cntrl; @@ -427,6 +451,9 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + /* start health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return 0; err_unprepare: @@ -442,6 +469,7 @@ static void mhi_pci_remove(struct pci_dev *pdev) struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + del_timer(&mhi_pdev->health_check_timer); cancel_work_sync(&mhi_pdev->recovery_work); if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { @@ -459,6 +487,8 @@ static void mhi_pci_reset_prepare(struct pci_dev *pdev) dev_info(&pdev->dev, "reset\n"); + del_timer(&mhi_pdev->health_check_timer); + /* Clean up MHI state */ if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { mhi_power_down(mhi_cntrl, false); @@ -502,6 +532,7 @@ static void mhi_pci_reset_done(struct pci_dev *pdev) } set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); } static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev, @@ -562,6 +593,7 @@ static int __maybe_unused mhi_pci_suspend(struct device *dev) struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + del_timer(&mhi_pdev->health_check_timer); cancel_work_sync(&mhi_pdev->recovery_work); /* Transition to M3 state */ @@ -597,6 +629,9 @@ static int __maybe_unused mhi_pci_resume(struct device *dev) goto err_recovery; } + /* Resume health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return 0; err_recovery: From 84026a5bbc113fb6cef0326ada0e6f5e4d95026c Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:57 +0100 Subject: [PATCH 008/633] mhi: pci_generic: Increase controller timeout value On cold boot, device can take slightly more than 5 seconds to start. Increase the timeout to prevent MHI power-up issues. Signed-off-by: Loic Poulain Reviewed-by: Hemant Kumar Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 3eac41072ad2..cbf28c3ab3d4 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -162,7 +162,7 @@ static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { .max_channels = 128, - .timeout_ms = 5000, + .timeout_ms = 8000, .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels), .ch_cfg = modem_qcom_v1_mhi_channels, .num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events), From 4da3d07db8ae325e8e01ad85e28e9e60c58bcc14 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:58 +0100 Subject: [PATCH 009/633] mhi: pci_generic: Add diag channels Add support for Diag over MHI. Qualcomm Diag is the qualcomm diagnostics interface that can be used to collect modem logs, events, traces, etc. It can be used by tools such QPST or QXDM. This patch adds the DIAG channels and a dedicated event ring. Reviewed-by Hemant Kumar Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index cbf28c3ab3d4..5104084d9c08 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -142,22 +142,26 @@ struct mhi_pci_dev_info { } static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { + MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1), + MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1), MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1), - MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3), }; static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { /* first ring is control+data ring */ MHI_EVENT_CONFIG_CTRL(0), + /* DIAG dedicated event ring */ + MHI_EVENT_CONFIG_DATA(1), /* Hardware channels request dedicated hardware event rings */ - MHI_EVENT_CONFIG_HW_DATA(1, 100), - MHI_EVENT_CONFIG_HW_DATA(2, 101) + MHI_EVENT_CONFIG_HW_DATA(2, 100), + MHI_EVENT_CONFIG_HW_DATA(3, 101) }; static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { From ec751369d6fbc9f84176e1530b11cbf387262b48 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 4 Jan 2021 17:14:59 +0100 Subject: [PATCH 010/633] mhi: pci_generic: Set irq moderation value to 1ms for hw channels MHI hardware channels are usually the hardware accelerated data path e.g. IP packets path for modems. This path needs to be optimized for low latency and high throughput. After several tests on FN980m SDX55 based modem, it seems 1ms is a good default irq_moderation value: - It allows to reach the maximum download throughput - It introduces limited latency (5ms is too high) - It prevents interrupt flooding Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 5104084d9c08..c13de0f2053e 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -130,7 +130,7 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ .num_elements = 256, \ - .irq_moderation_ms = 5, \ + .irq_moderation_ms = 1, \ .irq = (ev_ring) + 1, \ .priority = 1, \ .mode = MHI_DB_BRST_DISABLE, \ From 62feb14ee8a374814f0e905b24ddf6cdcbbe4159 Mon Sep 17 00:00:00 2001 From: Jun Nie Date: Fri, 4 Dec 2020 15:53:41 +0800 Subject: [PATCH 011/633] interconnect: qcom: Consolidate interconnect RPM support Add RPM based interconnect driver implements the set and aggregate functionalities that translates bandwidth requests into RPM messages. These modules provide a common set of functionalities for all Qualcomm RPM based interconnect providers and should help reduce code duplication when adding new providers. Signed-off-by: Jun Nie Link: https://lore.kernel.org/r/20201204075345.5161-2-jun.nie@linaro.org Signed-off-by: Georgi Djakov --- drivers/interconnect/qcom/Makefile | 2 +- drivers/interconnect/qcom/icc-rpm.c | 191 ++++++++++++++++++++++ drivers/interconnect/qcom/icc-rpm.h | 73 +++++++++ drivers/interconnect/qcom/msm8916.c | 241 ++-------------------------- 4 files changed, 275 insertions(+), 232 deletions(-) create mode 100644 drivers/interconnect/qcom/icc-rpm.c create mode 100644 drivers/interconnect/qcom/icc-rpm.h diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile index cf628f7990cd..916d7bbe55b7 100644 --- a/drivers/interconnect/qcom/Makefile +++ b/drivers/interconnect/qcom/Makefile @@ -10,7 +10,7 @@ qnoc-sc7180-objs := sc7180.o qnoc-sdm845-objs := sdm845.o qnoc-sm8150-objs := sm8150.o qnoc-sm8250-objs := sm8250.o -icc-smd-rpm-objs := smd-rpm.o +icc-smd-rpm-objs := smd-rpm.o icc-rpm.o obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8916) += qnoc-msm8916.o diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c new file mode 100644 index 000000000000..cc6095492cbe --- /dev/null +++ b/drivers/interconnect/qcom/icc-rpm.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Linaro Ltd + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "smd-rpm.h" +#include "icc-rpm.h" + +static int qcom_icc_set(struct icc_node *src, struct icc_node *dst) +{ + struct qcom_icc_provider *qp; + struct qcom_icc_node *qn; + struct icc_provider *provider; + struct icc_node *n; + u64 sum_bw; + u64 max_peak_bw; + u64 rate; + u32 agg_avg = 0; + u32 agg_peak = 0; + int ret, i; + + qn = src->data; + provider = src->provider; + qp = to_qcom_provider(provider); + + list_for_each_entry(n, &provider->nodes, node_list) + provider->aggregate(n, 0, n->avg_bw, n->peak_bw, + &agg_avg, &agg_peak); + + sum_bw = icc_units_to_bps(agg_avg); + max_peak_bw = icc_units_to_bps(agg_peak); + + /* send bandwidth request message to the RPM processor */ + if (qn->mas_rpm_id != -1) { + ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, + RPM_BUS_MASTER_REQ, + qn->mas_rpm_id, + sum_bw); + if (ret) { + pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", + qn->mas_rpm_id, ret); + return ret; + } + } + + if (qn->slv_rpm_id != -1) { + ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, + RPM_BUS_SLAVE_REQ, + qn->slv_rpm_id, + sum_bw); + if (ret) { + pr_err("qcom_icc_rpm_smd_send slv error %d\n", + ret); + return ret; + } + } + + rate = max(sum_bw, max_peak_bw); + + do_div(rate, qn->buswidth); + + if (qn->rate == rate) + return 0; + + for (i = 0; i < qp->num_clks; i++) { + ret = clk_set_rate(qp->bus_clks[i].clk, rate); + if (ret) { + pr_err("%s clk_set_rate error: %d\n", + qp->bus_clks[i].id, ret); + return ret; + } + } + + qn->rate = rate; + + return 0; +} + +int qnoc_probe(struct platform_device *pdev, size_t cd_size, int cd_num, + const struct clk_bulk_data *cd) +{ + struct device *dev = &pdev->dev; + const struct qcom_icc_desc *desc; + struct icc_onecell_data *data; + struct icc_provider *provider; + struct qcom_icc_node **qnodes; + struct qcom_icc_provider *qp; + struct icc_node *node; + size_t num_nodes, i; + int ret; + + /* wait for the RPM proxy */ + if (!qcom_icc_rpm_smd_available()) + return -EPROBE_DEFER; + + desc = of_device_get_match_data(dev); + if (!desc) + return -EINVAL; + + qnodes = desc->nodes; + num_nodes = desc->num_nodes; + + qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), + GFP_KERNEL); + if (!data) + return -ENOMEM; + + qp->bus_clks = devm_kmemdup(dev, cd, cd_size, + GFP_KERNEL); + if (!qp->bus_clks) + return -ENOMEM; + + qp->num_clks = cd_num; + ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); + if (ret) + return ret; + + ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); + if (ret) + return ret; + + provider = &qp->provider; + INIT_LIST_HEAD(&provider->nodes); + provider->dev = dev; + provider->set = qcom_icc_set; + provider->aggregate = icc_std_aggregate; + provider->xlate = of_icc_xlate_onecell; + provider->data = data; + + ret = icc_provider_add(provider); + if (ret) { + dev_err(dev, "error adding interconnect provider: %d\n", ret); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + return ret; + } + + for (i = 0; i < num_nodes; i++) { + size_t j; + + node = icc_node_create(qnodes[i]->id); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto err; + } + + node->name = qnodes[i]->name; + node->data = qnodes[i]; + icc_node_add(node, provider); + + for (j = 0; j < qnodes[i]->num_links; j++) + icc_link_create(node, qnodes[i]->links[j]); + + data->nodes[i] = node; + } + data->num_nodes = num_nodes; + + platform_set_drvdata(pdev, qp); + + return 0; +err: + icc_nodes_remove(provider); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + icc_provider_del(provider); + + return ret; +} +EXPORT_SYMBOL(qnoc_probe); + +int qnoc_remove(struct platform_device *pdev) +{ + struct qcom_icc_provider *qp = platform_get_drvdata(pdev); + + icc_nodes_remove(&qp->provider); + clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); + return icc_provider_del(&qp->provider); +} +EXPORT_SYMBOL(qnoc_remove); diff --git a/drivers/interconnect/qcom/icc-rpm.h b/drivers/interconnect/qcom/icc-rpm.h new file mode 100644 index 000000000000..79a6f68249c1 --- /dev/null +++ b/drivers/interconnect/qcom/icc-rpm.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Linaro Ltd + */ + +#ifndef __DRIVERS_INTERCONNECT_QCOM_ICC_RPM_H +#define __DRIVERS_INTERCONNECT_QCOM_ICC_RPM_H + +#define RPM_BUS_MASTER_REQ 0x73616d62 +#define RPM_BUS_SLAVE_REQ 0x766c7362 + +#define QCOM_MAX_LINKS 12 + +#define to_qcom_provider(_provider) \ + container_of(_provider, struct qcom_icc_provider, provider) + +/** + * struct qcom_icc_provider - Qualcomm specific interconnect provider + * @provider: generic interconnect provider + * @bus_clks: the clk_bulk_data table of bus clocks + * @num_clks: the total number of clk_bulk_data entries + */ +struct qcom_icc_provider { + struct icc_provider provider; + struct clk_bulk_data *bus_clks; + int num_clks; +}; + +/** + * struct qcom_icc_node - Qualcomm specific interconnect nodes + * @name: the node name used in debugfs + * @id: a unique node identifier + * @links: an array of nodes where we can go next while traversing + * @num_links: the total number of @links + * @buswidth: width of the interconnect between a node and the bus (bytes) + * @mas_rpm_id: RPM id for devices that are bus masters + * @slv_rpm_id: RPM id for devices that are bus slaves + * @rate: current bus clock rate in Hz + */ +struct qcom_icc_node { + unsigned char *name; + u16 id; + u16 links[QCOM_MAX_LINKS]; + u16 num_links; + u16 buswidth; + int mas_rpm_id; + int slv_rpm_id; + u64 rate; +}; + +struct qcom_icc_desc { + struct qcom_icc_node **nodes; + size_t num_nodes; +}; + +#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ + ...) \ + static struct qcom_icc_node _name = { \ + .name = #_name, \ + .id = _id, \ + .buswidth = _buswidth, \ + .mas_rpm_id = _mas_rpm_id, \ + .slv_rpm_id = _slv_rpm_id, \ + .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ + .links = { __VA_ARGS__ }, \ + } + + +int qnoc_probe(struct platform_device *pdev, size_t cd_size, int cd_num, + const struct clk_bulk_data *cd); +int qnoc_remove(struct platform_device *pdev); + +#endif diff --git a/drivers/interconnect/qcom/msm8916.c b/drivers/interconnect/qcom/msm8916.c index e8371d40ab8d..fc3689c8947a 100644 --- a/drivers/interconnect/qcom/msm8916.c +++ b/drivers/interconnect/qcom/msm8916.c @@ -15,9 +15,7 @@ #include #include "smd-rpm.h" - -#define RPM_BUS_MASTER_REQ 0x73616d62 -#define RPM_BUS_SLAVE_REQ 0x766c7362 +#include "icc-rpm.h" enum { MSM8916_BIMC_SNOC_MAS = 1, @@ -107,67 +105,11 @@ enum { MSM8916_SNOC_PNOC_SLV, }; -#define to_msm8916_provider(_provider) \ - container_of(_provider, struct msm8916_icc_provider, provider) - static const struct clk_bulk_data msm8916_bus_clocks[] = { { .id = "bus" }, { .id = "bus_a" }, }; -/** - * struct msm8916_icc_provider - Qualcomm specific interconnect provider - * @provider: generic interconnect provider - * @bus_clks: the clk_bulk_data table of bus clocks - * @num_clks: the total number of clk_bulk_data entries - */ -struct msm8916_icc_provider { - struct icc_provider provider; - struct clk_bulk_data *bus_clks; - int num_clks; -}; - -#define MSM8916_MAX_LINKS 8 - -/** - * struct msm8916_icc_node - Qualcomm specific interconnect nodes - * @name: the node name used in debugfs - * @id: a unique node identifier - * @links: an array of nodes where we can go next while traversing - * @num_links: the total number of @links - * @buswidth: width of the interconnect between a node and the bus (bytes) - * @mas_rpm_id: RPM ID for devices that are bus masters - * @slv_rpm_id: RPM ID for devices that are bus slaves - * @rate: current bus clock rate in Hz - */ -struct msm8916_icc_node { - unsigned char *name; - u16 id; - u16 links[MSM8916_MAX_LINKS]; - u16 num_links; - u16 buswidth; - int mas_rpm_id; - int slv_rpm_id; - u64 rate; -}; - -struct msm8916_icc_desc { - struct msm8916_icc_node **nodes; - size_t num_nodes; -}; - -#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ - ...) \ - static struct msm8916_icc_node _name = { \ - .name = #_name, \ - .id = _id, \ - .buswidth = _buswidth, \ - .mas_rpm_id = _mas_rpm_id, \ - .slv_rpm_id = _slv_rpm_id, \ - .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ - .links = { __VA_ARGS__ }, \ - } - DEFINE_QNODE(bimc_snoc_mas, MSM8916_BIMC_SNOC_MAS, 8, -1, -1, MSM8916_BIMC_SNOC_SLV); DEFINE_QNODE(bimc_snoc_slv, MSM8916_BIMC_SNOC_SLV, 8, -1, -1, MSM8916_SNOC_INT_0, MSM8916_SNOC_INT_1); DEFINE_QNODE(mas_apss, MSM8916_MASTER_AMPSS_M0, 8, -1, -1, MSM8916_SLAVE_EBI_CH0, MSM8916_BIMC_SNOC_MAS, MSM8916_SLAVE_AMPSS_L2); @@ -254,7 +196,7 @@ DEFINE_QNODE(snoc_int_bimc, MSM8916_SNOC_INT_BIMC, 8, 101, 132, MSM8916_SNOC_BIM DEFINE_QNODE(snoc_pcnoc_mas, MSM8916_SNOC_PNOC_MAS, 8, -1, -1, MSM8916_SNOC_PNOC_SLV); DEFINE_QNODE(snoc_pcnoc_slv, MSM8916_SNOC_PNOC_SLV, 8, -1, -1, MSM8916_PNOC_INT_0); -static struct msm8916_icc_node *msm8916_snoc_nodes[] = { +static struct qcom_icc_node *msm8916_snoc_nodes[] = { [BIMC_SNOC_SLV] = &bimc_snoc_slv, [MASTER_JPEG] = &mas_jpeg, [MASTER_MDP_PORT0] = &mas_mdp, @@ -283,12 +225,12 @@ static struct msm8916_icc_node *msm8916_snoc_nodes[] = { [SNOC_QDSS_INT] = &qdss_int, }; -static struct msm8916_icc_desc msm8916_snoc = { +static struct qcom_icc_desc msm8916_snoc = { .nodes = msm8916_snoc_nodes, .num_nodes = ARRAY_SIZE(msm8916_snoc_nodes), }; -static struct msm8916_icc_node *msm8916_bimc_nodes[] = { +static struct qcom_icc_node *msm8916_bimc_nodes[] = { [BIMC_SNOC_MAS] = &bimc_snoc_mas, [MASTER_AMPSS_M0] = &mas_apss, [MASTER_GRAPHICS_3D] = &mas_gfx, @@ -300,12 +242,12 @@ static struct msm8916_icc_node *msm8916_bimc_nodes[] = { [SNOC_BIMC_1_SLV] = &snoc_bimc_1_slv, }; -static struct msm8916_icc_desc msm8916_bimc = { +static struct qcom_icc_desc msm8916_bimc = { .nodes = msm8916_bimc_nodes, .num_nodes = ARRAY_SIZE(msm8916_bimc_nodes), }; -static struct msm8916_icc_node *msm8916_pcnoc_nodes[] = { +static struct qcom_icc_node *msm8916_pcnoc_nodes[] = { [MASTER_BLSP_1] = &mas_blsp_1, [MASTER_DEHR] = &mas_dehr, [MASTER_LPASS] = &mas_audio, @@ -358,178 +300,15 @@ static struct msm8916_icc_node *msm8916_pcnoc_nodes[] = { [SNOC_PCNOC_SLV] = &snoc_pcnoc_slv, }; -static struct msm8916_icc_desc msm8916_pcnoc = { +static struct qcom_icc_desc msm8916_pcnoc = { .nodes = msm8916_pcnoc_nodes, .num_nodes = ARRAY_SIZE(msm8916_pcnoc_nodes), }; -static int msm8916_icc_set(struct icc_node *src, struct icc_node *dst) -{ - struct msm8916_icc_provider *qp; - struct msm8916_icc_node *qn; - u64 sum_bw, max_peak_bw, rate; - u32 agg_avg = 0, agg_peak = 0; - struct icc_provider *provider; - struct icc_node *n; - int ret, i; - - qn = src->data; - provider = src->provider; - qp = to_msm8916_provider(provider); - - list_for_each_entry(n, &provider->nodes, node_list) - provider->aggregate(n, 0, n->avg_bw, n->peak_bw, - &agg_avg, &agg_peak); - - sum_bw = icc_units_to_bps(agg_avg); - max_peak_bw = icc_units_to_bps(agg_peak); - - /* send bandwidth request message to the RPM processor */ - if (qn->mas_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_MASTER_REQ, - qn->mas_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", - qn->mas_rpm_id, ret); - return ret; - } - } - - if (qn->slv_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_SLAVE_REQ, - qn->slv_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send slv error %d\n", - ret); - return ret; - } - } - - rate = max(sum_bw, max_peak_bw); - - do_div(rate, qn->buswidth); - - if (qn->rate == rate) - return 0; - - for (i = 0; i < qp->num_clks; i++) { - ret = clk_set_rate(qp->bus_clks[i].clk, rate); - if (ret) { - pr_err("%s clk_set_rate error: %d\n", - qp->bus_clks[i].id, ret); - return ret; - } - } - - qn->rate = rate; - - return 0; -} - static int msm8916_qnoc_probe(struct platform_device *pdev) { - const struct msm8916_icc_desc *desc; - struct msm8916_icc_node **qnodes; - struct msm8916_icc_provider *qp; - struct device *dev = &pdev->dev; - struct icc_onecell_data *data; - struct icc_provider *provider; - struct icc_node *node; - size_t num_nodes, i; - int ret; - - /* wait for the RPM proxy */ - if (!qcom_icc_rpm_smd_available()) - return -EPROBE_DEFER; - - desc = of_device_get_match_data(dev); - if (!desc) - return -EINVAL; - - qnodes = desc->nodes; - num_nodes = desc->num_nodes; - - qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - - data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), - GFP_KERNEL); - if (!data) - return -ENOMEM; - - qp->bus_clks = devm_kmemdup(dev, msm8916_bus_clocks, - sizeof(msm8916_bus_clocks), GFP_KERNEL); - if (!qp->bus_clks) - return -ENOMEM; - - qp->num_clks = ARRAY_SIZE(msm8916_bus_clocks); - ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - provider = &qp->provider; - INIT_LIST_HEAD(&provider->nodes); - provider->dev = dev; - provider->set = msm8916_icc_set; - provider->aggregate = icc_std_aggregate; - provider->xlate = of_icc_xlate_onecell; - provider->data = data; - - ret = icc_provider_add(provider); - if (ret) { - dev_err(dev, "error adding interconnect provider: %d\n", ret); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return ret; - } - - for (i = 0; i < num_nodes; i++) { - size_t j; - - node = icc_node_create(qnodes[i]->id); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - goto err; - } - - node->name = qnodes[i]->name; - node->data = qnodes[i]; - icc_node_add(node, provider); - - for (j = 0; j < qnodes[i]->num_links; j++) - icc_link_create(node, qnodes[i]->links[j]); - - data->nodes[i] = node; - } - data->num_nodes = num_nodes; - - platform_set_drvdata(pdev, qp); - - return 0; - -err: - icc_nodes_remove(provider); - icc_provider_del(provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - - return ret; -} - -static int msm8916_qnoc_remove(struct platform_device *pdev) -{ - struct msm8916_icc_provider *qp = platform_get_drvdata(pdev); - - icc_nodes_remove(&qp->provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + return qnoc_probe(pdev, sizeof(msm8916_bus_clocks), + ARRAY_SIZE(msm8916_bus_clocks), msm8916_bus_clocks); } static const struct of_device_id msm8916_noc_of_match[] = { @@ -542,7 +321,7 @@ MODULE_DEVICE_TABLE(of, msm8916_noc_of_match); static struct platform_driver msm8916_noc_driver = { .probe = msm8916_qnoc_probe, - .remove = msm8916_qnoc_remove, + .remove = qnoc_remove, .driver = { .name = "qnoc-msm8916", .of_match_table = msm8916_noc_of_match, From dfbd988f1ce6c139bca5b79fbcc946748f5f880b Mon Sep 17 00:00:00 2001 From: Jun Nie Date: Fri, 4 Dec 2020 15:53:42 +0800 Subject: [PATCH 012/633] interconnect: qcom: qcs404: use shared code Use shared code for aggregate functionalities and probe function to remove duplicated code. Signed-off-by: Jun Nie Link: https://lore.kernel.org/r/20201204075345.5161-3-jun.nie@linaro.org Signed-off-by: Georgi Djakov --- drivers/interconnect/qcom/qcs404.c | 244 ++--------------------------- 1 file changed, 9 insertions(+), 235 deletions(-) diff --git a/drivers/interconnect/qcom/qcs404.c b/drivers/interconnect/qcom/qcs404.c index 9820709b43db..36a7e30a00be 100644 --- a/drivers/interconnect/qcom/qcs404.c +++ b/drivers/interconnect/qcom/qcs404.c @@ -9,15 +9,12 @@ #include #include #include -#include -#include #include -#include +#include + #include "smd-rpm.h" - -#define RPM_BUS_MASTER_REQ 0x73616d62 -#define RPM_BUS_SLAVE_REQ 0x766c7362 +#include "icc-rpm.h" enum { QCS404_MASTER_AMPSS_M0 = 1, @@ -95,67 +92,11 @@ enum { QCS404_SLAVE_LPASS, }; -#define to_qcom_provider(_provider) \ - container_of(_provider, struct qcom_icc_provider, provider) - -static const struct clk_bulk_data bus_clocks[] = { +static const struct clk_bulk_data qcs404_bus_clocks[] = { { .id = "bus" }, { .id = "bus_a" }, }; -/** - * struct qcom_icc_provider - Qualcomm specific interconnect provider - * @provider: generic interconnect provider - * @bus_clks: the clk_bulk_data table of bus clocks - * @num_clks: the total number of clk_bulk_data entries - */ -struct qcom_icc_provider { - struct icc_provider provider; - struct clk_bulk_data *bus_clks; - int num_clks; -}; - -#define QCS404_MAX_LINKS 12 - -/** - * struct qcom_icc_node - Qualcomm specific interconnect nodes - * @name: the node name used in debugfs - * @id: a unique node identifier - * @links: an array of nodes where we can go next while traversing - * @num_links: the total number of @links - * @buswidth: width of the interconnect between a node and the bus (bytes) - * @mas_rpm_id: RPM id for devices that are bus masters - * @slv_rpm_id: RPM id for devices that are bus slaves - * @rate: current bus clock rate in Hz - */ -struct qcom_icc_node { - unsigned char *name; - u16 id; - u16 links[QCS404_MAX_LINKS]; - u16 num_links; - u16 buswidth; - int mas_rpm_id; - int slv_rpm_id; - u64 rate; -}; - -struct qcom_icc_desc { - struct qcom_icc_node **nodes; - size_t num_nodes; -}; - -#define DEFINE_QNODE(_name, _id, _buswidth, _mas_rpm_id, _slv_rpm_id, \ - ...) \ - static struct qcom_icc_node _name = { \ - .name = #_name, \ - .id = _id, \ - .buswidth = _buswidth, \ - .mas_rpm_id = _mas_rpm_id, \ - .slv_rpm_id = _slv_rpm_id, \ - .num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })), \ - .links = { __VA_ARGS__ }, \ - } - DEFINE_QNODE(mas_apps_proc, QCS404_MASTER_AMPSS_M0, 8, 0, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); DEFINE_QNODE(mas_oxili, QCS404_MASTER_GRAPHICS_3D, 8, -1, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); DEFINE_QNODE(mas_mdp, QCS404_MASTER_MDP_PORT0, 8, -1, -1, QCS404_SLAVE_EBI_CH0, QCS404_BIMC_SNOC_SLV); @@ -327,178 +268,11 @@ static struct qcom_icc_desc qcs404_snoc = { .num_nodes = ARRAY_SIZE(qcs404_snoc_nodes), }; -static int qcom_icc_set(struct icc_node *src, struct icc_node *dst) + +static int qcs404_qnoc_probe(struct platform_device *pdev) { - struct qcom_icc_provider *qp; - struct qcom_icc_node *qn; - struct icc_provider *provider; - struct icc_node *n; - u64 sum_bw; - u64 max_peak_bw; - u64 rate; - u32 agg_avg = 0; - u32 agg_peak = 0; - int ret, i; - - qn = src->data; - provider = src->provider; - qp = to_qcom_provider(provider); - - list_for_each_entry(n, &provider->nodes, node_list) - provider->aggregate(n, 0, n->avg_bw, n->peak_bw, - &agg_avg, &agg_peak); - - sum_bw = icc_units_to_bps(agg_avg); - max_peak_bw = icc_units_to_bps(agg_peak); - - /* send bandwidth request message to the RPM processor */ - if (qn->mas_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_MASTER_REQ, - qn->mas_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send mas %d error %d\n", - qn->mas_rpm_id, ret); - return ret; - } - } - - if (qn->slv_rpm_id != -1) { - ret = qcom_icc_rpm_smd_send(QCOM_SMD_RPM_ACTIVE_STATE, - RPM_BUS_SLAVE_REQ, - qn->slv_rpm_id, - sum_bw); - if (ret) { - pr_err("qcom_icc_rpm_smd_send slv error %d\n", - ret); - return ret; - } - } - - rate = max(sum_bw, max_peak_bw); - - do_div(rate, qn->buswidth); - - if (qn->rate == rate) - return 0; - - for (i = 0; i < qp->num_clks; i++) { - ret = clk_set_rate(qp->bus_clks[i].clk, rate); - if (ret) { - pr_err("%s clk_set_rate error: %d\n", - qp->bus_clks[i].id, ret); - return ret; - } - } - - qn->rate = rate; - - return 0; -} - -static int qnoc_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - const struct qcom_icc_desc *desc; - struct icc_onecell_data *data; - struct icc_provider *provider; - struct qcom_icc_node **qnodes; - struct qcom_icc_provider *qp; - struct icc_node *node; - size_t num_nodes, i; - int ret; - - /* wait for the RPM proxy */ - if (!qcom_icc_rpm_smd_available()) - return -EPROBE_DEFER; - - desc = of_device_get_match_data(dev); - if (!desc) - return -EINVAL; - - qnodes = desc->nodes; - num_nodes = desc->num_nodes; - - qp = devm_kzalloc(dev, sizeof(*qp), GFP_KERNEL); - if (!qp) - return -ENOMEM; - - data = devm_kzalloc(dev, struct_size(data, nodes, num_nodes), - GFP_KERNEL); - if (!data) - return -ENOMEM; - - qp->bus_clks = devm_kmemdup(dev, bus_clocks, sizeof(bus_clocks), - GFP_KERNEL); - if (!qp->bus_clks) - return -ENOMEM; - - qp->num_clks = ARRAY_SIZE(bus_clocks); - ret = devm_clk_bulk_get(dev, qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - ret = clk_bulk_prepare_enable(qp->num_clks, qp->bus_clks); - if (ret) - return ret; - - provider = &qp->provider; - INIT_LIST_HEAD(&provider->nodes); - provider->dev = dev; - provider->set = qcom_icc_set; - provider->aggregate = icc_std_aggregate; - provider->xlate = of_icc_xlate_onecell; - provider->data = data; - - ret = icc_provider_add(provider); - if (ret) { - dev_err(dev, "error adding interconnect provider: %d\n", ret); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return ret; - } - - for (i = 0; i < num_nodes; i++) { - size_t j; - - node = icc_node_create(qnodes[i]->id); - if (IS_ERR(node)) { - ret = PTR_ERR(node); - goto err; - } - - node->name = qnodes[i]->name; - node->data = qnodes[i]; - icc_node_add(node, provider); - - dev_dbg(dev, "registered node %s\n", node->name); - - /* populate links */ - for (j = 0; j < qnodes[i]->num_links; j++) - icc_link_create(node, qnodes[i]->links[j]); - - data->nodes[i] = node; - } - data->num_nodes = num_nodes; - - platform_set_drvdata(pdev, qp); - - return 0; -err: - icc_nodes_remove(provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - icc_provider_del(provider); - - return ret; -} - -static int qnoc_remove(struct platform_device *pdev) -{ - struct qcom_icc_provider *qp = platform_get_drvdata(pdev); - - icc_nodes_remove(&qp->provider); - clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + return qnoc_probe(pdev, sizeof(qcs404_bus_clocks), + ARRAY_SIZE(qcs404_bus_clocks), qcs404_bus_clocks); } static const struct of_device_id qcs404_noc_of_match[] = { @@ -510,7 +284,7 @@ static const struct of_device_id qcs404_noc_of_match[] = { MODULE_DEVICE_TABLE(of, qcs404_noc_of_match); static struct platform_driver qcs404_noc_driver = { - .probe = qnoc_probe, + .probe = qcs404_qnoc_probe, .remove = qnoc_remove, .driver = { .name = "qnoc-qcs404", From 4187f9c16b7daf375dc2ad1e51c0782f8a9e4f7c Mon Sep 17 00:00:00 2001 From: Jun Nie Date: Fri, 4 Dec 2020 15:53:43 +0800 Subject: [PATCH 013/633] dt-bindings: interconnect: single yaml file for RPM interconnect drivers MSM8916 and QCS404 bindings are almost identical, so combine them into one. This will make it easier to add interconnect bindings for more SoC with RPM. Signed-off-by: Jun Nie Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201204075345.5161-4-jun.nie@linaro.org Signed-off-by: Georgi Djakov --- .../bindings/interconnect/qcom,qcs404.yaml | 77 ------------------- .../{qcom,msm8916.yaml => qcom,rpm.yaml} | 18 +++-- 2 files changed, 11 insertions(+), 84 deletions(-) delete mode 100644 Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml rename Documentation/devicetree/bindings/interconnect/{qcom,msm8916.yaml => qcom,rpm.yaml} (81%) diff --git a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml b/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml deleted file mode 100644 index 3fbb8785fbc9..000000000000 --- a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/interconnect/qcom,qcs404.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Qualcomm QCS404 Network-On-Chip interconnect - -maintainers: - - Georgi Djakov - -description: | - The Qualcomm QCS404 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. - -properties: - reg: - maxItems: 1 - - compatible: - enum: - - qcom,qcs404-bimc - - qcom,qcs404-pcnoc - - qcom,qcs404-snoc - - '#interconnect-cells': - const: 1 - - clock-names: - items: - - const: bus - - const: bus_a - - clocks: - items: - - description: Bus Clock - - description: Bus A Clock - -required: - - compatible - - reg - - '#interconnect-cells' - - clock-names - - clocks - -additionalProperties: false - -examples: - - | - #include - - bimc: interconnect@400000 { - reg = <0x00400000 0x80000>; - compatible = "qcom,qcs404-bimc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_BIMC_CLK>, - <&rpmcc RPM_SMD_BIMC_A_CLK>; - }; - - pnoc: interconnect@500000 { - reg = <0x00500000 0x15080>; - compatible = "qcom,qcs404-pcnoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_PNOC_CLK>, - <&rpmcc RPM_SMD_PNOC_A_CLK>; - }; - - snoc: interconnect@580000 { - reg = <0x00580000 0x23080>; - compatible = "qcom,qcs404-snoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_SNOC_CLK>, - <&rpmcc RPM_SMD_SNOC_A_CLK>; - }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml similarity index 81% rename from Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml rename to Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml index e1009ae4e8f7..d720b68b3ead 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml @@ -1,27 +1,31 @@ # SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/interconnect/qcom,msm8916.yaml# +$id: http://devicetree.org/schemas/interconnect/qcom,rpm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Qualcomm MSM8916 Network-On-Chip interconnect +title: Qualcomm RPM Network-On-Chip Interconnect maintainers: - Georgi Djakov description: | - The Qualcomm MSM8916 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. + RPM interconnect providers support system bandwidth requirements through + RPM processor. The provider is able to communicate with the RPM through + the RPM shared memory device. properties: + reg: + maxItems: 1 + compatible: enum: - qcom,msm8916-bimc - qcom,msm8916-pcnoc - qcom,msm8916-snoc - - reg: - maxItems: 1 + - qcom,qcs404-bimc + - qcom,qcs404-pcnoc + - qcom,qcs404-snoc '#interconnect-cells': const: 1 From 4ec908d2104026c67e4eb0fee722ed6893622763 Mon Sep 17 00:00:00 2001 From: Jun Nie Date: Fri, 4 Dec 2020 15:53:44 +0800 Subject: [PATCH 014/633] dt-bindings: interconnect: Add Qualcomm MSM8939 DT bindings The Qualcomm MSM8939 platform has several bus fabrics that could be controlled and tuned dynamically according to the bandwidth demand. Signed-off-by: Jun Nie Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201204075345.5161-5-jun.nie@linaro.org Signed-off-by: Georgi Djakov --- .../bindings/interconnect/qcom,rpm.yaml | 4 + .../dt-bindings/interconnect/qcom,msm8939.h | 105 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 include/dt-bindings/interconnect/qcom,msm8939.h diff --git a/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml index d720b68b3ead..983d71fb5399 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml @@ -23,6 +23,10 @@ properties: - qcom,msm8916-bimc - qcom,msm8916-pcnoc - qcom,msm8916-snoc + - qcom,msm8939-bimc + - qcom,msm8939-pcnoc + - qcom,msm8939-snoc + - qcom,msm8939-snoc-mm - qcom,qcs404-bimc - qcom,qcs404-pcnoc - qcom,qcs404-snoc diff --git a/include/dt-bindings/interconnect/qcom,msm8939.h b/include/dt-bindings/interconnect/qcom,msm8939.h new file mode 100644 index 000000000000..c22369a4b9f5 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,msm8939.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Qualcomm interconnect IDs + * + * Copyright (c) 2020, Linaro Ltd. + * Author: Jun Nie + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MSM8939_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_MSM8939_H + +#define BIMC_SNOC_SLV 0 +#define MASTER_QDSS_BAM 1 +#define MASTER_QDSS_ETR 2 +#define MASTER_SNOC_CFG 3 +#define PCNOC_SNOC_SLV 4 +#define SLAVE_APSS 5 +#define SLAVE_CATS_128 6 +#define SLAVE_OCMEM_64 7 +#define SLAVE_IMEM 8 +#define SLAVE_QDSS_STM 9 +#define SLAVE_SRVC_SNOC 10 +#define SNOC_BIMC_0_MAS 11 +#define SNOC_BIMC_1_MAS 12 +#define SNOC_BIMC_2_MAS 13 +#define SNOC_INT_0 14 +#define SNOC_INT_1 15 +#define SNOC_INT_BIMC 16 +#define SNOC_PCNOC_MAS 17 +#define SNOC_QDSS_INT 18 + +#define MASTER_VIDEO_P0 0 +#define MASTER_JPEG 1 +#define MASTER_VFE 2 +#define MASTER_MDP_PORT0 3 +#define MASTER_MDP_PORT1 4 +#define MASTER_CPP 5 +#define SNOC_MM_INT_0 6 +#define SNOC_MM_INT_1 7 +#define SNOC_MM_INT_2 8 + +#define BIMC_SNOC_MAS 0 +#define MASTER_AMPSS_M0 1 +#define MASTER_GRAPHICS_3D 2 +#define MASTER_TCU0 3 +#define SLAVE_AMPSS_L2 4 +#define SLAVE_EBI_CH0 5 +#define SNOC_BIMC_0_SLV 6 +#define SNOC_BIMC_1_SLV 7 +#define SNOC_BIMC_2_SLV 8 + +#define MASTER_BLSP_1 0 +#define MASTER_DEHR 1 +#define MASTER_LPASS 2 +#define MASTER_CRYPTO_CORE0 3 +#define MASTER_SDCC_1 4 +#define MASTER_SDCC_2 5 +#define MASTER_SPDM 6 +#define MASTER_USB_HS1 7 +#define MASTER_USB_HS2 8 +#define PCNOC_INT_0 9 +#define PCNOC_INT_1 10 +#define PCNOC_MAS_0 11 +#define PCNOC_MAS_1 12 +#define PCNOC_SLV_0 13 +#define PCNOC_SLV_1 14 +#define PCNOC_SLV_2 15 +#define PCNOC_SLV_3 16 +#define PCNOC_SLV_4 17 +#define PCNOC_SLV_8 18 +#define PCNOC_SLV_9 19 +#define PCNOC_SNOC_MAS 20 +#define SLAVE_BIMC_CFG 21 +#define SLAVE_BLSP_1 22 +#define SLAVE_BOOT_ROM 23 +#define SLAVE_CAMERA_CFG 24 +#define SLAVE_CLK_CTL 25 +#define SLAVE_CRYPTO_0_CFG 26 +#define SLAVE_DEHR_CFG 27 +#define SLAVE_DISPLAY_CFG 28 +#define SLAVE_GRAPHICS_3D_CFG 29 +#define SLAVE_IMEM_CFG 30 +#define SLAVE_LPASS 31 +#define SLAVE_MPM 32 +#define SLAVE_MSG_RAM 33 +#define SLAVE_MSS 34 +#define SLAVE_PDM 35 +#define SLAVE_PMIC_ARB 36 +#define SLAVE_PCNOC_CFG 37 +#define SLAVE_PRNG 38 +#define SLAVE_QDSS_CFG 39 +#define SLAVE_RBCPR_CFG 40 +#define SLAVE_SDCC_1 41 +#define SLAVE_SDCC_2 42 +#define SLAVE_SECURITY 43 +#define SLAVE_SNOC_CFG 44 +#define SLAVE_SPDM 45 +#define SLAVE_TCSR 46 +#define SLAVE_TLMM 47 +#define SLAVE_USB_HS1 48 +#define SLAVE_USB_HS2 49 +#define SLAVE_VENUS_CFG 50 +#define SNOC_PCNOC_SLV 51 + +#endif From 6c6fe5d3dc5e2be4118ee67e0a70fe7882a89397 Mon Sep 17 00:00:00 2001 From: Jun Nie Date: Fri, 4 Dec 2020 15:53:45 +0800 Subject: [PATCH 015/633] interconnect: qcom: Add MSM8939 interconnect provider driver Add driver for the Qualcomm interconnect buses found in MSM8939 based platforms. The topology consists of four NoCs that are controlled by a remote processor that collects the aggregated bandwidth for each master-slave pairs. Signed-off-by: Jun Nie Link: https://lore.kernel.org/r/20201204075345.5161-6-jun.nie@linaro.org Signed-off-by: Georgi Djakov --- drivers/interconnect/qcom/Kconfig | 9 + drivers/interconnect/qcom/Makefile | 2 + drivers/interconnect/qcom/msm8939.c | 355 ++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 drivers/interconnect/qcom/msm8939.c diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig index a8f93ba265f8..a469470b4e4f 100644 --- a/drivers/interconnect/qcom/Kconfig +++ b/drivers/interconnect/qcom/Kconfig @@ -17,6 +17,15 @@ config INTERCONNECT_QCOM_MSM8916 This is a driver for the Qualcomm Network-on-Chip on msm8916-based platforms. +config INTERCONNECT_QCOM_MSM8939 + tristate "Qualcomm MSM8939 interconnect driver" + depends on INTERCONNECT_QCOM + depends on QCOM_SMD_RPM + select INTERCONNECT_QCOM_SMD_RPM + help + This is a driver for the Qualcomm Network-on-Chip on msm8939-based + platforms. + config INTERCONNECT_QCOM_MSM8974 tristate "Qualcomm MSM8974 interconnect driver" depends on INTERCONNECT_QCOM diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile index 916d7bbe55b7..709f65d2447d 100644 --- a/drivers/interconnect/qcom/Makefile +++ b/drivers/interconnect/qcom/Makefile @@ -2,6 +2,7 @@ icc-bcm-voter-objs := bcm-voter.o qnoc-msm8916-objs := msm8916.o +qnoc-msm8939-objs := msm8939.o qnoc-msm8974-objs := msm8974.o icc-osm-l3-objs := osm-l3.o qnoc-qcs404-objs := qcs404.o @@ -14,6 +15,7 @@ icc-smd-rpm-objs := smd-rpm.o icc-rpm.o obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8916) += qnoc-msm8916.o +obj-$(CONFIG_INTERCONNECT_QCOM_MSM8939) += qnoc-msm8939.o obj-$(CONFIG_INTERCONNECT_QCOM_MSM8974) += qnoc-msm8974.o obj-$(CONFIG_INTERCONNECT_QCOM_OSM_L3) += icc-osm-l3.o obj-$(CONFIG_INTERCONNECT_QCOM_QCS404) += qnoc-qcs404.o diff --git a/drivers/interconnect/qcom/msm8939.c b/drivers/interconnect/qcom/msm8939.c new file mode 100644 index 000000000000..dfbec30ed149 --- /dev/null +++ b/drivers/interconnect/qcom/msm8939.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Linaro Ltd + * Author: Jun Nie + * With reference of msm8916 interconnect driver of Georgi Djakov. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "smd-rpm.h" +#include "icc-rpm.h" + +enum { + MSM8939_BIMC_SNOC_MAS = 1, + MSM8939_BIMC_SNOC_SLV, + MSM8939_MASTER_AMPSS_M0, + MSM8939_MASTER_LPASS, + MSM8939_MASTER_BLSP_1, + MSM8939_MASTER_DEHR, + MSM8939_MASTER_GRAPHICS_3D, + MSM8939_MASTER_JPEG, + MSM8939_MASTER_MDP_PORT0, + MSM8939_MASTER_MDP_PORT1, + MSM8939_MASTER_CPP, + MSM8939_MASTER_CRYPTO_CORE0, + MSM8939_MASTER_SDCC_1, + MSM8939_MASTER_SDCC_2, + MSM8939_MASTER_QDSS_BAM, + MSM8939_MASTER_QDSS_ETR, + MSM8939_MASTER_SNOC_CFG, + MSM8939_MASTER_SPDM, + MSM8939_MASTER_TCU0, + MSM8939_MASTER_USB_HS1, + MSM8939_MASTER_USB_HS2, + MSM8939_MASTER_VFE, + MSM8939_MASTER_VIDEO_P0, + MSM8939_SNOC_MM_INT_0, + MSM8939_SNOC_MM_INT_1, + MSM8939_SNOC_MM_INT_2, + MSM8939_PNOC_INT_0, + MSM8939_PNOC_INT_1, + MSM8939_PNOC_MAS_0, + MSM8939_PNOC_MAS_1, + MSM8939_PNOC_SLV_0, + MSM8939_PNOC_SLV_1, + MSM8939_PNOC_SLV_2, + MSM8939_PNOC_SLV_3, + MSM8939_PNOC_SLV_4, + MSM8939_PNOC_SLV_8, + MSM8939_PNOC_SLV_9, + MSM8939_PNOC_SNOC_MAS, + MSM8939_PNOC_SNOC_SLV, + MSM8939_SNOC_QDSS_INT, + MSM8939_SLAVE_AMPSS_L2, + MSM8939_SLAVE_APSS, + MSM8939_SLAVE_LPASS, + MSM8939_SLAVE_BIMC_CFG, + MSM8939_SLAVE_BLSP_1, + MSM8939_SLAVE_BOOT_ROM, + MSM8939_SLAVE_CAMERA_CFG, + MSM8939_SLAVE_CATS_128, + MSM8939_SLAVE_OCMEM_64, + MSM8939_SLAVE_CLK_CTL, + MSM8939_SLAVE_CRYPTO_0_CFG, + MSM8939_SLAVE_DEHR_CFG, + MSM8939_SLAVE_DISPLAY_CFG, + MSM8939_SLAVE_EBI_CH0, + MSM8939_SLAVE_GRAPHICS_3D_CFG, + MSM8939_SLAVE_IMEM_CFG, + MSM8939_SLAVE_IMEM, + MSM8939_SLAVE_MPM, + MSM8939_SLAVE_MSG_RAM, + MSM8939_SLAVE_MSS, + MSM8939_SLAVE_PDM, + MSM8939_SLAVE_PMIC_ARB, + MSM8939_SLAVE_PNOC_CFG, + MSM8939_SLAVE_PRNG, + MSM8939_SLAVE_QDSS_CFG, + MSM8939_SLAVE_QDSS_STM, + MSM8939_SLAVE_RBCPR_CFG, + MSM8939_SLAVE_SDCC_1, + MSM8939_SLAVE_SDCC_2, + MSM8939_SLAVE_SECURITY, + MSM8939_SLAVE_SNOC_CFG, + MSM8939_SLAVE_SPDM, + MSM8939_SLAVE_SRVC_SNOC, + MSM8939_SLAVE_TCSR, + MSM8939_SLAVE_TLMM, + MSM8939_SLAVE_USB_HS1, + MSM8939_SLAVE_USB_HS2, + MSM8939_SLAVE_VENUS_CFG, + MSM8939_SNOC_BIMC_0_MAS, + MSM8939_SNOC_BIMC_0_SLV, + MSM8939_SNOC_BIMC_1_MAS, + MSM8939_SNOC_BIMC_1_SLV, + MSM8939_SNOC_BIMC_2_MAS, + MSM8939_SNOC_BIMC_2_SLV, + MSM8939_SNOC_INT_0, + MSM8939_SNOC_INT_1, + MSM8939_SNOC_INT_BIMC, + MSM8939_SNOC_PNOC_MAS, + MSM8939_SNOC_PNOC_SLV, +}; + +static const struct clk_bulk_data msm8939_bus_clocks[] = { + { .id = "bus" }, + { .id = "bus_a" }, +}; + +DEFINE_QNODE(bimc_snoc_mas, MSM8939_BIMC_SNOC_MAS, 8, -1, -1, MSM8939_BIMC_SNOC_SLV); +DEFINE_QNODE(bimc_snoc_slv, MSM8939_BIMC_SNOC_SLV, 16, -1, 2, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_1); +DEFINE_QNODE(mas_apss, MSM8939_MASTER_AMPSS_M0, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_audio, MSM8939_MASTER_LPASS, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_blsp_1, MSM8939_MASTER_BLSP_1, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_dehr, MSM8939_MASTER_DEHR, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_gfx, MSM8939_MASTER_GRAPHICS_3D, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_jpeg, MSM8939_MASTER_JPEG, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_mdp0, MSM8939_MASTER_MDP_PORT0, 16, -1, -1, MSM8939_SNOC_MM_INT_1, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_mdp1, MSM8939_MASTER_MDP_PORT1, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_cpp, MSM8939_MASTER_CPP, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_pcnoc_crypto_0, MSM8939_MASTER_CRYPTO_CORE0, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_pcnoc_sdcc_1, MSM8939_MASTER_SDCC_1, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_pcnoc_sdcc_2, MSM8939_MASTER_SDCC_2, 8, -1, -1, MSM8939_PNOC_INT_1); +DEFINE_QNODE(mas_qdss_bam, MSM8939_MASTER_QDSS_BAM, 8, -1, -1, MSM8939_SNOC_QDSS_INT); +DEFINE_QNODE(mas_qdss_etr, MSM8939_MASTER_QDSS_ETR, 8, -1, -1, MSM8939_SNOC_QDSS_INT); +DEFINE_QNODE(mas_snoc_cfg, MSM8939_MASTER_SNOC_CFG, 4, 20, -1, MSM8939_SLAVE_SRVC_SNOC); +DEFINE_QNODE(mas_spdm, MSM8939_MASTER_SPDM, 4, -1, -1, MSM8939_PNOC_MAS_0); +DEFINE_QNODE(mas_tcu0, MSM8939_MASTER_TCU0, 16, -1, -1, MSM8939_SLAVE_EBI_CH0, MSM8939_BIMC_SNOC_MAS, MSM8939_SLAVE_AMPSS_L2); +DEFINE_QNODE(mas_usb_hs1, MSM8939_MASTER_USB_HS1, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_usb_hs2, MSM8939_MASTER_USB_HS2, 4, -1, -1, MSM8939_PNOC_MAS_1); +DEFINE_QNODE(mas_vfe, MSM8939_MASTER_VFE, 16, -1, -1, MSM8939_SNOC_MM_INT_1, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mas_video, MSM8939_MASTER_VIDEO_P0, 16, -1, -1, MSM8939_SNOC_MM_INT_0, MSM8939_SNOC_MM_INT_2); +DEFINE_QNODE(mm_int_0, MSM8939_SNOC_MM_INT_0, 16, -1, -1, MSM8939_SNOC_BIMC_2_MAS); +DEFINE_QNODE(mm_int_1, MSM8939_SNOC_MM_INT_1, 16, -1, -1, MSM8939_SNOC_BIMC_1_MAS); +DEFINE_QNODE(mm_int_2, MSM8939_SNOC_MM_INT_2, 16, -1, -1, MSM8939_SNOC_INT_0); +DEFINE_QNODE(pcnoc_int_0, MSM8939_PNOC_INT_0, 8, -1, -1, MSM8939_PNOC_SNOC_MAS, MSM8939_PNOC_SLV_0, MSM8939_PNOC_SLV_1, MSM8939_PNOC_SLV_2, MSM8939_PNOC_SLV_3, MSM8939_PNOC_SLV_4, MSM8939_PNOC_SLV_8, MSM8939_PNOC_SLV_9); +DEFINE_QNODE(pcnoc_int_1, MSM8939_PNOC_INT_1, 8, -1, -1, MSM8939_PNOC_SNOC_MAS); +DEFINE_QNODE(pcnoc_m_0, MSM8939_PNOC_MAS_0, 8, -1, -1, MSM8939_PNOC_INT_0); +DEFINE_QNODE(pcnoc_m_1, MSM8939_PNOC_MAS_1, 8, -1, -1, MSM8939_PNOC_SNOC_MAS); +DEFINE_QNODE(pcnoc_s_0, MSM8939_PNOC_SLV_0, 4, -1, -1, MSM8939_SLAVE_CLK_CTL, MSM8939_SLAVE_TLMM, MSM8939_SLAVE_TCSR, MSM8939_SLAVE_SECURITY, MSM8939_SLAVE_MSS); +DEFINE_QNODE(pcnoc_s_1, MSM8939_PNOC_SLV_1, 4, -1, -1, MSM8939_SLAVE_IMEM_CFG, MSM8939_SLAVE_CRYPTO_0_CFG, MSM8939_SLAVE_MSG_RAM, MSM8939_SLAVE_PDM, MSM8939_SLAVE_PRNG); +DEFINE_QNODE(pcnoc_s_2, MSM8939_PNOC_SLV_2, 4, -1, -1, MSM8939_SLAVE_SPDM, MSM8939_SLAVE_BOOT_ROM, MSM8939_SLAVE_BIMC_CFG, MSM8939_SLAVE_PNOC_CFG, MSM8939_SLAVE_PMIC_ARB); +DEFINE_QNODE(pcnoc_s_3, MSM8939_PNOC_SLV_3, 4, -1, -1, MSM8939_SLAVE_MPM, MSM8939_SLAVE_SNOC_CFG, MSM8939_SLAVE_RBCPR_CFG, MSM8939_SLAVE_QDSS_CFG, MSM8939_SLAVE_DEHR_CFG); +DEFINE_QNODE(pcnoc_s_4, MSM8939_PNOC_SLV_4, 4, -1, -1, MSM8939_SLAVE_VENUS_CFG, MSM8939_SLAVE_CAMERA_CFG, MSM8939_SLAVE_DISPLAY_CFG); +DEFINE_QNODE(pcnoc_s_8, MSM8939_PNOC_SLV_8, 4, -1, -1, MSM8939_SLAVE_USB_HS1, MSM8939_SLAVE_SDCC_1, MSM8939_SLAVE_BLSP_1); +DEFINE_QNODE(pcnoc_s_9, MSM8939_PNOC_SLV_9, 4, -1, -1, MSM8939_SLAVE_SDCC_2, MSM8939_SLAVE_LPASS, MSM8939_SLAVE_USB_HS2); +DEFINE_QNODE(pcnoc_snoc_mas, MSM8939_PNOC_SNOC_MAS, 8, 29, -1, MSM8939_PNOC_SNOC_SLV); +DEFINE_QNODE(pcnoc_snoc_slv, MSM8939_PNOC_SNOC_SLV, 8, -1, 45, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_BIMC, MSM8939_SNOC_INT_1); +DEFINE_QNODE(qdss_int, MSM8939_SNOC_QDSS_INT, 8, -1, -1, MSM8939_SNOC_INT_0, MSM8939_SNOC_INT_BIMC); +DEFINE_QNODE(slv_apps_l2, MSM8939_SLAVE_AMPSS_L2, 16, -1, -1, 0); +DEFINE_QNODE(slv_apss, MSM8939_SLAVE_APSS, 4, -1, 20, 0); +DEFINE_QNODE(slv_audio, MSM8939_SLAVE_LPASS, 4, -1, -1, 0); +DEFINE_QNODE(slv_bimc_cfg, MSM8939_SLAVE_BIMC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_blsp_1, MSM8939_SLAVE_BLSP_1, 4, -1, -1, 0); +DEFINE_QNODE(slv_boot_rom, MSM8939_SLAVE_BOOT_ROM, 4, -1, -1, 0); +DEFINE_QNODE(slv_camera_cfg, MSM8939_SLAVE_CAMERA_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_cats_0, MSM8939_SLAVE_CATS_128, 16, -1, 106, 0); +DEFINE_QNODE(slv_cats_1, MSM8939_SLAVE_OCMEM_64, 8, -1, 107, 0); +DEFINE_QNODE(slv_clk_ctl, MSM8939_SLAVE_CLK_CTL, 4, -1, -1, 0); +DEFINE_QNODE(slv_crypto_0_cfg, MSM8939_SLAVE_CRYPTO_0_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_dehr_cfg, MSM8939_SLAVE_DEHR_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_display_cfg, MSM8939_SLAVE_DISPLAY_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_ebi_ch0, MSM8939_SLAVE_EBI_CH0, 16, -1, 0, 0); +DEFINE_QNODE(slv_gfx_cfg, MSM8939_SLAVE_GRAPHICS_3D_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_imem_cfg, MSM8939_SLAVE_IMEM_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_imem, MSM8939_SLAVE_IMEM, 8, -1, 26, 0); +DEFINE_QNODE(slv_mpm, MSM8939_SLAVE_MPM, 4, -1, -1, 0); +DEFINE_QNODE(slv_msg_ram, MSM8939_SLAVE_MSG_RAM, 4, -1, -1, 0); +DEFINE_QNODE(slv_mss, MSM8939_SLAVE_MSS, 4, -1, -1, 0); +DEFINE_QNODE(slv_pdm, MSM8939_SLAVE_PDM, 4, -1, -1, 0); +DEFINE_QNODE(slv_pmic_arb, MSM8939_SLAVE_PMIC_ARB, 4, -1, -1, 0); +DEFINE_QNODE(slv_pcnoc_cfg, MSM8939_SLAVE_PNOC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_prng, MSM8939_SLAVE_PRNG, 4, -1, -1, 0); +DEFINE_QNODE(slv_qdss_cfg, MSM8939_SLAVE_QDSS_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_qdss_stm, MSM8939_SLAVE_QDSS_STM, 4, -1, 30, 0); +DEFINE_QNODE(slv_rbcpr_cfg, MSM8939_SLAVE_RBCPR_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_sdcc_1, MSM8939_SLAVE_SDCC_1, 4, -1, -1, 0); +DEFINE_QNODE(slv_sdcc_2, MSM8939_SLAVE_SDCC_2, 4, -1, -1, 0); +DEFINE_QNODE(slv_security, MSM8939_SLAVE_SECURITY, 4, -1, -1, 0); +DEFINE_QNODE(slv_snoc_cfg, MSM8939_SLAVE_SNOC_CFG, 4, -1, -1, 0); +DEFINE_QNODE(slv_spdm, MSM8939_SLAVE_SPDM, 4, -1, -1, 0); +DEFINE_QNODE(slv_srvc_snoc, MSM8939_SLAVE_SRVC_SNOC, 8, -1, 29, 0); +DEFINE_QNODE(slv_tcsr, MSM8939_SLAVE_TCSR, 4, -1, -1, 0); +DEFINE_QNODE(slv_tlmm, MSM8939_SLAVE_TLMM, 4, -1, -1, 0); +DEFINE_QNODE(slv_usb_hs1, MSM8939_SLAVE_USB_HS1, 4, -1, -1, 0); +DEFINE_QNODE(slv_usb_hs2, MSM8939_SLAVE_USB_HS2, 4, -1, -1, 0); +DEFINE_QNODE(slv_venus_cfg, MSM8939_SLAVE_VENUS_CFG, 4, -1, -1, 0); +DEFINE_QNODE(snoc_bimc_0_mas, MSM8939_SNOC_BIMC_0_MAS, 16, 3, -1, MSM8939_SNOC_BIMC_0_SLV); +DEFINE_QNODE(snoc_bimc_0_slv, MSM8939_SNOC_BIMC_0_SLV, 16, -1, 24, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_bimc_1_mas, MSM8939_SNOC_BIMC_1_MAS, 16, 76, -1, MSM8939_SNOC_BIMC_1_SLV); +DEFINE_QNODE(snoc_bimc_1_slv, MSM8939_SNOC_BIMC_1_SLV, 16, -1, 104, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_bimc_2_mas, MSM8939_SNOC_BIMC_2_MAS, 16, -1, -1, MSM8939_SNOC_BIMC_2_SLV); +DEFINE_QNODE(snoc_bimc_2_slv, MSM8939_SNOC_BIMC_2_SLV, 16, -1, -1, MSM8939_SLAVE_EBI_CH0); +DEFINE_QNODE(snoc_int_0, MSM8939_SNOC_INT_0, 8, 99, 130, MSM8939_SLAVE_QDSS_STM, MSM8939_SLAVE_IMEM, MSM8939_SNOC_PNOC_MAS); +DEFINE_QNODE(snoc_int_1, MSM8939_SNOC_INT_1, 8, 100, 131, MSM8939_SLAVE_APSS, MSM8939_SLAVE_CATS_128, MSM8939_SLAVE_OCMEM_64); +DEFINE_QNODE(snoc_int_bimc, MSM8939_SNOC_INT_BIMC, 8, 101, 132, MSM8939_SNOC_BIMC_1_MAS); +DEFINE_QNODE(snoc_pcnoc_mas, MSM8939_SNOC_PNOC_MAS, 8, -1, -1, MSM8939_SNOC_PNOC_SLV); +DEFINE_QNODE(snoc_pcnoc_slv, MSM8939_SNOC_PNOC_SLV, 8, -1, -1, MSM8939_PNOC_INT_0); + +static struct qcom_icc_node *msm8939_snoc_nodes[] = { + [BIMC_SNOC_SLV] = &bimc_snoc_slv, + [MASTER_QDSS_BAM] = &mas_qdss_bam, + [MASTER_QDSS_ETR] = &mas_qdss_etr, + [MASTER_SNOC_CFG] = &mas_snoc_cfg, + [PCNOC_SNOC_SLV] = &pcnoc_snoc_slv, + [SLAVE_APSS] = &slv_apss, + [SLAVE_CATS_128] = &slv_cats_0, + [SLAVE_OCMEM_64] = &slv_cats_1, + [SLAVE_IMEM] = &slv_imem, + [SLAVE_QDSS_STM] = &slv_qdss_stm, + [SLAVE_SRVC_SNOC] = &slv_srvc_snoc, + [SNOC_BIMC_0_MAS] = &snoc_bimc_0_mas, + [SNOC_BIMC_1_MAS] = &snoc_bimc_1_mas, + [SNOC_BIMC_2_MAS] = &snoc_bimc_2_mas, + [SNOC_INT_0] = &snoc_int_0, + [SNOC_INT_1] = &snoc_int_1, + [SNOC_INT_BIMC] = &snoc_int_bimc, + [SNOC_PCNOC_MAS] = &snoc_pcnoc_mas, + [SNOC_QDSS_INT] = &qdss_int, +}; + +static struct qcom_icc_desc msm8939_snoc = { + .nodes = msm8939_snoc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_snoc_nodes), +}; + +static struct qcom_icc_node *msm8939_snoc_mm_nodes[] = { + [MASTER_VIDEO_P0] = &mas_video, + [MASTER_JPEG] = &mas_jpeg, + [MASTER_VFE] = &mas_vfe, + [MASTER_MDP_PORT0] = &mas_mdp0, + [MASTER_MDP_PORT1] = &mas_mdp1, + [MASTER_CPP] = &mas_cpp, + [SNOC_MM_INT_0] = &mm_int_0, + [SNOC_MM_INT_1] = &mm_int_1, + [SNOC_MM_INT_2] = &mm_int_2, +}; + +static struct qcom_icc_desc msm8939_snoc_mm = { + .nodes = msm8939_snoc_mm_nodes, + .num_nodes = ARRAY_SIZE(msm8939_snoc_mm_nodes), +}; + +static struct qcom_icc_node *msm8939_bimc_nodes[] = { + [BIMC_SNOC_MAS] = &bimc_snoc_mas, + [MASTER_AMPSS_M0] = &mas_apss, + [MASTER_GRAPHICS_3D] = &mas_gfx, + [MASTER_TCU0] = &mas_tcu0, + [SLAVE_AMPSS_L2] = &slv_apps_l2, + [SLAVE_EBI_CH0] = &slv_ebi_ch0, + [SNOC_BIMC_0_SLV] = &snoc_bimc_0_slv, + [SNOC_BIMC_1_SLV] = &snoc_bimc_1_slv, + [SNOC_BIMC_2_SLV] = &snoc_bimc_2_slv, +}; + +static struct qcom_icc_desc msm8939_bimc = { + .nodes = msm8939_bimc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_bimc_nodes), +}; + +static struct qcom_icc_node *msm8939_pcnoc_nodes[] = { + [MASTER_BLSP_1] = &mas_blsp_1, + [MASTER_DEHR] = &mas_dehr, + [MASTER_LPASS] = &mas_audio, + [MASTER_CRYPTO_CORE0] = &mas_pcnoc_crypto_0, + [MASTER_SDCC_1] = &mas_pcnoc_sdcc_1, + [MASTER_SDCC_2] = &mas_pcnoc_sdcc_2, + [MASTER_SPDM] = &mas_spdm, + [MASTER_USB_HS1] = &mas_usb_hs1, + [MASTER_USB_HS2] = &mas_usb_hs2, + [PCNOC_INT_0] = &pcnoc_int_0, + [PCNOC_INT_1] = &pcnoc_int_1, + [PCNOC_MAS_0] = &pcnoc_m_0, + [PCNOC_MAS_1] = &pcnoc_m_1, + [PCNOC_SLV_0] = &pcnoc_s_0, + [PCNOC_SLV_1] = &pcnoc_s_1, + [PCNOC_SLV_2] = &pcnoc_s_2, + [PCNOC_SLV_3] = &pcnoc_s_3, + [PCNOC_SLV_4] = &pcnoc_s_4, + [PCNOC_SLV_8] = &pcnoc_s_8, + [PCNOC_SLV_9] = &pcnoc_s_9, + [PCNOC_SNOC_MAS] = &pcnoc_snoc_mas, + [SLAVE_BIMC_CFG] = &slv_bimc_cfg, + [SLAVE_BLSP_1] = &slv_blsp_1, + [SLAVE_BOOT_ROM] = &slv_boot_rom, + [SLAVE_CAMERA_CFG] = &slv_camera_cfg, + [SLAVE_CLK_CTL] = &slv_clk_ctl, + [SLAVE_CRYPTO_0_CFG] = &slv_crypto_0_cfg, + [SLAVE_DEHR_CFG] = &slv_dehr_cfg, + [SLAVE_DISPLAY_CFG] = &slv_display_cfg, + [SLAVE_GRAPHICS_3D_CFG] = &slv_gfx_cfg, + [SLAVE_IMEM_CFG] = &slv_imem_cfg, + [SLAVE_LPASS] = &slv_audio, + [SLAVE_MPM] = &slv_mpm, + [SLAVE_MSG_RAM] = &slv_msg_ram, + [SLAVE_MSS] = &slv_mss, + [SLAVE_PDM] = &slv_pdm, + [SLAVE_PMIC_ARB] = &slv_pmic_arb, + [SLAVE_PCNOC_CFG] = &slv_pcnoc_cfg, + [SLAVE_PRNG] = &slv_prng, + [SLAVE_QDSS_CFG] = &slv_qdss_cfg, + [SLAVE_RBCPR_CFG] = &slv_rbcpr_cfg, + [SLAVE_SDCC_1] = &slv_sdcc_1, + [SLAVE_SDCC_2] = &slv_sdcc_2, + [SLAVE_SECURITY] = &slv_security, + [SLAVE_SNOC_CFG] = &slv_snoc_cfg, + [SLAVE_SPDM] = &slv_spdm, + [SLAVE_TCSR] = &slv_tcsr, + [SLAVE_TLMM] = &slv_tlmm, + [SLAVE_USB_HS1] = &slv_usb_hs1, + [SLAVE_USB_HS2] = &slv_usb_hs2, + [SLAVE_VENUS_CFG] = &slv_venus_cfg, + [SNOC_PCNOC_SLV] = &snoc_pcnoc_slv, +}; + +static struct qcom_icc_desc msm8939_pcnoc = { + .nodes = msm8939_pcnoc_nodes, + .num_nodes = ARRAY_SIZE(msm8939_pcnoc_nodes), +}; + +static int msm8939_qnoc_probe(struct platform_device *pdev) +{ + return qnoc_probe(pdev, sizeof(msm8939_bus_clocks), + ARRAY_SIZE(msm8939_bus_clocks), msm8939_bus_clocks); +} + +static const struct of_device_id msm8939_noc_of_match[] = { + { .compatible = "qcom,msm8939-bimc", .data = &msm8939_bimc }, + { .compatible = "qcom,msm8939-pcnoc", .data = &msm8939_pcnoc }, + { .compatible = "qcom,msm8939-snoc", .data = &msm8939_snoc }, + { .compatible = "qcom,msm8939-snoc-mm", .data = &msm8939_snoc_mm }, + { } +}; +MODULE_DEVICE_TABLE(of, msm8939_noc_of_match); + +static struct platform_driver msm8939_noc_driver = { + .probe = msm8939_qnoc_probe, + .remove = qnoc_remove, + .driver = { + .name = "qnoc-msm8939", + .of_match_table = msm8939_noc_of_match, + }, +}; +module_platform_driver(msm8939_noc_driver); +MODULE_AUTHOR("Jun Nie "); +MODULE_DESCRIPTION("Qualcomm MSM8939 NoC driver"); +MODULE_LICENSE("GPL v2"); From 22a9e57fccfea0cf9a066935dc6b81b866aaca55 Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Thu, 24 Dec 2020 21:24:46 +0800 Subject: [PATCH 016/633] misc: ocxl: use DEFINE_MUTEX() for mutex lock mutex lock can be initialized automatically with DEFINE_MUTEX() rather than explicitly calling mutex_init(). Acked-by: Frederic Barrat Signed-off-by: Zheng Yongjun Link: https://lore.kernel.org/r/20201224132446.31286-1-zhengyongjun3@huawei.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/ocxl/file.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index 4d1b44de1492..e70525eedaae 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -15,7 +15,7 @@ static dev_t ocxl_dev; static struct class *ocxl_class; -static struct mutex minors_idr_lock; +static DEFINE_MUTEX(minors_idr_lock); static struct idr minors_idr; static struct ocxl_file_info *find_and_get_file_info(dev_t devno) @@ -588,7 +588,6 @@ int ocxl_file_init(void) { int rc; - mutex_init(&minors_idr_lock); idr_init(&minors_idr); rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl"); From 157576d552339edc7988c79f37f4a335eaa0eaf2 Mon Sep 17 00:00:00 2001 From: Alexandre Belloni Date: Mon, 28 Dec 2020 21:44:13 +0100 Subject: [PATCH 017/633] misc: remove atmel_tclib There is no driver depending on atmel_tclib anymore. Remove this driver. Signed-off-by: Alexandre Belloni Link: https://lore.kernel.org/r/20201228204413.2677762-1-alexandre.belloni@bootlin.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/Kconfig | 8 -- drivers/misc/Makefile | 1 - drivers/misc/atmel_tclib.c | 200 ------------------------------------- 3 files changed, 209 deletions(-) delete mode 100644 drivers/misc/atmel_tclib.c diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index fafa8b0d8099..e90c2524e46c 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -50,14 +50,6 @@ config AD525X_DPOT_SPI To compile this driver as a module, choose M here: the module will be called ad525x_dpot-spi. -config ATMEL_TCLIB - bool "Atmel AT32/AT91 Timer/Counter Library" - depends on ARCH_AT91 - help - Select this if you want a library to allocate the Timer/Counter - blocks found on many Atmel processors. This facilitates using - these blocks by different drivers despite processor differences. - config DUMMY_IRQ tristate "Dummy IRQ handler" help diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index d23231e73330..f65e8b18ecd8 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -10,7 +10,6 @@ obj-$(CONFIG_AD525X_DPOT_I2C) += ad525x_dpot-i2c.o obj-$(CONFIG_AD525X_DPOT_SPI) += ad525x_dpot-spi.o obj-$(CONFIG_INTEL_MID_PTI) += pti.o obj-$(CONFIG_ATMEL_SSC) += atmel-ssc.o -obj-$(CONFIG_ATMEL_TCLIB) += atmel_tclib.o obj-$(CONFIG_DUMMY_IRQ) += dummy-irq.o obj-$(CONFIG_ICS932S401) += ics932s401.o obj-$(CONFIG_LKDTM) += lkdtm/ diff --git a/drivers/misc/atmel_tclib.c b/drivers/misc/atmel_tclib.c deleted file mode 100644 index 7de7840f613c..000000000000 --- a/drivers/misc/atmel_tclib.c +++ /dev/null @@ -1,200 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * This is a thin library to solve the problem of how to portably allocate - * one of the TC blocks. For simplicity, it doesn't currently expect to - * share individual timers between different drivers. - */ - -#if defined(CONFIG_AVR32) -/* AVR32 has these divide PBB */ -const u8 atmel_tc_divisors[5] = { 0, 4, 8, 16, 32, }; -EXPORT_SYMBOL(atmel_tc_divisors); - -#elif defined(CONFIG_ARCH_AT91) -/* AT91 has these divide MCK */ -const u8 atmel_tc_divisors[5] = { 2, 8, 32, 128, 0, }; -EXPORT_SYMBOL(atmel_tc_divisors); - -#endif - -static DEFINE_SPINLOCK(tc_list_lock); -static LIST_HEAD(tc_list); - -/** - * atmel_tc_alloc - allocate a specified TC block - * @block: which block to allocate - * - * Caller allocates a block. If it is available, a pointer to a - * pre-initialized struct atmel_tc is returned. The caller can access - * the registers directly through the "regs" field. - */ -struct atmel_tc *atmel_tc_alloc(unsigned block) -{ - struct atmel_tc *tc; - struct platform_device *pdev = NULL; - - spin_lock(&tc_list_lock); - list_for_each_entry(tc, &tc_list, node) { - if (tc->allocated) - continue; - - if ((tc->pdev->dev.of_node && tc->id == block) || - (tc->pdev->id == block)) { - pdev = tc->pdev; - tc->allocated = true; - break; - } - } - spin_unlock(&tc_list_lock); - - return pdev ? tc : NULL; -} -EXPORT_SYMBOL_GPL(atmel_tc_alloc); - -/** - * atmel_tc_free - release a specified TC block - * @tc: Timer/counter block that was returned by atmel_tc_alloc() - * - * This reverses the effect of atmel_tc_alloc(), invalidating the resource - * returned by that routine and making the TC available to other drivers. - */ -void atmel_tc_free(struct atmel_tc *tc) -{ - spin_lock(&tc_list_lock); - if (tc->allocated) - tc->allocated = false; - spin_unlock(&tc_list_lock); -} -EXPORT_SYMBOL_GPL(atmel_tc_free); - -#if defined(CONFIG_OF) -static struct atmel_tcb_config tcb_rm9200_config = { - .counter_width = 16, -}; - -static struct atmel_tcb_config tcb_sam9x5_config = { - .counter_width = 32, -}; - -static const struct of_device_id atmel_tcb_dt_ids[] = { - { - .compatible = "atmel,at91rm9200-tcb", - .data = &tcb_rm9200_config, - }, { - .compatible = "atmel,at91sam9x5-tcb", - .data = &tcb_sam9x5_config, - }, { - /* sentinel */ - } -}; - -MODULE_DEVICE_TABLE(of, atmel_tcb_dt_ids); -#endif - -static int __init tc_probe(struct platform_device *pdev) -{ - struct atmel_tc *tc; - struct clk *clk; - int irq; - unsigned int i; - - if (of_get_child_count(pdev->dev.of_node)) - return -EBUSY; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return -EINVAL; - - tc = devm_kzalloc(&pdev->dev, sizeof(struct atmel_tc), GFP_KERNEL); - if (!tc) - return -ENOMEM; - - tc->pdev = pdev; - - clk = devm_clk_get(&pdev->dev, "t0_clk"); - if (IS_ERR(clk)) - return PTR_ERR(clk); - - tc->slow_clk = devm_clk_get(&pdev->dev, "slow_clk"); - if (IS_ERR(tc->slow_clk)) - return PTR_ERR(tc->slow_clk); - - tc->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tc->regs)) - return PTR_ERR(tc->regs); - - /* Now take SoC information if available */ - if (pdev->dev.of_node) { - const struct of_device_id *match; - match = of_match_node(atmel_tcb_dt_ids, pdev->dev.of_node); - if (match) - tc->tcb_config = match->data; - - tc->id = of_alias_get_id(tc->pdev->dev.of_node, "tcb"); - } else { - tc->id = pdev->id; - } - - tc->clk[0] = clk; - tc->clk[1] = devm_clk_get(&pdev->dev, "t1_clk"); - if (IS_ERR(tc->clk[1])) - tc->clk[1] = clk; - tc->clk[2] = devm_clk_get(&pdev->dev, "t2_clk"); - if (IS_ERR(tc->clk[2])) - tc->clk[2] = clk; - - tc->irq[0] = irq; - tc->irq[1] = platform_get_irq(pdev, 1); - if (tc->irq[1] < 0) - tc->irq[1] = irq; - tc->irq[2] = platform_get_irq(pdev, 2); - if (tc->irq[2] < 0) - tc->irq[2] = irq; - - for (i = 0; i < 3; i++) - writel(ATMEL_TC_ALL_IRQ, tc->regs + ATMEL_TC_REG(i, IDR)); - - spin_lock(&tc_list_lock); - list_add_tail(&tc->node, &tc_list); - spin_unlock(&tc_list_lock); - - platform_set_drvdata(pdev, tc); - - return 0; -} - -static void tc_shutdown(struct platform_device *pdev) -{ - int i; - struct atmel_tc *tc = platform_get_drvdata(pdev); - - for (i = 0; i < 3; i++) - writel(ATMEL_TC_ALL_IRQ, tc->regs + ATMEL_TC_REG(i, IDR)); -} - -static struct platform_driver tc_driver = { - .driver = { - .name = "atmel_tcb", - .of_match_table = of_match_ptr(atmel_tcb_dt_ids), - }, - .shutdown = tc_shutdown, -}; - -static int __init tc_init(void) -{ - return platform_driver_probe(&tc_driver, tc_probe); -} -arch_initcall(tc_init); From 38d98d73be9f5f90b63d56d73afffa6c9de59c02 Mon Sep 17 00:00:00 2001 From: Ricky Wu Date: Wed, 30 Dec 2020 14:39:53 +0800 Subject: [PATCH 018/633] misc: rtsx: remove unused function removed unused function 'rtsx_pci_disable_aspm' Signed-off-by: Ricky Wu Link: https://lore.kernel.org/r/20201230063953.10972-1-ricky_wu@realtek.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cardreader/rtsx_pcr.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index 2aa6648fa41f..d782754fa346 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -59,12 +59,6 @@ static const struct pci_device_id rtsx_pci_ids[] = { MODULE_DEVICE_TABLE(pci, rtsx_pci_ids); -static inline void rtsx_pci_disable_aspm(struct rtsx_pcr *pcr) -{ - pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0); -} - static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency) { rtsx_pci_write_register(pcr, MSGTXDATA0, From 4c5a6a7b71431f39bd80f6c3a14a429602a5555f Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 15 Dec 2020 06:53:06 -0800 Subject: [PATCH 019/633] greybus: remove h from printk format specifier See Documentation/core-api/printk-formats.rst. h should no longer be used in the format specifier for printk. Reviewed-by: Alex Elder Signed-off-by: Tom Rix Link: https://lore.kernel.org/r/20201215145306.1901598-1-trix@redhat.com Signed-off-by: Greg Kroah-Hartman --- drivers/greybus/greybus_trace.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/greybus/greybus_trace.h b/drivers/greybus/greybus_trace.h index 1bc9f1275c65..616a3bd61aa6 100644 --- a/drivers/greybus/greybus_trace.h +++ b/drivers/greybus/greybus_trace.h @@ -40,7 +40,7 @@ DECLARE_EVENT_CLASS(gb_message, __entry->result = message->header->result; ), - TP_printk("size=%hu operation_id=0x%04x type=0x%02x result=0x%02x", + TP_printk("size=%u operation_id=0x%04x type=0x%02x result=0x%02x", __entry->size, __entry->operation_id, __entry->type, __entry->result) ); @@ -317,7 +317,7 @@ DECLARE_EVENT_CLASS(gb_interface, __entry->mode_switch = intf->mode_switch; ), - TP_printk("intf_id=%hhu device_id=%hhu module_id=%hhu D=%d J=%d A=%d E=%d M=%d", + TP_printk("intf_id=%u device_id=%u module_id=%u D=%d J=%d A=%d E=%d M=%d", __entry->id, __entry->device_id, __entry->module_id, __entry->disconnected, __entry->ejected, __entry->active, __entry->enabled, __entry->mode_switch) @@ -391,7 +391,7 @@ DECLARE_EVENT_CLASS(gb_module, __entry->disconnected = module->disconnected; ), - TP_printk("hd_bus_id=%d module_id=%hhu num_interfaces=%zu disconnected=%d", + TP_printk("hd_bus_id=%d module_id=%u num_interfaces=%zu disconnected=%d", __entry->hd_bus_id, __entry->module_id, __entry->num_interfaces, __entry->disconnected) ); From df2f392c61b6f12290480df000b9cc4dcae07dc2 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 5 Jan 2021 17:56:03 -0600 Subject: [PATCH 020/633] rpmsg: glink: fix some kerneldoc comments The kerneldoc comments for the do_cleanup_msg and cleanup_done_msg structures describe the fields, but don't prefix the field names with "@". Add those, to get rid of some W=1 build warnings on an x86_64 architecture build. Signed-off-by: Alex Elder Link: https://lore.kernel.org/r/20210105235603.32663-1-elder@linaro.org Signed-off-by: Bjorn Andersson --- drivers/rpmsg/qcom_glink_ssr.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/rpmsg/qcom_glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c index dcd1ce616974..1fc2bf5354f5 100644 --- a/drivers/rpmsg/qcom_glink_ssr.c +++ b/drivers/rpmsg/qcom_glink_ssr.c @@ -12,11 +12,11 @@ /** * struct do_cleanup_msg - The data structure for an SSR do_cleanup message - * version: The G-Link SSR protocol version - * command: The G-Link SSR command - do_cleanup - * seq_num: Sequence number - * name_len: Length of the name of the subsystem being restarted - * name: G-Link edge name of the subsystem being restarted + * @version: The G-Link SSR protocol version + * @command: The G-Link SSR command - do_cleanup + * @seq_num: Sequence number + * @name_len: Length of the name of the subsystem being restarted + * @name: G-Link edge name of the subsystem being restarted */ struct do_cleanup_msg { __le32 version; @@ -28,9 +28,9 @@ struct do_cleanup_msg { /** * struct cleanup_done_msg - The data structure for an SSR cleanup_done message - * version: The G-Link SSR protocol version - * response: The G-Link SSR response to a do_cleanup command, cleanup_done - * seq_num: Sequence number + * @version: The G-Link SSR protocol version + * @response: The G-Link SSR response to a do_cleanup command, cleanup_done + * @seq_num: Sequence number */ struct cleanup_done_msg { __le32 version; From 8527efc59d45d7135460daac36054f2f213ac08a Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Tue, 5 Jan 2021 19:59:05 -0800 Subject: [PATCH 021/633] rpmsg: glink: Guard qcom_glink_ssr_notify() with correct config The qcom_glink_ssr_notify() function doesn't relate to the SMEM specific GLINK config, but the common RPMSG_QCOM_GLINK config. Update the guard to properly reflect this. Reviewed-by: Alex Elder Reported-by: kernel test robot Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210106035905.4153692-1-bjorn.andersson@linaro.org Signed-off-by: Bjorn Andersson --- include/linux/rpmsg/qcom_glink.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/rpmsg/qcom_glink.h b/include/linux/rpmsg/qcom_glink.h index daded9fddf36..22fc3a69b683 100644 --- a/include/linux/rpmsg/qcom_glink.h +++ b/include/linux/rpmsg/qcom_glink.h @@ -7,12 +7,17 @@ struct qcom_glink; +#if IS_ENABLED(CONFIG_RPMSG_QCOM_GLINK) +void qcom_glink_ssr_notify(const char *ssr_name); +#else +static inline void qcom_glink_ssr_notify(const char *ssr_name) {} +#endif + #if IS_ENABLED(CONFIG_RPMSG_QCOM_GLINK_SMEM) struct qcom_glink *qcom_glink_smem_register(struct device *parent, struct device_node *node); void qcom_glink_smem_unregister(struct qcom_glink *glink); -void qcom_glink_ssr_notify(const char *ssr_name); #else @@ -24,7 +29,6 @@ qcom_glink_smem_register(struct device *parent, } static inline void qcom_glink_smem_unregister(struct qcom_glink *glink) {} -static inline void qcom_glink_ssr_notify(const char *ssr_name) {} #endif #endif From 3e35772bc1e42287c8f4c70055deb5e3f5a3e8b5 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 5 Jan 2021 17:55:28 -0600 Subject: [PATCH 022/633] rpmsg: glink: add include of header file With an x86_64 architecture W=1 build, qcom_glink_ssr_notify() is reported as having no previous prototype. The prototype is found in "qcom_glink.h", so we just need "qcom_glink_ssr.c" to include that file. Signed-off-by: Alex Elder Link: https://lore.kernel.org/r/20210105235528.32538-1-elder@linaro.org Signed-off-by: Bjorn Andersson --- drivers/rpmsg/qcom_glink_ssr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/rpmsg/qcom_glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c index 1fc2bf5354f5..dea929c6045d 100644 --- a/drivers/rpmsg/qcom_glink_ssr.c +++ b/drivers/rpmsg/qcom_glink_ssr.c @@ -8,6 +8,7 @@ #include #include #include +#include #include /** From 235ae89b66767652f9e9e10946b8a38caf6a26f6 Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Tue, 29 Dec 2020 21:50:12 +0800 Subject: [PATCH 023/633] soundwire: intel: Use kzalloc for allocating only one thing Use kzalloc rather than kcalloc(1,...) The semantic patch that makes this change is as follows: (http://coccinelle.lip6.fr/) // @@ @@ - kcalloc(1, + kzalloc( ...) // Signed-off-by: Zheng Yongjun Link: https://lore.kernel.org/r/20201229135012.23472-1-zhengyongjun3@huawei.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 66adb258a425..10b60b7b17bb 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -967,7 +967,7 @@ static int intel_hw_params(struct snd_pcm_substream *substream, } /* Port configuration */ - pconfig = kcalloc(1, sizeof(*pconfig), GFP_KERNEL); + pconfig = kzalloc(sizeof(*pconfig), GFP_KERNEL); if (!pconfig) { ret = -ENOMEM; goto error; From 3e265f836e9d62ccc4820157dc6a34d7685ba41d Mon Sep 17 00:00:00 2001 From: Matthew Gerlach Date: Wed, 6 Jan 2021 20:37:07 -0800 Subject: [PATCH 024/633] fpga: dfl: refactor cci_enumerate_feature_devs() In preparation of looking for dfls based on a vendor specific pci capability, move the code for the default method of finding the first dfl at offset 0 of Bar 0 to its own function. Acked-by: Wu Hao Signed-off-by: Matthew Gerlach Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-2-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/fpga/dfl-pci.c | 84 +++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index a2203d03c9e2..5100695e27cd 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -119,49 +119,20 @@ static int *cci_pci_create_irq_table(struct pci_dev *pcidev, unsigned int nvec) return table; } -/* enumerate feature devices under pci device */ -static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +/* default method of finding dfls starting at offset 0 of bar 0 */ +static int find_dfls_by_default(struct pci_dev *pcidev, + struct dfl_fpga_enum_info *info) { - struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); - int port_num, bar, i, nvec, ret = 0; - struct dfl_fpga_enum_info *info; - struct dfl_fpga_cdev *cdev; + int port_num, bar, i, ret = 0; resource_size_t start, len; void __iomem *base; - int *irq_table; u32 offset; u64 v; - /* allocate enumeration info via pci_dev */ - info = dfl_fpga_enum_info_alloc(&pcidev->dev); - if (!info) - return -ENOMEM; - - /* add irq info for enumeration if the device support irq */ - nvec = cci_pci_alloc_irq(pcidev); - if (nvec < 0) { - dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); - ret = nvec; - goto enum_info_free_exit; - } else if (nvec) { - irq_table = cci_pci_create_irq_table(pcidev, nvec); - if (!irq_table) { - ret = -ENOMEM; - goto irq_free_exit; - } - - ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); - kfree(irq_table); - if (ret) - goto irq_free_exit; - } - - /* start to find Device Feature List in Bar 0 */ + /* start to find Device Feature List from Bar 0 */ base = cci_pci_ioremap_bar0(pcidev); - if (!base) { - ret = -ENOMEM; - goto irq_free_exit; - } + if (!base) + return -ENOMEM; /* * PF device has FME and Ports/AFUs, and VF device only has one @@ -208,12 +179,51 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev) dfl_fpga_enum_info_add_dfl(info, start, len); } else { ret = -ENODEV; - goto irq_free_exit; } /* release I/O mappings for next step enumeration */ pcim_iounmap_regions(pcidev, BIT(0)); + return ret; +} + +/* enumerate feature devices under pci device */ +static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +{ + struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); + struct dfl_fpga_enum_info *info; + struct dfl_fpga_cdev *cdev; + int nvec, ret = 0; + int *irq_table; + + /* allocate enumeration info via pci_dev */ + info = dfl_fpga_enum_info_alloc(&pcidev->dev); + if (!info) + return -ENOMEM; + + /* add irq info for enumeration if the device support irq */ + nvec = cci_pci_alloc_irq(pcidev); + if (nvec < 0) { + dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); + ret = nvec; + goto enum_info_free_exit; + } else if (nvec) { + irq_table = cci_pci_create_irq_table(pcidev, nvec); + if (!irq_table) { + ret = -ENOMEM; + goto irq_free_exit; + } + + ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); + kfree(irq_table); + if (ret) + goto irq_free_exit; + } + + ret = find_dfls_by_default(pcidev, info); + if (ret) + goto irq_free_exit; + /* start enumeration with prepared enumeration information */ cdev = dfl_fpga_feature_devs_enumerate(info); if (IS_ERR(cdev)) { From fa41d10589be124404492b5181a818a509d8cb1c Mon Sep 17 00:00:00 2001 From: Matthew Gerlach Date: Wed, 6 Jan 2021 20:37:08 -0800 Subject: [PATCH 025/633] fpga: dfl-pci: locate DFLs by PCIe vendor specific capability A PCIe vendor specific extended capability is introduced by Intel to specify the start of a number of DFLs. Signed-off-by: Matthew Gerlach Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-3-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/fpga/dfl.rst | 27 ++++++++++++ drivers/fpga/dfl-pci.c | 87 +++++++++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/Documentation/fpga/dfl.rst b/Documentation/fpga/dfl.rst index 0404fe6ffc74..ea8cefc18bdb 100644 --- a/Documentation/fpga/dfl.rst +++ b/Documentation/fpga/dfl.rst @@ -501,6 +501,33 @@ Developer only needs to provide a sub feature driver with matched feature id. FME Partial Reconfiguration Sub Feature driver (see drivers/fpga/dfl-fme-pr.c) could be a reference. +Location of DFLs on a PCI Device +=========================== +The original method for finding a DFL on a PCI device assumed the start of the +first DFL to offset 0 of bar 0. If the first node of the DFL is an FME, +then further DFLs in the port(s) are specified in FME header registers. +Alternatively, a PCIe vendor specific capability structure can be used to +specify the location of all the DFLs on the device, providing flexibility +for the type of starting node in the DFL. Intel has reserved the +VSEC ID of 0x43 for this purpose. The vendor specific +data begins with a 4 byte vendor specific register for the number of DFLs followed 4 byte +Offset/BIR vendor specific registers for each DFL. Bits 2:0 of Offset/BIR register +indicates the BAR, and bits 31:3 form the 8 byte aligned offset where bits 2:0 are +zero. + + +----------------------------+ + |31 Number of DFLS 0| + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + . . . + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + +Being able to specify more than one DFL per BAR has been considered, but it +was determined the use case did not provide value. Specifying a single DFL +per BAR simplifies the implementation and allows for extra error checking. Open discussion =============== diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index 5100695e27cd..04e47e266f26 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -27,6 +27,14 @@ #define DRV_VERSION "0.8" #define DRV_NAME "dfl-pci" +#define PCI_VSEC_ID_INTEL_DFLS 0x43 + +#define PCI_VNDR_DFLS_CNT 0x8 +#define PCI_VNDR_DFLS_RES 0xc + +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) +#define PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3) + struct cci_drvdata { struct dfl_fpga_cdev *cdev; /* container device */ }; @@ -119,6 +127,80 @@ static int *cci_pci_create_irq_table(struct pci_dev *pcidev, unsigned int nvec) return table; } +static int find_dfls_by_vsec(struct pci_dev *pcidev, struct dfl_fpga_enum_info *info) +{ + u32 bir, offset, vndr_hdr, dfl_cnt, dfl_res; + int dfl_res_off, i, bars, voff = 0; + resource_size_t start, len; + + while ((voff = pci_find_next_ext_capability(pcidev, voff, PCI_EXT_CAP_ID_VNDR))) { + vndr_hdr = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_HEADER, &vndr_hdr); + + if (PCI_VNDR_HEADER_ID(vndr_hdr) == PCI_VSEC_ID_INTEL_DFLS && + pcidev->vendor == PCI_VENDOR_ID_INTEL) + break; + } + + if (!voff) { + dev_dbg(&pcidev->dev, "%s no DFL VSEC found\n", __func__); + return -ENODEV; + } + + dfl_cnt = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_DFLS_CNT, &dfl_cnt); + if (dfl_cnt > PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s too many DFLs %d > %d\n", + __func__, dfl_cnt, PCI_STD_NUM_BARS); + return -EINVAL; + } + + dfl_res_off = voff + PCI_VNDR_DFLS_RES; + if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) { + dev_err(&pcidev->dev, "%s DFL VSEC too big for PCIe config space\n", + __func__); + return -EINVAL; + } + + for (i = 0, bars = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) { + dfl_res = GENMASK(31, 0); + pci_read_config_dword(pcidev, dfl_res_off, &dfl_res); + + bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK; + if (bir >= PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s bad bir number %d\n", + __func__, bir); + return -EINVAL; + } + + if (bars & BIT(bir)) { + dev_err(&pcidev->dev, "%s DFL for BAR %d already specified\n", + __func__, bir); + return -EINVAL; + } + + bars |= BIT(bir); + + len = pci_resource_len(pcidev, bir); + offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK; + if (offset >= len) { + dev_err(&pcidev->dev, "%s bad offset %u >= %pa\n", + __func__, offset, &len); + return -EINVAL; + } + + dev_dbg(&pcidev->dev, "%s BAR %d offset 0x%x\n", __func__, bir, offset); + + len -= offset; + + start = pci_resource_start(pcidev, bir) + offset; + + dfl_fpga_enum_info_add_dfl(info, start, len); + } + + return 0; +} + /* default method of finding dfls starting at offset 0 of bar 0 */ static int find_dfls_by_default(struct pci_dev *pcidev, struct dfl_fpga_enum_info *info) @@ -220,7 +302,10 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev) goto irq_free_exit; } - ret = find_dfls_by_default(pcidev, info); + ret = find_dfls_by_vsec(pcidev, info); + if (ret == -ENODEV) + ret = find_dfls_by_default(pcidev, info); + if (ret) goto irq_free_exit; From e08b9e6d87cca2bd8f427d109d22970906aaf6f8 Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:09 -0800 Subject: [PATCH 026/633] fpga: dfl: fix the definitions of type & feature_id for dfl devices The value of the field dfl_device.type comes from the 12 bits register field DFH_ID according to DFL spec. So this patch changes the definition of the type field to u16. Also it is not necessary to illustrate the valid bits of the type field in comments. Instead we should explicitly define the possible values in the enumeration type for it, because they are shared by hardware spec. We should not let the compiler decide these values. Similar changes are also applied to dfl_device.feature_id. This patch also fixed the MODALIAS format according to the changes above. Reviewed-by: Tom Rix Signed-off-by: Xu Yilun Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-4-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/fpga/dfl.c | 3 +-- drivers/fpga/dfl.h | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index b450870b75ed..5a6ba3b2fa05 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -298,8 +298,7 @@ static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env) { struct dfl_device *ddev = to_dfl_dev(dev); - /* The type has 4 valid bits and feature_id has 12 valid bits */ - return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X", + return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X", ddev->type, ddev->feature_id); } diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index 5dc758f655b7..ac373b1fcff9 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -520,19 +520,19 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, * enum dfl_id_type - define the DFL FIU types */ enum dfl_id_type { - FME_ID, - PORT_ID, + FME_ID = 0, + PORT_ID = 1, DFL_ID_MAX, }; /** * struct dfl_device_id - dfl device identifier - * @type: contains 4 bits DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: contains 12 bits feature identifier local to its DFL FIU type. + * @type: DFL FIU type of the device. See enum dfl_id_type. + * @feature_id: feature identifier local to its DFL FIU type. * @driver_data: driver specific data. */ struct dfl_device_id { - u8 type; + u16 type; u16 feature_id; unsigned long driver_data; }; @@ -543,7 +543,7 @@ struct dfl_device_id { * @dev: generic device interface. * @id: id of the dfl device. * @type: type of DFL FIU of the device. See enum dfl_id_type. - * @feature_id: 16 bits feature identifier local to its DFL FIU type. + * @feature_id: feature identifier local to its DFL FIU type. * @mmio_res: mmio resource of this dfl device. * @irqs: list of Linux IRQ numbers of this dfl device. * @num_irqs: number of IRQs supported by this dfl device. @@ -553,7 +553,7 @@ struct dfl_device_id { struct dfl_device { struct device dev; int id; - u8 type; + u16 type; u16 feature_id; struct resource mmio_res; int *irqs; From 9326eecd9365a5b82220a4011592f7c0209566fc Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:10 -0800 Subject: [PATCH 027/633] fpga: dfl: move dfl_device_id to mod_devicetable.h In order to support MODULE_DEVICE_TABLE() for dfl device driver, this patch moves struct dfl_device_id to mod_devicetable.h Some brief description for DFL (Device Feature List) is added to make the DFL known to the whole kernel. Reviewed-by: Tom Rix Acked-by: Wu Hao Signed-off-by: Xu Yilun Signed-off-by: Wu Hao Signed-off-by: Matthew Gerlach Signed-off-by: Russ Weight Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-5-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/fpga/dfl.h | 13 +------------ include/linux/mod_devicetable.h | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index ac373b1fcff9..549c7900dcfd 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -525,18 +526,6 @@ enum dfl_id_type { DFL_ID_MAX, }; -/** - * struct dfl_device_id - dfl device identifier - * @type: DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - u16 type; - u16 feature_id; - unsigned long driver_data; -}; - /** * struct dfl_device - represent an dfl device on dfl bus * diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index c425290b21e2..b8dae34eca10 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -846,4 +846,28 @@ struct auxiliary_device_id { kernel_ulong_t driver_data; }; +/* + * DFL (Device Feature List) + * + * DFL defines a linked list of feature headers within the device MMIO space to + * provide an extensible way of adding features. Software can walk through these + * predefined data structures to enumerate features. It is now used in the FPGA. + * See Documentation/fpga/dfl.rst for more information. + * + * The dfl bus type is introduced to match the individual feature devices (dfl + * devices) for specific dfl drivers. + */ + +/** + * struct dfl_device_id - dfl device identifier + * @type: DFL FIU type of the device. See enum dfl_id_type. + * @feature_id: feature identifier local to its DFL FIU type. + * @driver_data: driver specific data. + */ +struct dfl_device_id { + __u16 type; + __u16 feature_id; + kernel_ulong_t driver_data; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ From 4a224acec5971653bf0e8b6e1d2d0df72a7d57f7 Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:11 -0800 Subject: [PATCH 028/633] fpga: dfl: add dfl bus support to MODULE_DEVICE_TABLE() Device Feature List (DFL) is a linked list of feature headers within the device MMIO space. It is used by FPGA to enumerate multiple sub features within it. Each feature can be uniquely identified by DFL type and feature id, which can be read out from feature headers. A dfl bus helps DFL framework modularize DFL device drivers for different sub features. The dfl bus matches its devices and drivers by DFL type and feature id. This patch adds dfl bus support to MODULE_DEVICE_TABLE() by adding info about struct dfl_device_id in devicetable-offsets.c and add a dfl entry point in file2alias.c. Acked-by: Wu Hao Signed-off-by: Xu Yilun Signed-off-by: Wu Hao Signed-off-by: Matthew Gerlach Signed-off-by: Russ Weight Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-6-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- scripts/mod/devicetable-offsets.c | 4 ++++ scripts/mod/file2alias.c | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index e377f52dbfa3..1b14f3cde4e5 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -246,5 +246,9 @@ int main(void) DEVID(auxiliary_device_id); DEVID_FIELD(auxiliary_device_id, name); + DEVID(dfl_device_id); + DEVID_FIELD(dfl_device_id, type); + DEVID_FIELD(dfl_device_id, feature_id); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index fb4827027536..7ebabeb1e9c9 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1375,6 +1375,18 @@ static int do_auxiliary_entry(const char *filename, void *symval, char *alias) return 1; } +/* Looks like: dfl:tNfN */ +static int do_dfl_entry(const char *filename, void *symval, char *alias) +{ + DEF_FIELD(symval, dfl_device_id, type); + DEF_FIELD(symval, dfl_device_id, feature_id); + + sprintf(alias, "dfl:t%04Xf%04X", type, feature_id); + + add_wildcard(alias); + return 1; +} + /* Does namelen bytes of name exactly match the symbol? */ static bool sym_is(const char *name, unsigned namelen, const char *symbol) { @@ -1450,6 +1462,7 @@ static const struct devtable devtable[] = { {"wmi", SIZE_wmi_device_id, do_wmi_entry}, {"mhi", SIZE_mhi_device_id, do_mhi_entry}, {"auxiliary", SIZE_auxiliary_device_id, do_auxiliary_entry}, + {"dfl", SIZE_dfl_device_id, do_dfl_entry}, }; /* Create MODULE_ALIAS() statements. From ecc1641aca658ed4140751748f84985ffb6cce28 Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:12 -0800 Subject: [PATCH 029/633] fpga: dfl: move dfl bus related APIs to include/linux/dfl.h Now the dfl drivers could be made as independent modules and put in different folders according to their functionalities. In order for scattered dfl device drivers to include dfl bus APIs, move the dfl bus APIs to a new header file in the public folder. [mdf@kernel.org: Fixed up header guards to match filename] Reviewed-by: Tom Rix Acked-by: Wu Hao Signed-off-by: Xu Yilun Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20210107043714.991646-7-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 1 + drivers/fpga/dfl.c | 1 + drivers/fpga/dfl.h | 72 ------------------------------------- include/linux/dfl.h | 86 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 72 deletions(-) create mode 100644 include/linux/dfl.h diff --git a/MAINTAINERS b/MAINTAINERS index 546aa66428c9..a044b58eb9ac 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6957,6 +6957,7 @@ S: Maintained F: Documentation/ABI/testing/sysfs-bus-dfl F: Documentation/fpga/dfl.rst F: drivers/fpga/dfl* +F: include/linux/dfl.h F: include/uapi/linux/fpga-dfl.h FPGA MANAGER FRAMEWORK diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 5a6ba3b2fa05..511b20ff35a3 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -10,6 +10,7 @@ * Wu Hao * Xiao Guangrong */ +#include #include #include #include diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index 549c7900dcfd..2b82c96ba56c 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -517,76 +517,4 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, struct dfl_feature *feature, unsigned long arg); -/** - * enum dfl_id_type - define the DFL FIU types - */ -enum dfl_id_type { - FME_ID = 0, - PORT_ID = 1, - DFL_ID_MAX, -}; - -/** - * struct dfl_device - represent an dfl device on dfl bus - * - * @dev: generic device interface. - * @id: id of the dfl device. - * @type: type of DFL FIU of the device. See enum dfl_id_type. - * @feature_id: feature identifier local to its DFL FIU type. - * @mmio_res: mmio resource of this dfl device. - * @irqs: list of Linux IRQ numbers of this dfl device. - * @num_irqs: number of IRQs supported by this dfl device. - * @cdev: pointer to DFL FPGA container device this dfl device belongs to. - * @id_entry: matched id entry in dfl driver's id table. - */ -struct dfl_device { - struct device dev; - int id; - u16 type; - u16 feature_id; - struct resource mmio_res; - int *irqs; - unsigned int num_irqs; - struct dfl_fpga_cdev *cdev; - const struct dfl_device_id *id_entry; -}; - -/** - * struct dfl_driver - represent an dfl device driver - * - * @drv: driver model structure. - * @id_table: pointer to table of device IDs the driver is interested in. - * { } member terminated. - * @probe: mandatory callback for device binding. - * @remove: callback for device unbinding. - */ -struct dfl_driver { - struct device_driver drv; - const struct dfl_device_id *id_table; - - int (*probe)(struct dfl_device *dfl_dev); - void (*remove)(struct dfl_device *dfl_dev); -}; - -#define to_dfl_dev(d) container_of(d, struct dfl_device, dev) -#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv) - -/* - * use a macro to avoid include chaining to get THIS_MODULE. - */ -#define dfl_driver_register(drv) \ - __dfl_driver_register(drv, THIS_MODULE) -int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner); -void dfl_driver_unregister(struct dfl_driver *dfl_drv); - -/* - * module_dfl_driver() - Helper macro for drivers that don't do - * anything special in module init/exit. This eliminates a lot of - * boilerplate. Each module may only use this macro once, and - * calling it replaces module_init() and module_exit(). - */ -#define module_dfl_driver(__dfl_driver) \ - module_driver(__dfl_driver, dfl_driver_register, \ - dfl_driver_unregister) - #endif /* __FPGA_DFL_H */ diff --git a/include/linux/dfl.h b/include/linux/dfl.h new file mode 100644 index 000000000000..6cc10982351a --- /dev/null +++ b/include/linux/dfl.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Header file for DFL driver and device API + * + * Copyright (C) 2020 Intel Corporation, Inc. + */ + +#ifndef __LINUX_DFL_H +#define __LINUX_DFL_H + +#include +#include + +/** + * enum dfl_id_type - define the DFL FIU types + */ +enum dfl_id_type { + FME_ID = 0, + PORT_ID = 1, + DFL_ID_MAX, +}; + +/** + * struct dfl_device - represent an dfl device on dfl bus + * + * @dev: generic device interface. + * @id: id of the dfl device. + * @type: type of DFL FIU of the device. See enum dfl_id_type. + * @feature_id: feature identifier local to its DFL FIU type. + * @mmio_res: mmio resource of this dfl device. + * @irqs: list of Linux IRQ numbers of this dfl device. + * @num_irqs: number of IRQs supported by this dfl device. + * @cdev: pointer to DFL FPGA container device this dfl device belongs to. + * @id_entry: matched id entry in dfl driver's id table. + */ +struct dfl_device { + struct device dev; + int id; + u16 type; + u16 feature_id; + struct resource mmio_res; + int *irqs; + unsigned int num_irqs; + struct dfl_fpga_cdev *cdev; + const struct dfl_device_id *id_entry; +}; + +/** + * struct dfl_driver - represent an dfl device driver + * + * @drv: driver model structure. + * @id_table: pointer to table of device IDs the driver is interested in. + * { } member terminated. + * @probe: mandatory callback for device binding. + * @remove: callback for device unbinding. + */ +struct dfl_driver { + struct device_driver drv; + const struct dfl_device_id *id_table; + + int (*probe)(struct dfl_device *dfl_dev); + void (*remove)(struct dfl_device *dfl_dev); +}; + +#define to_dfl_dev(d) container_of(d, struct dfl_device, dev) +#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv) + +/* + * use a macro to avoid include chaining to get THIS_MODULE. + */ +#define dfl_driver_register(drv) \ + __dfl_driver_register(drv, THIS_MODULE) +int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner); +void dfl_driver_unregister(struct dfl_driver *dfl_drv); + +/* + * module_dfl_driver() - Helper macro for drivers that don't do + * anything special in module init/exit. This eliminates a lot of + * boilerplate. Each module may only use this macro once, and + * calling it replaces module_init() and module_exit(). + */ +#define module_dfl_driver(__dfl_driver) \ + module_driver(__dfl_driver, dfl_driver_register, \ + dfl_driver_unregister) + +#endif /* __LINUX_DFL_H */ From 56172ab35338e3bb13c6bff65dea96b12e8c41ea Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:13 -0800 Subject: [PATCH 030/633] fpga: dfl: add support for N3000 Nios private feature This patch adds support for the Nios handshake private feature on Intel PAC (Programmable Acceleration Card) N3000. The Nios is the embedded processor on the FPGA card. This private feature provides a handshake interface to FPGA Nios firmware, which receives retimer configuration command from host and executes via an internal SPI master (spi-altera). When Nios finishes the configuration, host takes over the ownership of the SPI master to control an Intel MAX10 BMC (Board Management Controller) Chip on the SPI bus. For Nios firmware handshake part, this driver requests the retimer configuration for Nios firmware on probe, and adds some sysfs nodes for user to query the onboard retimer's working mode and Nios firmware version. For SPI part, this driver adds a spi-altera platform device as well as the MAX10 BMC spi slave info. A spi-altera driver will be matched to handle the following SPI work. [mdf@kernel.org: Fixed up ABI doc kernel release] Reviewed-by: Tom Rix Signed-off-by: Xu Yilun Signed-off-by: Wu Hao Signed-off-by: Matthew Gerlach Signed-off-by: Russ Weight Signed-off-by: YueHaibing Link: https://lore.kernel.org/r/20210107043714.991646-8-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- .../testing/sysfs-bus-dfl-devices-n3000-nios | 47 ++ MAINTAINERS | 2 +- drivers/fpga/Kconfig | 11 + drivers/fpga/Makefile | 2 + drivers/fpga/dfl-n3000-nios.c | 588 ++++++++++++++++++ 5 files changed, 649 insertions(+), 1 deletion(-) create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios create mode 100644 drivers/fpga/dfl-n3000-nios.c diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file mode 100644 index 000000000000..5335d742bcaf --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios @@ -0,0 +1,47 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/fec_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the FEC mode of the 25G links of the + ethernet retimers configured by Nios firmware. "rs" for Reed + Solomon FEC, "kr" for Fire Code FEC, "no" for NO FEC. + "not supported" if the FEC mode setting is not supported, this + happens when the Nios firmware version major < 3, or no link is + configured to 25G. + Format: string + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_A_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer A configured by the Nios firmware. The value is + read out from shared registers filled by the Nios firmware. Now + the values could be: + + - "0": Reset + - "1": 4x10G + - "2": 4x25G + - "3": 2x25G + - "4": 2x25G+2x10G + - "5": 1x25G + + If the Nios firmware is updated in future to support more + retimer modes, more enumeration value is expected. + Format: 0x%x + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_B_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer B configured by the Nios firmware. The value format + is the same as retimer_A_mode. + +What: /sys/bus/dfl/devices/dfl_dev.X/nios_fw_version +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the version of the Nios firmware in the + FPGA. Its format is "major.minor.patch". + Format: %x.%x.%x diff --git a/MAINTAINERS b/MAINTAINERS index a044b58eb9ac..aebb26a7d8f2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6954,7 +6954,7 @@ M: Wu Hao R: Tom Rix L: linux-fpga@vger.kernel.org S: Maintained -F: Documentation/ABI/testing/sysfs-bus-dfl +F: Documentation/ABI/testing/sysfs-bus-dfl* F: Documentation/fpga/dfl.rst F: drivers/fpga/dfl* F: include/linux/dfl.h diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig index 5645226ca3ce..5ff9438b7b46 100644 --- a/drivers/fpga/Kconfig +++ b/drivers/fpga/Kconfig @@ -192,6 +192,17 @@ config FPGA_DFL_AFU to the FPGA infrastructure via a Port. There may be more than one Port/AFU per DFL based FPGA device. +config FPGA_DFL_NIOS_INTEL_PAC_N3000 + tristate "FPGA DFL NIOS Driver for Intel PAC N3000" + depends on FPGA_DFL + select REGMAP + help + This is the driver for the N3000 Nios private feature on Intel + PAC (Programmable Acceleration Card) N3000. It communicates + with the embedded Nios processor to configure the retimers on + the card. It also instantiates the SPI master (spi-altera) for + the card's BMC (Board Management Controller). + config FPGA_DFL_PCI tristate "FPGA DFL PCIe Device Driver" depends on PCI && FPGA_DFL diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile index d8e21dfc6778..18dc9885883a 100644 --- a/drivers/fpga/Makefile +++ b/drivers/fpga/Makefile @@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o dfl-afu-objs += dfl-afu-error.o +obj-$(CONFIG_FPGA_DFL_NIOS_INTEL_PAC_N3000) += dfl-n3000-nios.o + # Drivers for FPGAs which implement DFL obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o diff --git a/drivers/fpga/dfl-n3000-nios.c b/drivers/fpga/dfl-n3000-nios.c new file mode 100644 index 000000000000..7a95366f6516 --- /dev/null +++ b/drivers/fpga/dfl-n3000-nios.c @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for Nios private feature on Intel PAC (Programmable + * Acceleration Card) N3000 + * + * Copyright (C) 2019-2020 Intel Corporation, Inc. + * + * Authors: + * Wu Hao + * Xu Yilun + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * N3000 Nios private feature registers, named as NIOS_SPI_XX on spec. + * NS is the abbreviation of NIOS_SPI. + */ +#define N3000_NS_PARAM 0x8 +#define N3000_NS_PARAM_SHIFT_MODE_MSK BIT_ULL(1) +#define N3000_NS_PARAM_SHIFT_MODE_MSB 0 +#define N3000_NS_PARAM_SHIFT_MODE_LSB 1 +#define N3000_NS_PARAM_DATA_WIDTH GENMASK_ULL(7, 2) +#define N3000_NS_PARAM_NUM_CS GENMASK_ULL(13, 8) +#define N3000_NS_PARAM_CLK_POL BIT_ULL(14) +#define N3000_NS_PARAM_CLK_PHASE BIT_ULL(15) +#define N3000_NS_PARAM_PERIPHERAL_ID GENMASK_ULL(47, 32) + +#define N3000_NS_CTRL 0x10 +#define N3000_NS_CTRL_WR_DATA GENMASK_ULL(31, 0) +#define N3000_NS_CTRL_ADDR GENMASK_ULL(44, 32) +#define N3000_NS_CTRL_CMD_MSK GENMASK_ULL(63, 62) +#define N3000_NS_CTRL_CMD_NOP 0 +#define N3000_NS_CTRL_CMD_RD 1 +#define N3000_NS_CTRL_CMD_WR 2 + +#define N3000_NS_STAT 0x18 +#define N3000_NS_STAT_RD_DATA GENMASK_ULL(31, 0) +#define N3000_NS_STAT_RW_VAL BIT_ULL(32) + +/* Nios handshake registers, indirect access */ +#define N3000_NIOS_INIT 0x1000 +#define N3000_NIOS_INIT_DONE BIT(0) +#define N3000_NIOS_INIT_START BIT(1) +/* Mode for retimer A, link 0, the same below */ +#define N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK GENMASK(9, 8) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK GENMASK(11, 10) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK GENMASK(13, 12) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK GENMASK(15, 14) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK GENMASK(17, 16) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK GENMASK(19, 18) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK GENMASK(21, 20) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK GENMASK(23, 22) +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO 0x0 +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR 0x1 +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS 0x2 + +#define N3000_NIOS_FW_VERSION 0x1004 +#define N3000_NIOS_FW_VERSION_PATCH GENMASK(23, 20) +#define N3000_NIOS_FW_VERSION_MINOR GENMASK(27, 24) +#define N3000_NIOS_FW_VERSION_MAJOR GENMASK(31, 28) + +/* The retimers we use on Intel PAC N3000 is Parkvale, abbreviated to PKVL */ +#define N3000_NIOS_PKVL_A_MODE_STS 0x1020 +#define N3000_NIOS_PKVL_B_MODE_STS 0x1024 +#define N3000_NIOS_PKVL_MODE_STS_GROUP_MSK GENMASK(15, 8) +#define N3000_NIOS_PKVL_MODE_STS_GROUP_OK 0x0 +#define N3000_NIOS_PKVL_MODE_STS_ID_MSK GENMASK(7, 0) +/* When GROUP MASK field == GROUP_OK */ +#define N3000_NIOS_PKVL_MODE_ID_RESET 0x0 +#define N3000_NIOS_PKVL_MODE_ID_4X10G 0x1 +#define N3000_NIOS_PKVL_MODE_ID_4X25G 0x2 +#define N3000_NIOS_PKVL_MODE_ID_2X25G 0x3 +#define N3000_NIOS_PKVL_MODE_ID_2X25G_2X10G 0x4 +#define N3000_NIOS_PKVL_MODE_ID_1X25G 0x5 + +#define N3000_NIOS_REGBUS_RETRY_COUNT 10000 /* loop count */ + +#define N3000_NIOS_INIT_TIMEOUT 10000000 /* usec */ +#define N3000_NIOS_INIT_TIME_INTV 100000 /* usec */ + +#define N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL \ + (N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS)) + +struct n3000_nios { + void __iomem *base; + struct regmap *regmap; + struct device *dev; + struct platform_device *altera_spi; +}; + +static ssize_t nios_fw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + return sysfs_emit(buf, "%x.%x.%x\n", + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MINOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_PATCH, val)); +} +static DEVICE_ATTR_RO(nios_fw_version); + +#define IS_MODE_STATUS_OK(mode_stat) \ + (FIELD_GET(N3000_NIOS_PKVL_MODE_STS_GROUP_MSK, (mode_stat)) == \ + N3000_NIOS_PKVL_MODE_STS_GROUP_OK) + +#define IS_RETIMER_FEC_SUPPORTED(retimer_mode) \ + ((retimer_mode) != N3000_NIOS_PKVL_MODE_ID_RESET && \ + (retimer_mode) != N3000_NIOS_PKVL_MODE_ID_4X10G) + +static int get_retimer_mode(struct n3000_nios *nn, unsigned int mode_stat_reg, + unsigned int *retimer_mode) +{ + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, mode_stat_reg, &val); + if (ret) + return ret; + + if (!IS_MODE_STATUS_OK(val)) + return -EFAULT; + + *retimer_mode = FIELD_GET(N3000_NIOS_PKVL_MODE_STS_ID_MSK, val); + + return 0; +} + +static ssize_t retimer_A_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_A_mode); + +static ssize_t retimer_B_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_B_mode); + +static ssize_t fec_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int val, retimer_a_mode, retimer_b_mode, fec_modes; + struct n3000_nios *nn = dev_get_drvdata(dev); + int ret; + + /* FEC mode setting is not supported in early FW versions */ + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) < 3) + return sysfs_emit(buf, "not supported\n"); + + /* If no 25G links, FEC mode setting is not supported either */ + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &retimer_a_mode); + if (ret) + return ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &retimer_b_mode); + if (ret) + return ret; + + if (!IS_RETIMER_FEC_SUPPORTED(retimer_a_mode) && + !IS_RETIMER_FEC_SUPPORTED(retimer_b_mode)) + return sysfs_emit(buf, "not supported\n"); + + /* get the valid FEC mode for 25G links */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* + * FEC mode should always be the same for all links, as we set them + * in this way. + */ + fec_modes = (val & N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL); + if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL) + return sysfs_emit(buf, "no\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL) + return sysfs_emit(buf, "kr\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL) + return sysfs_emit(buf, "rs\n"); + + return -EFAULT; +} +static DEVICE_ATTR_RO(fec_mode); + +static struct attribute *n3000_nios_attrs[] = { + &dev_attr_nios_fw_version.attr, + &dev_attr_retimer_A_mode.attr, + &dev_attr_retimer_B_mode.attr, + &dev_attr_fec_mode.attr, + NULL, +}; +ATTRIBUTE_GROUPS(n3000_nios); + +static int n3000_nios_init_done_check(struct n3000_nios *nn) +{ + unsigned int val, state_a, state_b; + struct device *dev = nn->dev; + int ret, ret2; + + /* + * The SPI is shared by the Nios core inside the FPGA, Nios will use + * this SPI master to do some one time initialization after power up, + * and then release the control to OS. The driver needs to poll on + * INIT_DONE to see when driver could take the control. + * + * Please note that after Nios firmware version 3.0.0, INIT_START is + * introduced, so driver needs to trigger START firstly and then check + * INIT_DONE. + */ + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + /* + * If Nios version register is totally uninitialized(== 0x0), then the + * Nios firmware is missing. So host could take control of SPI master + * safely, but initialization work for Nios is not done. To restore the + * card, we need to reprogram a new Nios firmware via the BMC chip on + * SPI bus. So the driver doesn't error out, it continues to create the + * spi controller device and spi_board_info for BMC. + */ + if (val == 0) { + dev_err(dev, "Nios version reg = 0x%x, skip INIT_DONE check, but the retimer may be uninitialized\n", + val); + return 0; + } + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) >= 3) { + /* read NIOS_INIT to check if retimer initialization is done */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* check if retimers are initialized already */ + if (val & (N3000_NIOS_INIT_DONE | N3000_NIOS_INIT_START)) + goto nios_init_done; + + /* configure FEC mode per module param */ + val = N3000_NIOS_INIT_START; + + /* + * When the retimer is to be set to 10G mode, there is no FEC + * mode setting, so the REQ_FEC_MODE field will be ignored by + * Nios firmware in this case. But we should still fill the FEC + * mode field cause host could not get the retimer working mode + * until the Nios init is done. + * + * For now the driver doesn't support the retimer FEC mode + * switching per user's request. It is always set to Reed + * Solomon FEC. + * + * The driver will set the same FEC mode for all links. + */ + val |= N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL; + + ret = regmap_write(nn->regmap, N3000_NIOS_INIT, val); + if (ret) + return ret; + } + +nios_init_done: + /* polls on NIOS_INIT_DONE */ + ret = regmap_read_poll_timeout(nn->regmap, N3000_NIOS_INIT, val, + val & N3000_NIOS_INIT_DONE, + N3000_NIOS_INIT_TIME_INTV, + N3000_NIOS_INIT_TIMEOUT); + if (ret) + dev_err(dev, "NIOS_INIT_DONE %s\n", + (ret == -ETIMEDOUT) ? "timed out" : "check error"); + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_A_MODE_STS, &state_a); + if (ret2) + return ret2; + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_B_MODE_STS, &state_b); + if (ret2) + return ret2; + + if (!ret) { + /* + * After INIT_DONE is detected, it still needs to check if the + * Nios firmware reports any error during the retimer + * configuration. + */ + if (IS_MODE_STATUS_OK(state_a) && IS_MODE_STATUS_OK(state_b)) + return 0; + + /* + * If the retimer configuration is failed, the Nios firmware + * will still release the spi controller for host to + * communicate with the BMC. It makes possible for people to + * reprogram a new Nios firmware and restore the card. So the + * driver doesn't error out, it continues to create the spi + * controller device and spi_board_info for BMC. + */ + dev_err(dev, "NIOS_INIT_DONE OK, but err on retimer init\n"); + } + + dev_err(nn->dev, "PKVL_A_MODE_STS 0x%x\n", state_a); + dev_err(nn->dev, "PKVL_B_MODE_STS 0x%x\n", state_b); + + return ret; +} + +static struct spi_board_info m10_n3000_info = { + .modalias = "m10-n3000", + .max_speed_hz = 12500000, + .bus_num = 0, + .chip_select = 0, +}; + +static int create_altera_spi_controller(struct n3000_nios *nn) +{ + struct altera_spi_platform_data pdata = { 0 }; + struct platform_device_info pdevinfo = { 0 }; + void __iomem *base = nn->base; + u64 v; + + v = readq(base + N3000_NS_PARAM); + + pdata.mode_bits = SPI_CS_HIGH; + if (FIELD_GET(N3000_NS_PARAM_CLK_POL, v)) + pdata.mode_bits |= SPI_CPOL; + if (FIELD_GET(N3000_NS_PARAM_CLK_PHASE, v)) + pdata.mode_bits |= SPI_CPHA; + + pdata.num_chipselect = FIELD_GET(N3000_NS_PARAM_NUM_CS, v); + pdata.bits_per_word_mask = + SPI_BPW_RANGE_MASK(1, FIELD_GET(N3000_NS_PARAM_DATA_WIDTH, v)); + + pdata.num_devices = 1; + pdata.devices = &m10_n3000_info; + + dev_dbg(nn->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__, + pdata.num_chipselect, pdata.bits_per_word_mask, + pdata.mode_bits); + + pdevinfo.name = "subdev_spi_altera"; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.parent = nn->dev; + pdevinfo.data = &pdata; + pdevinfo.size_data = sizeof(pdata); + + nn->altera_spi = platform_device_register_full(&pdevinfo); + return PTR_ERR_OR_ZERO(nn->altera_spi); +} + +static void destroy_altera_spi_controller(struct n3000_nios *nn) +{ + platform_device_unregister(nn->altera_spi); +} + +static int n3000_nios_poll_stat_timeout(void __iomem *base, u64 *v) +{ + int loops; + + /* + * We don't use the time based timeout here for performance. + * + * The regbus read/write is on the critical path of Intel PAC N3000 + * image programing. The time based timeout checking will add too much + * overhead on it. Usually the state changes in 1 or 2 loops on the + * test server, and we set 10000 times loop here for safety. + */ + for (loops = N3000_NIOS_REGBUS_RETRY_COUNT; loops > 0 ; loops--) { + *v = readq(base + N3000_NS_STAT); + if (*v & N3000_NS_STAT_RW_VAL) + break; + cpu_relax(); + } + + return (loops > 0) ? 0 : -ETIMEDOUT; +} + +static int n3000_nios_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_WR) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg) | + FIELD_PREP(N3000_NS_CTRL_WR_DATA, val); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to write reg 0x%x val 0x%x: %d\n", + reg, val, ret); + + return ret; +} + +static int n3000_nios_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_RD) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to read reg 0x%x: %d\n", reg, ret); + else + *val = FIELD_GET(N3000_NS_STAT_RD_DATA, v); + + return ret; +} + +static const struct regmap_config n3000_nios_regbus_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + + .reg_write = n3000_nios_reg_write, + .reg_read = n3000_nios_reg_read, +}; + +static int n3000_nios_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct n3000_nios *nn; + int ret; + + nn = devm_kzalloc(dev, sizeof(*nn), GFP_KERNEL); + if (!nn) + return -ENOMEM; + + dev_set_drvdata(&ddev->dev, nn); + + nn->dev = dev; + + nn->base = devm_ioremap_resource(&ddev->dev, &ddev->mmio_res); + if (IS_ERR(nn->base)) + return PTR_ERR(nn->base); + + nn->regmap = devm_regmap_init(dev, NULL, nn, &n3000_nios_regbus_cfg); + if (IS_ERR(nn->regmap)) + return PTR_ERR(nn->regmap); + + ret = n3000_nios_init_done_check(nn); + if (ret) + return ret; + + ret = create_altera_spi_controller(nn); + if (ret) + dev_err(dev, "altera spi controller create failed: %d\n", ret); + + return ret; +} + +static void n3000_nios_remove(struct dfl_device *ddev) +{ + struct n3000_nios *nn = dev_get_drvdata(&ddev->dev); + + destroy_altera_spi_controller(nn); +} + +#define FME_FEATURE_ID_N3000_NIOS 0xd + +static const struct dfl_device_id n3000_nios_ids[] = { + { FME_ID, FME_FEATURE_ID_N3000_NIOS }, + { } +}; +MODULE_DEVICE_TABLE(dfl, n3000_nios_ids); + +static struct dfl_driver n3000_nios_driver = { + .drv = { + .name = "dfl-n3000-nios", + .dev_groups = n3000_nios_groups, + }, + .id_table = n3000_nios_ids, + .probe = n3000_nios_probe, + .remove = n3000_nios_remove, +}; + +module_dfl_driver(n3000_nios_driver); + +MODULE_DESCRIPTION("Driver for Nios private feature on Intel PAC N3000"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); From 477dfdccfcae4665f073260446199933369cd50e Mon Sep 17 00:00:00 2001 From: Xu Yilun Date: Wed, 6 Jan 2021 20:37:14 -0800 Subject: [PATCH 031/633] memory: dfl-emif: add the DFL EMIF private feature driver This driver is for the EMIF private feature implemented under FPGA Device Feature List (DFL) framework. It is used to expose memory interface status information as well as memory clearing control. The purpose of memory clearing block is to zero out all private memory when FPGA is to be reprogrammed. This gives users a reliable method to prevent potential data leakage. [mdf@kernel.org: Fixed up ABI doc] Reviewed-by: Tom Rix Acked-by: Krzysztof Kozlowski Signed-off-by: Xu Yilun Signed-off-by: Russ Weight Link: https://lore.kernel.org/r/20210107043714.991646-9-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-bus-dfl-devices-emif | 25 +++ drivers/memory/Kconfig | 9 + drivers/memory/Makefile | 2 + drivers/memory/dfl-emif.c | 207 ++++++++++++++++++ 4 files changed, 243 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-bus-dfl-devices-emif create mode 100644 drivers/memory/dfl-emif.c diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif new file mode 100644 index 000000000000..817d14126d4d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif @@ -0,0 +1,25 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/infX_cal_fail +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the calibration failed on this + memory interface. "1" for calibration failure, "0" for OK. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_init_done +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the initialization completed on + this memory interface. "1" for initialization complete, "0" + for not yet. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_clear +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Write-only. Writing "1" to this file will zero out all memory + data in this memory interface. Writing of other values is + invalid. + Format: %u diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index 3ea6913df176..3c9a9882ef28 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -137,6 +137,15 @@ config TI_EMIF_SRAM sequence so this driver provides several relocatable PM functions for the SoC PM code to use. +config FPGA_DFL_EMIF + tristate "FPGA DFL EMIF Driver" + depends on FPGA_DFL && HAS_IOMEM + help + This driver is for the EMIF private feature implemented under + FPGA Device Feature List (DFL) framework. It is used to expose + memory interface status information as well as memory clearing + control. + config MVEBU_DEVBUS bool "Marvell EBU Device Bus Controller" default y if PLAT_ORION diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index e71cf7b99641..bc7663ed1c25 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -28,6 +28,8 @@ obj-$(CONFIG_STM32_FMC2_EBI) += stm32-fmc2-ebi.o obj-$(CONFIG_SAMSUNG_MC) += samsung/ obj-$(CONFIG_TEGRA_MC) += tegra/ obj-$(CONFIG_TI_EMIF_SRAM) += ti-emif-sram.o +obj-$(CONFIG_FPGA_DFL_EMIF) += dfl-emif.o + ti-emif-sram-objs := ti-emif-pm.o ti-emif-sram-pm.o AFLAGS_ti-emif-sram-pm.o :=-Wa,-march=armv7-a diff --git a/drivers/memory/dfl-emif.c b/drivers/memory/dfl-emif.c new file mode 100644 index 000000000000..3f719816771d --- /dev/null +++ b/drivers/memory/dfl-emif.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for EMIF private feature + * + * Copyright (C) 2020 Intel Corporation, Inc. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FME_FEATURE_ID_EMIF 0x9 + +#define EMIF_STAT 0x8 +#define EMIF_STAT_INIT_DONE_SFT 0 +#define EMIF_STAT_CALC_FAIL_SFT 8 +#define EMIF_STAT_CLEAR_BUSY_SFT 16 +#define EMIF_CTRL 0x10 +#define EMIF_CTRL_CLEAR_EN_SFT 0 +#define EMIF_CTRL_CLEAR_EN_MSK GENMASK_ULL(3, 0) + +#define EMIF_POLL_INVL 10000 /* us */ +#define EMIF_POLL_TIMEOUT 5000000 /* us */ + +struct dfl_emif { + struct device *dev; + void __iomem *base; + spinlock_t lock; /* Serialises access to EMIF_CTRL reg */ +}; + +struct emif_attr { + struct device_attribute attr; + u32 shift; + u32 index; +}; + +#define to_emif_attr(dev_attr) \ + container_of(dev_attr, struct emif_attr, attr) + +static ssize_t emif_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 val; + + val = readq(de->base + EMIF_STAT); + + return sysfs_emit(buf, "%u\n", + !!(val & BIT_ULL(eattr->shift + eattr->index))); +} + +static ssize_t emif_clear_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct emif_attr *eattr = to_emif_attr(attr); + struct dfl_emif *de = dev_get_drvdata(dev); + u64 clear_busy_msk, clear_en_msk, val; + void __iomem *base = de->base; + + if (!sysfs_streq(buf, "1")) + return -EINVAL; + + clear_busy_msk = BIT_ULL(EMIF_STAT_CLEAR_BUSY_SFT + eattr->index); + clear_en_msk = BIT_ULL(EMIF_CTRL_CLEAR_EN_SFT + eattr->index); + + spin_lock(&de->lock); + /* The CLEAR_EN field is WO, but other fields are RW */ + val = readq(base + EMIF_CTRL); + val &= ~EMIF_CTRL_CLEAR_EN_MSK; + val |= clear_en_msk; + writeq(val, base + EMIF_CTRL); + spin_unlock(&de->lock); + + if (readq_poll_timeout(base + EMIF_STAT, val, + !(val & clear_busy_msk), + EMIF_POLL_INVL, EMIF_POLL_TIMEOUT)) { + dev_err(de->dev, "timeout, fail to clear\n"); + return -ETIMEDOUT; + } + + return count; +} + +#define emif_state_attr(_name, _shift, _index) \ + static struct emif_attr emif_attr_##inf##_index##_##_name = \ + { .attr = __ATTR(inf##_index##_##_name, 0444, \ + emif_state_show, NULL), \ + .shift = (_shift), .index = (_index) } + +#define emif_clear_attr(_index) \ + static struct emif_attr emif_attr_##inf##_index##_clear = \ + { .attr = __ATTR(inf##_index##_clear, 0200, \ + NULL, emif_clear_store), \ + .index = (_index) } + +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 0); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 1); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 2); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 3); + +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 0); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 1); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 2); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 3); + +emif_clear_attr(0); +emif_clear_attr(1); +emif_clear_attr(2); +emif_clear_attr(3); + +static struct attribute *dfl_emif_attrs[] = { + &emif_attr_inf0_init_done.attr.attr, + &emif_attr_inf0_cal_fail.attr.attr, + &emif_attr_inf0_clear.attr.attr, + + &emif_attr_inf1_init_done.attr.attr, + &emif_attr_inf1_cal_fail.attr.attr, + &emif_attr_inf1_clear.attr.attr, + + &emif_attr_inf2_init_done.attr.attr, + &emif_attr_inf2_cal_fail.attr.attr, + &emif_attr_inf2_clear.attr.attr, + + &emif_attr_inf3_init_done.attr.attr, + &emif_attr_inf3_cal_fail.attr.attr, + &emif_attr_inf3_clear.attr.attr, + + NULL, +}; + +static umode_t dfl_emif_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct dfl_emif *de = dev_get_drvdata(kobj_to_dev(kobj)); + struct emif_attr *eattr = container_of(attr, struct emif_attr, + attr.attr); + u64 val; + + /* + * This device supports upto 4 memory interfaces, but not all + * interfaces are used on different platforms. The read out value of + * CLEAN_EN field (which is a bitmap) could tell how many interfaces + * are available. + */ + val = FIELD_GET(EMIF_CTRL_CLEAR_EN_MSK, readq(de->base + EMIF_CTRL)); + + return (val & BIT_ULL(eattr->index)) ? attr->mode : 0; +} + +static const struct attribute_group dfl_emif_group = { + .is_visible = dfl_emif_visible, + .attrs = dfl_emif_attrs, +}; + +static const struct attribute_group *dfl_emif_groups[] = { + &dfl_emif_group, + NULL, +}; + +static int dfl_emif_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct dfl_emif *de; + + de = devm_kzalloc(dev, sizeof(*de), GFP_KERNEL); + if (!de) + return -ENOMEM; + + de->base = devm_ioremap_resource(dev, &ddev->mmio_res); + if (IS_ERR(de->base)) + return PTR_ERR(de->base); + + de->dev = dev; + spin_lock_init(&de->lock); + dev_set_drvdata(dev, de); + + return 0; +} + +static const struct dfl_device_id dfl_emif_ids[] = { + { FME_ID, FME_FEATURE_ID_EMIF }, + { } +}; +MODULE_DEVICE_TABLE(dfl, dfl_emif_ids); + +static struct dfl_driver dfl_emif_driver = { + .drv = { + .name = "dfl-emif", + .dev_groups = dfl_emif_groups, + }, + .id_table = dfl_emif_ids, + .probe = dfl_emif_probe, +}; +module_dfl_driver(dfl_emif_driver); + +MODULE_DESCRIPTION("DFL EMIF driver"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); From a1cd0d4d8678c3e7afda7819597fc4d3fb886f9e Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Wed, 6 Jan 2021 11:21:32 +0100 Subject: [PATCH 032/633] dt-bindings: remoteproc: qcom,wcnss: Add qcom,wcn3660b compatible WCN3660B is a variant of WCN3660, but with the same regulator requirements as WCN3620/WCN3680. Add a new qcom,wcn3660b compatible to describe it from device trees. Signed-off-by: Stephan Gerhold Link: https://lore.kernel.org/r/20210106102134.59801-2-stephan@gerhold.net Signed-off-by: Bjorn Andersson --- Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt index cc0b7fc1c29b..da09c0d79ac0 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt @@ -80,6 +80,7 @@ and its resource dependencies. It is described by the following properties: Definition: must be one of: "qcom,wcn3620", "qcom,wcn3660", + "qcom,wcn3660b", "qcom,wcn3680" - clocks: From 8cc8eeffd058f3e7e2d8710a514ffcbc2bd69d28 Mon Sep 17 00:00:00 2001 From: Stephan Gerhold Date: Wed, 6 Jan 2021 11:21:33 +0100 Subject: [PATCH 033/633] remoteproc: qcom_wcnss: Add qcom,wcn3660b compatible WCN3660B is a variant of WCN3660, but with the same regulator requirements as WCN3620/WCN3680. As far as qcom_wcnss_iris is concerned we can just use qcom,wcn3680 (wcn3680_data). However, a separate compatible is needed for WCN3660B because the wcn36xx driver uses it to enable chip-specific functionality. In particular, it enables 802.11ac for qcom,wcn3680 which is not supported by WCN3660B. Signed-off-by: Stephan Gerhold Link: https://lore.kernel.org/r/20210106102134.59801-3-stephan@gerhold.net Signed-off-by: Bjorn Andersson --- drivers/remoteproc/qcom_wcnss_iris.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c index 0e0ae1e764ea..169acd305ae3 100644 --- a/drivers/remoteproc/qcom_wcnss_iris.c +++ b/drivers/remoteproc/qcom_wcnss_iris.c @@ -160,6 +160,7 @@ static int qcom_iris_remove(struct platform_device *pdev) static const struct of_device_id iris_of_match[] = { { .compatible = "qcom,wcn3620", .data = &wcn3620_data }, { .compatible = "qcom,wcn3660", .data = &wcn3660_data }, + { .compatible = "qcom,wcn3660b", .data = &wcn3680_data }, { .compatible = "qcom,wcn3680", .data = &wcn3680_data }, {} }; From 9e19f44d7f081e4c05d64cfcc412ab1fa43fd2bb Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Thu, 17 Dec 2020 11:04:00 +0800 Subject: [PATCH 034/633] remoteproc: qcom: add more help text qcom options With these more help text added, hopefully it's easier to understand the distinctions of these qcom remoteproc drivers. Signed-off-by: Shawn Guo Link: https://lore.kernel.org/r/20201217030400.6235-1-shawn.guo@linaro.org Signed-off-by: Bjorn Andersson --- drivers/remoteproc/Kconfig | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 9e7efe542f69..bb31b115fdbd 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -162,7 +162,9 @@ config QCOM_Q6V5_ADSP select QCOM_RPROC_COMMON help Say y here to support the Peripheral Image Loader - for the Qualcomm Technology Inc. ADSP remote processors. + for the non-TrustZone part of Qualcomm Technology Inc. ADSP and CDSP + remote processors. The TrustZone part is handled by QCOM_Q6V5_PAS + driver. config QCOM_Q6V5_MSS tristate "Qualcomm Hexagon V5 self-authenticating modem subsystem support" @@ -179,7 +181,8 @@ config QCOM_Q6V5_MSS select QCOM_SCM help Say y here to support the Qualcomm self-authenticating modem - subsystem based on Hexagon V5. + subsystem based on Hexagon V5. The TrustZone based system is + handled by QCOM_Q6V5_PAS driver. config QCOM_Q6V5_PAS tristate "Qualcomm Hexagon v5 Peripheral Authentication Service support" @@ -197,7 +200,9 @@ config QCOM_Q6V5_PAS help Say y here to support the TrustZone based Peripheral Image Loader for the Qualcomm Hexagon v5 based remote processors. This is commonly - used to control subsystems such as ADSP, Compute and Sensor. + used to control subsystems such as ADSP (Audio DSP), + CDSP (Compute DSP), MPSS (Modem Peripheral SubSystem), and + SLPI (Sensor Low Power Island). config QCOM_Q6V5_WCSS tristate "Qualcomm Hexagon based WCSS Peripheral Image Loader" @@ -214,7 +219,8 @@ config QCOM_Q6V5_WCSS select QCOM_SCM help Say y here to support the Qualcomm Peripheral Image Loader for the - Hexagon V5 based WCSS remote processors. + Hexagon V5 based WCSS remote processors on e.g. IPQ8074. This is + a non-TrustZone wireless subsystem. config QCOM_SYSMON tristate "Qualcomm sysmon driver" @@ -243,8 +249,10 @@ config QCOM_WCNSS_PIL select QCOM_RPROC_COMMON select QCOM_SCM help - Say y here to support the Peripheral Image Loader for the Qualcomm - Wireless Connectivity Subsystem. + Say y here to support the Peripheral Image Loader for loading WCNSS + firmware and boot the core on e.g. MSM8974, MSM8916. The firmware is + verified and booted with the help of the Peripheral Authentication + System (PAS) in TrustZone. config ST_REMOTEPROC tristate "ST remoteproc support" From cf34838d591f1fce426fb7823c2f1890b6d1ccc8 Mon Sep 17 00:00:00 2001 From: Arnaud Pouliquen Date: Thu, 17 Dec 2020 15:41:25 +0100 Subject: [PATCH 035/633] remoteproc: stm32: improve debug using dev_err_probe When possible use dev_err_probe help to properly deal with the PROBE_DEFER error. The benefit is that DEFER issue will be logged in the devices_deferred debugfs file. Signed-off-by: Arnaud Pouliquen Link: https://lore.kernel.org/r/20201217144125.12903-1-arnaud.pouliquen@foss.st.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/stm32_rproc.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index a180aeae9675..ccb3c14a0023 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -370,8 +370,13 @@ static int stm32_rproc_request_mbox(struct rproc *rproc) ddata->mb[i].chan = mbox_request_channel_byname(cl, name); if (IS_ERR(ddata->mb[i].chan)) { - if (PTR_ERR(ddata->mb[i].chan) == -EPROBE_DEFER) + if (PTR_ERR(ddata->mb[i].chan) == -EPROBE_DEFER) { + dev_err_probe(dev->parent, + PTR_ERR(ddata->mb[i].chan), + "failed to request mailbox %s\n", + name); goto err_probe; + } dev_warn(dev, "cannot get %s mbox\n", name); ddata->mb[i].chan = NULL; } @@ -592,15 +597,14 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev, irq = platform_get_irq(pdev, 0); if (irq == -EPROBE_DEFER) - return -EPROBE_DEFER; + return dev_err_probe(dev, irq, "failed to get interrupt\n"); if (irq > 0) { err = devm_request_irq(dev, irq, stm32_rproc_wdg, 0, dev_name(dev), pdev); - if (err) { - dev_err(dev, "failed to request wdg irq\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "failed to request wdg irq\n"); ddata->wdg_irq = irq; @@ -613,10 +617,9 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev, } ddata->rst = devm_reset_control_get_by_index(dev, 0); - if (IS_ERR(ddata->rst)) { - dev_err(dev, "failed to get mcu reset\n"); - return PTR_ERR(ddata->rst); - } + if (IS_ERR(ddata->rst)) + return dev_err_probe(dev, PTR_ERR(ddata->rst), + "failed to get mcu_reset\n"); /* * if platform is secured the hold boot bit must be written by From 13613a2246bf531f5fc04e8e62e8f21a3d39bf1c Mon Sep 17 00:00:00 2001 From: Aswath Govindraju Date: Thu, 7 Jan 2021 22:09:53 +0530 Subject: [PATCH 036/633] misc: eeprom_93xx46: Fix module alias to enable module autoprobe Fix module autoprobe by correcting module alias to match the string from /sys/class/.../spi1.0/modalias content. Fixes: 06b4501e88ad ("misc/eeprom: add driver for microwire 93xx46 EEPROMs") Signed-off-by: Aswath Govindraju Link: https://lore.kernel.org/r/20210107163957.28664-2-a-govindraju@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/eeprom/eeprom_93xx46.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 7c45f82b4302..206d920dc92f 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -511,4 +511,4 @@ module_spi_driver(eeprom_93xx46_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs"); MODULE_AUTHOR("Anatolij Gustschin "); -MODULE_ALIAS("spi:93xx46"); +MODULE_ALIAS("spi:eeprom-93xx46"); From 47771f1715bf05b69b040bf784a2607a5dc13c62 Mon Sep 17 00:00:00 2001 From: Aswath Govindraju Date: Thu, 7 Jan 2021 22:09:54 +0530 Subject: [PATCH 037/633] misc: eeprom_93xx46: Enable module autoprobe for microchip 93LC46B eeprom Add module alias to enable autoprobe for microchip 93LC46B eeprom by using /sys/class/.../spi1.0/modalias content. Signed-off-by: Aswath Govindraju Link: https://lore.kernel.org/r/20210107163957.28664-3-a-govindraju@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/eeprom/eeprom_93xx46.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 206d920dc92f..849cfea9e294 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -512,3 +512,4 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs"); MODULE_AUTHOR("Anatolij Gustschin "); MODULE_ALIAS("spi:eeprom-93xx46"); +MODULE_ALIAS("spi:93lc46b"); From 59b26d2e96c473b7b65cfe19381444d034e91715 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 16 Dec 2020 18:10:15 +0200 Subject: [PATCH 038/633] bus: fsl-mc: add missing __iomem attribute This pointer to an i/o register block is missing the __iomem attribute, so add it. The issue was found by 0-day sparse run. Reported-by: kernel test robot Signed-off-by: Laurentiu Tudor Link: https://lore.kernel.org/r/20201216161015.29060-1-laurentiu.tudor@nxp.com Signed-off-by: Greg Kroah-Hartman --- drivers/bus/fsl-mc/fsl-mc-bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index b8e6acdf932e..5a8fc68d6109 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -41,7 +41,7 @@ struct fsl_mc { struct fsl_mc_device *root_mc_bus_dev; u8 num_translation_ranges; struct fsl_mc_addr_translation_range *translation_ranges; - void *fsl_mc_regs; + void __iomem *fsl_mc_regs; }; /** From f5187329d07247393413b7a4bc32e16b35849e0f Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Mon, 28 Dec 2020 21:51:35 +0800 Subject: [PATCH 039/633] fpga: Use DEFINE_SPINLOCK() for spinlock spinlock can be initialized automatically with DEFINE_SPINLOCK() rather than explicitly calling spin_lock_init(). Signed-off-by: Zheng Yongjun Reviewed-by: Tom Rix Signed-off-by: Moritz Fischer --- drivers/fpga/fpga-bridge.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 2deccacc3aa7..e9266b2a357f 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -17,7 +17,7 @@ static DEFINE_IDA(fpga_bridge_ida); static struct class *fpga_bridge_class; /* Lock for adding/removing bridges to linked lists*/ -static spinlock_t bridge_list_lock; +static DEFINE_SPINLOCK(bridge_list_lock); /** * fpga_bridge_enable - Enable transactions on the bridge @@ -479,8 +479,6 @@ static void fpga_bridge_dev_release(struct device *dev) static int __init fpga_bridge_dev_init(void) { - spin_lock_init(&bridge_list_lock); - fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge"); if (IS_ERR(fpga_bridge_class)) return PTR_ERR(fpga_bridge_class); From e41d4c011706c97f2faea6ef2e46e51f52d8f715 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sat, 9 Jan 2021 00:54:14 +0100 Subject: [PATCH 040/633] fpga: dfl: fme: Constify static attribute_group structs The only usage of these is to put their addresses in arrays of pointers to const attribute_groups. Make them const to allow the compiler to put them in read-only memory. Signed-off-by: Rikard Falkeborn Reviewed-by: Tom Rix Signed-off-by: Moritz Fischer --- drivers/fpga/dfl-fme-perf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 531266287eee..4299145ef347 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -192,7 +192,7 @@ static struct attribute *fme_perf_cpumask_attrs[] = { NULL, }; -static struct attribute_group fme_perf_cpumask_group = { +static const struct attribute_group fme_perf_cpumask_group = { .attrs = fme_perf_cpumask_attrs, }; @@ -225,7 +225,7 @@ static struct attribute *fme_perf_format_attrs[] = { NULL, }; -static struct attribute_group fme_perf_format_group = { +static const struct attribute_group fme_perf_format_group = { .name = "format", .attrs = fme_perf_format_attrs, }; @@ -239,7 +239,7 @@ static struct attribute *fme_perf_events_attrs_empty[] = { NULL, }; -static struct attribute_group fme_perf_events_group = { +static const struct attribute_group fme_perf_events_group = { .name = "events", .attrs = fme_perf_events_attrs_empty, }; From 729e3a669d1b62e9876a671ac03ccba399a23b68 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sat, 9 Jan 2021 10:53:53 +0100 Subject: [PATCH 041/633] PCI: Decline to resize resources if boot config must be preserved The _DSM #5 method in the ACPI host bridge object tells us whether the OS must preserve the resource assignments done by firmware. If this is the case, we should not permit drivers to resize BARs on the fly. Make pci_resize_resource() take this into account. Link: https://lore.kernel.org/r/20210109095353.13417-1-ardb@kernel.org Signed-off-by: Ard Biesheuvel Signed-off-by: Bjorn Helgaas Cc: stable@vger.kernel.org # v5.4+ --- drivers/pci/setup-res.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 43eda101fcf4..7f1acb3918d0 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -410,10 +410,16 @@ EXPORT_SYMBOL(pci_release_resource); int pci_resize_resource(struct pci_dev *dev, int resno, int size) { struct resource *res = dev->resource + resno; + struct pci_host_bridge *host; int old, ret; u32 sizes; u16 cmd; + /* Check if we must preserve the firmware's resource assignment */ + host = pci_find_host_bridge(dev->bus); + if (host->preserve_config) + return -ENOTSUPP; + /* Make sure the resource isn't assigned before resizing it. */ if (!(res->flags & IORESOURCE_UNSET)) return -EBUSY; From 39961bd6b70e5a5d7c4b5483ad8e1db6b5765c60 Mon Sep 17 00:00:00 2001 From: Chris Ruehl Date: Thu, 10 Dec 2020 16:04:54 +0800 Subject: [PATCH 042/633] phy: rockchip-emmc: emmc_phy_init() always return 0 rockchip_emmc_phy_init() return variable is not set with the error value if clk_get() failed. 'emmcclk' is optional, thus use clk_get_optional() and if the return value != NULL make error processing and set the return code accordingly. Fixes: 52c0624a10cce phy: rockchip-emmc: Set phyctrl_frqsel based on card clock Signed-off-by: Chris Ruehl Reviewed-by: Douglas Anderson Link: https://lore.kernel.org/r/20201210080454.17379-1-chris.ruehl@gtsys.com.hk Signed-off-by: Vinod Koul --- drivers/phy/rockchip/phy-rockchip-emmc.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/phy/rockchip/phy-rockchip-emmc.c b/drivers/phy/rockchip/phy-rockchip-emmc.c index 1e424f263e7a..496d199852af 100644 --- a/drivers/phy/rockchip/phy-rockchip-emmc.c +++ b/drivers/phy/rockchip/phy-rockchip-emmc.c @@ -248,15 +248,17 @@ static int rockchip_emmc_phy_init(struct phy *phy) * - SDHCI driver to get the PHY * - SDHCI driver to init the PHY * - * The clock is optional, so upon any error we just set to NULL. + * The clock is optional, using clk_get_optional() to get the clock + * and do error processing if the return value != NULL * * NOTE: we don't do anything special for EPROBE_DEFER here. Given the * above expected use case, EPROBE_DEFER isn't sensible to expect, so * it's just like any other error. */ - rk_phy->emmcclk = clk_get(&phy->dev, "emmcclk"); + rk_phy->emmcclk = clk_get_optional(&phy->dev, "emmcclk"); if (IS_ERR(rk_phy->emmcclk)) { - dev_dbg(&phy->dev, "Error getting emmcclk: %d\n", ret); + ret = PTR_ERR(rk_phy->emmcclk); + dev_err(&phy->dev, "Error getting emmcclk: %d\n", ret); rk_phy->emmcclk = NULL; } From aaf316de3bba4b152a1212e5581a57938a57b4f4 Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Thu, 10 Dec 2020 19:11:56 +0800 Subject: [PATCH 043/633] phy: cpcap-usb: remove unneeded conversion to bool Fix the following warning: drivers/phy/motorola/phy-cpcap-usb.c:146:31-36: WARNING: conversion to bool not needed here. Signed-off-by: Tian Tao Link: https://lore.kernel.org/r/1607598716-16108-1-git-send-email-tiantao6@hisilicon.com Signed-off-by: Vinod Koul --- drivers/phy/motorola/phy-cpcap-usb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index 4728e2bff662..6ee478bc5211 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -143,7 +143,7 @@ static bool cpcap_usb_vbus_valid(struct cpcap_phy_ddata *ddata) error = iio_read_channel_processed(ddata->vbus, &value); if (error >= 0) - return value > 3900 ? true : false; + return value > 3900; dev_err(ddata->dev, "error reading VBUS: %i\n", error); From 88d9f40c4b715aa8bf8d08a5188e032e9426219d Mon Sep 17 00:00:00 2001 From: Chris Ruehl Date: Tue, 15 Dec 2020 09:44:07 +0800 Subject: [PATCH 044/633] devicetree: phy: rockchip-emmc optional add vendor prefix Update the documentation and add the vendor prefix to the optional properties referred in vendor-prefixes.yaml. Follow up with commit 8b5c2b45b8f0a ("phy: rockchip: set pulldown for strobe line in dts") commit a8cef928276bb ("phy: rockchip-emmc: output tap delay dt property") Signed-off-by: Chris Ruehl Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201215014409.905-2-chris.ruehl@gtsys.com.hk Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/rockchip-emmc-phy.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt index 00aa2d349e55..57d28c0d5696 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt +++ b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt @@ -16,11 +16,11 @@ Optional properties: - drive-impedance-ohm: Specifies the drive impedance in Ohm. Possible values are 33, 40, 50, 66 and 100. If not set, the default value of 50 will be applied. - - enable-strobe-pulldown: Enable internal pull-down for the strobe line. - If not set, pull-down is not used. - - output-tapdelay-select: Specifies the phyctrl_otapdlysec register. - If not set, the register defaults to 0x4. - Maximum value 0xf. + - rockchip,enable-strobe-pulldown: Enable internal pull-down for the strobe + line. If not set, pull-down is not used. + - rockchip,output-tapdelay-select: Specifies the phyctrl_otapdlysec register. + If not set, the register defaults to 0x4. + Maximum value 0xf. Example: From c188365402f65fbc1d1561aeaf4a20f09b0b4f22 Mon Sep 17 00:00:00 2001 From: Chris Ruehl Date: Tue, 15 Dec 2020 09:44:08 +0800 Subject: [PATCH 045/633] phy: rockchip: emmc, add vendor prefix to dts properties Update the implementation add "rockchip," vendor prefix for the optional dts properties. Prefix referred from vendor-prefixes.yaml. Follow up with commit 8b5c2b45b8f0a ("phy: rockchip: set pulldown for strobe line in dts") commit a8cef928276bb ("phy: rockchip-emmc: output tap delay dt property") Signed-off-by: Chris Ruehl Link: https://lore.kernel.org/r/20201215014409.905-3-chris.ruehl@gtsys.com.hk Signed-off-by: Vinod Koul --- drivers/phy/rockchip/phy-rockchip-emmc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/phy/rockchip/phy-rockchip-emmc.c b/drivers/phy/rockchip/phy-rockchip-emmc.c index 496d199852af..20023f6eb994 100644 --- a/drivers/phy/rockchip/phy-rockchip-emmc.c +++ b/drivers/phy/rockchip/phy-rockchip-emmc.c @@ -382,10 +382,10 @@ static int rockchip_emmc_phy_probe(struct platform_device *pdev) if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val)) rk_phy->drive_impedance = convert_drive_impedance_ohm(pdev, val); - if (of_property_read_bool(dev->of_node, "enable-strobe-pulldown")) + if (of_property_read_bool(dev->of_node, "rockchip,enable-strobe-pulldown")) rk_phy->enable_strobe_pulldown = PHYCTRL_REN_STRB_ENABLE; - if (!of_property_read_u32(dev->of_node, "output-tapdelay-select", &val)) { + if (!of_property_read_u32(dev->of_node, "rockchip,output-tapdelay-select", &val)) { if (val <= PHYCTRL_OTAPDLYSEL_MAXVALUE) rk_phy->output_tapdelay_select = val; else From d14f4cce9340a6586512a0eb6bc680dedeaaef14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 16 Dec 2020 15:33:04 +0100 Subject: [PATCH 046/633] phy: phy-brcm-usb: improve getting OF matching data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Use of_device_get_match_data() helper to simplify the code 2. Check for NULL as a good practice Signed-off-by: Rafał Miłecki Acked-by: Florian Fainelli Link: https://lore.kernel.org/r/20201216143305.12179-1-zajec5@gmail.com Signed-off-by: Vinod Koul --- drivers/phy/broadcom/phy-brcm-usb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c index 99fbc7e4138b..65fe592dd7f0 100644 --- a/drivers/phy/broadcom/phy-brcm-usb.c +++ b/drivers/phy/broadcom/phy-brcm-usb.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -427,7 +428,6 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) struct device_node *dn = pdev->dev.of_node; int err; const char *mode; - const struct of_device_id *match; void (*dvr_init)(struct brcm_usb_init_params *params); const struct match_chip_info *info; struct regmap *rmap; @@ -441,8 +441,9 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) priv->ini.family_id = brcmstb_get_family_id(); priv->ini.product_id = brcmstb_get_product_id(); - match = of_match_node(brcm_usb_dt_ids, dev->of_node); - info = match->data; + info = of_device_get_match_data(&pdev->dev); + if (!info) + return -ENOENT; dvr_init = info->init_func; (*dvr_init)(&priv->ini); From 915f1d230e5292bc2156a9997bcb19d9e632f10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 16 Dec 2020 15:33:05 +0100 Subject: [PATCH 047/633] phy: phy-brcm-usb: specify init function format at struct level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is slightly cleaner solution that assures noone assings a wrong function to the pointer. Signed-off-by: Rafał Miłecki Acked-by: Florian Fainelli Link: https://lore.kernel.org/r/20201216143305.12179-2-zajec5@gmail.com Signed-off-by: Vinod Koul --- drivers/phy/broadcom/phy-brcm-usb.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c index 65fe592dd7f0..c7d751b7c144 100644 --- a/drivers/phy/broadcom/phy-brcm-usb.c +++ b/drivers/phy/broadcom/phy-brcm-usb.c @@ -35,7 +35,7 @@ struct value_to_name_map { }; struct match_chip_info { - void *init_func; + void (*init_func)(struct brcm_usb_init_params *params); u8 required_regs[BRCM_REGS_MAX + 1]; u8 optional_reg; }; @@ -428,7 +428,6 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) struct device_node *dn = pdev->dev.of_node; int err; const char *mode; - void (*dvr_init)(struct brcm_usb_init_params *params); const struct match_chip_info *info; struct regmap *rmap; int x; @@ -444,8 +443,8 @@ static int brcm_usb_phy_probe(struct platform_device *pdev) info = of_device_get_match_data(&pdev->dev); if (!info) return -ENOENT; - dvr_init = info->init_func; - (*dvr_init)(&priv->ini); + + info->init_func(&priv->ini); dev_dbg(dev, "Best mapping table is for %s\n", priv->ini.family_name); From 266df28f9ac16b0dff553d78bc3fb1c084b96b9d Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Thu, 17 Dec 2020 14:04:59 +0300 Subject: [PATCH 048/633] phy: cadence-torrent: Fix error code in cdns_torrent_phy_probe() This error path should return -EINVAL, but currently it returns success. Fixes: d09945eacad0 ("phy: cadence-torrent: Check total lane count for all subnodes is within limit") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/X9s7Wxq+b6ls0q7o@mwanda Signed-off-by: Vinod Koul --- drivers/phy/cadence/phy-cadence-torrent.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c index f310e15d94cb..591a15834b48 100644 --- a/drivers/phy/cadence/phy-cadence-torrent.c +++ b/drivers/phy/cadence/phy-cadence-torrent.c @@ -2298,6 +2298,7 @@ static int cdns_torrent_phy_probe(struct platform_device *pdev) if (total_num_lanes > MAX_NUM_LANES) { dev_err(dev, "Invalid lane configuration\n"); + ret = -EINVAL; goto put_lnk_rst; } From a9dfa098b7d65e226b0c17809f2f0e39d7ade5ed Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 23 Dec 2020 12:45:05 +0000 Subject: [PATCH 049/633] phy: ingenic: Remove useless field .version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the useless field .version from the private structure, which is set but never read. Signed-off-by: Paul Cercueil Reviewed-by: 周琰杰 (Zhou Yanjie) Link: https://lore.kernel.org/r/20201223124505.40792-1-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/phy/ingenic/phy-ingenic-usb.c | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/drivers/phy/ingenic/phy-ingenic-usb.c b/drivers/phy/ingenic/phy-ingenic-usb.c index 4d1587d82286..ea127b177f46 100644 --- a/drivers/phy/ingenic/phy-ingenic-usb.c +++ b/drivers/phy/ingenic/phy-ingenic-usb.c @@ -82,18 +82,7 @@ #define USBPCR1_PORT_RST BIT(21) #define USBPCR1_WORD_IF_16BIT BIT(19) -enum ingenic_usb_phy_version { - ID_JZ4770, - ID_JZ4775, - ID_JZ4780, - ID_X1000, - ID_X1830, - ID_X2000, -}; - struct ingenic_soc_info { - enum ingenic_usb_phy_version version; - void (*usb_phy_init)(struct phy *phy); }; @@ -300,38 +289,26 @@ static void x2000_usb_phy_init(struct phy *phy) } static const struct ingenic_soc_info jz4770_soc_info = { - .version = ID_JZ4770, - .usb_phy_init = jz4770_usb_phy_init, }; static const struct ingenic_soc_info jz4775_soc_info = { - .version = ID_JZ4775, - .usb_phy_init = jz4775_usb_phy_init, }; static const struct ingenic_soc_info jz4780_soc_info = { - .version = ID_JZ4780, - .usb_phy_init = jz4780_usb_phy_init, }; static const struct ingenic_soc_info x1000_soc_info = { - .version = ID_X1000, - .usb_phy_init = x1000_usb_phy_init, }; static const struct ingenic_soc_info x1830_soc_info = { - .version = ID_X1830, - .usb_phy_init = x1830_usb_phy_init, }; static const struct ingenic_soc_info x2000_soc_info = { - .version = ID_X2000, - .usb_phy_init = x2000_usb_phy_init, }; From a8ec9e048bf3b748b9c4bb679c6513a9d8be36c0 Mon Sep 17 00:00:00 2001 From: Chunfeng Yun Date: Fri, 25 Dec 2020 15:52:50 +0800 Subject: [PATCH 050/633] dt-bindings: phy: convert phy-mtk-xsphy.txt to YAML schema Convert phy-mtk-xsphy.txt to YAML schema mediatek,xsphy.yaml Signed-off-by: Chunfeng Yun Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201225075258.33352-3-chunfeng.yun@mediatek.com Signed-off-by: Vinod Koul --- .../bindings/phy/mediatek,xsphy.yaml | 199 ++++++++++++++++++ .../devicetree/bindings/phy/phy-mtk-xsphy.txt | 109 ---------- 2 files changed, 199 insertions(+), 109 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml delete mode 100644 Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt diff --git a/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml new file mode 100644 index 000000000000..598fd2b95c29 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,xsphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek XS-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The XS-PHY controller supports physical layer functionality for USB3.1 + GEN2 controller on MediaTek SoCs. + + Banks layout of xsphy + ---------------------------------- + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u2 port2 0x2000 MISC + ... + u31 common 0x3000 DIG_GLB + 0x3100 PHYA_GLB + u31 port0 0x3400 DIG_LN_TOP + 0x3500 DIG_LN_TX0 + 0x3600 DIG_LN_RX0 + 0x3700 DIG_LN_DAIF + 0x3800 PHYA_LN + u31 port1 0x3a00 DIG_LN_TOP + 0x3b00 DIG_LN_TX0 + 0x3c00 DIG_LN_RX0 + 0x3d00 DIG_LN_DAIF + 0x3e00 PHYA_LN + ... + DIG_GLB & PHYA_GLB are shared by U31 ports. + +properties: + $nodename: + pattern: "^xs-phy@[0-9a-f]+$" + + compatible: + items: + - enum: + - mediatek,mt3611-xsphy + - mediatek,mt3612-xsphy + - const: mediatek,xsphy + + reg: + description: + Register shared by multiple U3 ports, exclude port's private register, + if only U2 ports provided, shouldn't use the property. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 17 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + + clock-names: + items: + - const: ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,efuse-intr: + description: + The selection of Internal Resistor (U2/U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 63 + + mediatek,efuse-tx-imp: + description: + The selection of TX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,efuse-rx-imp: + description: + The selection of RX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + required: + - reg + - clocks + - clock-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + + u3phy: xs-phy@11c40000 { + compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; + reg = <0x11c43000 0x0200>; + mediatek,src-ref-clk-mhz = <26>; + mediatek,src-coef = <17>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11c40000 { + reg = <0x11c40000 0x0400>; + clocks = <&clk48m>; + clock-names = "ref"; + mediatek,eye-src = <4>; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11c43000 { + reg = <0x11c43400 0x0500>; + clocks = <&clk26m>; + clock-names = "ref"; + mediatek,efuse-intr = <28>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt deleted file mode 100644 index e7caefa0b9c2..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt +++ /dev/null @@ -1,109 +0,0 @@ -MediaTek XS-PHY binding --------------------------- - -The XS-PHY controller supports physical layer functionality for USB3.1 -GEN2 controller on MediaTek SoCs. - -Required properties (controller (parent) node): - - compatible : should be "mediatek,-xsphy", "mediatek,xsphy", - soc-model is the name of SoC, such as mt3611 etc; - when using "mediatek,xsphy" compatible string, you need SoC specific - ones in addition, one of: - - "mediatek,mt3611-xsphy" - - - #address-cells, #size-cells : should use the same values as the root node - - ranges: must be present - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple U3 ports, - exclude port's private register, if only U2 ports provided, - shouldn't use the property. - - mediatek,src-ref-clk-mhz : u32, frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : u32, coefficient for slew rate calibrate, depends on - SoC process - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : must contain - "ref": 48M reference clock for HighSpeed analog phy; and 26M - reference clock for SuperSpeedPlus analog phy, sometimes is - 24M, 25M or 27M, depended on platform. -- #phy-cells : should be 1 - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - -The following optional properties are only for debug or HQA test -Optional properties (PHY_TYPE_USB2 port (child) node): -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,efuse-intr : u32, the selection of Internal Resistor - -Optional properties (PHY_TYPE_USB3 port (child) node): -- mediatek,efuse-intr : u32, the selection of Internal Resistor -- mediatek,efuse-tx-imp : u32, the selection of TX Impedance -- mediatek,efuse-rx-imp : u32, the selection of RX Impedance - -Banks layout of xsphy -------------------------------------------------------------- -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u2 port2 0x2000 MISC - ... -u31 common 0x3000 DIG_GLB - 0x3100 PHYA_GLB -u31 port0 0x3400 DIG_LN_TOP - 0x3500 DIG_LN_TX0 - 0x3600 DIG_LN_RX0 - 0x3700 DIG_LN_DAIF - 0x3800 PHYA_LN -u31 port1 0x3a00 DIG_LN_TOP - 0x3b00 DIG_LN_TX0 - 0x3c00 DIG_LN_RX0 - 0x3d00 DIG_LN_DAIF - 0x3e00 PHYA_LN - ... - -DIG_GLB & PHYA_GLB are shared by U31 ports. - -Example: - -u3phy: usb-phy@11c40000 { - compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; - reg = <0 0x11c43000 0 0x0200>; - mediatek,src-ref-clk-mhz = <26>; - mediatek,src-coef = <17>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11c40000 { - reg = <0 0x11c40000 0 0x0400>; - clocks = <&clk48m>; - clock-names = "ref"; - mediatek,eye-src = <4>; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11c43000 { - reg = <0 0x11c43400 0 0x0500>; - clocks = <&clk26m>; - clock-names = "ref"; - mediatek,efuse-intr = <28>; - #phy-cells = <1>; - }; -}; From cbdf8f5080173fcf5d521c0e04403144b12b29eb Mon Sep 17 00:00:00 2001 From: Chunfeng Yun Date: Fri, 25 Dec 2020 15:52:51 +0800 Subject: [PATCH 051/633] dt-bindings: phy: convert phy-mtk-tphy.txt to YAML schema Convert phy-mtk-tphy.txt to YAML schema mediatek,tphy.yaml Signed-off-by: Chunfeng Yun Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20201225075258.33352-4-chunfeng.yun@mediatek.com Signed-off-by: Vinod Koul --- .../bindings/phy/mediatek,tphy.yaml | 260 ++++++++++++++++++ .../devicetree/bindings/phy/phy-mtk-tphy.txt | 162 ----------- 2 files changed, 260 insertions(+), 162 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/mediatek,tphy.yaml delete mode 100644 Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt diff --git a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml new file mode 100644 index 000000000000..602e6ff45785 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,tphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek T-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The T-PHY controller supports physical layer functionality for a number of + controllers on MediaTek SoCs, includes USB2.0, USB3.0, PCIe and SATA. + + Layout differences of banks between T-PHY V1 (mt8173/mt2701) and + T-PHY V2 (mt2712) when works on USB mode: + ----------------------------------- + Version 1: + port offset bank + shared 0x0000 SPLLC + 0x0100 FMREG + u2 port0 0x0800 U2PHY_COM + u3 port0 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 U2PHY_COM + u3 port1 0x1100 U3PHYD + 0x1200 U3PHYD_BANK2 + 0x1300 U3PHYA + 0x1400 U3PHYA_DA + u2 port2 0x1800 U2PHY_COM + ... + + Version 2: + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u3 port0 0x0700 SPLLC + 0x0800 CHIP + 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u3 port1 0x1700 SPLLC + 0x1800 CHIP + 0x1900 U3PHYD + 0x1a00 U3PHYD_BANK2 + 0x1b00 U3PHYA + 0x1c00 U3PHYA_DA + u2 port2 0x2000 MISC + ... + + SPLLC shared by u3 ports and FMREG shared by u2 ports on V1 are put back + into each port; a new bank MISC for u2 ports and CHIP for u3 ports are + added on V2. + +properties: + $nodename: + pattern: "^t-phy@[0-9a-f]+$" + + compatible: + oneOf: + - items: + - enum: + - mediatek,mt2701-tphy + - mediatek,mt7623-tphy + - mediatek,mt7622-tphy + - mediatek,mt8516-tphy + - const: mediatek,generic-tphy-v1 + - items: + - enum: + - mediatek,mt2712-tphy + - mediatek,mt7629-tphy + - mediatek,mt8183-tphy + - const: mediatek,generic-tphy-v2 + - const: mediatek,mt2701-u3phy + deprecated: true + - const: mediatek,mt2712-u3phy + deprecated: true + - const: mediatek,mt8173-u3phy + + reg: + description: + Register shared by multiple ports, exclude port's private register. + It is needed for T-PHY V1, such as mt2701 and mt8173, but not for + T-PHY V2, such as mt2712. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + # Used with non-empty value if optional 'reg' is not provided. + # The format of the value is an arbitrary number of triplets of + # (child-bus-address, parent-bus-address, length). + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 28 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + - description: Reference clock of analog phy + description: + Uses both clocks if the clock of analog and digital phys are + separated, otherwise uses "ref" clock only if needed. + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: ref + - const: da_ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + - PHY_TYPE_PCIE + - PHY_TYPE_SATA + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,intr: + description: + The selection of internal resistor (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,discth: + description: + The selection of disconnect threshold (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 15 + + mediatek,bc12: + description: + Specify the flag to enable BC1.2 if support it + type: boolean + + required: + - reg + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + usb@11271000 { + compatible = "mediatek,mt8173-mtu3", "mediatek,mtu3"; + reg = <0x11271000 0x3000>, <0x11280700 0x0100>; + reg-names = "mac", "ippc"; + phys = <&u2port0 PHY_TYPE_USB2>, + <&u3port0 PHY_TYPE_USB3>, + <&u2port1 PHY_TYPE_USB2>; + interrupts = ; + clocks = <&topckgen CLK_TOP_USB30_SEL>; + clock-names = "sys_ck"; + }; + + t-phy@11290000 { + compatible = "mediatek,mt8173-u3phy"; + reg = <0x11290000 0x800>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11290800 { + reg = <0x11290800 0x100>; + clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>, <&clk48m>; + clock-names = "ref", "da_ref"; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11290900 { + reg = <0x11290900 0x700>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + }; + + u2port1: usb-phy@11291000 { + reg = <0x11291000 0x100>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt deleted file mode 100644 index dd75b676b71d..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt +++ /dev/null @@ -1,162 +0,0 @@ -MediaTek T-PHY binding --------------------------- - -T-phy controller supports physical layer functionality for a number of -controllers on MediaTek SoCs, such as, USB2.0, USB3.0, PCIe, and SATA. - -Required properties (controller (parent) node): - - compatible : should be one of - "mediatek,generic-tphy-v1" - "mediatek,generic-tphy-v2" - "mediatek,mt2701-u3phy" (deprecated) - "mediatek,mt2712-u3phy" (deprecated) - "mediatek,mt8173-u3phy"; - make use of "mediatek,generic-tphy-v1" on mt2701 instead and - "mediatek,generic-tphy-v2" on mt2712 instead. - -- #address-cells: the number of cells used to represent physical - base addresses. -- #size-cells: the number of cells used to represent the size of an address. -- ranges: the address mapping relationship to the parent, defined with - - empty value: if optional 'reg' is used. - - non-empty value: if optional 'reg' is not used. should set - the child's base address to 0, the physical address - within parent's address space, and the length of - the address map. - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple ports, - exclude port's private register. It is needed on mt2701 - and mt8173, but not on mt2712. - - mediatek,src-ref-clk-mhz : frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : coefficient for slew rate calibrate, depends on - SoC process - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- #phy-cells : should be 1 (See second example) - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - - PHY_TYPE_PCIE - - PHY_TYPE_SATA - -Optional properties (PHY_TYPE_USB2 port (child) node): -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : may contain - "ref": 48M reference clock for HighSpeed (digital) phy; and 26M - reference clock for SuperSpeed (digital) phy, sometimes is - 24M, 25M or 27M, depended on platform. - "da_ref": the reference clock of analog phy, used if the clocks - of analog and digital phys are separated, otherwise uses - "ref" clock only if needed. - -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,bc12 : bool, enable BC12 of u2phy if support it -- mediatek,discth : u32, the selection of disconnect threshold -- mediatek,intr : u32, the selection of internal R (resistance) - -Example: - -u3phy: usb-phy@11290000 { - compatible = "mediatek,mt8173-u3phy"; - reg = <0 0x11290000 0 0x800>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11290800 { - reg = <0 0x11290800 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11290900 { - reg = <0 0x11290800 0 0x700>; - clocks = <&clk26m>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u2port1: usb-phy@11291000 { - reg = <0 0x11291000 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; -}; - -Specifying phy control of devices ---------------------------------- - -Device nodes should specify the configuration required in their "phys" -property, containing a phandle to the phy port node and a device type; -phy-names for each port are optional. - -Example: - -#include - -usb30: usb@11270000 { - ... - phys = <&u2port0 PHY_TYPE_USB2>, <&u3port0 PHY_TYPE_USB3>; - phy-names = "usb2-0", "usb3-0"; - ... -}; - - -Layout differences of banks between mt8173/mt2701 and mt2712 -------------------------------------------------------------- -mt8173 and mt2701: -port offset bank -shared 0x0000 SPLLC - 0x0100 FMREG -u2 port0 0x0800 U2PHY_COM -u3 port0 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 U2PHY_COM -u3 port1 0x1100 U3PHYD - 0x1200 U3PHYD_BANK2 - 0x1300 U3PHYA - 0x1400 U3PHYA_DA -u2 port2 0x1800 U2PHY_COM - ... - -mt2712: -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u3 port0 0x0700 SPLLC - 0x0800 CHIP - 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u3 port1 0x1700 SPLLC - 0x1800 CHIP - 0x1900 U3PHYD - 0x1a00 U3PHYD_BANK2 - 0x1b00 U3PHYA - 0x1c00 U3PHYA_DA -u2 port2 0x2000 MISC - ... - - SPLLC shared by u3 ports and FMREG shared by u2 ports on -mt8173/mt2701 are put back into each port; a new bank MISC for -u2 ports and CHIP for u3 ports are added on mt2712. From 67038ec1bdfbb36a54d24cd5dec108aec2cf3827 Mon Sep 17 00:00:00 2001 From: Chunfeng Yun Date: Fri, 25 Dec 2020 15:52:52 +0800 Subject: [PATCH 052/633] dt-bindings: phy: convert phy-mtk-ufs.txt to YAML schema Convert phy-mtk-ufs.txt to YAML schema mediatek,ufs-phy.yaml Signed-off-by: Chunfeng Yun Reviewed-by: Rob Herring Reviewed-by: Stanley Chu Cc: Stanley Chu Link: https://lore.kernel.org/r/20201225075258.33352-5-chunfeng.yun@mediatek.com Signed-off-by: Vinod Koul --- .../bindings/phy/mediatek,ufs-phy.yaml | 64 +++++++++++++++++++ .../devicetree/bindings/phy/phy-mtk-ufs.txt | 38 ----------- 2 files changed, 64 insertions(+), 38 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml delete mode 100644 Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt diff --git a/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml new file mode 100644 index 000000000000..3a9be82e7f13 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,ufs-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Universal Flash Storage (UFS) M-PHY binding + +maintainers: + - Stanley Chu + - Chunfeng Yun + +description: | + UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. + Each UFS M-PHY node should have its own node. + To bind UFS M-PHY with UFS host controller, the controller node should + contain a phandle reference to UFS M-PHY node. + +properties: + $nodename: + pattern: "^ufs-phy@[0-9a-f]+$" + + compatible: + const: mediatek,mt8183-ufsphy + + reg: + maxItems: 1 + + clocks: + items: + - description: Unipro core control clock. + - description: M-PHY core control clock. + + clock-names: + items: + - const: unipro + - const: mp + + "#phy-cells": + const: 0 + +required: + - compatible + - reg + - "#phy-cells" + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + ufsphy: ufs-phy@11fa0000 { + compatible = "mediatek,mt8183-ufsphy"; + reg = <0x11fa0000 0xc000>; + clocks = <&infracfg CLK_INFRA_UNIPRO_SCK>, + <&infracfg CLK_INFRA_UFS_MP_SAP_BCLK>; + clock-names = "unipro", "mp"; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt b/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt deleted file mode 100644 index 5789029a1d42..000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt +++ /dev/null @@ -1,38 +0,0 @@ -MediaTek Universal Flash Storage (UFS) M-PHY binding --------------------------------------------------------- - -UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. -Each UFS M-PHY node should have its own node. - -To bind UFS M-PHY with UFS host controller, the controller node should -contain a phandle reference to UFS M-PHY node. - -Required properties for UFS M-PHY nodes: -- compatible : Compatible list, contains the following controller: - "mediatek,mt8183-ufsphy" for ufs phy - persent on MT81xx chipsets. -- reg : Address and length of the UFS M-PHY register set. -- #phy-cells : This property shall be set to 0. -- clocks : List of phandle and clock specifier pairs. -- clock-names : List of clock input name strings sorted in the same - order as the clocks property. Following clocks are - mandatory. - "unipro": Unipro core control clock. - "mp": M-PHY core control clock. - -Example: - - ufsphy: phy@11fa0000 { - compatible = "mediatek,mt8183-ufsphy"; - reg = <0 0x11fa0000 0 0xc000>; - #phy-cells = <0>; - - clocks = <&infracfg_ao INFRACFG_AO_UNIPRO_SCK_CG>, - <&infracfg_ao INFRACFG_AO_UFS_MP_SAP_BCLK_CG>; - clock-names = "unipro", "mp"; - }; - - ufshci@11270000 { - ... - phys = <&ufsphy>; - }; From 5ada755de9db0053740e73ea264393a20fc5eded Mon Sep 17 00:00:00 2001 From: Chunfeng Yun Date: Fri, 25 Dec 2020 15:52:53 +0800 Subject: [PATCH 053/633] dt-bindings: phy: convert HDMI PHY binding to YAML schema Convert HDMI PHY binding to YAML schema mediatek,hdmi-phy.yaml Signed-off-by: Chunfeng Yun Reviewed-by: Rob Herring Reviewed-by: Chun-Kuang Hu Cc: Chun-Kuang Hu Cc: Philipp Zabel Link: https://lore.kernel.org/r/20201225075258.33352-6-chunfeng.yun@mediatek.com Signed-off-by: Vinod Koul --- .../display/mediatek/mediatek,hdmi.txt | 18 +--- .../bindings/phy/mediatek,hdmi-phy.yaml | 92 +++++++++++++++++++ 2 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt index 6b1c586403e4..b284ca51b913 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt @@ -53,23 +53,7 @@ Required properties: HDMI PHY ======== - -The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel -output and drives the HDMI pads. - -Required properties: -- compatible: "mediatek,-hdmi-phy" -- the supported chips are mt2701, mt7623 and mt8173 -- reg: Physical base address and length of the module's registers -- clocks: PLL reference clock -- clock-names: must contain "pll_ref" -- clock-output-names: must be "hdmitx_dig_cts" on mt8173 -- #phy-cells: must be <0> -- #clock-cells: must be <0> - -Optional properties: -- mediatek,ibias: TX DRV bias current for <1.65Gbps, defaults to 0xa -- mediatek,ibias_up: TX DRV bias current for >1.65Gbps, defaults to 0x1c +See phy/mediatek,hdmi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml new file mode 100644 index 000000000000..4752517a1446 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,hdmi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek High Definition Multimedia Interface (HDMI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: | + The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel + output and drives the HDMI pads. + +properties: + $nodename: + pattern: "^hdmi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-hdmi-phy + - mediatek,mt7623-hdmi-phy + - mediatek,mt8173-hdmi-phy + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-names: + items: + - const: pll_ref + + clock-output-names: + items: + - const: hdmitx_dig_cts + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + mediatek,ibias: + description: + TX DRV bias current for < 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0xa + + mediatek,ibias_up: + description: + TX DRV bias current for >= 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0x1c + +required: + - compatible + - reg + - clocks + - clock-names + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + hdmi_phy: hdmi-phy@10209100 { + compatible = "mediatek,mt8173-hdmi-phy"; + reg = <0x10209100 0x24>; + clocks = <&apmixedsys CLK_APMIXED_HDMI_REF>; + clock-names = "pll_ref"; + clock-output-names = "hdmitx_dig_cts"; + mediatek,ibias = <0xa>; + mediatek,ibias_up = <0x1c>; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... From dc8423a879b169e992e9abc3b00ba4e6c53783ab Mon Sep 17 00:00:00 2001 From: Chunfeng Yun Date: Fri, 25 Dec 2020 15:52:54 +0800 Subject: [PATCH 054/633] dt-bindings: phy: convert MIPI DSI PHY binding to YAML schema Convert MIPI DSI PHY binding to YAML schema mediatek,dsi-phy.yaml Signed-off-by: Chunfeng Yun Reviewed-by: Rob Herring Reviewed-by: Chun-Kuang Hu Cc: Chun-Kuang Hu Cc: Philipp Zabel Link: https://lore.kernel.org/r/20201225075258.33352-7-chunfeng.yun@mediatek.com Signed-off-by: Vinod Koul --- .../display/mediatek/mediatek,dsi.txt | 18 +--- .../bindings/phy/mediatek,dsi-phy.yaml | 85 +++++++++++++++++++ 2 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt index f06f24d405a5..8238a86686be 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt @@ -22,23 +22,7 @@ Required properties: MIPI TX Configuration Module ============================ -The MIPI TX configuration module controls the MIPI D-PHY. - -Required properties: -- compatible: "mediatek,-mipi-tx" -- the supported chips are mt2701, 7623, mt8173 and mt8183. -- reg: Physical base address and length of the controller's registers -- clocks: PLL reference clock -- clock-output-names: name of the output clock line to the DSI encoder -- #clock-cells: must be <0>; -- #phy-cells: must be <0>. - -Optional properties: -- drive-strength-microamp: adjust driving current, should be 3000 ~ 6000. And - the step is 200. -- nvmem-cells: A phandle to the calibration data provided by a nvmem device. If - unspecified default values shall be used. -- nvmem-cell-names: Should be "calibration-data" +See phy/mediatek,dsi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml new file mode 100644 index 000000000000..71d4acea1f66 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,dsi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MIPI Display Serial Interface (DSI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: The MIPI DSI PHY supports up to 4-lane output. + +properties: + $nodename: + pattern: "^dsi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-mipi-tx + - mediatek,mt7623-mipi-tx + - mediatek,mt8173-mipi-tx + - mediatek,mt8183-mipi-tx + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + nvmem-cells: + maxItems: 1 + description: A phandle to the calibration data provided by a nvmem device, + if unspecified, default values shall be used. + + nvmem-cell-names: + items: + - const: calibration-data + + drive-strength-microamp: + description: adjust driving current + multipleOf: 200 + minimum: 2000 + maximum: 6000 + default: 4600 + +required: + - compatible + - reg + - clocks + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + dsi-phy@10215000 { + compatible = "mediatek,mt8173-mipi-tx"; + reg = <0x10215000 0x1000>; + clocks = <&clk26m>; + clock-output-names = "mipi_tx0_pll"; + drive-strength-microamp = <4000>; + nvmem-cells= <&mipi_tx_calibration>; + nvmem-cell-names = "calibration-data"; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... From 36acd5e24e3000691fb8d1ee31cf959cb1582d35 Mon Sep 17 00:00:00 2001 From: Mathias Kresin Date: Thu, 7 Jan 2021 23:49:01 +0100 Subject: [PATCH 055/633] phy: lantiq: rcu-usb2: wait after clock enable Commit 65dc2e725286 ("usb: dwc2: Update Core Reset programming flow.") revealed that the phy isn't ready immediately after enabling it's clocks. The dwc2_check_core_version() fails and the dwc2 usb driver errors out. Add a short delay to let the phy get up and running. There isn't any documentation how much time is required, the value was chosen based on tests. Signed-off-by: Mathias Kresin Acked-by: Hauke Mehrtens Acked-by: Martin Blumenstingl Cc: # v5.7+ Link: https://lore.kernel.org/r/20210107224901.2102479-1-dev@kresin.me Signed-off-by: Vinod Koul --- drivers/phy/lantiq/phy-lantiq-rcu-usb2.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c index a7d126192cf1..29d246ea24b4 100644 --- a/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c +++ b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c @@ -124,8 +124,16 @@ static int ltq_rcu_usb2_phy_power_on(struct phy *phy) reset_control_deassert(priv->phy_reset); ret = clk_prepare_enable(priv->phy_gate_clk); - if (ret) + if (ret) { dev_err(dev, "failed to enable PHY gate\n"); + return ret; + } + + /* + * at least the xrx200 usb2 phy requires some extra time to be + * operational after enabling the clock + */ + usleep_range(100, 200); return ret; } From 6d54623a5627569503082bbbeb6f475f3cbabf1a Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Tue, 12 Jan 2021 09:38:04 +0800 Subject: [PATCH 056/633] phy: mediatek: Mark mtk_mipi_tx_driver with static keyword Fix the following sparse warning: drivers/phy/mediatek/phy-mtk-mipi-dsi.c:237:24: warning: symbol 'mtk_mipi_tx_driver' was not declared. Should it be static? Signed-off-by: Zou Wei Link: https://lore.kernel.org/r/1610415484-92497-1-git-send-email-zou_wei@huawei.com Signed-off-by: Vinod Koul --- drivers/phy/mediatek/phy-mtk-mipi-dsi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c index 18c481251f04..eeb357bfb0bb 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c @@ -234,7 +234,7 @@ static const struct of_device_id mtk_mipi_tx_match[] = { { }, }; -struct platform_driver mtk_mipi_tx_driver = { +static struct platform_driver mtk_mipi_tx_driver = { .probe = mtk_mipi_tx_probe, .remove = mtk_mipi_tx_remove, .driver = { From b39069a482ade0c5e18c407c3218ba1aeed371b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 6 Jan 2021 21:58:36 +0100 Subject: [PATCH 057/633] dt-bindings: phy: brcm, brcmstb-usb-phy: convert to the json-schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes that require mentioning: 1. interrupt-names Name "wakeup" was changed to the "wake". It matches example and what Linux driver looks for in the first place 2. brcm,ipp and brcm,ioc Both were described as booleans with 0 / 1 values. In examples they were integers and Linux checks for int as well. Both got uint32. 3. Added minimal description Signed-off-by: Rafał Miłecki Reviewed-by: Florian Fainelli Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20210106205838.10964-1-zajec5@gmail.com Signed-off-by: Vinod Koul --- .../bindings/phy/brcm,brcmstb-usb-phy.txt | 86 -------- .../bindings/phy/brcm,brcmstb-usb-phy.yaml | 193 ++++++++++++++++++ 2 files changed, 193 insertions(+), 86 deletions(-) delete mode 100644 Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt create mode 100644 Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt deleted file mode 100644 index 698aacbdcfc4..000000000000 --- a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt +++ /dev/null @@ -1,86 +0,0 @@ -Broadcom STB USB PHY - -Required properties: -- compatible: should be one of - "brcm,brcmstb-usb-phy" - "brcm,bcm7216-usb-phy" - "brcm,bcm7211-usb-phy" - -- reg and reg-names properties requirements are specific to the - compatible string. - "brcm,brcmstb-usb-phy": - - reg: 1 or 2 offset and length pairs. One for the base CTRL registers - and an optional pair for systems with USB 3.x support - - reg-names: not specified - "brcm,bcm7216-usb-phy": - - reg: 3 offset and length pairs for CTRL, XHCI_EC and XHCI_GBL - registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl" - "brcm,bcm7211-usb-phy": - - reg: 5 offset and length pairs for CTRL, XHCI_EC, XHCI_GBL, - USB_PHY and USB_MDIO registers and an optional pair - for the BDC registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl", "usb_phy", "usb_mdio", "bdc_ec" - -- #phy-cells: Shall be 1 as it expects one argument for setting - the type of the PHY. Possible values are: - - PHY_TYPE_USB2 for USB1.1/2.0 PHY - - PHY_TYPE_USB3 for USB3.x PHY - -Optional Properties: -- clocks : clock phandles. -- clock-names: String, clock name. -- interrupts: wakeup interrupt -- interrupt-names: "wakeup" -- brcm,ipp: Boolean, Invert Port Power. - Possible values are: 0 (Don't invert), 1 (Invert) -- brcm,ioc: Boolean, Invert Over Current detection. - Possible values are: 0 (Don't invert), 1 (Invert) -- dr_mode: String, PHY Device mode. - Possible values are: "host", "peripheral ", "drd" or "typec-pd" - If this property is not defined, the phy will default to "host" mode. -- brcm,syscon-piarbctl: phandle to syscon for handling config registers -NOTE: one or both of the following two properties must be set -- brcm,has-xhci: Boolean indicating the phy has an XHCI phy. -- brcm,has-eohci: Boolean indicating the phy has an EHCI/OHCI phy. - - -Example: - -usbphy_0: usb-phy@f0470200 { - reg = <0xf0470200 0xb8>, - <0xf0471940 0x6c0>; - compatible = "brcm,brcmstb-usb-phy"; - #phy-cells = <1>; - dr_mode = "host" - brcm,ioc = <1>; - brcm,ipp = <1>; - brcm,has-xhci; - brcm,has-eohci; - clocks = <&usb20>, <&usb30>; - clock-names = "sw_usb", "sw_usb3"; -}; - -usb-phy@29f0200 { - reg = <0x29f0200 0x200>, - <0x29c0880 0x30>, - <0x29cc100 0x534>, - <0x2808000 0x24>, - <0x2980080 0x8>; - reg-names = "ctrl", - "xhci_ec", - "xhci_gbl", - "usb_phy", - "usb_mdio"; - brcm,ioc = <0x0>; - brcm,ipp = <0x0>; - compatible = "brcm,bcm7211-usb-phy"; - interrupts = <0x30>; - interrupt-parent = <&vpu_intr1_nosec_intc>; - interrupt-names = "wake"; - #phy-cells = <0x1>; - brcm,has-xhci; - syscon-piarbctl = <&syscon_piarbctl>; - clocks = <&scmi_clk 256>; - clock-names = "sw_usb"; -}; diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml new file mode 100644 index 000000000000..a5780beadf97 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/brcm,brcmstb-usb-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom STB USB PHY + +description: Broadcom's PHY that handles EHCI/OHCI and/or XHCI + +maintainers: + - Al Cooper + - Rafał Miłecki + +properties: + compatible: + enum: + - brcm,bcm7211-usb-phy + - brcm,bcm7216-usb-phy + - brcm,brcmstb-usb-phy + + reg: + minItems: 1 + maxItems: 6 + items: + - description: the base CTRL register + - description: XHCI EC register + - description: XHCI GBL register + - description: USB PHY register + - description: USB MDIO register + - description: BDC register + + reg-names: + minItems: 1 + maxItems: 6 + items: + - const: ctrl + - const: xhci_ec + - const: xhci_gbl + - const: usb_phy + - const: usb_mdio + - const: bdc_ec + + clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: sw_usb + - const: sw_usb3 + + interrupts: + description: wakeup interrupt + + interrupt-names: + const: wake + + brcm,ipp: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Port Power + minimum: 0 + maximum: 1 + + brcm,ioc: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Over Current detection + minimum: 0 + maximum: 1 + + dr_mode: + description: PHY Device mode. If this property is not defined, the PHY will + default to "host" mode. + enum: + - host + - peripheral + - drd + - typec-pd + + brcm,syscon-piarbctl: + description: phandle to syscon for handling config registers + $ref: /schemas/types.yaml#/definitions/phandle + + brcm,has-xhci: + description: Indicates the PHY has an XHCI PHY. + type: boolean + + brcm,has-eohci: + description: Indicates the PHY has an EHCI/OHCI PHY. + type: boolean + + "#phy-cells": + description: | + Cell allows setting the type of the PHY. Possible values are: + - PHY_TYPE_USB2 for USB1.1/2.0 PHY + - PHY_TYPE_USB3 for USB3.x PHY + const: 1 + +required: + - reg + - "#phy-cells" + +anyOf: + - required: + - brcm,has-xhci + - required: + - brcm,has-eohci + +allOf: + - if: + properties: + compatible: + contains: + const: brcm,brcmstb-usb-phy + then: + properties: + reg: + minItems: 1 + maxItems: 2 + - if: + properties: + compatible: + contains: + const: brcm,bcm7211-usb-phy + then: + properties: + reg: + minItems: 5 + maxItems: 6 + reg-names: + minItems: 5 + maxItems: 6 + - if: + properties: + compatible: + contains: + const: brcm,bcm7216-usb-phy + then: + properties: + reg: + minItems: 3 + maxItems: 3 + reg-names: + minItems: 3 + maxItems: 3 + +additionalProperties: false + +examples: + - | + #include + + usb-phy@f0470200 { + compatible = "brcm,brcmstb-usb-phy"; + reg = <0xf0470200 0xb8>, + <0xf0471940 0x6c0>; + #phy-cells = <1>; + dr_mode = "host"; + brcm,ioc = <1>; + brcm,ipp = <1>; + brcm,has-xhci; + brcm,has-eohci; + clocks = <&usb20>, <&usb30>; + clock-names = "sw_usb", "sw_usb3"; + }; + - | + #include + + usb-phy@29f0200 { + compatible = "brcm,bcm7211-usb-phy"; + reg = <0x29f0200 0x200>, + <0x29c0880 0x30>, + <0x29cc100 0x534>, + <0x2808000 0x24>, + <0x2980080 0x8>; + reg-names = "ctrl", + "xhci_ec", + "xhci_gbl", + "usb_phy", + "usb_mdio"; + brcm,ioc = <0x0>; + brcm,ipp = <0x0>; + interrupts = <0x30>; + interrupt-parent = <&vpu_intr1_nosec_intc>; + interrupt-names = "wake"; + #phy-cells = <0x1>; + brcm,has-xhci; + brcm,syscon-piarbctl = <&syscon_piarbctl>; + clocks = <&scmi_clk 256>; + clock-names = "sw_usb"; + }; From 46b616c1574def7a1629bdeded3d44e76382f950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 6 Jan 2021 21:58:37 +0100 Subject: [PATCH 058/633] dt-bindings: phy: brcm, brcmstb-usb-phy: add BCM4908 binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BCM4908 uses the same PHY and may require just a slightly different programming. Signed-off-by: Rafał Miłecki Acked-by: Florian Fainelli Acked-by: Rob Herring Link: https://lore.kernel.org/r/20210106205838.10964-2-zajec5@gmail.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml index a5780beadf97..0497368d1fca 100644 --- a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml @@ -15,6 +15,7 @@ maintainers: properties: compatible: enum: + - brcm,bcm4908-usb-phy - brcm,bcm7211-usb-phy - brcm,bcm7216-usb-phy - brcm,brcmstb-usb-phy @@ -113,7 +114,9 @@ allOf: properties: compatible: contains: - const: brcm,brcmstb-usb-phy + enum: + - const: brcm,bcm4908-usb-phy + - const: brcm,brcmstb-usb-phy then: properties: reg: From 4b402fa8e0b7817f3e3738d7828038f114e6899e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Wed, 6 Jan 2021 21:58:38 +0100 Subject: [PATCH 059/633] phy: phy-brcm-usb: support PHY on the BCM4908 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BCM4908 seems to have slightly different registers but works when programmed just like the STB one. Signed-off-by: Rafał Miłecki Acked-by: Florian Fainelli Link: https://lore.kernel.org/r/20210106205838.10964-3-zajec5@gmail.com Signed-off-by: Vinod Koul --- drivers/phy/broadcom/Kconfig | 3 ++- drivers/phy/broadcom/phy-brcm-usb.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig index a1f1a9c90d0d..09256339bd04 100644 --- a/drivers/phy/broadcom/Kconfig +++ b/drivers/phy/broadcom/Kconfig @@ -91,10 +91,11 @@ config PHY_BRCM_SATA config PHY_BRCM_USB tristate "Broadcom STB USB PHY driver" - depends on ARCH_BRCMSTB || COMPILE_TEST + depends on ARCH_BCM4908 || ARCH_BRCMSTB || COMPILE_TEST depends on OF select GENERIC_PHY select SOC_BRCMSTB + default ARCH_BCM4908 default ARCH_BRCMSTB help Enable this to support the Broadcom STB USB PHY. diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c index c7d751b7c144..116fb23aebd9 100644 --- a/drivers/phy/broadcom/phy-brcm-usb.c +++ b/drivers/phy/broadcom/phy-brcm-usb.c @@ -286,6 +286,10 @@ static const struct match_chip_info chip_info_7445 = { }; static const struct of_device_id brcm_usb_dt_ids[] = { + { + .compatible = "brcm,bcm4908-usb-phy", + .data = &chip_info_7445, + }, { .compatible = "brcm,bcm7216-usb-phy", .data = &chip_info_7216, From 34168172eb9f8e444caa843eca59813e60379f91 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 13 Jan 2021 11:59:25 +0100 Subject: [PATCH 060/633] dt-bindings: phy: update phy-cadence-sierra.yaml reference Changeset ba2bf1f090eb ("dt-bindings: phy: Add Cadence Sierra PHY bindings in YAML format") renamed: Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt to: Documentation/devicetree/bindings/phy/phy-cadence-sierra.yaml. Update its cross-reference accordingly. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/3550b08d4e8312e7d4a247a3515a93a5f0fd04c5.1610535350.git.mchehab+huawei@kernel.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml index c33e9bc79521..bbbd85501ada 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml @@ -151,7 +151,7 @@ patternProperties: WIZ node should have '1' subnode for the SERDES. It could be either Sierra SERDES or Torrent SERDES. Sierra SERDES should follow the bindings specified in - Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt + Documentation/devicetree/bindings/phy/phy-cadence-sierra.yaml Torrent SERDES should follow the bindings specified in Documentation/devicetree/bindings/phy/phy-cadence-torrent.yaml From 00a9f71760376749e0857b43a8573803b1a075dc Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:20 +0100 Subject: [PATCH 061/633] dt-bindings: phy: phy-stm32-usbphyc: move PLL supplies to parent node PLL block requires to be powered with 1v1 and 1v8 supplies to catch ENABLE signal. Currently, supplies are managed through phy_ops .power_on/off, and PLL activation/deactivation is managed through phy_ops .init/exit. The sequence of phy_ops .power_on/.phy_init, .power_off/.exit is USB drivers dependent. To ensure a good behavior of the PLL, supplies have to be managed at PLL activation/deactivation. That means the supplies need to be put in usbphyc parent node and not in phy children nodes. Signed-off-by: Amelie Delaunay Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20210105090525.23164-2-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- .../bindings/phy/phy-stm32-usbphyc.yaml | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml index 0ba61979b970..46df6786727a 100644 --- a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml +++ b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml @@ -45,6 +45,12 @@ properties: "#size-cells": const: 0 + vdda1v1-supply: + description: regulator providing 1V1 power supply to the PLL block + + vdda1v8-supply: + description: regulator providing 1V8 power supply to the PLL block + #Required child nodes: patternProperties: @@ -61,12 +67,6 @@ patternProperties: phy-supply: description: regulator providing 3V3 power supply to the PHY. - vdda1v1-supply: - description: regulator providing 1V1 power supply to the PLL block - - vdda1v8-supply: - description: regulator providing 1V8 power supply to the PLL block - "#phy-cells": enum: [ 0x0, 0x1 ] @@ -90,8 +90,6 @@ patternProperties: required: - reg - phy-supply - - vdda1v1-supply - - vdda1v8-supply - "#phy-cells" additionalProperties: false @@ -102,6 +100,8 @@ required: - clocks - "#address-cells" - "#size-cells" + - vdda1v1-supply + - vdda1v8-supply - usb-phy@0 - usb-phy@1 @@ -116,22 +116,20 @@ examples: reg = <0x5a006000 0x1000>; clocks = <&rcc USBPHY_K>; resets = <&rcc USBPHY_R>; + vdda1v1-supply = <®11>; + vdda1v8-supply = <®18>; #address-cells = <1>; #size-cells = <0>; usbphyc_port0: usb-phy@0 { reg = <0>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <0>; }; usbphyc_port1: usb-phy@1 { reg = <1>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <1>; }; }; From 613a475f0be10930474b5cf67c1c9aaa0c992798 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:21 +0100 Subject: [PATCH 062/633] phy: stm32: manage 1v1 and 1v8 supplies at pll activation/deactivation PLL block requires to be powered with 1v1 and 1v8 supplies to catch ENABLE signal. Currently, supplies are managed through phy_ops .power_on/off, and PLL activation/deactivation is managed through phy_ops .init/exit. The sequence of phy_ops .power_on/.phy_init, .power_off/.exit is USB drivers dependent. To ensure a good behavior of the PLL, supplies have to be managed at PLL activation/deactivation. That means the supplies need to be put in usbphyc node and not in phy children nodes. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20210105090525.23164-3-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 118 +++++++++++++---------------- 1 file changed, 54 insertions(+), 64 deletions(-) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index a54317e96c41..c78a2c7947ce 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -58,7 +58,6 @@ struct pll_params { struct stm32_usbphyc_phy { struct phy *phy; struct stm32_usbphyc *usbphyc; - struct regulator_bulk_data supplies[NUM_SUPPLIES]; u32 index; bool active; }; @@ -70,6 +69,7 @@ struct stm32_usbphyc { struct reset_control *rst; struct stm32_usbphyc_phy **phys; int nphys; + struct regulator_bulk_data supplies[NUM_SUPPLIES]; int switch_setup; }; @@ -153,39 +153,6 @@ static bool stm32_usbphyc_has_one_phy_active(struct stm32_usbphyc *usbphyc) return false; } -static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) -{ - void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; - bool pllen = (readl_relaxed(pll_reg) & PLLEN); - int ret; - - /* Check if one phy port has already configured the pll */ - if (pllen && stm32_usbphyc_has_one_phy_active(usbphyc)) - return 0; - - if (pllen) { - stm32_usbphyc_clr_bits(pll_reg, PLLEN); - /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ - udelay(PLL_PWR_DOWN_TIME_US); - } - - ret = stm32_usbphyc_pll_init(usbphyc); - if (ret) - return ret; - - stm32_usbphyc_set_bits(pll_reg, PLLEN); - - /* Wait for maximum lock time */ - udelay(PLL_LOCK_TIME_US); - - if (!(readl_relaxed(pll_reg) & PLLEN)) { - dev_err(usbphyc->dev, "PLLEN not set\n"); - return -EIO; - } - - return 0; -} - static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) { void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; @@ -203,7 +170,49 @@ static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) return -EIO; } + return regulator_bulk_disable(NUM_SUPPLIES, usbphyc->supplies); +} + +static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) +{ + void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; + bool pllen = readl_relaxed(pll_reg) & PLLEN; + int ret; + + /* Check if one phy port has already configured the pll */ + if (pllen && stm32_usbphyc_has_one_phy_active(usbphyc)) + return 0; + + if (pllen) { + ret = stm32_usbphyc_pll_disable(usbphyc); + if (ret) + return ret; + } + + ret = regulator_bulk_enable(NUM_SUPPLIES, usbphyc->supplies); + if (ret) + return ret; + + ret = stm32_usbphyc_pll_init(usbphyc); + if (ret) + goto reg_disable; + + stm32_usbphyc_set_bits(pll_reg, PLLEN); + /* Wait for maximum lock time */ + udelay(PLL_LOCK_TIME_US); + + if (!(readl_relaxed(pll_reg) & PLLEN)) { + dev_err(usbphyc->dev, "PLLEN not set\n"); + ret = -EIO; + goto reg_disable; + } + return 0; + +reg_disable: + regulator_bulk_disable(NUM_SUPPLIES, usbphyc->supplies); + + return ret; } static int stm32_usbphyc_phy_init(struct phy *phy) @@ -231,25 +240,9 @@ static int stm32_usbphyc_phy_exit(struct phy *phy) return stm32_usbphyc_pll_disable(usbphyc); } -static int stm32_usbphyc_phy_power_on(struct phy *phy) -{ - struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); - - return regulator_bulk_enable(NUM_SUPPLIES, usbphyc_phy->supplies); -} - -static int stm32_usbphyc_phy_power_off(struct phy *phy) -{ - struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); - - return regulator_bulk_disable(NUM_SUPPLIES, usbphyc_phy->supplies); -} - static const struct phy_ops stm32_usbphyc_phy_ops = { .init = stm32_usbphyc_phy_init, .exit = stm32_usbphyc_phy_exit, - .power_on = stm32_usbphyc_phy_power_on, - .power_off = stm32_usbphyc_phy_power_off, .owner = THIS_MODULE, }; @@ -313,7 +306,7 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) struct device_node *child, *np = dev->of_node; struct phy_provider *phy_provider; u32 version; - int ret, port = 0; + int ret, i, port = 0; usbphyc = devm_kzalloc(dev, sizeof(*usbphyc), GFP_KERNEL); if (!usbphyc) @@ -355,11 +348,20 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) goto clk_disable; } + for (i = 0; i < NUM_SUPPLIES; i++) + usbphyc->supplies[i].supply = supplies_names[i]; + + ret = devm_regulator_bulk_get(dev, NUM_SUPPLIES, usbphyc->supplies); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get regulators: %d\n", ret); + goto clk_disable; + } + for_each_child_of_node(np, child) { struct stm32_usbphyc_phy *usbphyc_phy; struct phy *phy; u32 index; - int i; phy = devm_phy_create(dev, child, &stm32_usbphyc_phy_ops); if (IS_ERR(phy)) { @@ -377,18 +379,6 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) goto put_child; } - for (i = 0; i < NUM_SUPPLIES; i++) - usbphyc_phy->supplies[i].supply = supplies_names[i]; - - ret = devm_regulator_bulk_get(&phy->dev, NUM_SUPPLIES, - usbphyc_phy->supplies); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(&phy->dev, - "failed to get regulators: %d\n", ret); - goto put_child; - } - ret = of_property_read_u32(child, "reg", &index); if (ret || index > usbphyc->nphys) { dev_err(&phy->dev, "invalid reg property: %d\n", ret); From 04edf6d6e22b76aacb23d545a9fe642573f73c9b Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:22 +0100 Subject: [PATCH 063/633] phy: stm32: replace regulator_bulk* by multiple regulator_* Due to async_schedule_domain call in regulator_bulk_enable, scheduling while atomic bug can raise if regulator_bulk_enable is called under atomic context. To avoid this issue, this patch replaces all regulator_bulk* by regulator_ per regulators. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20210105090525.23164-4-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 71 ++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index c78a2c7947ce..8ef97c8806ff 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -36,13 +36,6 @@ #define MINREV GENMASK(3, 0) #define MAJREV GENMASK(7, 4) -static const char * const supplies_names[] = { - "vdda1v1", /* 1V1 */ - "vdda1v8", /* 1V8 */ -}; - -#define NUM_SUPPLIES ARRAY_SIZE(supplies_names) - #define PLL_LOCK_TIME_US 100 #define PLL_PWR_DOWN_TIME_US 5 #define PLL_FVCO_MHZ 2880 @@ -69,7 +62,8 @@ struct stm32_usbphyc { struct reset_control *rst; struct stm32_usbphyc_phy **phys; int nphys; - struct regulator_bulk_data supplies[NUM_SUPPLIES]; + struct regulator *vdda1v1; + struct regulator *vdda1v8; int switch_setup; }; @@ -83,6 +77,41 @@ static inline void stm32_usbphyc_clr_bits(void __iomem *reg, u32 bits) writel_relaxed(readl_relaxed(reg) & ~bits, reg); } +static int stm32_usbphyc_regulators_enable(struct stm32_usbphyc *usbphyc) +{ + int ret; + + ret = regulator_enable(usbphyc->vdda1v1); + if (ret) + return ret; + + ret = regulator_enable(usbphyc->vdda1v8); + if (ret) + goto vdda1v1_disable; + + return 0; + +vdda1v1_disable: + regulator_disable(usbphyc->vdda1v1); + + return ret; +} + +static int stm32_usbphyc_regulators_disable(struct stm32_usbphyc *usbphyc) +{ + int ret; + + ret = regulator_disable(usbphyc->vdda1v8); + if (ret) + return ret; + + ret = regulator_disable(usbphyc->vdda1v1); + if (ret) + return ret; + + return 0; +} + static void stm32_usbphyc_get_pll_params(u32 clk_rate, struct pll_params *pll_params) { @@ -170,7 +199,7 @@ static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) return -EIO; } - return regulator_bulk_disable(NUM_SUPPLIES, usbphyc->supplies); + return stm32_usbphyc_regulators_disable(usbphyc); } static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) @@ -189,7 +218,7 @@ static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) return ret; } - ret = regulator_bulk_enable(NUM_SUPPLIES, usbphyc->supplies); + ret = stm32_usbphyc_regulators_enable(usbphyc); if (ret) return ret; @@ -210,7 +239,7 @@ static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) return 0; reg_disable: - regulator_bulk_disable(NUM_SUPPLIES, usbphyc->supplies); + stm32_usbphyc_regulators_disable(usbphyc); return ret; } @@ -306,7 +335,7 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) struct device_node *child, *np = dev->of_node; struct phy_provider *phy_provider; u32 version; - int ret, i, port = 0; + int ret, port = 0; usbphyc = devm_kzalloc(dev, sizeof(*usbphyc), GFP_KERNEL); if (!usbphyc) @@ -348,13 +377,19 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) goto clk_disable; } - for (i = 0; i < NUM_SUPPLIES; i++) - usbphyc->supplies[i].supply = supplies_names[i]; - - ret = devm_regulator_bulk_get(dev, NUM_SUPPLIES, usbphyc->supplies); - if (ret) { + usbphyc->vdda1v1 = devm_regulator_get(dev, "vdda1v1"); + if (IS_ERR(usbphyc->vdda1v1)) { + ret = PTR_ERR(usbphyc->vdda1v1); if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulators: %d\n", ret); + dev_err(dev, "failed to get vdda1v1 supply: %d\n", ret); + goto clk_disable; + } + + usbphyc->vdda1v8 = devm_regulator_get(dev, "vdda1v8"); + if (IS_ERR(usbphyc->vdda1v8)) { + ret = PTR_ERR(usbphyc->vdda1v8); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to get vdda1v8 supply: %d\n", ret); goto clk_disable; } From 56bf858edd17bd4a320aca1a10aea7dbfb0c6699 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:23 +0100 Subject: [PATCH 064/633] phy: stm32: ensure pll is disabled before phys creation To ensure a good balancing of regulators, force PLL disable either by reset or by clearing the PLLEN bit. If waiting the powerdown pulse delay isn't enough, return -EPROBE_DEFER instead of polling the PLLEN bit, which will be low at the next probe. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20210105090525.23164-5-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index 8ef97c8806ff..33367a325612 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -334,7 +334,7 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *child, *np = dev->of_node; struct phy_provider *phy_provider; - u32 version; + u32 pllen, version; int ret, port = 0; usbphyc = devm_kzalloc(dev, sizeof(*usbphyc), GFP_KERNEL); @@ -366,6 +366,19 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) ret = PTR_ERR(usbphyc->rst); if (ret == -EPROBE_DEFER) goto clk_disable; + + stm32_usbphyc_clr_bits(usbphyc->base + STM32_USBPHYC_PLL, PLLEN); + } + + /* + * Wait for minimum width of powerdown pulse (ENABLE = Low): + * we have to ensure the PLL is disabled before phys initialization. + */ + if (readl_relaxed_poll_timeout(usbphyc->base + STM32_USBPHYC_PLL, + pllen, !(pllen & PLLEN), 5, 50)) { + dev_warn(usbphyc->dev, "PLL not reset\n"); + ret = -EPROBE_DEFER; + goto clk_disable; } usbphyc->switch_setup = -EINVAL; From 649627245cc439fddef2096bf646cb3558e13803 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:24 +0100 Subject: [PATCH 065/633] phy: stm32: ensure phy are no more active when removing the driver To ensure a good balancing of regulators, and allow PLL disabling when the driver is removed, call stm32_usbphyc_phy_exit on each ports to set phys inactive and disable PLL. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20210105090525.23164-6-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index 33367a325612..8b11d95b2c20 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -470,6 +470,12 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) static int stm32_usbphyc_remove(struct platform_device *pdev) { struct stm32_usbphyc *usbphyc = dev_get_drvdata(&pdev->dev); + int port; + + /* Ensure PHYs are not active, to allow PLL disabling */ + for (port = 0; port < usbphyc->nphys; port++) + if (usbphyc->phys[port]->active) + stm32_usbphyc_phy_exit(usbphyc->phys[port]->phy); clk_disable_unprepare(usbphyc->clk); From 5b1af71280abd82efbe28cd28d553363dfde0a34 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Tue, 5 Jan 2021 10:05:25 +0100 Subject: [PATCH 066/633] phy: stm32: rework PLL Lock detection USBPHYC has a register per phy to control and monitor the debug interface of the HS PHY through a digital debug access. With this register, it is possible to know if PLL Lock input to phy is high. That means the PLL is ready for HS operation. Instead of using an hard-coded delay after PLL enable and PLL disable, use this bit to ensure good operating of the HS PHY. Also use an atomic counter (n_pll_cons) to count the actual number of PLL consumers and get rid of stm32_usbphyc_has_one_phy_active. The boolean active in the usbphyc_phy structure is kept, because we need to know in remove if a phy_exit is required to properly disable the PLL. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20210105090525.23164-7-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 88 ++++++++++++++++++------------ 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index 8b11d95b2c20..d08fbb180e43 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -17,6 +17,7 @@ #define STM32_USBPHYC_PLL 0x0 #define STM32_USBPHYC_MISC 0x8 +#define STM32_USBPHYC_MONITOR(X) (0x108 + ((X) * 0x100)) #define STM32_USBPHYC_VERSION 0x3F4 /* STM32_USBPHYC_PLL bit fields */ @@ -32,12 +33,16 @@ /* STM32_USBPHYC_MISC bit fields */ #define SWITHOST BIT(0) +/* STM32_USBPHYC_MONITOR bit fields */ +#define STM32_USBPHYC_MON_OUT GENMASK(3, 0) +#define STM32_USBPHYC_MON_SEL GENMASK(8, 4) +#define STM32_USBPHYC_MON_SEL_LOCKP 0x1F +#define STM32_USBPHYC_MON_OUT_LOCKP BIT(3) + /* STM32_USBPHYC_VERSION bit fields */ #define MINREV GENMASK(3, 0) #define MAJREV GENMASK(7, 4) -#define PLL_LOCK_TIME_US 100 -#define PLL_PWR_DOWN_TIME_US 5 #define PLL_FVCO_MHZ 2880 #define PLL_INFF_MIN_RATE_HZ 19200000 #define PLL_INFF_MAX_RATE_HZ 38400000 @@ -64,6 +69,7 @@ struct stm32_usbphyc { int nphys; struct regulator *vdda1v1; struct regulator *vdda1v8; + atomic_t n_pll_cons; int switch_setup; }; @@ -171,35 +177,27 @@ static int stm32_usbphyc_pll_init(struct stm32_usbphyc *usbphyc) return 0; } -static bool stm32_usbphyc_has_one_phy_active(struct stm32_usbphyc *usbphyc) +static int __stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) { - int i; + void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; + u32 pllen; - for (i = 0; i < usbphyc->nphys; i++) - if (usbphyc->phys[i]->active) - return true; + stm32_usbphyc_clr_bits(pll_reg, PLLEN); - return false; + /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ + if (readl_relaxed_poll_timeout(pll_reg, pllen, !(pllen & PLLEN), 5, 50)) + dev_err(usbphyc->dev, "PLL not reset\n"); + + return stm32_usbphyc_regulators_disable(usbphyc); } static int stm32_usbphyc_pll_disable(struct stm32_usbphyc *usbphyc) { - void __iomem *pll_reg = usbphyc->base + STM32_USBPHYC_PLL; - - /* Check if other phy port active */ - if (stm32_usbphyc_has_one_phy_active(usbphyc)) + /* Check if a phy port is still active or clk48 in use */ + if (atomic_dec_return(&usbphyc->n_pll_cons) > 0) return 0; - stm32_usbphyc_clr_bits(pll_reg, PLLEN); - /* Wait for minimum width of powerdown pulse (ENABLE = Low) */ - udelay(PLL_PWR_DOWN_TIME_US); - - if (readl_relaxed(pll_reg) & PLLEN) { - dev_err(usbphyc->dev, "PLL not reset\n"); - return -EIO; - } - - return stm32_usbphyc_regulators_disable(usbphyc); + return __stm32_usbphyc_pll_disable(usbphyc); } static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) @@ -208,39 +206,43 @@ static int stm32_usbphyc_pll_enable(struct stm32_usbphyc *usbphyc) bool pllen = readl_relaxed(pll_reg) & PLLEN; int ret; - /* Check if one phy port has already configured the pll */ - if (pllen && stm32_usbphyc_has_one_phy_active(usbphyc)) + /* + * Check if a phy port or clk48 prepare has configured the pll + * and ensure the PLL is enabled + */ + if (atomic_inc_return(&usbphyc->n_pll_cons) > 1 && pllen) return 0; if (pllen) { - ret = stm32_usbphyc_pll_disable(usbphyc); + /* + * PLL shouldn't be enabled without known consumer, + * disable it and reinit n_pll_cons + */ + dev_warn(usbphyc->dev, "PLL enabled without known consumers\n"); + + ret = __stm32_usbphyc_pll_disable(usbphyc); if (ret) return ret; } ret = stm32_usbphyc_regulators_enable(usbphyc); if (ret) - return ret; + goto dec_n_pll_cons; ret = stm32_usbphyc_pll_init(usbphyc); if (ret) goto reg_disable; stm32_usbphyc_set_bits(pll_reg, PLLEN); - /* Wait for maximum lock time */ - udelay(PLL_LOCK_TIME_US); - - if (!(readl_relaxed(pll_reg) & PLLEN)) { - dev_err(usbphyc->dev, "PLLEN not set\n"); - ret = -EIO; - goto reg_disable; - } return 0; reg_disable: stm32_usbphyc_regulators_disable(usbphyc); +dec_n_pll_cons: + atomic_dec(&usbphyc->n_pll_cons); + return ret; } @@ -248,15 +250,33 @@ static int stm32_usbphyc_phy_init(struct phy *phy) { struct stm32_usbphyc_phy *usbphyc_phy = phy_get_drvdata(phy); struct stm32_usbphyc *usbphyc = usbphyc_phy->usbphyc; + u32 reg_mon = STM32_USBPHYC_MONITOR(usbphyc_phy->index); + u32 monsel = FIELD_PREP(STM32_USBPHYC_MON_SEL, + STM32_USBPHYC_MON_SEL_LOCKP); + u32 monout; int ret; ret = stm32_usbphyc_pll_enable(usbphyc); if (ret) return ret; + /* Check that PLL Lock input to PHY is High */ + writel_relaxed(monsel, usbphyc->base + reg_mon); + ret = readl_relaxed_poll_timeout(usbphyc->base + reg_mon, monout, + (monout & STM32_USBPHYC_MON_OUT_LOCKP), + 100, 1000); + if (ret) { + dev_err(usbphyc->dev, "PLL Lock input to PHY is Low (val=%x)\n", + (u32)(monout & STM32_USBPHYC_MON_OUT)); + goto pll_disable; + } + usbphyc_phy->active = true; return 0; + +pll_disable: + return stm32_usbphyc_pll_disable(usbphyc); } static int stm32_usbphyc_phy_exit(struct phy *phy) From 4540b9fbd8ebb21bb3735796d300a1589ee5fbf2 Mon Sep 17 00:00:00 2001 From: Aswath Govindraju Date: Wed, 13 Jan 2021 10:42:52 +0530 Subject: [PATCH 067/633] misc: eeprom_93xx46: Add module alias to avoid breaking support for non device tree users Module alias "spi:93xx46" is used by non device tree users like drivers/misc/eeprom/digsy_mtc_eeprom.c and removing it will break support for them. Fix this by adding back the module alias "spi:93xx46". Fixes: 13613a2246bf ("misc: eeprom_93xx46: Fix module alias to enable module autoprobe") Signed-off-by: Aswath Govindraju Link: https://lore.kernel.org/r/20210113051253.15061-1-a-govindraju@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/eeprom/eeprom_93xx46.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 849cfea9e294..e130a0d858e9 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -511,5 +511,6 @@ module_spi_driver(eeprom_93xx46_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs"); MODULE_AUTHOR("Anatolij Gustschin "); +MODULE_ALIAS("spi:93xx46"); MODULE_ALIAS("spi:eeprom-93xx46"); MODULE_ALIAS("spi:93lc46b"); From 20612d2428c3cdd191d45d548e930f41785f62cc Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 11 Jan 2021 12:21:13 +0100 Subject: [PATCH 068/633] fpga: dfl-pci: rectify ReST formatting Commit fa41d10589be ("fpga: dfl-pci: locate DFLs by PCIe vendor specific capability") provides documentation to the FPGA Device Feature List (DFL) Framework Overview, but introduced new documentation warnings: ./Documentation/fpga/dfl.rst: 505: WARNING: Title underline too short. 523: WARNING: Unexpected indentation. 523: WARNING: Blank line required after table. 524: WARNING: Block quote ends without a blank line; unexpected unindent. Rectify ReST formatting in ./Documentation/fpga/dfl.rst. Tested-by: Tom Rix Acked-by: Moritz Fischer Acked-by: Matthew Gerlach Signed-off-by: Lukas Bulwahn Link: https://lore.kernel.org/r/20210111112113.27242-1-lukas.bulwahn@gmail.com Signed-off-by: Greg Kroah-Hartman --- Documentation/fpga/dfl.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/fpga/dfl.rst b/Documentation/fpga/dfl.rst index ea8cefc18bdb..c41ac76ffaae 100644 --- a/Documentation/fpga/dfl.rst +++ b/Documentation/fpga/dfl.rst @@ -502,7 +502,7 @@ FME Partial Reconfiguration Sub Feature driver (see drivers/fpga/dfl-fme-pr.c) could be a reference. Location of DFLs on a PCI Device -=========================== +================================ The original method for finding a DFL on a PCI device assumed the start of the first DFL to offset 0 of bar 0. If the first node of the DFL is an FME, then further DFLs in the port(s) are specified in FME header registers. @@ -514,6 +514,7 @@ data begins with a 4 byte vendor specific register for the number of DFLs follow Offset/BIR vendor specific registers for each DFL. Bits 2:0 of Offset/BIR register indicates the BAR, and bits 31:3 form the 8 byte aligned offset where bits 2:0 are zero. +:: +----------------------------+ |31 Number of DFLS 0| From c4e0fec2f7ee013dbf86445394ff47f719408f99 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Fri, 16 Oct 2020 14:04:31 +0200 Subject: [PATCH 069/633] PCI: rcar: Always allocate MSI addresses in 32bit space This fixes MSI operation on legacy PCI cards, which cannot issue 64bit MSIs. The R-Car controller only has one MSI trigger address instead of two, one for 64bit and one for 32bit MSI, set the address to 32bit PCIe space so that legacy PCI cards can also trigger MSIs. Link: https://lore.kernel.org/r/20201016120431.7062-1-marek.vasut@gmail.com Fixes: 290c1fb35860 ("PCI: rcar: Add MSI support for PCIe") Tested-by: Yoshihiro Shimoda Tested-by: Geert Uytterhoeven Signed-off-by: Marek Vasut Signed-off-by: Lorenzo Pieralisi Reviewed-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Cc: Bjorn Helgaas Cc: Geert Uytterhoeven Cc: Lorenzo Pieralisi Cc: Wolfram Sang Cc: Yoshihiro Shimoda Cc: linux-renesas-soc@vger.kernel.org --- drivers/pci/controller/pcie-rcar-host.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 4d1c4b24e537..a728e8f9ad3c 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -735,7 +735,7 @@ static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) } /* setup MSI data target */ - msi->pages = __get_free_pages(GFP_KERNEL, 0); + msi->pages = __get_free_pages(GFP_KERNEL | GFP_DMA32, 0); rcar_pcie_hw_enable_msi(host); return 0; From aa4731c8b5f41bbc7931f44a68e803fcca576dd3 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Mon, 11 Jan 2021 17:00:09 +0530 Subject: [PATCH 070/633] dt-bindings: phy: qcom,qmp: Add SDX55 USB PHY binding Add devicetree YAML binding for Qualcomm QMP Super Speed (SS) PHY found in SDX55. Signed-off-by: Manivannan Sadhasivam Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/20210111113010.32056-2-manivannan.sadhasivam@linaro.org Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/qcom,qmp-phy.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index ec05db374645..0f00d82461fd 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -34,6 +34,7 @@ properties: - qcom,sm8250-qmp-gen3x1-pcie-phy - qcom,sm8250-qmp-gen3x2-pcie-phy - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sdx55-qmp-usb3-uni-phy reg: items: @@ -131,6 +132,32 @@ allOf: items: - const: phy - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sdx55-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: Phy config clock. + - description: 19.2 MHz ref clk. + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common - if: properties: compatible: From 86ef5a79d6bbc755fd764318bd501ee6583961f1 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Mon, 11 Jan 2021 17:00:10 +0530 Subject: [PATCH 071/633] phy: qcom-qmp: Add support for SDX55 QMP PHY Add support for USB3 QMP PHY found in SDX55 platform. SDX55 uses version 4.0.0 of the QMP PHY IP and doesn't make use of "com_aux" clock. Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20210111113010.32056-3-manivannan.sadhasivam@linaro.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 83 +++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index 0939a9e9d448..bdcb8bf6225d 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -1974,6 +1974,53 @@ static const struct qmp_phy_init_tbl sm8250_qmp_gen3x2_pcie_pcs_misc_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCIE_POWER_STATE_CONFIG4, 0x07), }; +static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V4_TX_RCV_DETECT_LVL_2, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_LANE_MODE_1, 0xd5), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_LANE_MODE_2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_PI_QEC_CTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V4_TX_RES_CODE_LANE_OFFSET_TX, 0x08), +}; + +static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH4, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH3, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH2, 0xbf), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_LOW, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH4, 0xb4), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH3, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH2, 0x5c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_LOW, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH1, 0x048), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FO_GAIN, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL2, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4a), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_EN_TIMER, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_DCC_CTRL1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V4_RX_GM_CAL, 0x1f), +}; + /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { /* phy-type - PCIE/UFS/USB */ @@ -2183,6 +2230,11 @@ static const char * const sdm845_ufs_phy_clk_l[] = { "ref", "ref_aux", }; +/* usb3 phy on sdx55 doesn't have com_aux clock */ +static const char * const qmp_v4_sdx55_usbphy_clk_l[] = { + "aux", "cfg_ahb", "ref" +}; + /* list of resets */ static const char * const msm8996_pciephy_reset_l[] = { "phy", "common", "cfg", @@ -2824,6 +2876,34 @@ static const struct qmp_phy_cfg sm8250_usb3_uniphy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; +static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), + .tx_tbl = sdx55_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sdx55_usb3_uniphy_tx_tbl), + .rx_tbl = sdx55_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sdx55_usb3_uniphy_rx_tbl), + .pcs_tbl = sm8250_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8250_usb3_uniphy_pcs_tbl), + .clk_list = qmp_v4_sdx55_usbphy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_sdx55_usbphy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3_uniphy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + static void qcom_qmp_phy_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], @@ -4173,6 +4253,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { }, { .compatible = "qcom,sm8250-qmp-modem-pcie-phy", .data = &sm8250_qmp_gen3x2_pciephy_cfg, + }, { + .compatible = "qcom,sdx55-qmp-usb3-uni-phy", + .data = &sdx55_usb3_uniphy_cfg, }, { }, }; From 3f0ea2360e4826b3aba42f9892bda15b1e38bdf4 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Fri, 15 Jan 2021 22:24:33 +0100 Subject: [PATCH 072/633] PCI: altera-msi: Remove IRQ handler and data in one go Call irq_set_chained_handler_and_data() to clear the chained handler and the handler's data under irq_desc->lock. See also 2cf5a03cb29d ("PCI/keystone: Fix race in installing chained IRQ handler"). Link: https://lore.kernel.org/r/20210115212435.19940-1-martin@kaiser.cx Signed-off-by: Martin Kaiser Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/pcie-altera-msi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index e1636f7714ca..42691dd8ebef 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -204,8 +204,7 @@ static int altera_msi_remove(struct platform_device *pdev) struct altera_msi *msi = platform_get_drvdata(pdev); msi_writel(msi, 0, MSI_INTMASK); - irq_set_chained_handler(msi->irq, NULL); - irq_set_handler_data(msi->irq, NULL); + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); altera_free_domains(msi); From ad1cc6b75a79471d14b64c32bf13067659ba2bac Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Fri, 15 Jan 2021 22:24:34 +0100 Subject: [PATCH 073/633] PCI: dwc: Remove IRQ handler and data in one go Call irq_set_chained_handler_and_data() to clear the chained handler and the handler's data under irq_desc->lock. See also 2cf5a03cb29d ("PCI/keystone: Fix race in installing chained IRQ handler"). Link: https://lore.kernel.org/r/20210115212435.19940-2-martin@kaiser.cx Signed-off-by: Martin Kaiser Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/dwc/pcie-designware-host.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 8a84c005f32b..3837fff48944 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -258,10 +258,8 @@ int dw_pcie_allocate_domains(struct pcie_port *pp) static void dw_pcie_free_msi(struct pcie_port *pp) { - if (pp->msi_irq) { - irq_set_chained_handler(pp->msi_irq, NULL); - irq_set_handler_data(pp->msi_irq, NULL); - } + if (pp->msi_irq) + irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL); irq_domain_remove(pp->msi_domain); irq_domain_remove(pp->irq_domain); From a93c00e5f975f23592895b7e83f35de2d36b7633 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Fri, 15 Jan 2021 22:24:35 +0100 Subject: [PATCH 074/633] PCI: xgene-msi: Fix race in installing chained irq handler Fix a race where a pending interrupt could be received and the handler called before the handler's data has been setup, by converting to irq_set_chained_handler_and_data(). See also 2cf5a03cb29d ("PCI/keystone: Fix race in installing chained IRQ handler"). Based on the mail discussion, it seems ok to drop the error handling. Link: https://lore.kernel.org/r/20210115212435.19940-3-martin@kaiser.cx Signed-off-by: Martin Kaiser Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/pci-xgene-msi.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c index 2470782cb01a..1c34c897a7e2 100644 --- a/drivers/pci/controller/pci-xgene-msi.c +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -384,13 +384,9 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu) if (!msi_group->gic_irq) continue; - irq_set_chained_handler(msi_group->gic_irq, - xgene_msi_isr); - err = irq_set_handler_data(msi_group->gic_irq, msi_group); - if (err) { - pr_err("failed to register GIC IRQ handler\n"); - return -EINVAL; - } + irq_set_chained_handler_and_data(msi_group->gic_irq, + xgene_msi_isr, msi_group); + /* * Statically allocate MSI GIC IRQs to each CPU core. * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated From f435ce7ebf8c6fa7d962a34842a57500d6db5733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Thu, 10 Dec 2020 19:04:20 +0100 Subject: [PATCH 075/633] dt-bindings: PCI: brcmstb: add BCM4908 binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BCM4908 is a SoC family with PCIe controller sharing design with the one for STB. BCM4908 has different power management and memory controller so few tweaks are required. PERST# signal on BCM4908 is handled by an external MISC block so it needs specifying a reset phandle. Link: https://lore.kernel.org/r/20201210180421.7230-2-zajec5@gmail.com Signed-off-by: Rafał Miłecki Signed-off-by: Lorenzo Pieralisi Reviewed-by: Rob Herring Acked-by: Florian Fainelli --- .../bindings/pci/brcm,stb-pcie.yaml | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 807694b4f41f..f90557f6deb8 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -14,6 +14,7 @@ properties: items: - enum: - brcm,bcm2711-pcie # The Raspberry Pi 4 + - brcm,bcm4908-pcie - brcm,bcm7211-pcie # Broadcom STB version of RPi4 - brcm,bcm7278-pcie # Broadcom 7278 Arm - brcm,bcm7216-pcie # Broadcom 7216 Arm @@ -63,15 +64,6 @@ properties: aspm-no-l0s: true - resets: - description: for "brcm,bcm7216-pcie", must be a valid reset - phandle pointing to the RESCAL reset controller provider node. - $ref: "/schemas/types.yaml#/definitions/phandle" - - reset-names: - items: - - const: rescal - brcm,scb-sizes: description: u64 giving the 64bit PCIe memory viewport size of a memory controller. There may be up to @@ -98,12 +90,39 @@ required: allOf: - $ref: /schemas/pci/pci-bus.yaml# + - if: + properties: + compatible: + contains: + const: brcm,bcm4908-pcie + then: + properties: + resets: + items: + - description: reset controller handling the PERST# signal + + reset-names: + items: + - const: perst + + required: + - resets + - reset-names - if: properties: compatible: contains: const: brcm,bcm7216-pcie then: + properties: + resets: + items: + - description: phandle pointing to the RESCAL reset controller + + reset-names: + items: + - const: rescal + required: - resets - reset-names From 0cdfaceb9889b69d0230b82ae91c46ed0b33fc27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Thu, 10 Dec 2020 19:04:21 +0100 Subject: [PATCH 076/633] PCI: brcmstb: support BCM4908 with external PERST# signal controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BCM4908 uses external MISC block for controlling PERST# signal. Use it as a reset controller. Link: https://lore.kernel.org/r/20201210180421.7230-3-zajec5@gmail.com Signed-off-by: Rafał Miłecki Signed-off-by: Lorenzo Pieralisi Acked-by: Florian Fainelli --- drivers/pci/controller/Kconfig | 2 +- drivers/pci/controller/pcie-brcmstb.c | 32 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa..d44c70bb88f6 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -273,7 +273,7 @@ config VMD config PCIE_BRCMSTB tristate "Broadcom Brcmstb PCIe host controller" - depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST + depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN default ARCH_BRCMSTB diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index d41257f43a8f..0d21c83bc3be 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -97,6 +97,7 @@ #define PCIE_MISC_REVISION 0x406c #define BRCM_PCIE_HW_REV_33 0x0303 +#define BRCM_PCIE_HW_REV_3_20 0x0320 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 @@ -187,6 +188,7 @@ struct brcm_pcie; static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val); +static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val); static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val); @@ -203,6 +205,7 @@ enum { enum pcie_type { GENERIC, + BCM4908, BCM7278, BCM2711, }; @@ -227,6 +230,13 @@ static const struct pcie_cfg_data generic_cfg = { .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, }; +static const struct pcie_cfg_data bcm4908_cfg = { + .offsets = pcie_offsets, + .type = BCM4908, + .perst_set = brcm_pcie_perst_set_4908, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, +}; + static const int pcie_offset_bcm7278[] = { [RGR1_SW_INIT_1] = 0xc010, [EXT_CFG_INDEX] = 0x9000, @@ -279,6 +289,7 @@ struct brcm_pcie { const int *reg_offsets; enum pcie_type type; struct reset_control *rescal; + struct reset_control *perst_reset; int num_memc; u64 memc_size[PCIE_BRCM_MAX_MEMC]; u32 hw_rev; @@ -735,6 +746,17 @@ static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); } +static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) +{ + if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n")) + return; + + if (val) + reset_control_assert(pcie->perst_reset); + else + reset_control_deassert(pcie->perst_reset); +} + static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) { u32 tmp; @@ -1194,6 +1216,7 @@ static int brcm_pcie_remove(struct platform_device *pdev) static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, + { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, { .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg }, @@ -1250,6 +1273,11 @@ static int brcm_pcie_probe(struct platform_device *pdev) clk_disable_unprepare(pcie->clk); return PTR_ERR(pcie->rescal); } + pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst"); + if (IS_ERR(pcie->perst_reset)) { + clk_disable_unprepare(pcie->clk); + return PTR_ERR(pcie->perst_reset); + } ret = reset_control_deassert(pcie->rescal); if (ret) @@ -1267,6 +1295,10 @@ static int brcm_pcie_probe(struct platform_device *pdev) goto fail; pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); + if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { + dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n"); + goto fail; + } msi_np = of_parse_phandle(pcie->np, "msi-parent", 0); if (pci_msi_enabled() && msi_np == pcie->np) { From ff591f7490cffef49c022e802b64499edb6137b6 Mon Sep 17 00:00:00 2001 From: Lad Prabhakar Date: Tue, 29 Dec 2020 17:08:48 +0000 Subject: [PATCH 077/633] PCI: Drop PCIE_RCAR config option All the defconfig files have replaced PCIE_RCAR config option with PCIE_RCAR_HOST config option which built the same driver, so we can now safely drop PCIE_RCAR config option. Link: https://lore.kernel.org/r/20201229170848.18482-1-prabhakar.mahadev-lad.rj@bp.renesas.com Signed-off-by: Lad Prabhakar Signed-off-by: Lorenzo Pieralisi Reviewed-by: Geert Uytterhoeven --- drivers/pci/controller/Kconfig | 9 --------- 1 file changed, 9 deletions(-) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa..0d98d8dd448b 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -55,15 +55,6 @@ config PCI_RCAR_GEN2 There are 3 internal PCI controllers available with a single built-in EHCI/OHCI host controller present on each one. -config PCIE_RCAR - bool "Renesas R-Car PCIe controller" - depends on ARCH_RENESAS || COMPILE_TEST - depends on PCI_MSI_IRQ_DOMAIN - select PCIE_RCAR_HOST - help - Say Y here if you want PCIe controller support on R-Car SoCs. - This option will be removed after arm64 defconfig is updated. - config PCIE_RCAR_HOST bool "Renesas R-Car PCIe host controller" depends on ARCH_RENESAS || COMPILE_TEST From 5ce6697a4460d06d4ea3ec8347974176591e1694 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Fri, 15 Jan 2021 22:15:32 +0100 Subject: [PATCH 078/633] PCI: brcmstb: Remove chained IRQ handler and data in one go Call irq_set_chained_handler_and_data() to clear the chained handler and the handler's data under irq_desc->lock. See also 2cf5a03cb29d ("PCI/keystone: Fix race in installing chained IRQ handler"). Link: https://lore.kernel.org/r/20210115211532.19837-1-martin@kaiser.cx Signed-off-by: Martin Kaiser Signed-off-by: Lorenzo Pieralisi Acked-by: Florian Fainelli Acked-by: Nicolas Saenz Julienne --- drivers/pci/controller/pcie-brcmstb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index d41257f43a8f..95f6dd93ceae 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -603,8 +603,7 @@ static void brcm_msi_remove(struct brcm_pcie *pcie) if (!msi) return; - irq_set_chained_handler(msi->irq, NULL); - irq_set_handler_data(msi->irq, NULL); + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); brcm_free_domains(msi); } From 0cff991179918f5618fdac7d65b7149bff6b3747 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 15 Jan 2021 14:16:51 +0800 Subject: [PATCH 079/633] soundwire: cadence: reduce timeout on transactions Currently the timeout for SoundWire individual transactions is 2s. This is too large in comparison with the enumeration and completion timeouts used in codec drivers. A command will typically be handled in less than 100us, so 500ms for the command completion is more than generous. Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115061651.9740-3-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/cadence_master.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index 9fa55164354a..f0b0ec173f8b 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -188,7 +188,7 @@ MODULE_PARM_DESC(cdns_mcp_int_mask, "Cadence MCP IntMask"); #define CDNS_PDI_CONFIG_PORT GENMASK(4, 0) /* Driver defaults */ -#define CDNS_TX_TIMEOUT 2000 +#define CDNS_TX_TIMEOUT 500 #define CDNS_SCP_RX_FIFOLEVEL 0x2 From 565e3afaefee3cfe13783c2b1e8d5ceb09023beb Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sun, 17 Jan 2021 23:16:22 +0100 Subject: [PATCH 080/633] soundwire: sysfs: Constify static struct attribute_group The only place sdw_slave_dev_attr_group is used is when its address is passed to devm_device_add_group() which takes a pointer to const struct attribute_group. Make it const to allow the compiler to put it in read-only memory. This makes all attribute_group structs in the file const. Done with the help of Coccinelle. Signed-off-by: Rikard Falkeborn Link: https://lore.kernel.org/r/20210117221622.34315-1-rikard.falkeborn@gmail.com Signed-off-by: Vinod Koul --- drivers/soundwire/sysfs_slave.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soundwire/sysfs_slave.c b/drivers/soundwire/sysfs_slave.c index b48b6617a396..3210359cd944 100644 --- a/drivers/soundwire/sysfs_slave.c +++ b/drivers/soundwire/sysfs_slave.c @@ -130,7 +130,7 @@ static struct attribute *slave_dev_attrs[] = { * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory * for device-level properties */ -static struct attribute_group sdw_slave_dev_attr_group = { +static const struct attribute_group sdw_slave_dev_attr_group = { .attrs = slave_dev_attrs, .name = "dev-properties", }; From c219624c50d57e45d0e9d5345eb1c47ab5523b85 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Sun, 17 Jan 2021 12:42:58 +0530 Subject: [PATCH 081/633] MAINTAINERS: soundwire: Add soundwire tree Somehow the soundwire MAINTAINERS did not have the tree details, so add it. Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20210117071258.2541484-1-vkoul@kernel.org Signed-off-by: Vinod Koul --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 546aa66428c9..10b234688f22 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16654,6 +16654,7 @@ R: Pierre-Louis Bossart R: Sanyog Kale L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git F: Documentation/driver-api/soundwire/ F: drivers/soundwire/ F: include/linux/soundwire/ From 6d5e7af1f6f5924de5dd1ebe97675c2363100878 Mon Sep 17 00:00:00 2001 From: Srinivas Kandagatla Date: Fri, 15 Jan 2021 16:25:59 +0000 Subject: [PATCH 082/633] soundwire: debugfs: use controller id instead of link_id link_id can be zero and if we have multiple controller instances in a system like Qualcomm debugfs will end-up with duplicate namespace resulting in incorrect debugfs entries. Using id should give a unique debugfs directory entry and should fix below warning too. "debugfs: Directory 'master-0' with parent 'soundwire' already present!" Fixes: bf03473d5bcc ("soundwire: add debugfs support") Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210115162559.20869-1-srinivas.kandagatla@linaro.org Signed-off-by: Vinod Koul --- drivers/soundwire/debugfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soundwire/debugfs.c b/drivers/soundwire/debugfs.c index b6cad0d59b7b..5f9efa42bb25 100644 --- a/drivers/soundwire/debugfs.c +++ b/drivers/soundwire/debugfs.c @@ -19,7 +19,7 @@ void sdw_bus_debugfs_init(struct sdw_bus *bus) return; /* create the debugfs master-N */ - snprintf(name, sizeof(name), "master-%d", bus->link_id); + snprintf(name, sizeof(name), "master-%d", bus->id); bus->debugfs = debugfs_create_dir(name, sdw_debugfs_root); } From ee3db942432c9f9410345fb9d56aa8f81fbec862 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Thu, 14 Jan 2021 11:02:48 +0800 Subject: [PATCH 083/633] soundwire: intel: don't return error when clock stop failed dev->power.runtime_error will be set to the return value of the runtime suspend callback function, and runtime resume function will return -EINVAL if dev->power.runtime_error is not 0. Somehow the codec rarely doesn't return an ACK to the clock prepare command. If we stop the runtime suspend process and return error, we will not be able to resume again. Likewise, if the codec lost sync and did not rejoin, the resume operation will also fail. As a result, the SoundWire bus can not be used anymore. This patch suggests to finish the runtime suspend process even if we fail to stop sdw bus clock. In the case where we do a hardware reset, the codecs will be reconfigured completely. In the case where we use the regular clock stop, the codecs keep their state and worst case will fall off the bus and reattach. The only drawback is that the power consumption may be higher and device-initiated interrupts may be lost, but at least audio function can still work after resume. Signed-off-by: Bard Liao Reviewed-by: Rander Wang Reviewed-by: Kai Vehmanen Reviewed-by: Pierre-Louis Bossart Link: https://lore.kernel.org/r/20210114030248.9005-1-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 10b60b7b17bb..a2d5cdaa9998 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -1673,10 +1673,12 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev) } else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET || !clock_stop_quirks) { + bool wake_enable = true; + ret = sdw_cdns_clock_stop(cdns, true); if (ret < 0) { dev_err(dev, "cannot enable clock stop on suspend\n"); - return ret; + wake_enable = false; } ret = sdw_cdns_enable_interrupt(cdns, false); @@ -1691,7 +1693,7 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev) return ret; } - intel_shim_wake(sdw, true); + intel_shim_wake(sdw, wake_enable); } else { dev_err(dev, "%s clock_stop_quirks %x unsupported\n", __func__, clock_stop_quirks); From c397efb77d814774285fe7db9613ca736b0c732e Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 15 Jan 2021 13:37:34 +0800 Subject: [PATCH 084/633] soundwire: use consistent format for Slave devID logs We mix decimal and hexadecimal values, this leads to confusions in dmesg logs and bug reports. Let's add a 0x prefix for all hexadecimal values and a format when more than 4 bits are used. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115053738.22630-2-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 5 ++--- drivers/soundwire/slave.c | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index d1e8c3a54976..3cc006bfae71 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -679,9 +679,8 @@ void sdw_extract_slave_id(struct sdw_bus *bus, id->class_id = SDW_CLASS_ID(addr); dev_dbg(bus->dev, - "SDW Slave class_id %x, part_id %x, mfg_id %x, unique_id %x, version %x\n", - id->class_id, id->part_id, id->mfg_id, - id->unique_id, id->sdw_version); + "SDW Slave class_id 0x%02x, mfg_id 0x%04x, part_id 0x%04x, unique_id 0x%x, version 0x%x\n", + id->class_id, id->mfg_id, id->part_id, id->unique_id, id->sdw_version); } static int sdw_program_device_num(struct sdw_bus *bus) diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c index a08f4081c1c4..180f38bd003b 100644 --- a/drivers/soundwire/slave.c +++ b/drivers/soundwire/slave.c @@ -163,15 +163,13 @@ int sdw_acpi_find_slaves(struct sdw_bus *bus) if (id.unique_id != id2.unique_id) { dev_dbg(bus->dev, - "Valid unique IDs %x %x for Slave mfg %x part %d\n", - id.unique_id, id2.unique_id, - id.mfg_id, id.part_id); + "Valid unique IDs 0x%x 0x%x for Slave mfg_id 0x%04x, part_id 0x%04x\n", + id.unique_id, id2.unique_id, id.mfg_id, id.part_id); ignore_unique_id = false; } else { dev_err(bus->dev, - "Invalid unique IDs %x %x for Slave mfg %x part %d\n", - id.unique_id, id2.unique_id, - id.mfg_id, id.part_id); + "Invalid unique IDs 0x%x 0x%x for Slave mfg_id 0x%04x, part_id 0x%04x\n", + id.unique_id, id2.unique_id, id.mfg_id, id.part_id); return -ENODEV; } } From 6f206833cec60fbab6a86bd886ef40b917e7d50d Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 15 Jan 2021 13:37:35 +0800 Subject: [PATCH 085/633] soundwire: cadence: add status in dev_dbg 'State change' log The existing debug log only mentions a state change, without providing any details. For integration and stress-tests, it's helpful to see in the dmesg log the reason for the state change. The value is intended for power users and isn't converted as human-readable values. But for the record each device has a 4-bit status: BIT(0): Unattached BIT(1): Attached BIT(2): Alert BIT(3): Reserved (should not happen) Example: [ 121.891288] intel-sdw intel-sdw.0: Slave status change: 0x2 << this shows a Device0 Attached [ 121.891295] soundwire sdw-master-0: Slave attached, programming device number [ 121.891629] soundwire sdw-master-0: SDW Slave Addr: 30025d071101 [ 121.891632] soundwire sdw-master-0: SDW Slave class_id 1, part_id 711, mfg_id 25d, unique_id 0, version 3 [ 121.892011] intel-sdw intel-sdw.0: Msg ignored for Slave 0 [ 121.892013] soundwire sdw-master-0: No more devices to enumerate [ 121.892200] intel-sdw intel-sdw.0: Slave status change: 0x21 << this shows the device now Attached as Device1 and Unattached as Device0, i.e. a successful enumeration. Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115053738.22630-3-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/cadence_master.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index f0b0ec173f8b..3a363ba4bfef 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -734,21 +734,18 @@ static void cdns_read_response(struct sdw_cdns *cdns) } static int cdns_update_slave_status(struct sdw_cdns *cdns, - u32 slave0, u32 slave1) + u64 slave_intstat) { enum sdw_slave_status status[SDW_MAX_DEVICES + 1]; bool is_slave = false; - u64 slave; u32 mask; int i, set_status; - /* combine the two status */ - slave = ((u64)slave1 << 32) | slave0; memset(status, 0, sizeof(status)); for (i = 0; i <= SDW_MAX_DEVICES; i++) { - mask = (slave >> (i * CDNS_MCP_SLAVE_STATUS_NUM)) & - CDNS_MCP_SLAVE_STATUS_BITS; + mask = (slave_intstat >> (i * CDNS_MCP_SLAVE_STATUS_NUM)) & + CDNS_MCP_SLAVE_STATUS_BITS; if (!mask) continue; @@ -918,13 +915,17 @@ static void cdns_update_slave_status_work(struct work_struct *work) struct sdw_cdns *cdns = container_of(work, struct sdw_cdns, work); u32 slave0, slave1; - - dev_dbg_ratelimited(cdns->dev, "Slave status change\n"); + u64 slave_intstat; slave0 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT0); slave1 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT1); - cdns_update_slave_status(cdns, slave0, slave1); + /* combine the two status */ + slave_intstat = ((u64)slave1 << 32) | slave0; + + dev_dbg_ratelimited(cdns->dev, "Slave status change: 0x%llx\n", slave_intstat); + + cdns_update_slave_status(cdns, slave_intstat); cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT0, slave0); cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT1, slave1); From ec47518742c6c495ec4b4e2bb11c313ef732fc04 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 15 Jan 2021 13:37:36 +0800 Subject: [PATCH 086/633] soundwire: bus: add more details to track failed transfers The current error log does not provide details on the type of transfers and which address/count was requested. All this information can help locate in which parts of the configuration process an error occurred. Co-developed-by: Pierre-Louis Bossart Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115053738.22630-4-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 3cc006bfae71..6e1c988f3845 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -267,8 +267,10 @@ static int sdw_transfer_unlocked(struct sdw_bus *bus, struct sdw_msg *msg) ret = do_transfer(bus, msg); if (ret != 0 && ret != -ENODATA) - dev_err(bus->dev, "trf on Slave %d failed:%d\n", - msg->dev_num, ret); + dev_err(bus->dev, "trf on Slave %d failed:%d %s addr %x count %d\n", + msg->dev_num, ret, + (msg->flags & SDW_MSG_FLAG_WRITE) ? "write" : "read", + msg->addr, msg->len); if (msg->page) sdw_reset_page(bus, msg->dev_num); From db9d9f944f95e7f3aa60ac00cbd502415152c421 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 15 Jan 2021 13:37:37 +0800 Subject: [PATCH 087/633] soundwire: cadence: fix ACK/NAK handling The existing code reports a NAK only when ACK=0 This is not aligned with the SoundWire 1.x specifications. Table 32 in the SoundWire 1.2 specification shows that a Device shall not set NAK=1 if ACK=1. But Table 33 shows the Combined Response may very well be NAK=1/ACK=1, e.g. if another Device than the one addressed reports a parity error. NAK=1 signals a 'Command_Aborted', regardless of the ACK bit value. Move the tests for NAK so that the NAK=1/ACK=1 combination is properly detected according to the specification. Fixes: 956baa1992f9a ('soundwire: cdns: Add sdw_master_ops and IO transfer support') Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115053738.22630-5-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/cadence_master.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index 3a363ba4bfef..be6c9eaacef9 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -484,10 +484,10 @@ cdns_fill_msg_resp(struct sdw_cdns *cdns, if (!(cdns->response_buf[i] & CDNS_MCP_RESP_ACK)) { no_ack = 1; dev_dbg_ratelimited(cdns->dev, "Msg Ack not received\n"); - if (cdns->response_buf[i] & CDNS_MCP_RESP_NACK) { - nack = 1; - dev_err_ratelimited(cdns->dev, "Msg NACK received\n"); - } + } + if (cdns->response_buf[i] & CDNS_MCP_RESP_NACK) { + nack = 1; + dev_err_ratelimited(cdns->dev, "Msg NACK received\n"); } } From 9a0c798c7a98d7c9db2d24c12f93ff72fa6ce7c4 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 15 Jan 2021 13:37:38 +0800 Subject: [PATCH 088/633] soundwire: cadence: adjust verbosity in response handling There are too many logs on startup, e.g. [ 8811.851497] cdns_fill_msg_resp: 2 callbacks suppressed [ 8811.851497] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851498] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851499] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851499] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851500] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851500] intel-sdw intel-sdw.0: Msg Ack not received [ 8811.851502] intel-sdw intel-sdw.0: Msg ignored for Slave 0 [ 8811.851503] soundwire sdw-master-0: No more devices to enumerate We can skip the 'Msg Ack not received' since it's typical of the enumeration end, and conversely add the information on which command fails. Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210115053738.22630-6-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/cadence_master.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index be6c9eaacef9..d05442e646a3 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -483,11 +483,11 @@ cdns_fill_msg_resp(struct sdw_cdns *cdns, for (i = 0; i < count; i++) { if (!(cdns->response_buf[i] & CDNS_MCP_RESP_ACK)) { no_ack = 1; - dev_dbg_ratelimited(cdns->dev, "Msg Ack not received\n"); + dev_vdbg(cdns->dev, "Msg Ack not received, cmd %d\n", i); } if (cdns->response_buf[i] & CDNS_MCP_RESP_NACK) { nack = 1; - dev_err_ratelimited(cdns->dev, "Msg NACK received\n"); + dev_err_ratelimited(cdns->dev, "Msg NACK received, cmd %d\n", i); } } From c149ced37667da5c0ca32c7ca67b9fe38fa07562 Mon Sep 17 00:00:00 2001 From: Jack Pham Date: Fri, 15 Jan 2021 09:47:20 -0800 Subject: [PATCH 089/633] dt-bindings: phy: qcom,qmp: Add SM8150, SM8250 and SM8350 USB PHY bindings Add the compatible strings for the USB3 PHYs found on SM8150, SM8250 and SM8350 SoCs. These require separate subschemas due to the different required clock entries. Note the SM8150 and SM8250 compatibles have already been in place in the dts as well as the driver implementation but were missing from the documentation. Signed-off-by: Jack Pham Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210115174723.7424-2-jackp@codeaurora.org Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/qcom,qmp-phy.yaml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index 0f00d82461fd..f578677886c4 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -30,15 +30,24 @@ properties: - qcom,sdm845-qmp-ufs-phy - qcom,sdm845-qmp-usb3-uni-phy - qcom,sm8150-qmp-ufs-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy - qcom,sm8250-qmp-ufs-phy - qcom,sm8250-qmp-gen3x1-pcie-phy - qcom,sm8250-qmp-gen3x2-pcie-phy - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-uni-phy - qcom,sdx55-qmp-usb3-uni-phy reg: + minItems: 1 + maxItems: 2 items: - description: Address and length of PHY's common serdes block. + - description: Address and length of PHY's DP_COM control block. "#clock-cells": enum: [ 1, 2 ] @@ -312,6 +321,64 @@ allOf: reset-names: items: - const: phy + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk source. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: ref + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common examples: - | From 10c744d48d7f01b0d0c954d8d71aebd07705a7b9 Mon Sep 17 00:00:00 2001 From: Jack Pham Date: Fri, 15 Jan 2021 09:47:21 -0800 Subject: [PATCH 090/633] phy: qcom-qmp: Add SM8350 USB QMP PHYs Add support for the USB DP & UNI PHYs found on SM8350. These use version 5.0.0 of the QMP PHY IP and thus require new "V5" definitions of the register offset macros for the QSERDES RX and TX blocks. The QSERDES common and QPHY PCS blocks' register offsets are largely unchanged from V4 so some of the existing macros can be reused. Signed-off-by: Jack Pham Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210115174723.7424-3-jackp@codeaurora.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 212 ++++++++++++++++++++++++++++ drivers/phy/qualcomm/phy-qcom-qmp.h | 100 +++++++++++++ 2 files changed, 312 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index bdcb8bf6225d..a679b5fb7b48 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -212,6 +212,15 @@ static const unsigned int qmp_v4_usb3_uniphy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = 0x614, }; +static const unsigned int sm8350_usb3_uniphy_regs_layout[QPHY_LAYOUT_SIZE] = { + [QPHY_SW_RESET] = 0x00, + [QPHY_START_CTRL] = 0x44, + [QPHY_PCS_STATUS] = 0x14, + [QPHY_PCS_POWER_DOWN_CONTROL] = 0x40, + [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = 0x1008, + [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = 0x1014, +}; + static const unsigned int sdm845_ufsphy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_START_CTRL] = 0x00, [QPHY_PCS_READY_STATUS] = 0x160, @@ -2021,6 +2030,144 @@ static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_rx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V4_RX_GM_CAL, 0x1f), }; +static const struct qmp_phy_init_tbl sm8350_usb3_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_TX, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_RX, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0x35), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_4, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_5, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RCV_DETECT_LVL_2, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PI_QEC_CTRL, 0x21), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0xbb), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xbb), + QMP_PHY_INIT_CFG_LANE(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3d, 1), + QMP_PHY_INIT_CFG_LANE(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3c, 2), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0xdb), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xd2), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x13), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xa9), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_EN_TIMER, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_AUX_DATA_TCOARSE_TFINE, 0xa0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DCC_CTRL1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_GM_CAL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VTH_CODE, 0x10), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_L, 0x40), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_H, 0x00), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0xa5), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_2, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_4, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PI_QEC_CTRL, 0x21), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0xbd), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xa9), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xe4), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_GM_CAL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_ENABLES, 0x00), +}; + +static const struct qmp_phy_init_tbl sm8350_usb3_uniphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_UNI_RXEQTRAINING_DFE_TIME_S2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_UNI_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21), +}; + /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { /* phy-type - PCIE/UFS/USB */ @@ -2904,6 +3051,65 @@ static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; +static const struct qmp_phy_cfg sm8350_usb3phy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), + .tx_tbl = sm8350_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_usb3_tx_tbl), + .rx_tbl = sm8350_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_usb3_rx_tbl), + .pcs_tbl = sm8350_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_usb3_pcs_tbl), + .clk_list = qmp_v4_sm8250_usbphy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_sm8250_usbphy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, + + .has_phy_dp_com_ctrl = true, + .is_dual_lane_phy = true, +}; + +static const struct qmp_phy_cfg sm8350_usb3_uniphy_cfg = { + .type = PHY_TYPE_USB3, + .nlanes = 1, + + .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), + .tx_tbl = sm8350_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_tx_tbl), + .rx_tbl = sm8350_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_rx_tbl), + .pcs_tbl = sm8350_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_usb3_uniphy_pcs_tbl), + .clk_list = qmp_v4_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = sm8350_usb3_uniphy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + static void qcom_qmp_phy_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], @@ -4256,6 +4462,12 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { }, { .compatible = "qcom,sdx55-qmp-usb3-uni-phy", .data = &sdx55_usb3_uniphy_cfg, + }, { + .compatible = "qcom,sm8350-qmp-usb3-phy", + .data = &sm8350_usb3phy_cfg, + }, { + .compatible = "qcom,sm8350-qmp-usb3-uni-phy", + .data = &sm8350_usb3_uniphy_cfg, }, { }, }; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index db92a461dd2e..dff7be5a1cc1 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -824,4 +824,104 @@ #define QPHY_V4_PCS_PCIE_PRESET_P10_PRE 0xbc #define QPHY_V4_PCS_PCIE_PRESET_P10_POST 0xe0 +/* Only for QMP V5 PHY - TX registers */ +#define QSERDES_V5_TX_RES_CODE_LANE_TX 0x34 +#define QSERDES_V5_TX_RES_CODE_LANE_RX 0x38 +#define QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX 0x3c +#define QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX 0x40 +#define QSERDES_V5_TX_LANE_MODE_1 0x84 +#define QSERDES_V5_TX_LANE_MODE_2 0x88 +#define QSERDES_V5_TX_LANE_MODE_3 0x8c +#define QSERDES_V5_TX_LANE_MODE_4 0x90 +#define QSERDES_V5_TX_LANE_MODE_5 0x94 +#define QSERDES_V5_TX_RCV_DETECT_LVL_2 0xa4 +#define QSERDES_V5_TX_TRAN_DRVR_EMP_EN 0xc0 +#define QSERDES_V5_TX_PI_QEC_CTRL 0xe4 + +/* Only for QMP V5 PHY - RX registers */ +#define QSERDES_V5_RX_UCDR_FO_GAIN 0x008 +#define QSERDES_V5_RX_UCDR_SO_GAIN 0x014 +#define QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN 0x030 +#define QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE 0x034 +#define QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW 0x03c +#define QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH 0x040 +#define QSERDES_V5_RX_UCDR_PI_CONTROLS 0x044 +#define QSERDES_V5_RX_UCDR_PI_CTRL2 0x048 +#define QSERDES_V5_RX_UCDR_SB2_THRESH1 0x04c +#define QSERDES_V5_RX_UCDR_SB2_THRESH2 0x050 +#define QSERDES_V5_RX_UCDR_SB2_GAIN1 0x054 +#define QSERDES_V5_RX_UCDR_SB2_GAIN2 0x058 +#define QSERDES_V5_RX_AUX_DATA_TCOARSE_TFINE 0x060 +#define QSERDES_V5_RX_RCLK_AUXDATA_SEL 0x064 +#define QSERDES_V5_RX_AC_JTAG_ENABLE 0x068 +#define QSERDES_V5_RX_AC_JTAG_MODE 0x078 +#define QSERDES_V5_RX_RX_TERM_BW 0x080 +#define QSERDES_V5_RX_VGA_CAL_CNTRL1 0x0d4 +#define QSERDES_V5_RX_VGA_CAL_CNTRL2 0x0d8 +#define QSERDES_V5_RX_GM_CAL 0x0dc +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL1 0x0e8 +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2 0x0ec +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3 0x0f0 +#define QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4 0x0f4 +#define QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW 0x0f8 +#define QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH 0x0fc +#define QSERDES_V5_RX_RX_IDAC_MEASURE_TIME 0x100 +#define QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x110 +#define QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2 0x114 +#define QSERDES_V5_RX_SIGDET_ENABLES 0x118 +#define QSERDES_V5_RX_SIGDET_CNTRL 0x11c +#define QSERDES_V5_RX_SIGDET_LVL 0x120 +#define QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL 0x124 +#define QSERDES_V5_RX_RX_BAND 0x128 +#define QSERDES_V5_RX_RX_MODE_00_LOW 0x15c +#define QSERDES_V5_RX_RX_MODE_00_HIGH 0x160 +#define QSERDES_V5_RX_RX_MODE_00_HIGH2 0x164 +#define QSERDES_V5_RX_RX_MODE_00_HIGH3 0x168 +#define QSERDES_V5_RX_RX_MODE_00_HIGH4 0x16c +#define QSERDES_V5_RX_RX_MODE_01_LOW 0x170 +#define QSERDES_V5_RX_RX_MODE_01_HIGH 0x174 +#define QSERDES_V5_RX_RX_MODE_01_HIGH2 0x178 +#define QSERDES_V5_RX_RX_MODE_01_HIGH3 0x17c +#define QSERDES_V5_RX_RX_MODE_01_HIGH4 0x180 +#define QSERDES_V5_RX_RX_MODE_10_LOW 0x184 +#define QSERDES_V5_RX_RX_MODE_10_HIGH 0x188 +#define QSERDES_V5_RX_RX_MODE_10_HIGH2 0x18c +#define QSERDES_V5_RX_RX_MODE_10_HIGH3 0x190 +#define QSERDES_V5_RX_RX_MODE_10_HIGH4 0x194 +#define QSERDES_V5_RX_DFE_EN_TIMER 0x1a0 +#define QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET 0x1a4 +#define QSERDES_V5_RX_DCC_CTRL1 0x1a8 +#define QSERDES_V5_RX_VTH_CODE 0x1b0 + +/* Only for QMP V5 PHY - USB3 have different offsets than V4 */ +#define QPHY_V5_PCS_USB3_POWER_STATE_CONFIG1 0x300 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_STATUS 0x304 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_CTRL 0x308 +#define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_CTRL2 0x30c +#define QPHY_V5_PCS_USB3_LFPS_RXTERM_IRQ_SOURCE_STATUS 0x310 +#define QPHY_V5_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR 0x314 +#define QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL 0x318 +#define QPHY_V5_PCS_USB3_LFPS_TX_ECSTART 0x31c +#define QPHY_V5_PCS_USB3_LFPS_PER_TIMER_VAL 0x320 +#define QPHY_V5_PCS_USB3_LFPS_TX_END_CNT_U3_START 0x324 +#define QPHY_V5_PCS_USB3_LFPS_CONFIG1 0x328 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_LOCK_TIME 0x32c +#define QPHY_V5_PCS_USB3_RXEQTRAINING_WAIT_TIME 0x330 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_CTLE_TIME 0x334 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_WAIT_TIME_S2 0x338 +#define QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2 0x33c +#define QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_L 0x340 +#define QPHY_V5_PCS_USB3_RCVR_DTCT_DLY_U3_H 0x344 +#define QPHY_V5_PCS_USB3_ARCVR_DTCT_EN_PERIOD 0x348 +#define QPHY_V5_PCS_USB3_ARCVR_DTCT_CM_DLY 0x34c +#define QPHY_V5_PCS_USB3_TXONESZEROS_RUN_LENGTH 0x350 +#define QPHY_V5_PCS_USB3_ALFPS_DEGLITCH_VAL 0x354 +#define QPHY_V5_PCS_USB3_SIGDET_STARTUP_TIMER_VAL 0x358 +#define QPHY_V5_PCS_USB3_TEST_CONTROL 0x35c +#define QPHY_V5_PCS_USB3_RXTERMINATION_DLY_SEL 0x360 + +/* Only for QMP V5 PHY - UNI has 0x1000 offset for PCS_USB3 regs */ +#define QPHY_V5_PCS_USB3_UNI_LFPS_DET_HIGH_COUNT_VAL 0x1018 +#define QPHY_V5_PCS_USB3_UNI_RXEQTRAINING_DFE_TIME_S2 0x103c + #endif From fcba632d8148ab3ec19f6c7dd015cca866357d7e Mon Sep 17 00:00:00 2001 From: Jack Pham Date: Fri, 15 Jan 2021 09:47:22 -0800 Subject: [PATCH 091/633] dt-bindings: phy: qcom,usb-snps-femto-v2: Add SM8250 and SM8350 bindings Add the compatible strings for the USB2 PHYs found on QCOM SM8250 & SM8350 SoCs. Note that the SM8250 compatible is already in use in the dts and driver implementation but was missing from the documentation. Signed-off-by: Jack Pham Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210115174723.7424-4-jackp@codeaurora.org Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml index 4949a2851532..ee77c6458326 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml @@ -17,6 +17,8 @@ properties: enum: - qcom,usb-snps-hs-7nm-phy - qcom,sm8150-usb-hs-phy + - qcom,sm8250-usb-hs-phy + - qcom,sm8350-usb-hs-phy - qcom,usb-snps-femto-v2-phy reg: From 26e6d50e93129af85a759820c1608be8b470f229 Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Thu, 14 Jan 2021 18:47:16 +0100 Subject: [PATCH 092/633] phy: qcom-qusb2: Allow specifying default clock scheme The TCSR's PHY_CLK_SCHEME register is not available on all SoC models, but some may still use a differential reference clock. In preparation for these SoCs, add a se_clk_scheme_default configuration entry and declare it to true for all currently supported SoCs (retaining the previous defaults. This patch brings no functional changes. Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210114174718.398638-1-angelogioacchino.delregno@somainline.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qusb2.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index 109792203baf..8fcfea2a8f1f 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -245,6 +245,9 @@ struct qusb2_phy_cfg { /* true if PHY has PLL_CORE_INPUT_OVERRIDE register to reset PLL */ bool has_pll_override; + + /* true if PHY default clk scheme is single-ended */ + bool se_clk_scheme_default; }; static const struct qusb2_phy_cfg msm8996_phy_cfg = { @@ -253,6 +256,7 @@ static const struct qusb2_phy_cfg msm8996_phy_cfg = { .regs = msm8996_regs_layout, .has_pll_test = true, + .se_clk_scheme_default = true, .disable_ctrl = (CLAMP_N_EN | FREEZIO_N | POWER_DOWN), .mask_core_ready = PLL_LOCKED, .autoresume_en = BIT(3), @@ -266,6 +270,7 @@ static const struct qusb2_phy_cfg msm8998_phy_cfg = { .disable_ctrl = POWER_DOWN, .mask_core_ready = CORE_READY_STATUS, .has_pll_override = true, + .se_clk_scheme_default = true, .autoresume_en = BIT(0), .update_tune1_with_efuse = true, }; @@ -279,6 +284,7 @@ static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = { POWER_DOWN), .mask_core_ready = CORE_READY_STATUS, .has_pll_override = true, + .se_clk_scheme_default = true, .autoresume_en = BIT(0), .update_tune1_with_efuse = true, }; @@ -701,8 +707,13 @@ static int qusb2_phy_init(struct phy *phy) /* Required to get phy pll lock successfully */ usleep_range(150, 160); - /* Default is single-ended clock on msm8996 */ - qphy->has_se_clk_scheme = true; + /* + * Not all the SoCs have got a readable TCSR_PHY_CLK_SCHEME + * register in the TCSR so, if there's none, use the default + * value hardcoded in the configuration. + */ + qphy->has_se_clk_scheme = cfg->se_clk_scheme_default; + /* * read TCSR_PHY_CLK_SCHEME register to check if single-ended * clock scheme is selected. If yes, then disable differential From b0e7f781fc9958bfb57527a49c5a1eb1f423439c Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Thu, 14 Jan 2021 18:47:17 +0100 Subject: [PATCH 093/633] phy: qcom-qusb2: Add configuration for SDM660 The SDM660 SoC uses the same configuration as MSM8996, but the clock scheme uses a differential reference clock and none of the SoCs in this series (630, 636 and others) have got a usable PHY_CLK_SCHEME register in the TCSR for clk scheme detection. Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210114174718.398638-2-angelogioacchino.delregno@somainline.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qusb2.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index 8fcfea2a8f1f..719e0888ed87 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -289,6 +289,18 @@ static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = { .update_tune1_with_efuse = true, }; +static const struct qusb2_phy_cfg sdm660_phy_cfg = { + .tbl = msm8996_init_tbl, + .tbl_num = ARRAY_SIZE(msm8996_init_tbl), + .regs = msm8996_regs_layout, + + .has_pll_test = true, + .se_clk_scheme_default = false, + .disable_ctrl = (CLAMP_N_EN | FREEZIO_N | POWER_DOWN), + .mask_core_ready = PLL_LOCKED, + .autoresume_en = BIT(3), +}; + static const char * const qusb2_phy_vreg_names[] = { "vdda-pll", "vdda-phy-dpdm", }; @@ -829,6 +841,9 @@ static const struct of_device_id qusb2_phy_of_match_table[] = { }, { .compatible = "qcom,msm8998-qusb2-phy", .data = &msm8998_phy_cfg, + }, { + .compatible = "qcom,sdm660-qusb2-phy", + .data = &sdm660_phy_cfg, }, { /* * Deprecated. Only here to support legacy device From 71edb0b4fa0e3bc248df564ba01a0f7c41607c8e Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Thu, 14 Jan 2021 18:47:18 +0100 Subject: [PATCH 094/633] dt-bindings: phy: qcom-qusb2: Document SDM660 compatible Support for the SDM630/660 series of SoCs was added to the driver: document the qcom,sdm660-qusb2-phy compatible here. Signed-off-by: AngeloGioacchino Del Regno Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210114174718.398638-3-angelogioacchino.delregno@somainline.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index d457fb6a4779..582abbbd8b32 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -21,6 +21,7 @@ properties: - qcom,ipq8074-qusb2-phy - qcom,msm8996-qusb2-phy - qcom,msm8998-qusb2-phy + - qcom,sdm660-qusb2-phy - items: - enum: - qcom,sc7180-qusb2-phy From 48b7de6687f6ed597ef7162151f1c35469775545 Mon Sep 17 00:00:00 2001 From: Christian Vogel Date: Wed, 13 Jan 2021 20:50:17 +0100 Subject: [PATCH 095/633] w1/w1.c: w1 address crc quick for DS28E04 eeproms Onewire addresses are 64bit family(8bit), unique_id(48bit), crc(8bit) (LSBt to MSB) and self-consistent: crc = crc8(family, unique). DS28E04-100 4096-Bit Addressable 1-Wire EEPROM with PIO have strap pins to set 7 LSB of the address, unfortunately without updating the crc part of the address. It is only consistent if all strap pins float high. [see datasheet 19-6134; Rev 12/11 page 6: 64-bit device id number] We therefore introduce a special handling of family 0x1c (DS28E04) to check address consistency with 7 LSBs of the unique_id set to 1. Acked-by: Evgeniy Polyakov Signed-off-by: Christian Vogel Link: https://lore.kernel.org/r/20210113195018.7498-2-vogelchr@vogel.cx Signed-off-by: Greg Kroah-Hartman --- drivers/w1/w1.c | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c index 15a2ee32f116..f2ae2e563dc5 100644 --- a/drivers/w1/w1.c +++ b/drivers/w1/w1.c @@ -25,6 +25,8 @@ #include "w1_netlink.h" #define W1_FAMILY_DEFAULT 0 +#define W1_FAMILY_DS28E04 0x1C /* for crc quirk */ + static int w1_timeout = 10; module_param_named(timeout, w1_timeout, int, 0); @@ -913,11 +915,44 @@ void w1_reconnect_slaves(struct w1_family *f, int attach) mutex_unlock(&w1_mlock); } +static int w1_addr_crc_is_valid(struct w1_master *dev, u64 rn) +{ + u64 rn_le = cpu_to_le64(rn); + struct w1_reg_num *tmp = (struct w1_reg_num *)&rn; + u8 crc; + + crc = w1_calc_crc8((u8 *)&rn_le, 7); + + /* quirk: + * DS28E04 (1w eeprom) has strapping pins to change + * address, but will not update the crc. So normal rules + * for consistent w1 addresses are violated. We test + * with the 7 LSBs of the address forced high. + * + * (char*)&rn_le = { family, addr_lsb, ..., addr_msb, crc }. + */ + if (crc != tmp->crc && tmp->family == W1_FAMILY_DS28E04) { + u64 corr_le = rn_le; + + ((u8 *)&corr_le)[1] |= 0x7f; + crc = w1_calc_crc8((u8 *)&corr_le, 7); + + dev_info(&dev->dev, "DS28E04 crc workaround on %02x.%012llx.%02x\n", + tmp->family, (unsigned long long)tmp->id, tmp->crc); + } + + if (crc != tmp->crc) { + dev_dbg(&dev->dev, "w1 addr crc mismatch: %02x.%012llx.%02x != 0x%02x.\n", + tmp->family, (unsigned long long)tmp->id, tmp->crc, crc); + return 0; + } + return 1; +} + void w1_slave_found(struct w1_master *dev, u64 rn) { struct w1_slave *sl; struct w1_reg_num *tmp; - u64 rn_le = cpu_to_le64(rn); atomic_inc(&dev->refcnt); @@ -927,7 +962,7 @@ void w1_slave_found(struct w1_master *dev, u64 rn) if (sl) { set_bit(W1_SLAVE_ACTIVE, &sl->flags); } else { - if (rn && tmp->crc == w1_calc_crc8((u8 *)&rn_le, 7)) + if (rn && w1_addr_crc_is_valid(dev, rn)) w1_attach_slave_device(dev, tmp); } From e3fe0e89fec6965342434e7acae7ed6e6f021d08 Mon Sep 17 00:00:00 2001 From: Christian Vogel Date: Wed, 13 Jan 2021 20:50:18 +0100 Subject: [PATCH 096/633] w1/masters/ds2490: queue up found IDs during scan Queue up found IDs in a buffer and run the callback once for each found ID at the end. This is necessary because we hold the bus_mutex during the whole scan, and some of the "add-device" callbacks deadlock as they themselves want to mutex_lock(bus_mutex). Acked-by: Evgeniy Polyakov Signed-off-by: Christian Vogel Link: https://lore.kernel.org/r/20210113195018.7498-3-vogelchr@vogel.cx Signed-off-by: Greg Kroah-Hartman --- drivers/w1/masters/ds2490.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c index e17c8f70dcd0..cd8821580f71 100644 --- a/drivers/w1/masters/ds2490.c +++ b/drivers/w1/masters/ds2490.c @@ -688,12 +688,22 @@ static void ds9490r_search(void *data, struct w1_master *master, * packet size. */ const size_t bufsize = 2 * 64; - u64 *buf; + u64 *buf, *found_ids; buf = kmalloc(bufsize, GFP_KERNEL); if (!buf) return; + /* + * We are holding the bus mutex during the scan, but adding devices via the + * callback needs the bus to be unlocked. So we queue up found ids here. + */ + found_ids = kmalloc_array(master->max_slave_count, sizeof(u64), GFP_KERNEL); + if (!found_ids) { + kfree(buf); + return; + } + mutex_lock(&master->bus_mutex); /* address to start searching at */ @@ -729,13 +739,13 @@ static void ds9490r_search(void *data, struct w1_master *master, if (err < 0) break; for (i = 0; i < err/8; ++i) { - ++found; - if (found <= search_limit) - callback(master, buf[i]); + found_ids[found++] = buf[i]; /* can't know if there will be a discrepancy * value after until the next id */ - if (found == search_limit) + if (found == search_limit) { master->search_id = buf[i]; + break; + } } } @@ -759,9 +769,14 @@ static void ds9490r_search(void *data, struct w1_master *master, master->max_slave_count); set_bit(W1_WARN_MAX_COUNT, &master->flags); } + search_out: mutex_unlock(&master->bus_mutex); kfree(buf); + + for (i = 0; i < found; i++) /* run callback for all queued up IDs */ + callback(master, found_ids[i]); + kfree(found_ids); } #if 0 From 6805822954f0fd24de4d6febc4c84afdcb6f9baa Mon Sep 17 00:00:00 2001 From: Aswath Govindraju Date: Tue, 5 Jan 2021 16:28:11 +0530 Subject: [PATCH 097/633] Documentation: devicetree: Add new compatible string for eeprom microchip 93LC46B Add a new compatible string for eeprom microchip 93LC46B in eeprom-93xx46 dt-binding file as it belongs to the 93xx46 family of devices. Reviewed-by: Rob Herring Signed-off-by: Aswath Govindraju Link: https://lore.kernel.org/r/20210105105817.17644-2-a-govindraju@ti.com Signed-off-by: Greg Kroah-Hartman --- Documentation/devicetree/bindings/misc/eeprom-93xx46.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt index a8ebb4621f79..7b636b7a8311 100644 --- a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt +++ b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt @@ -4,6 +4,7 @@ Required properties: - compatible : shall be one of: "atmel,at93c46d" "eeprom-93xx46" + "microchip,93lc46b" - data-size : number of data bits per word (either 8 or 16) Optional properties: From f6f1f8e6e3eea25f539105d48166e91f0ab46dd1 Mon Sep 17 00:00:00 2001 From: Aswath Govindraju Date: Tue, 5 Jan 2021 16:28:12 +0530 Subject: [PATCH 098/633] misc: eeprom_93xx46: Add quirk to support Microchip 93LC46B eeprom A dummy zero bit is sent preceding the data during a read transfer by the Microchip 93LC46B eeprom (section 2.7 of[1]). This results in right shift of data during a read. In order to ignore this bit a quirk can be added to send an extra zero bit after the read address. Add a quirk to ignore the zero bit sent before data by adding a zero bit after the read address. [1] - https://www.mouser.com/datasheet/2/268/20001749K-277859.pdf Signed-off-by: Aswath Govindraju Link: https://lore.kernel.org/r/20210105105817.17644-3-a-govindraju@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/eeprom/eeprom_93xx46.c | 15 +++++++++++++++ include/linux/eeprom_93xx46.h | 2 ++ 2 files changed, 17 insertions(+) diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index e130a0d858e9..80114f4c80ad 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -35,6 +35,10 @@ static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = { EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH, }; +static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = { + .quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE, +}; + struct eeprom_93xx46_dev { struct spi_device *spi; struct eeprom_93xx46_platform_data *pdata; @@ -55,6 +59,11 @@ static inline bool has_quirk_instruction_length(struct eeprom_93xx46_dev *edev) return edev->pdata->quirks & EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH; } +static inline bool has_quirk_extra_read_cycle(struct eeprom_93xx46_dev *edev) +{ + return edev->pdata->quirks & EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE; +} + static int eeprom_93xx46_read(void *priv, unsigned int off, void *val, size_t count) { @@ -96,6 +105,11 @@ static int eeprom_93xx46_read(void *priv, unsigned int off, dev_dbg(&edev->spi->dev, "read cmd 0x%x, %d Hz\n", cmd_addr, edev->spi->max_speed_hz); + if (has_quirk_extra_read_cycle(edev)) { + cmd_addr <<= 1; + bits += 1; + } + spi_message_init(&m); t[0].tx_buf = (char *)&cmd_addr; @@ -363,6 +377,7 @@ static void select_deassert(void *context) static const struct of_device_id eeprom_93xx46_of_table[] = { { .compatible = "eeprom-93xx46", }, { .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, }, + { .compatible = "microchip,93lc46b", .data = µchip_93lc46b_data, }, {} }; MODULE_DEVICE_TABLE(of, eeprom_93xx46_of_table); diff --git a/include/linux/eeprom_93xx46.h b/include/linux/eeprom_93xx46.h index eec7928ff8fe..99580c22f91a 100644 --- a/include/linux/eeprom_93xx46.h +++ b/include/linux/eeprom_93xx46.h @@ -16,6 +16,8 @@ struct eeprom_93xx46_platform_data { #define EEPROM_93XX46_QUIRK_SINGLE_WORD_READ (1 << 0) /* Instructions such as EWEN are (addrlen + 2) in length. */ #define EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH (1 << 1) +/* Add extra cycle after address during a read */ +#define EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE BIT(2) /* * optional hooks to control additional logic From f49b6aeb5c45dea3a1b6ee6a842599147dfd5929 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Thu, 26 Nov 2020 16:06:41 +0100 Subject: [PATCH 099/633] bus: mhi: Ensure correct ring update ordering with memory barrier The ring element data, though being part of coherent memory, still need to be performed before updating the ring context to point to this new element. That can be guaranteed with a memory barrier (dma_wmb). Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/main.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 9b425402767c..8576b0f03f2c 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -111,7 +111,14 @@ void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, dma_addr_t db; db = ring->iommu_base + (ring->wp - ring->base); + + /* + * Writes to the new ring element must be visible to the hardware + * before letting h/w know there is new element to fetch. + */ + dma_wmb(); *ring->ctxt_wp = db; + mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, ring->db_addr, db); } From fcba4b2047a31a55e1cef73849363a3cf7c2736d Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Tue, 5 Jan 2021 17:44:35 +0100 Subject: [PATCH 100/633] mhi: unconstify mhi_event_config Some parameters may have to be determined at runtime. It is the case for the event ring MSI vector. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- include/linux/mhi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 54afcae1709a..48f5c9168ae0 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -279,7 +279,7 @@ struct mhi_controller_config { u32 num_channels; const struct mhi_channel_config *ch_cfg; u32 num_events; - const struct mhi_event_config *event_cfg; + struct mhi_event_config *event_cfg; bool use_bounce_buf; bool m2_no_db; }; From b91c3b30e2267265cd7e67cb3d0c99c48c02b001 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Tue, 5 Jan 2021 17:44:36 +0100 Subject: [PATCH 101/633] mhi: pci_generic: Fix shared MSI vector support When a shared MSI vector must be used (e.g. when VTd is disabled on x86_64), each event MSI vector must be set to the shared vector idx. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Reviewed-by: Hemant Kumar Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index c13de0f2053e..444693e3fd72 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -154,7 +154,7 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3), }; -static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { +static struct mhi_event_config modem_qcom_v1_mhi_events[] = { /* first ring is control+data ring */ MHI_EVENT_CONFIG_CTRL(0), /* DIAG dedicated event ring */ @@ -164,7 +164,7 @@ static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { MHI_EVENT_CONFIG_HW_DATA(3, 101) }; -static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { +static struct mhi_controller_config modem_qcom_v1_mhiv_config = { .max_channels = 128, .timeout_ms = 8000, .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels), @@ -295,8 +295,12 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl, } if (nr_vectors < mhi_cntrl->nr_irqs) { - dev_warn(&pdev->dev, "Not enough MSI vectors (%d/%d), use shared MSI\n", - nr_vectors, mhi_cntrl_config->num_events); + dev_warn(&pdev->dev, "using shared MSI\n"); + + /* Patch msi vectors, use only one (shared) */ + for (i = 0; i < mhi_cntrl_config->num_events; i++) + mhi_cntrl_config->event_cfg[i].irq = 0; + mhi_cntrl->nr_irqs = 1; } irq = devm_kcalloc(&pdev->dev, mhi_cntrl->nr_irqs, sizeof(int), GFP_KERNEL); From 6ffcc18d9c0b4522c95aab71ff3ff5a56e699945 Mon Sep 17 00:00:00 2001 From: Carl Huang Date: Mon, 4 Jan 2021 18:11:28 +0800 Subject: [PATCH 102/633] mhi: use irq_flags if controller driver configures it If controller driver has specified the irq_flags, mhi uses this specified irq_flags. Otherwise, mhi uses default irq_flags. The purpose of this change is to support one MSI vector for QCA6390. MHI will use one same MSI vector too in this scenario. In case of one MSI vector, IRQ_NO_BALANCING is needed when irq handler is requested. The reason is if irq migration happens, the msi_data may change too. However, the msi_data is already programmed to QCA6390 hardware during initialization phase. This msi_data inconsistence will result in crash in kernel. Another issue is in case of one MSI vector, IRQF_NO_SUSPEND will trigger WARNINGS because QCA6390 wants to disable the IRQ during the suspend. To avoid above two issues, QCA6390 driver specifies the irq_flags in case of one MSI vector when mhi_register_controller is called. Signed-off-by: Carl Huang Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/init.c | 9 +++++++-- include/linux/mhi.h | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index f0697f433c2f..aa575d3fb3ae 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -151,12 +151,17 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event = mhi_cntrl->mhi_event; struct device *dev = &mhi_cntrl->mhi_dev->dev; + unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND; int i, ret; + /* if controller driver has set irq_flags, use it */ + if (mhi_cntrl->irq_flags) + irq_flags = mhi_cntrl->irq_flags; + /* Setup BHI_INTVEC IRQ */ ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler, mhi_intvec_threaded_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "bhi", mhi_cntrl); if (ret) return ret; @@ -174,7 +179,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) ret = request_irq(mhi_cntrl->irq[mhi_event->irq], mhi_irq_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "mhi", mhi_event); if (ret) { dev_err(dev, "Error requesting irq:%d for ev:%d\n", diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 562862ff819c..b7138127664f 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -353,6 +353,7 @@ struct mhi_controller_config { * @fbc_download: MHI host needs to do complete image transfer (optional) * @pre_init: MHI host needs to do pre-initialization before power up * @wake_set: Device wakeup set flag + * @irq_flags: irq flags passed to request_irq (optional) * * Fields marked as (required) need to be populated by the controller driver * before calling mhi_register_controller(). For the fields marked as (optional) @@ -444,6 +445,7 @@ struct mhi_controller { bool fbc_download; bool pre_init; bool wake_set; + unsigned long irq_flags; }; /** From a8f75cb348fd52e7a5cf25991cdf9c89fb0cfd41 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Mon, 11 Jan 2021 17:56:35 +0100 Subject: [PATCH 103/633] mhi: core: Factorize mhi queuing Instead of duplicating queuing procedure in mhi_queue_dma(), mhi_queue_buf() and mhi_queue_skb(), add a new generic mhi_queue() as common helper. Note that the unified mhi_queue align pm_lock locking on mhi_queue_buf behavior, taking it with irqsave variant (vs _bh for former queue_skb and queue_dma version). Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/main.c | 168 ++++++++++-------------------------- 1 file changed, 47 insertions(+), 121 deletions(-) diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 8576b0f03f2c..b67c25491069 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -967,118 +967,88 @@ static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, return (tmp == ring->rp); } -int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct sk_buff *skb, size_t len, enum mhi_flags mflags) +static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, + enum dma_data_direction dir, enum mhi_flags mflags) { struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; struct mhi_ring *tre_ring = &mhi_chan->tre_ring; - struct mhi_buf_info buf_info = { }; + unsigned long flags; int ret; - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - read_unlock_bh(&mhi_cntrl->pm_lock); + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) return -EIO; + + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); + + ret = mhi_is_ring_full(mhi_cntrl, tre_ring); + if (unlikely(ret)) { + ret = -ENOMEM; + goto exit_unlock; } - /* we're in M3 or transitioning to M3 */ + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags); + if (unlikely(ret)) + goto exit_unlock; + + /* trigger M3 exit if necessary */ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) mhi_trigger_resume(mhi_cntrl); - /* Toggle wake to exit out of M2 */ + /* Assert dev_wake (to exit/prevent M1/M2)*/ mhi_cntrl->wake_toggle(mhi_cntrl); + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_inc(&mhi_cntrl->pending_pkts); + + if (unlikely(!MHI_DB_ACCESS_VALID(mhi_cntrl))) { + ret = -EIO; + goto exit_unlock; + } + + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + +exit_unlock: + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + + return ret; +} + +int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct sk_buff *skb, size_t len, enum mhi_flags mflags) +{ + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_buf_info buf_info = { }; + buf_info.v_addr = skb->data; buf_info.cb_buf = skb; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_skb); int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; - struct device *dev = &mhi_cntrl->mhi_dev->dev; - struct mhi_ring *tre_ring = &mhi_chan->tre_ring; struct mhi_buf_info buf_info = { }; - int ret; - - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - dev_err(dev, "MHI is not in activate state, PM state: %s\n", - to_mhi_pm_state_str(mhi_cntrl->pm_state)); - read_unlock_bh(&mhi_cntrl->pm_lock); - - return -EIO; - } - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); buf_info.p_addr = mhi_buf->dma_addr; buf_info.cb_buf = mhi_buf; buf_info.pre_mapped = true; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_dma); @@ -1132,57 +1102,13 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, void *buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : - mhi_dev->dl_chan; - struct mhi_ring *tre_ring; struct mhi_buf_info buf_info = { }; - unsigned long flags; - int ret; - - /* - * this check here only as a guard, it's always - * possible mhi can enter error while executing rest of function, - * which is not fatal so we do not need to hold pm_lock - */ - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) - return -EIO; - - tre_ring = &mhi_chan->tre_ring; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; buf_info.v_addr = buf; buf_info.cb_buf = buf; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) - return ret; - - read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - unsigned long flags; - - read_lock_irqsave(&mhi_chan->lock, flags); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_irqrestore(&mhi_chan->lock, flags); - } - - read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_buf); From de9427ca87cfa959abcd8bab7e38343b51219ffa Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 20 Jan 2021 16:07:29 +0100 Subject: [PATCH 104/633] PCI: Remove tango host controller driver The tango platform is getting removed, so the driver is no longer needed. Link: https://lore.kernel.org/r/20210120150800.1650898-1-arnd@kernel.org Signed-off-by: Arnd Bergmann Signed-off-by: Lorenzo Pieralisi Acked-by: Mans Rullgard Cc: Marc Gonzalez Cc: Mans Rullgard --- drivers/pci/controller/Kconfig | 14 -- drivers/pci/controller/Makefile | 1 - drivers/pci/controller/pcie-tango.c | 341 ---------------------------- 3 files changed, 356 deletions(-) delete mode 100644 drivers/pci/controller/pcie-tango.c diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa..8c85c16594f2 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -242,20 +242,6 @@ config PCIE_MEDIATEK Say Y here if you want to enable PCIe controller support on MediaTek SoCs. -config PCIE_TANGO_SMP8759 - bool "Tango SMP8759 PCIe controller (DANGEROUS)" - depends on ARCH_TANGO && PCI_MSI && OF - depends on BROKEN - select PCI_HOST_COMMON - help - Say Y here to enable PCIe controller support for Sigma Designs - Tango SMP8759-based systems. - - Note: The SMP8759 controller multiplexes PCI config and MMIO - accesses, and Linux doesn't provide a way to serialize them. - This can lead to data corruption if drivers perform concurrent - config and MMIO accesses. - config VMD depends on PCI_MSI && X86_64 && SRCU tristate "Intel Volume Management Device Driver" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 04c6edc285c5..b3a7912f8a5c 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -27,7 +27,6 @@ obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o obj-$(CONFIG_PCIE_ROCKCHIP_EP) += pcie-rockchip-ep.o obj-$(CONFIG_PCIE_ROCKCHIP_HOST) += pcie-rockchip-host.o obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o -obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o diff --git a/drivers/pci/controller/pcie-tango.c b/drivers/pci/controller/pcie-tango.c deleted file mode 100644 index 62a061f1d62e..000000000000 --- a/drivers/pci/controller/pcie-tango.c +++ /dev/null @@ -1,341 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include - -#define MSI_MAX 256 - -#define SMP8759_MUX 0x48 -#define SMP8759_TEST_OUT 0x74 -#define SMP8759_DOORBELL 0x7c -#define SMP8759_STATUS 0x80 -#define SMP8759_ENABLE 0xa0 - -struct tango_pcie { - DECLARE_BITMAP(used_msi, MSI_MAX); - u64 msi_doorbell; - spinlock_t used_msi_lock; - void __iomem *base; - struct irq_domain *dom; -}; - -static void tango_msi_isr(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct tango_pcie *pcie = irq_desc_get_handler_data(desc); - unsigned long status, base, virq, idx, pos = 0; - - chained_irq_enter(chip, desc); - spin_lock(&pcie->used_msi_lock); - - while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) { - base = round_down(pos, 32); - status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8); - for_each_set_bit(idx, &status, 32) { - virq = irq_find_mapping(pcie->dom, base + idx); - generic_handle_irq(virq); - } - pos = base + 32; - } - - spin_unlock(&pcie->used_msi_lock); - chained_irq_exit(chip, desc); -} - -static void tango_ack(struct irq_data *d) -{ - struct tango_pcie *pcie = d->chip_data; - u32 offset = (d->hwirq / 32) * 4; - u32 bit = BIT(d->hwirq % 32); - - writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset); -} - -static void update_msi_enable(struct irq_data *d, bool unmask) -{ - unsigned long flags; - struct tango_pcie *pcie = d->chip_data; - u32 offset = (d->hwirq / 32) * 4; - u32 bit = BIT(d->hwirq % 32); - u32 val; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset); - val = unmask ? val | bit : val & ~bit; - writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); -} - -static void tango_mask(struct irq_data *d) -{ - update_msi_enable(d, false); -} - -static void tango_unmask(struct irq_data *d) -{ - update_msi_enable(d, true); -} - -static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask, - bool force) -{ - return -EINVAL; -} - -static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) -{ - struct tango_pcie *pcie = d->chip_data; - msg->address_lo = lower_32_bits(pcie->msi_doorbell); - msg->address_hi = upper_32_bits(pcie->msi_doorbell); - msg->data = d->hwirq; -} - -static struct irq_chip tango_chip = { - .irq_ack = tango_ack, - .irq_mask = tango_mask, - .irq_unmask = tango_unmask, - .irq_set_affinity = tango_set_affinity, - .irq_compose_msi_msg = tango_compose_msi_msg, -}; - -static void msi_ack(struct irq_data *d) -{ - irq_chip_ack_parent(d); -} - -static void msi_mask(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void msi_unmask(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip msi_chip = { - .name = "MSI", - .irq_ack = msi_ack, - .irq_mask = msi_mask, - .irq_unmask = msi_unmask, -}; - -static struct msi_domain_info msi_dom_info = { - .flags = MSI_FLAG_PCI_MSIX - | MSI_FLAG_USE_DEF_DOM_OPS - | MSI_FLAG_USE_DEF_CHIP_OPS, - .chip = &msi_chip, -}; - -static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq, - unsigned int nr_irqs, void *args) -{ - struct tango_pcie *pcie = dom->host_data; - unsigned long flags; - int pos; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - pos = find_first_zero_bit(pcie->used_msi, MSI_MAX); - if (pos >= MSI_MAX) { - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); - return -ENOSPC; - } - __set_bit(pos, pcie->used_msi); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); - irq_domain_set_info(dom, virq, pos, &tango_chip, - pcie, handle_edge_irq, NULL, NULL); - - return 0; -} - -static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq, - unsigned int nr_irqs) -{ - unsigned long flags; - struct irq_data *d = irq_domain_get_irq_data(dom, virq); - struct tango_pcie *pcie = d->chip_data; - - spin_lock_irqsave(&pcie->used_msi_lock, flags); - __clear_bit(d->hwirq, pcie->used_msi); - spin_unlock_irqrestore(&pcie->used_msi_lock, flags); -} - -static const struct irq_domain_ops dom_ops = { - .alloc = tango_irq_domain_alloc, - .free = tango_irq_domain_free, -}; - -static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - struct pci_config_window *cfg = bus->sysdata; - struct tango_pcie *pcie = dev_get_drvdata(cfg->parent); - int ret; - - /* Reads in configuration space outside devfn 0 return garbage */ - if (devfn != 0) - return PCIBIOS_FUNC_NOT_SUPPORTED; - - /* - * PCI config and MMIO accesses are muxed. Linux doesn't have a - * mutual exclusion mechanism for config vs. MMIO accesses, so - * concurrent accesses may cause corruption. - */ - writel_relaxed(1, pcie->base + SMP8759_MUX); - ret = pci_generic_config_read(bus, devfn, where, size, val); - writel_relaxed(0, pcie->base + SMP8759_MUX); - - return ret; -} - -static int smp8759_config_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - struct pci_config_window *cfg = bus->sysdata; - struct tango_pcie *pcie = dev_get_drvdata(cfg->parent); - int ret; - - writel_relaxed(1, pcie->base + SMP8759_MUX); - ret = pci_generic_config_write(bus, devfn, where, size, val); - writel_relaxed(0, pcie->base + SMP8759_MUX); - - return ret; -} - -static const struct pci_ecam_ops smp8759_ecam_ops = { - .pci_ops = { - .map_bus = pci_ecam_map_bus, - .read = smp8759_config_read, - .write = smp8759_config_write, - } -}; - -static int tango_pcie_link_up(struct tango_pcie *pcie) -{ - void __iomem *test_out = pcie->base + SMP8759_TEST_OUT; - int i; - - writel_relaxed(16, test_out); - for (i = 0; i < 10; ++i) { - u32 ltssm_state = readl_relaxed(test_out) >> 8; - if ((ltssm_state & 0x1f) == 0xf) /* L0 */ - return 1; - usleep_range(3000, 4000); - } - - return 0; -} - -static int tango_pcie_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct tango_pcie *pcie; - struct resource *res; - struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node); - struct irq_domain *msi_dom, *irq_dom; - struct of_pci_range_parser parser; - struct of_pci_range range; - int virq, offset; - - dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n"); - add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); - - pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); - if (!pcie) - return -ENOMEM; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - pcie->base = devm_ioremap_resource(dev, res); - if (IS_ERR(pcie->base)) - return PTR_ERR(pcie->base); - - platform_set_drvdata(pdev, pcie); - - if (!tango_pcie_link_up(pcie)) - return -ENODEV; - - if (of_pci_dma_range_parser_init(&parser, dev->of_node) < 0) - return -ENOENT; - - if (of_pci_range_parser_one(&parser, &range) == NULL) - return -ENOENT; - - range.pci_addr += range.size; - pcie->msi_doorbell = range.pci_addr + res->start + SMP8759_DOORBELL; - - for (offset = 0; offset < MSI_MAX / 8; offset += 4) - writel_relaxed(0, pcie->base + SMP8759_ENABLE + offset); - - virq = platform_get_irq(pdev, 1); - if (virq < 0) - return virq; - - irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie); - if (!irq_dom) { - dev_err(dev, "Failed to create IRQ domain\n"); - return -ENOMEM; - } - - msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom); - if (!msi_dom) { - dev_err(dev, "Failed to create MSI domain\n"); - irq_domain_remove(irq_dom); - return -ENOMEM; - } - - pcie->dom = irq_dom; - spin_lock_init(&pcie->used_msi_lock); - irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie); - - return pci_host_common_probe(pdev); -} - -static const struct of_device_id tango_pcie_ids[] = { - { - .compatible = "sigma,smp8759-pcie", - .data = &smp8759_ecam_ops, - }, - { }, -}; - -static struct platform_driver tango_pcie_driver = { - .probe = tango_pcie_probe, - .driver = { - .name = KBUILD_MODNAME, - .of_match_table = tango_pcie_ids, - .suppress_bind_attrs = true, - }, -}; -builtin_platform_driver(tango_pcie_driver); - -/* - * The root complex advertises the wrong device class. - * Header Type 1 is for PCI-to-PCI bridges. - */ -static void tango_fixup_class(struct pci_dev *dev) -{ - dev->class = PCI_CLASS_BRIDGE_PCI << 8; -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_class); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_class); - -/* - * The root complex exposes a "fake" BAR, which is used to filter - * bus-to-system accesses. Only accesses within the range defined by this - * BAR are forwarded to the host, others are ignored. - * - * By default, the DMA framework expects an identity mapping, and DRAM0 is - * mapped at 0x80000000. - */ -static void tango_fixup_bar(struct pci_dev *dev) -{ - dev->non_compliant_bars = true; - pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, 0x80000000); -} -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0024, tango_fixup_bar); -DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SIGMA, 0x0028, tango_fixup_bar); From ae191d2e513ae5274224777ae67018a584074a28 Mon Sep 17 00:00:00 2001 From: Pan Bian Date: Wed, 20 Jan 2021 06:37:45 -0800 Subject: [PATCH 105/633] PCI: xilinx-cpm: Fix reference count leak on error path Also drop the reference count of the node on error path. Link: https://lore.kernel.org/r/20210120143745.699-1-bianpan2016@163.com Fixes: 508f610648b9 ("PCI: xilinx-cpm: Add Versal CPM Root Port driver") Signed-off-by: Pan Bian Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/pcie-xilinx-cpm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c index f92e0152e65e..67937facd90c 100644 --- a/drivers/pci/controller/pcie-xilinx-cpm.c +++ b/drivers/pci/controller/pcie-xilinx-cpm.c @@ -404,6 +404,7 @@ static int xilinx_cpm_pcie_init_irq_domain(struct xilinx_cpm_pcie_port *port) return 0; out: xilinx_cpm_free_irq_domains(port); + of_node_put(pcie_intc_node); dev_err(dev, "Failed to allocate IRQ domains\n"); return -ENOMEM; From 42814c438aac79746d310f413a27d5b0b959c5de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Wilczy=C5=84ski?= Date: Wed, 20 Jan 2021 18:48:10 +0000 Subject: [PATCH 106/633] PCI: mediatek: Add missing of_node_put() to fix reference leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The for_each_available_child_of_node helper internally makes use of the of_get_next_available_child() which performs an of_node_get() on each iteration when searching for next available child node. Should an available child node be found, then it would return a device node pointer with reference count incremented, thus early return from the middle of the loop requires an explicit of_node_put() to prevent reference count leak. To stop the reference leak, explicitly call of_node_put() before returning after an error occurred. Link: https://lore.kernel.org/r/20210120184810.3068794-1-kw@linux.com Signed-off-by: Krzysztof Wilczyński Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/pcie-mediatek.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index cf4c18f0c25a..23548b517e4b 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -1035,14 +1035,14 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) err = of_pci_get_devfn(child); if (err < 0) { dev_err(dev, "failed to parse devfn: %d\n", err); - return err; + goto error_put_node; } slot = PCI_SLOT(err); err = mtk_pcie_parse_port(pcie, child, slot); if (err) - return err; + goto error_put_node; } err = mtk_pcie_subsys_powerup(pcie); @@ -1058,6 +1058,9 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) mtk_pcie_subsys_powerdown(pcie); return 0; +error_put_node: + of_node_put(child); + return err; } static int mtk_pcie_probe(struct platform_device *pdev) From 882227626459ce7a24191f13c6d9749a00992059 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:15 -0800 Subject: [PATCH 107/633] bcm-vk: add bcm_vk UAPI Add user space api for bcm-vk driver. Provide ioctl api to load images and issue reset command to card. FW status registers in PCI BAR space also defined as part of API so that user space is able to interpret these memory locations as needed via direct PCIe access. Acked-by: Olof Johansson Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-2-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- include/uapi/linux/misc/bcm_vk.h | 84 ++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 include/uapi/linux/misc/bcm_vk.h diff --git a/include/uapi/linux/misc/bcm_vk.h b/include/uapi/linux/misc/bcm_vk.h new file mode 100644 index 000000000000..ec28e0bd46a9 --- /dev/null +++ b/include/uapi/linux/misc/bcm_vk.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef __UAPI_LINUX_MISC_BCM_VK_H +#define __UAPI_LINUX_MISC_BCM_VK_H + +#include +#include + +#define BCM_VK_MAX_FILENAME 64 + +struct vk_image { + __u32 type; /* Type of image */ +#define VK_IMAGE_TYPE_BOOT1 1 /* 1st stage (load to SRAM) */ +#define VK_IMAGE_TYPE_BOOT2 2 /* 2nd stage (load to DDR) */ + __u8 filename[BCM_VK_MAX_FILENAME]; /* Filename of image */ +}; + +struct vk_reset { + __u32 arg1; + __u32 arg2; +}; + +#define VK_MAGIC 0x5e + +/* Load image to Valkyrie */ +#define VK_IOCTL_LOAD_IMAGE _IOW(VK_MAGIC, 0x2, struct vk_image) + +/* Send Reset to Valkyrie */ +#define VK_IOCTL_RESET _IOW(VK_MAGIC, 0x4, struct vk_reset) + +/* + * Firmware Status accessed directly via BAR space + */ +#define VK_BAR_FWSTS 0x41c +#define VK_BAR_COP_FWSTS 0x428 +/* VK_FWSTS definitions */ +#define VK_FWSTS_RELOCATION_ENTRY (1UL << 0) +#define VK_FWSTS_RELOCATION_EXIT (1UL << 1) +#define VK_FWSTS_INIT_START (1UL << 2) +#define VK_FWSTS_ARCH_INIT_DONE (1UL << 3) +#define VK_FWSTS_PRE_KNL1_INIT_DONE (1UL << 4) +#define VK_FWSTS_PRE_KNL2_INIT_DONE (1UL << 5) +#define VK_FWSTS_POST_KNL_INIT_DONE (1UL << 6) +#define VK_FWSTS_INIT_DONE (1UL << 7) +#define VK_FWSTS_APP_INIT_START (1UL << 8) +#define VK_FWSTS_APP_INIT_DONE (1UL << 9) +#define VK_FWSTS_MASK 0xffffffff +#define VK_FWSTS_READY (VK_FWSTS_INIT_START | \ + VK_FWSTS_ARCH_INIT_DONE | \ + VK_FWSTS_PRE_KNL1_INIT_DONE | \ + VK_FWSTS_PRE_KNL2_INIT_DONE | \ + VK_FWSTS_POST_KNL_INIT_DONE | \ + VK_FWSTS_INIT_DONE | \ + VK_FWSTS_APP_INIT_START | \ + VK_FWSTS_APP_INIT_DONE) +/* Deinit */ +#define VK_FWSTS_APP_DEINIT_START (1UL << 23) +#define VK_FWSTS_APP_DEINIT_DONE (1UL << 24) +#define VK_FWSTS_DRV_DEINIT_START (1UL << 25) +#define VK_FWSTS_DRV_DEINIT_DONE (1UL << 26) +#define VK_FWSTS_RESET_DONE (1UL << 27) +#define VK_FWSTS_DEINIT_TRIGGERED (VK_FWSTS_APP_DEINIT_START | \ + VK_FWSTS_APP_DEINIT_DONE | \ + VK_FWSTS_DRV_DEINIT_START | \ + VK_FWSTS_DRV_DEINIT_DONE) +/* Last nibble for reboot reason */ +#define VK_FWSTS_RESET_REASON_SHIFT 28 +#define VK_FWSTS_RESET_REASON_MASK (0xf << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_SYS_PWRUP (0x0 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_MBOX_DB (0x1 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_M7_WDOG (0x2 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_TEMP (0x3 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_PCI_FLR (0x4 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_PCI_HOT (0x5 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_PCI_WARM (0x6 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_PCI_COLD (0x7 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_L1 (0x8 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_L0 (0x9 << VK_FWSTS_RESET_REASON_SHIFT) +#define VK_FWSTS_RESET_UNKNOWN (0xf << VK_FWSTS_RESET_REASON_SHIFT) + +#endif /* __UAPI_LINUX_MISC_BCM_VK_H */ From 522f692686a73e51300073c08e4107a8cd0b9854 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:16 -0800 Subject: [PATCH 108/633] misc: bcm-vk: add Broadcom VK driver Add initial version of Broadcom VK driver to enumerate PCI device IDs of Valkyrie and Viper device IDs. VK based cards provide real-time high performance, high throughput, low latency offload compute engine operations. They are used for multiple parallel offload tasks as: audio, video and image processing and crypto operations. Further commits add additional features to driver beyond probe/remove. Acked-by: Olof Johansson Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-3-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/Kconfig | 1 + drivers/misc/Makefile | 1 + drivers/misc/bcm-vk/Kconfig | 17 ++++ drivers/misc/bcm-vk/Makefile | 8 ++ drivers/misc/bcm-vk/bcm_vk.h | 29 +++++++ drivers/misc/bcm-vk/bcm_vk_dev.c | 141 +++++++++++++++++++++++++++++++ 6 files changed, 197 insertions(+) create mode 100644 drivers/misc/bcm-vk/Kconfig create mode 100644 drivers/misc/bcm-vk/Makefile create mode 100644 drivers/misc/bcm-vk/bcm_vk.h create mode 100644 drivers/misc/bcm-vk/bcm_vk_dev.c diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index e90c2524e46c..8085213913cc 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -470,6 +470,7 @@ source "drivers/misc/genwqe/Kconfig" source "drivers/misc/echo/Kconfig" source "drivers/misc/cxl/Kconfig" source "drivers/misc/ocxl/Kconfig" +source "drivers/misc/bcm-vk/Kconfig" source "drivers/misc/cardreader/Kconfig" source "drivers/misc/habanalabs/Kconfig" source "drivers/misc/uacce/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index f65e8b18ecd8..456aa8a4c896 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -50,6 +50,7 @@ obj-$(CONFIG_ECHO) += echo/ obj-$(CONFIG_CXL_BASE) += cxl/ obj-$(CONFIG_PCI_ENDPOINT_TEST) += pci_endpoint_test.o obj-$(CONFIG_OCXL) += ocxl/ +obj-$(CONFIG_BCM_VK) += bcm-vk/ obj-y += cardreader/ obj-$(CONFIG_PVPANIC) += pvpanic.o obj-$(CONFIG_HABANA_AI) += habanalabs/ diff --git a/drivers/misc/bcm-vk/Kconfig b/drivers/misc/bcm-vk/Kconfig new file mode 100644 index 000000000000..052f6f28b540 --- /dev/null +++ b/drivers/misc/bcm-vk/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Broadcom VK device +# +config BCM_VK + tristate "Support for Broadcom VK Accelerators" + depends on PCI_MSI + help + Select this option to enable support for Broadcom + VK Accelerators. VK is used for performing + multiple specific offload processing tasks in parallel. + Such offload tasks assist in such operations as video + transcoding, compression, and crypto tasks. + This driver enables userspace programs to access these + accelerators via /dev/bcm-vk.N devices. + + If unsure, say N. diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile new file mode 100644 index 000000000000..f8a7ac4c242f --- /dev/null +++ b/drivers/misc/bcm-vk/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Broadcom VK driver +# + +obj-$(CONFIG_BCM_VK) += bcm_vk.o +bcm_vk-objs := \ + bcm_vk_dev.o diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h new file mode 100644 index 000000000000..9152785199ab --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_H +#define BCM_VK_H + +#include + +#define DRV_MODULE_NAME "bcm-vk" + +/* VK device supports a maximum of 3 bars */ +#define MAX_BAR 3 + +enum pci_barno { + BAR_0 = 0, + BAR_1, + BAR_2 +}; + +#define BCM_VK_NUM_TTY 2 + +struct bcm_vk { + struct pci_dev *pdev; + void __iomem *bar[MAX_BAR]; +}; + +#endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c new file mode 100644 index 000000000000..14afe2477b97 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include +#include +#include +#include + +#include "bcm_vk.h" + +#define PCI_DEVICE_ID_VALKYRIE 0x5e87 +#define PCI_DEVICE_ID_VIPER 0x5e88 + +/* MSIX usages */ +#define VK_MSIX_MSGQ_MAX 3 +#define VK_MSIX_NOTF_MAX 1 +#define VK_MSIX_TTY_MAX BCM_VK_NUM_TTY +#define VK_MSIX_IRQ_MAX (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX + \ + VK_MSIX_TTY_MAX) +#define VK_MSIX_IRQ_MIN_REQ (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX) + +/* Number of bits set in DMA mask*/ +#define BCM_VK_DMA_BITS 64 + +static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + int err; + int i; + int irq; + struct bcm_vk *vk; + struct device *dev = &pdev->dev; + + vk = kzalloc(sizeof(*vk), GFP_KERNEL); + if (!vk) + return -ENOMEM; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Cannot enable PCI device\n"); + goto err_free_exit; + } + vk->pdev = pci_dev_get(pdev); + + err = pci_request_regions(pdev, DRV_MODULE_NAME); + if (err) { + dev_err(dev, "Cannot obtain PCI resources\n"); + goto err_disable_pdev; + } + + /* make sure DMA is good */ + err = dma_set_mask_and_coherent(&pdev->dev, + DMA_BIT_MASK(BCM_VK_DMA_BITS)); + if (err) { + dev_err(dev, "failed to set DMA mask\n"); + goto err_disable_pdev; + } + + pci_set_master(pdev); + pci_set_drvdata(pdev, vk); + + irq = pci_alloc_irq_vectors(pdev, + 1, + VK_MSIX_IRQ_MAX, + PCI_IRQ_MSI | PCI_IRQ_MSIX); + + if (irq < VK_MSIX_IRQ_MIN_REQ) { + dev_err(dev, "failed to get min %d MSIX interrupts, irq(%d)\n", + VK_MSIX_IRQ_MIN_REQ, irq); + err = (irq >= 0) ? -EINVAL : irq; + goto err_disable_pdev; + } + + if (irq != VK_MSIX_IRQ_MAX) + dev_warn(dev, "Number of IRQs %d allocated - requested(%d).\n", + irq, VK_MSIX_IRQ_MAX); + + for (i = 0; i < MAX_BAR; i++) { + /* multiple by 2 for 64 bit BAR mapping */ + vk->bar[i] = pci_ioremap_bar(pdev, i * 2); + if (!vk->bar[i]) { + dev_err(dev, "failed to remap BAR%d\n", i); + goto err_iounmap; + } + } + + return 0; + +err_iounmap: + for (i = 0; i < MAX_BAR; i++) { + if (vk->bar[i]) + pci_iounmap(pdev, vk->bar[i]); + } + pci_release_regions(pdev); + +err_disable_pdev: + pci_free_irq_vectors(pdev); + pci_disable_device(pdev); + pci_dev_put(pdev); + +err_free_exit: + kfree(vk); + + return err; +} + +static void bcm_vk_remove(struct pci_dev *pdev) +{ + int i; + struct bcm_vk *vk = pci_get_drvdata(pdev); + + for (i = 0; i < MAX_BAR; i++) { + if (vk->bar[i]) + pci_iounmap(pdev, vk->bar[i]); + } + + pci_release_regions(pdev); + pci_free_irq_vectors(pdev); + pci_disable_device(pdev); +} + +static const struct pci_device_id bcm_vk_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), }, + { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VIPER), }, + { } +}; +MODULE_DEVICE_TABLE(pci, bcm_vk_ids); + +static struct pci_driver pci_driver = { + .name = DRV_MODULE_NAME, + .id_table = bcm_vk_ids, + .probe = bcm_vk_probe, + .remove = bcm_vk_remove, +}; +module_pci_driver(pci_driver); + +MODULE_DESCRIPTION("Broadcom VK Host Driver"); +MODULE_AUTHOR("Scott Branden "); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION("1.0"); From 064ffc7c3939ebc9c152e17bd9f4496d7b318688 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:17 -0800 Subject: [PATCH 109/633] misc: bcm-vk: add autoload support Add support to load and boot images on card automatically. The kernel module parameter auto_load can be passed in as false to disable such support on probe. As well, nr_scratch_pages can be specified to allocate more or less scratch memory on init as needed for desired card operation. Co-developed-by: Desmond Yan Co-developed-by: James Hu Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: James Hu Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-4-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 250 +++++++++++ drivers/misc/bcm-vk/bcm_vk_dev.c | 723 +++++++++++++++++++++++++++++++ 2 files changed, 973 insertions(+) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 9152785199ab..c4fb61a84e41 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -6,10 +6,187 @@ #ifndef BCM_VK_H #define BCM_VK_H +#include #include +#include #define DRV_MODULE_NAME "bcm-vk" +/* + * Load Image is completed in two stages: + * + * 1) When the VK device boot-up, M7 CPU runs and executes the BootROM. + * The Secure Boot Loader (SBL) as part of the BootROM will run + * to open up ITCM for host to push BOOT1 image. + * SBL will authenticate the image before jumping to BOOT1 image. + * + * 2) Because BOOT1 image is a secured image, we also called it the + * Secure Boot Image (SBI). At second stage, SBI will initialize DDR + * and wait for host to push BOOT2 image to DDR. + * SBI will authenticate the image before jumping to BOOT2 image. + * + */ +/* Location of registers of interest in BAR0 */ + +/* Request register for Secure Boot Loader (SBL) download */ +#define BAR_CODEPUSH_SBL 0x400 +/* Start of ITCM */ +#define CODEPUSH_BOOT1_ENTRY 0x00400000 +#define CODEPUSH_MASK 0xfffff000 +#define CODEPUSH_BOOTSTART BIT(0) + +/* Boot Status register */ +#define BAR_BOOT_STATUS 0x404 + +#define SRAM_OPEN BIT(16) +#define DDR_OPEN BIT(17) + +/* Firmware loader progress status definitions */ +#define FW_LOADER_ACK_SEND_MORE_DATA BIT(18) +#define FW_LOADER_ACK_IN_PROGRESS BIT(19) +#define FW_LOADER_ACK_RCVD_ALL_DATA BIT(20) + +/* Boot1/2 is running in standalone mode */ +#define BOOT_STDALONE_RUNNING BIT(21) + +/* definitions for boot status register */ +#define BOOT_STATE_MASK (0xffffffff & \ + ~(FW_LOADER_ACK_SEND_MORE_DATA | \ + FW_LOADER_ACK_IN_PROGRESS | \ + BOOT_STDALONE_RUNNING)) + +#define BOOT_ERR_SHIFT 4 +#define BOOT_ERR_MASK (0xf << BOOT_ERR_SHIFT) +#define BOOT_PROG_MASK 0xf + +#define BROM_STATUS_NOT_RUN 0x2 +#define BROM_NOT_RUN (SRAM_OPEN | BROM_STATUS_NOT_RUN) +#define BROM_STATUS_COMPLETE 0x6 +#define BROM_RUNNING (SRAM_OPEN | BROM_STATUS_COMPLETE) +#define BOOT1_STATUS_COMPLETE 0x6 +#define BOOT1_RUNNING (DDR_OPEN | BOOT1_STATUS_COMPLETE) +#define BOOT2_STATUS_COMPLETE 0x6 +#define BOOT2_RUNNING (FW_LOADER_ACK_RCVD_ALL_DATA | \ + BOOT2_STATUS_COMPLETE) + +/* Boot request for Secure Boot Image (SBI) */ +#define BAR_CODEPUSH_SBI 0x408 +/* 64M mapped to BAR2 */ +#define CODEPUSH_BOOT2_ENTRY 0x60000000 + +#define BAR_CARD_STATUS 0x410 + +#define BAR_BOOT1_STDALONE_PROGRESS 0x420 +#define BOOT1_STDALONE_SUCCESS (BIT(13) | BIT(14)) +#define BOOT1_STDALONE_PROGRESS_MASK BOOT1_STDALONE_SUCCESS + +#define BAR_METADATA_VERSION 0x440 +#define BAR_OS_UPTIME 0x444 +#define BAR_CHIP_ID 0x448 +#define MAJOR_SOC_REV(_chip_id) (((_chip_id) >> 20) & 0xf) + +#define BAR_CARD_TEMPERATURE 0x45c + +#define BAR_CARD_VOLTAGE 0x460 + +#define BAR_CARD_ERR_LOG 0x464 + +#define BAR_CARD_ERR_MEM 0x468 + +#define BAR_CARD_PWR_AND_THRE 0x46c + +#define BAR_CARD_STATIC_INFO 0x470 + +#define BAR_INTF_VER 0x47c +#define BAR_INTF_VER_MAJOR_SHIFT 16 +#define BAR_INTF_VER_MASK 0xffff +/* + * major and minor semantic version numbers supported + * Please update as required on interface changes + */ +#define SEMANTIC_MAJOR 1 +#define SEMANTIC_MINOR 0 + +/* + * first door bell reg, ie for queue = 0. Only need the first one, as + * we will use the queue number to derive the others + */ +#define VK_BAR0_REGSEG_DB_BASE 0x484 +#define VK_BAR0_REGSEG_DB_REG_GAP 8 /* + * DB register gap, + * DB1 at 0x48c and DB2 at 0x494 + */ + +/* reset register and specific values */ +#define VK_BAR0_RESET_DB_NUM 3 +#define VK_BAR0_RESET_DB_SOFT 0xffffffff +#define VK_BAR0_RESET_DB_HARD 0xfffffffd +#define VK_BAR0_RESET_RAMPDUMP 0xa0000000 + +#define VK_BAR0_Q_DB_BASE(q_num) (VK_BAR0_REGSEG_DB_BASE + \ + ((q_num) * VK_BAR0_REGSEG_DB_REG_GAP)) +#define VK_BAR0_RESET_DB_BASE (VK_BAR0_REGSEG_DB_BASE + \ + (VK_BAR0_RESET_DB_NUM * VK_BAR0_REGSEG_DB_REG_GAP)) + +#define BAR_BOOTSRC_SELECT 0xc78 +/* BOOTSRC definitions */ +#define BOOTSRC_SOFT_ENABLE BIT(14) + +/* Card OS Firmware version size */ +#define BAR_FIRMWARE_TAG_SIZE 50 +#define FIRMWARE_STATUS_PRE_INIT_DONE 0x1f + +/* + * BAR1 + */ + +/* BAR1 message q definition */ + +/* indicate if msgq ctrl in BAR1 is populated */ +#define VK_BAR1_MSGQ_DEF_RDY 0x60c0 +/* ready marker value for the above location, normal boot2 */ +#define VK_BAR1_MSGQ_RDY_MARKER 0xbeefcafe +/* ready marker value for the above location, normal boot2 */ +#define VK_BAR1_DIAG_RDY_MARKER 0xdeadcafe +/* number of msgqs in BAR1 */ +#define VK_BAR1_MSGQ_NR 0x60c4 +/* BAR1 queue control structure offset */ +#define VK_BAR1_MSGQ_CTRL_OFF 0x60c8 + +/* BAR1 ucode and boot1 version tag */ +#define VK_BAR1_UCODE_VER_TAG 0x6170 +#define VK_BAR1_BOOT1_VER_TAG 0x61b0 +#define VK_BAR1_VER_TAG_SIZE 64 + +/* Memory to hold the DMA buffer memory address allocated for boot2 download */ +#define VK_BAR1_DMA_BUF_OFF_HI 0x61e0 +#define VK_BAR1_DMA_BUF_OFF_LO (VK_BAR1_DMA_BUF_OFF_HI + 4) +#define VK_BAR1_DMA_BUF_SZ (VK_BAR1_DMA_BUF_OFF_HI + 8) + +/* Scratch memory allocated on host for VK */ +#define VK_BAR1_SCRATCH_OFF_HI 0x61f0 +#define VK_BAR1_SCRATCH_OFF_LO (VK_BAR1_SCRATCH_OFF_HI + 4) +#define VK_BAR1_SCRATCH_SZ_ADDR (VK_BAR1_SCRATCH_OFF_HI + 8) +#define VK_BAR1_SCRATCH_DEF_NR_PAGES 32 + +/* BAR1 DAUTH info */ +#define VK_BAR1_DAUTH_BASE_ADDR 0x6200 +#define VK_BAR1_DAUTH_STORE_SIZE 0x48 +#define VK_BAR1_DAUTH_VALID_SIZE 0x8 +#define VK_BAR1_DAUTH_MAX 4 +#define VK_BAR1_DAUTH_STORE_ADDR(x) \ + (VK_BAR1_DAUTH_BASE_ADDR + \ + (x) * (VK_BAR1_DAUTH_STORE_SIZE + VK_BAR1_DAUTH_VALID_SIZE)) +#define VK_BAR1_DAUTH_VALID_ADDR(x) \ + (VK_BAR1_DAUTH_STORE_ADDR(x) + VK_BAR1_DAUTH_STORE_SIZE) + +/* BAR1 SOTP AUTH and REVID info */ +#define VK_BAR1_SOTP_REVID_BASE_ADDR 0x6340 +#define VK_BAR1_SOTP_REVID_SIZE 0x10 +#define VK_BAR1_SOTP_REVID_MAX 2 +#define VK_BAR1_SOTP_REVID_ADDR(x) \ + (VK_BAR1_SOTP_REVID_BASE_ADDR + (x) * VK_BAR1_SOTP_REVID_SIZE) + /* VK device supports a maximum of 3 bars */ #define MAX_BAR 3 @@ -21,9 +198,82 @@ enum pci_barno { #define BCM_VK_NUM_TTY 2 +/* DAUTH related info */ +struct bcm_vk_dauth_key { + char store[VK_BAR1_DAUTH_STORE_SIZE]; + char valid[VK_BAR1_DAUTH_VALID_SIZE]; +}; + +struct bcm_vk_dauth_info { + struct bcm_vk_dauth_key keys[VK_BAR1_DAUTH_MAX]; +}; + struct bcm_vk { struct pci_dev *pdev; void __iomem *bar[MAX_BAR]; + + struct bcm_vk_dauth_info dauth_info; + + int devid; /* dev id allocated */ + + struct workqueue_struct *wq_thread; + struct work_struct wq_work; /* work queue for deferred job */ + unsigned long wq_offload[1]; /* various flags on wq requested */ + void *tdma_vaddr; /* test dma segment virtual addr */ + dma_addr_t tdma_addr; /* test dma segment bus addr */ }; +/* wq offload work items bits definitions */ +enum bcm_vk_wq_offload_flags { + BCM_VK_WQ_DWNLD_PEND = 0, + BCM_VK_WQ_DWNLD_AUTO = 1, +}; + +/* + * check if PCIe interface is down on read. Use it when it is + * certain that _val should never be all ones. + */ +#define BCM_VK_INTF_IS_DOWN(val) ((val) == 0xffffffff) + +static inline u32 vkread32(struct bcm_vk *vk, enum pci_barno bar, u64 offset) +{ + return readl(vk->bar[bar] + offset); +} + +static inline void vkwrite32(struct bcm_vk *vk, + u32 value, + enum pci_barno bar, + u64 offset) +{ + writel(value, vk->bar[bar] + offset); +} + +static inline u8 vkread8(struct bcm_vk *vk, enum pci_barno bar, u64 offset) +{ + return readb(vk->bar[bar] + offset); +} + +static inline void vkwrite8(struct bcm_vk *vk, + u8 value, + enum pci_barno bar, + u64 offset) +{ + writeb(value, vk->bar[bar] + offset); +} + +static inline bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk) +{ + u32 rdy_marker = 0; + u32 fw_status; + + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + + if ((fw_status & VK_FWSTS_READY) == VK_FWSTS_READY) + rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); + + return (rdy_marker == VK_BAR1_MSGQ_RDY_MARKER); +} + +int bcm_vk_auto_load_all_images(struct bcm_vk *vk); + #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 14afe2477b97..adc3103c7012 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -3,16 +3,72 @@ * Copyright 2018-2020 Broadcom. */ +#include #include +#include +#include #include #include #include +#include #include "bcm_vk.h" #define PCI_DEVICE_ID_VALKYRIE 0x5e87 #define PCI_DEVICE_ID_VIPER 0x5e88 +static DEFINE_IDA(bcm_vk_ida); + +enum soc_idx { + VALKYRIE_A0 = 0, + VALKYRIE_B0, + VIPER, + VK_IDX_INVALID +}; + +enum img_idx { + IMG_PRI = 0, + IMG_SEC, + IMG_PER_TYPE_MAX +}; + +struct load_image_entry { + const u32 image_type; + const char *image_name[IMG_PER_TYPE_MAX]; +}; + +#define NUM_BOOT_STAGES 2 +/* default firmware images names */ +static const struct load_image_entry image_tab[][NUM_BOOT_STAGES] = { + [VALKYRIE_A0] = { + {VK_IMAGE_TYPE_BOOT1, {"vk_a0-boot1.bin", "vk-boot1.bin"}}, + {VK_IMAGE_TYPE_BOOT2, {"vk_a0-boot2.bin", "vk-boot2.bin"}} + }, + [VALKYRIE_B0] = { + {VK_IMAGE_TYPE_BOOT1, {"vk_b0-boot1.bin", "vk-boot1.bin"}}, + {VK_IMAGE_TYPE_BOOT2, {"vk_b0-boot2.bin", "vk-boot2.bin"}} + }, + + [VIPER] = { + {VK_IMAGE_TYPE_BOOT1, {"vp-boot1.bin", ""}}, + {VK_IMAGE_TYPE_BOOT2, {"vp-boot2.bin", ""}} + }, +}; + +/* Location of memory base addresses of interest in BAR1 */ +/* Load Boot1 to start of ITCM */ +#define BAR1_CODEPUSH_BASE_BOOT1 0x100000 + +/* Allow minimum 1s for Load Image timeout responses */ +#define LOAD_IMAGE_TIMEOUT_MS (1 * MSEC_PER_SEC) + +/* Image startup timeouts */ +#define BOOT1_STARTUP_TIMEOUT_MS (5 * MSEC_PER_SEC) +#define BOOT2_STARTUP_TIMEOUT_MS (10 * MSEC_PER_SEC) + +/* 1ms wait for checking the transfer complete status */ +#define TXFR_COMPLETE_TIMEOUT_MS 1 + /* MSIX usages */ #define VK_MSIX_MSGQ_MAX 3 #define VK_MSIX_NOTF_MAX 1 @@ -24,13 +80,570 @@ /* Number of bits set in DMA mask*/ #define BCM_VK_DMA_BITS 64 +/* Ucode boot wait time */ +#define BCM_VK_UCODE_BOOT_US (100 * USEC_PER_MSEC) +/* 50% margin */ +#define BCM_VK_UCODE_BOOT_MAX_US ((BCM_VK_UCODE_BOOT_US * 3) >> 1) + +/* deinit time for the card os after receiving doorbell */ +#define BCM_VK_DEINIT_TIME_MS (2 * MSEC_PER_SEC) + +/* + * module parameters + */ +static bool auto_load = true; +module_param(auto_load, bool, 0444); +MODULE_PARM_DESC(auto_load, + "Load images automatically at PCIe probe time.\n"); +static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES; +module_param(nr_scratch_pages, uint, 0444); +MODULE_PARM_DESC(nr_scratch_pages, + "Number of pre allocated DMAable coherent pages.\n"); + +static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 reg; + u16 major, minor; + int ret = 0; + + /* read interface register */ + reg = vkread32(vk, BAR_0, BAR_INTF_VER); + major = (reg >> BAR_INTF_VER_MAJOR_SHIFT) & BAR_INTF_VER_MASK; + minor = reg & BAR_INTF_VER_MASK; + + /* + * if major number is 0, it is pre-release and it would be allowed + * to continue, else, check versions accordingly + */ + if (!major) { + dev_warn(dev, "Pre-release major.minor=%d.%d - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + } else if (major != SEMANTIC_MAJOR) { + dev_err(dev, + "Intf major.minor=%d.%d rejected - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + ret = -EPFNOSUPPORT; + } else { + dev_dbg(dev, + "Intf major.minor=%d.%d passed - drv %d.%d\n", + major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + } + return ret; +} + +static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar, + u64 offset, u32 mask, u32 value, + unsigned long timeout_ms) +{ + struct device *dev = &vk->pdev->dev; + unsigned long start_time; + unsigned long timeout; + u32 rd_val, boot_status; + + start_time = jiffies; + timeout = start_time + msecs_to_jiffies(timeout_ms); + + do { + rd_val = vkread32(vk, bar, offset); + dev_dbg(dev, "BAR%d Offset=0x%llx: 0x%x\n", + bar, offset, rd_val); + + /* check for any boot err condition */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (boot_status & BOOT_ERR_MASK) { + dev_err(dev, "Boot Err 0x%x, progress 0x%x after %d ms\n", + (boot_status & BOOT_ERR_MASK) >> BOOT_ERR_SHIFT, + boot_status & BOOT_PROG_MASK, + jiffies_to_msecs(jiffies - start_time)); + return -EFAULT; + } + + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + + cpu_relax(); + cond_resched(); + } while ((rd_val & mask) != value); + + return 0; +} + +static int bcm_vk_sync_card_info(struct bcm_vk *vk) +{ + u32 rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); + + /* check for marker, but allow diags mode to skip sync */ + if (!bcm_vk_msgq_marker_valid(vk)) + return (rdy_marker == VK_BAR1_DIAG_RDY_MARKER ? 0 : -EINVAL); + + /* + * Write down scratch addr which is used for DMA. For + * signed part, BAR1 is accessible only after boot2 has come + * up + */ + if (vk->tdma_addr) { + vkwrite32(vk, (u64)vk->tdma_addr >> 32, BAR_1, + VK_BAR1_SCRATCH_OFF_HI); + vkwrite32(vk, (u32)vk->tdma_addr, BAR_1, + VK_BAR1_SCRATCH_OFF_LO); + vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1, + VK_BAR1_SCRATCH_SZ_ADDR); + } + return 0; +} + +static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp, + dma_addr_t host_buf_addr, u32 buf_size) +{ + /* update the dma address to the card */ + vkwrite32(vk, (u64)host_buf_addr >> 32, BAR_1, + VK_BAR1_DMA_BUF_OFF_HI); + vkwrite32(vk, (u32)host_buf_addr, BAR_1, + VK_BAR1_DMA_BUF_OFF_LO); + vkwrite32(vk, buf_size, BAR_1, VK_BAR1_DMA_BUF_SZ); +} + +static int bcm_vk_load_image_by_type(struct bcm_vk *vk, u32 load_type, + const char *filename) +{ + struct device *dev = &vk->pdev->dev; + const struct firmware *fw = NULL; + void *bufp = NULL; + size_t max_buf, offset; + int ret; + u64 offset_codepush; + u32 codepush; + u32 value; + dma_addr_t boot_dma_addr; + bool is_stdalone; + + if (load_type == VK_IMAGE_TYPE_BOOT1) { + /* + * After POR, enable VK soft BOOTSRC so bootrom do not clear + * the pushed image (the TCM memories). + */ + value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT); + value |= BOOTSRC_SOFT_ENABLE; + vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT); + + codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY; + offset_codepush = BAR_CODEPUSH_SBL; + + /* Write a 1 to request SRAM open bit */ + vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush); + + /* Wait for VK to respond */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN, + SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret); + goto err_buf_out; + } + + max_buf = SZ_256K; + bufp = dma_alloc_coherent(dev, + max_buf, + &boot_dma_addr, GFP_KERNEL); + if (!bufp) { + dev_err(dev, "Error allocating 0x%zx\n", max_buf); + ret = -ENOMEM; + goto err_buf_out; + } + } else if (load_type == VK_IMAGE_TYPE_BOOT2) { + codepush = CODEPUSH_BOOT2_ENTRY; + offset_codepush = BAR_CODEPUSH_SBI; + + /* Wait for VK to respond */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN, + DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "boot2 wait DDR open error - ret(%d)\n", + ret); + goto err_buf_out; + } + + max_buf = SZ_4M; + bufp = dma_alloc_coherent(dev, + max_buf, + &boot_dma_addr, GFP_KERNEL); + if (!bufp) { + dev_err(dev, "Error allocating 0x%zx\n", max_buf); + ret = -ENOMEM; + goto err_buf_out; + } + + bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf); + } else { + dev_err(dev, "Error invalid image type 0x%x\n", load_type); + ret = -EINVAL; + goto err_buf_out; + } + + offset = 0; + ret = request_partial_firmware_into_buf(&fw, filename, dev, + bufp, max_buf, offset); + if (ret) { + dev_err(dev, "Error %d requesting firmware file: %s\n", + ret, filename); + goto err_firmware_out; + } + dev_dbg(dev, "size=0x%zx\n", fw->size); + if (load_type == VK_IMAGE_TYPE_BOOT1) + memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1, + bufp, + fw->size); + + dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush); + vkwrite32(vk, codepush, BAR_0, offset_codepush); + + if (load_type == VK_IMAGE_TYPE_BOOT1) { + u32 boot_status; + + /* wait until done */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, + BOOT1_RUNNING, + BOOT1_RUNNING, + BOOT1_STARTUP_TIMEOUT_MS); + + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + is_stdalone = !BCM_VK_INTF_IS_DOWN(boot_status) && + (boot_status & BOOT_STDALONE_RUNNING); + if (ret && !is_stdalone) { + dev_err(dev, + "Timeout %ld ms waiting for boot1 to come up - ret(%d)\n", + BOOT1_STARTUP_TIMEOUT_MS, ret); + goto err_firmware_out; + } else if (is_stdalone) { + u32 reg; + + reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS); + if ((reg & BOOT1_STDALONE_PROGRESS_MASK) == + BOOT1_STDALONE_SUCCESS) { + dev_info(dev, "Boot1 standalone success\n"); + ret = 0; + } else { + dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n", + BOOT1_STARTUP_TIMEOUT_MS); + ret = -EINVAL; + goto err_firmware_out; + } + } + } else if (load_type == VK_IMAGE_TYPE_BOOT2) { + unsigned long timeout; + + timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); + + /* To send more data to VK than max_buf allowed at a time */ + do { + /* + * Check for ack from card. when Ack is received, + * it means all the data is received by card. + * Exit the loop after ack is received. + */ + ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, + FW_LOADER_ACK_RCVD_ALL_DATA, + FW_LOADER_ACK_RCVD_ALL_DATA, + TXFR_COMPLETE_TIMEOUT_MS); + if (ret == 0) { + dev_dbg(dev, "Exit boot2 download\n"); + break; + } else if (ret == -EFAULT) { + dev_err(dev, "Error detected during ACK waiting"); + goto err_firmware_out; + } + + /* exit the loop, if there is no response from card */ + if (time_after(jiffies, timeout)) { + dev_err(dev, "Error. No reply from card\n"); + ret = -ETIMEDOUT; + goto err_firmware_out; + } + + /* Wait for VK to open BAR space to copy new data */ + ret = bcm_vk_wait(vk, BAR_0, offset_codepush, + codepush, 0, + TXFR_COMPLETE_TIMEOUT_MS); + if (ret == 0) { + offset += max_buf; + ret = request_partial_firmware_into_buf + (&fw, + filename, + dev, bufp, + max_buf, + offset); + if (ret) { + dev_err(dev, + "Error %d requesting firmware file: %s offset: 0x%zx\n", + ret, filename, offset); + goto err_firmware_out; + } + dev_dbg(dev, "size=0x%zx\n", fw->size); + dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", + codepush, offset_codepush); + vkwrite32(vk, codepush, BAR_0, offset_codepush); + /* reload timeout after every codepush */ + timeout = jiffies + + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); + } else if (ret == -EFAULT) { + dev_err(dev, "Error detected waiting for transfer\n"); + goto err_firmware_out; + } + } while (1); + + /* wait for fw status bits to indicate app ready */ + ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS, + VK_FWSTS_READY, + VK_FWSTS_READY, + BOOT2_STARTUP_TIMEOUT_MS); + if (ret < 0) { + dev_err(dev, "Boot2 not ready - ret(%d)\n", ret); + goto err_firmware_out; + } + + is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) & + BOOT_STDALONE_RUNNING; + if (!is_stdalone) { + ret = bcm_vk_intf_ver_chk(vk); + if (ret) { + dev_err(dev, "failure in intf version check\n"); + goto err_firmware_out; + } + + /* sync & channel other info */ + ret = bcm_vk_sync_card_info(vk); + if (ret) { + dev_err(dev, "Syncing Card Info failure\n"); + goto err_firmware_out; + } + } + } + +err_firmware_out: + release_firmware(fw); + +err_buf_out: + if (bufp) + dma_free_coherent(dev, max_buf, bufp, boot_dma_addr); + + return ret; +} + +static u32 bcm_vk_next_boot_image(struct bcm_vk *vk) +{ + u32 boot_status; + u32 fw_status; + u32 load_type = 0; /* default for unknown */ + + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + + if (!BCM_VK_INTF_IS_DOWN(boot_status) && (boot_status & SRAM_OPEN)) + load_type = VK_IMAGE_TYPE_BOOT1; + else if (boot_status == BOOT1_RUNNING) + load_type = VK_IMAGE_TYPE_BOOT2; + + /* Log status so that we know different stages */ + dev_info(&vk->pdev->dev, + "boot-status value for next image: 0x%x : fw-status 0x%x\n", + boot_status, fw_status); + + return load_type; +} + +static enum soc_idx get_soc_idx(struct bcm_vk *vk) +{ + struct pci_dev *pdev = vk->pdev; + enum soc_idx idx = VK_IDX_INVALID; + u32 rev; + static enum soc_idx const vk_soc_tab[] = { VALKYRIE_A0, VALKYRIE_B0 }; + + switch (pdev->device) { + case PCI_DEVICE_ID_VALKYRIE: + /* get the chip id to decide sub-class */ + rev = MAJOR_SOC_REV(vkread32(vk, BAR_0, BAR_CHIP_ID)); + if (rev < ARRAY_SIZE(vk_soc_tab)) { + idx = vk_soc_tab[rev]; + } else { + /* Default to A0 firmware for all other chip revs */ + idx = VALKYRIE_A0; + dev_warn(&pdev->dev, + "Rev %d not in image lookup table, default to idx=%d\n", + rev, idx); + } + break; + + case PCI_DEVICE_ID_VIPER: + idx = VIPER; + break; + + default: + dev_err(&pdev->dev, "no images for 0x%x\n", pdev->device); + } + return idx; +} + +static const char *get_load_fw_name(struct bcm_vk *vk, + const struct load_image_entry *entry) +{ + const struct firmware *fw; + struct device *dev = &vk->pdev->dev; + int ret; + unsigned long dummy; + int i; + + for (i = 0; i < IMG_PER_TYPE_MAX; i++) { + fw = NULL; + ret = request_partial_firmware_into_buf(&fw, + entry->image_name[i], + dev, &dummy, + sizeof(dummy), + 0); + release_firmware(fw); + if (!ret) + return entry->image_name[i]; + } + return NULL; +} + +int bcm_vk_auto_load_all_images(struct bcm_vk *vk) +{ + int i, ret = -1; + enum soc_idx idx; + struct device *dev = &vk->pdev->dev; + u32 curr_type; + const char *curr_name; + + idx = get_soc_idx(vk); + if (idx == VK_IDX_INVALID) + goto auto_load_all_exit; + + /* log a message to know the relative loading order */ + dev_dbg(dev, "Load All for device %d\n", vk->devid); + + for (i = 0; i < NUM_BOOT_STAGES; i++) { + curr_type = image_tab[idx][i].image_type; + if (bcm_vk_next_boot_image(vk) == curr_type) { + curr_name = get_load_fw_name(vk, &image_tab[idx][i]); + if (!curr_name) { + dev_err(dev, "No suitable firmware exists for type %d", + curr_type); + ret = -ENOENT; + goto auto_load_all_exit; + } + ret = bcm_vk_load_image_by_type(vk, curr_type, + curr_name); + dev_info(dev, "Auto load %s, ret %d\n", + curr_name, ret); + + if (ret) { + dev_err(dev, "Error loading default %s\n", + curr_name); + goto auto_load_all_exit; + } + } + } + +auto_load_all_exit: + return ret; +} + +static int bcm_vk_trigger_autoload(struct bcm_vk *vk) +{ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) + return -EPERM; + + set_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); + queue_work(vk->wq_thread, &vk->wq_work); + + return 0; +} + +/* + * deferred work queue for auto download. + */ +static void bcm_vk_wq_handler(struct work_struct *work) +{ + struct bcm_vk *vk = container_of(work, struct bcm_vk, wq_work); + + if (test_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload)) { + bcm_vk_auto_load_all_images(vk); + + /* + * at the end of operation, clear AUTO bit and pending + * bit + */ + clear_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + } +} + +static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val) +{ + vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE); +} + +static int bcm_vk_trigger_reset(struct bcm_vk *vk) +{ + u32 i; + u32 value, boot_status; + static const u32 bar0_reg_clr_list[] = { BAR_OS_UPTIME, + BAR_INTF_VER, + BAR_CARD_VOLTAGE, + BAR_CARD_TEMPERATURE, + BAR_CARD_PWR_AND_THRE }; + + /* make tag '\0' terminated */ + vkwrite32(vk, 0, BAR_1, VK_BAR1_BOOT1_VER_TAG); + + for (i = 0; i < VK_BAR1_DAUTH_MAX; i++) { + vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_STORE_ADDR(i)); + vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_VALID_ADDR(i)); + } + for (i = 0; i < VK_BAR1_SOTP_REVID_MAX; i++) + vkwrite32(vk, 0, BAR_1, VK_BAR1_SOTP_REVID_ADDR(i)); + + /* + * When boot request fails, the CODE_PUSH_OFFSET stays persistent. + * Allowing us to debug the failure. When we call reset, + * we should clear CODE_PUSH_OFFSET so ROM does not execute + * boot again (and fails again) and instead waits for a new + * codepush. And, if previous boot has encountered error, need + * to clear the entry values + */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (boot_status & BOOT_ERR_MASK) { + dev_info(&vk->pdev->dev, + "Card in boot error 0x%x, clear CODEPUSH val\n", + boot_status); + value = 0; + } else { + value = vkread32(vk, BAR_0, BAR_CODEPUSH_SBL); + value &= CODEPUSH_MASK; + } + vkwrite32(vk, value, BAR_0, BAR_CODEPUSH_SBL); + + /* reset fw_status with proper reason, and press db */ + vkwrite32(vk, VK_FWSTS_RESET_MBOX_DB, BAR_0, VK_BAR_FWSTS); + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_SOFT); + + /* clear other necessary registers records */ + for (i = 0; i < ARRAY_SIZE(bar0_reg_clr_list); i++) + vkwrite32(vk, 0, BAR_0, bar0_reg_clr_list[i]); + + return 0; +} + static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { int err; int i; + int id; int irq; + char name[20]; struct bcm_vk *vk; struct device *dev = &pdev->dev; + u32 boot_status; vk = kzalloc(sizeof(*vk), GFP_KERNEL); if (!vk) @@ -57,6 +670,18 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_disable_pdev; } + /* The tdma is a scratch area for some DMA testings. */ + if (nr_scratch_pages) { + vk->tdma_vaddr = dma_alloc_coherent + (dev, + nr_scratch_pages * PAGE_SIZE, + &vk->tdma_addr, GFP_KERNEL); + if (!vk->tdma_vaddr) { + err = -ENOMEM; + goto err_disable_pdev; + } + } + pci_set_master(pdev); pci_set_drvdata(pdev, vk); @@ -85,8 +710,53 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } + id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); + if (id < 0) { + err = id; + dev_err(dev, "unable to get id\n"); + goto err_iounmap; + } + + vk->devid = id; + snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id); + + INIT_WORK(&vk->wq_work, bcm_vk_wq_handler); + + /* create dedicated workqueue */ + vk->wq_thread = create_singlethread_workqueue(name); + if (!vk->wq_thread) { + dev_err(dev, "Fail to create workqueue thread\n"); + err = -ENOMEM; + goto err_ida_remove; + } + + /* sync other info */ + bcm_vk_sync_card_info(vk); + + /* + * lets trigger an auto download. We don't want to do it serially here + * because at probing time, it is not supposed to block for a long time. + */ + boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + if (auto_load) { + if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { + if (bcm_vk_trigger_autoload(vk)) + goto err_destroy_workqueue; + } else { + dev_err(dev, + "Auto-load skipped - BROM not in proper state (0x%x)\n", + boot_status); + } + } + return 0; +err_destroy_workqueue: + destroy_workqueue(vk->wq_thread); + +err_ida_remove: + ida_simple_remove(&bcm_vk_ida, id); + err_iounmap: for (i = 0; i < MAX_BAR; i++) { if (vk->bar[i]) @@ -95,6 +765,10 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_release_regions(pdev); err_disable_pdev: + if (vk->tdma_vaddr) + dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, + vk->tdma_vaddr, vk->tdma_addr); + pci_free_irq_vectors(pdev); pci_disable_device(pdev); pci_dev_put(pdev); @@ -110,6 +784,22 @@ static void bcm_vk_remove(struct pci_dev *pdev) int i; struct bcm_vk *vk = pci_get_drvdata(pdev); + /* + * Trigger a reset to card and wait enough time for UCODE to rerun, + * which re-initialize the card into its default state. + * This ensures when driver is re-enumerated it will start from + * a completely clean state. + */ + bcm_vk_trigger_reset(vk); + usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US); + + if (vk->tdma_vaddr) + dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, + vk->tdma_vaddr, vk->tdma_addr); + + cancel_work_sync(&vk->wq_work); + destroy_workqueue(vk->wq_thread); + for (i = 0; i < MAX_BAR; i++) { if (vk->bar[i]) pci_iounmap(pdev, vk->bar[i]); @@ -120,6 +810,38 @@ static void bcm_vk_remove(struct pci_dev *pdev) pci_disable_device(pdev); } +static void bcm_vk_shutdown(struct pci_dev *pdev) +{ + struct bcm_vk *vk = pci_get_drvdata(pdev); + u32 reg, boot_stat; + + reg = vkread32(vk, BAR_0, BAR_BOOT_STATUS); + boot_stat = reg & BOOT_STATE_MASK; + + if (boot_stat == BOOT1_RUNNING) { + /* simply trigger a reset interrupt to park it */ + bcm_vk_trigger_reset(vk); + } else if (boot_stat == BROM_NOT_RUN) { + int err; + u16 lnksta; + + /* + * The boot status only reflects boot condition since last reset + * As ucode will run only once to configure pcie, if multiple + * resets happen, we lost track if ucode has run or not. + * Here, read the current link speed and use that to + * sync up the bootstatus properly so that on reboot-back-up, + * it has the proper state to start with autoload + */ + err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + if (!err && + (lnksta & PCI_EXP_LNKSTA_CLS) != PCI_EXP_LNKSTA_CLS_2_5GB) { + reg |= BROM_STATUS_COMPLETE; + vkwrite32(vk, reg, BAR_0, BAR_BOOT_STATUS); + } + } +} + static const struct pci_device_id bcm_vk_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), }, { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VIPER), }, @@ -132,6 +854,7 @@ static struct pci_driver pci_driver = { .id_table = bcm_vk_ids, .probe = bcm_vk_probe, .remove = bcm_vk_remove, + .shutdown = bcm_vk_shutdown, }; module_pci_driver(pci_driver); From bfc53e01d221e7068d66a0a051762f8b1edcbf63 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:18 -0800 Subject: [PATCH 110/633] misc: bcm-vk: add misc device to Broadcom VK driver Add misc device base support to create and remove devnode. Additional misc functions for open/read/write/release/ioctl/sysfs, etc will be added in follow on commits to allow for individual review. Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-5-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 2 ++ drivers/misc/bcm-vk/bcm_vk_dev.c | 36 +++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index c4fb61a84e41..0a366db693c8 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -7,6 +7,7 @@ #define BCM_VK_H #include +#include #include #include @@ -214,6 +215,7 @@ struct bcm_vk { struct bcm_vk_dauth_info dauth_info; + struct miscdevice miscdev; int devid; /* dev id allocated */ struct workqueue_struct *wq_thread; diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index adc3103c7012..4ecd5b5f80d3 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -643,6 +644,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) char name[20]; struct bcm_vk *vk; struct device *dev = &pdev->dev; + struct miscdevice *misc_device; u32 boot_status; vk = kzalloc(sizeof(*vk), GFP_KERNEL); @@ -719,6 +721,19 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) vk->devid = id; snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id); + misc_device = &vk->miscdev; + misc_device->minor = MISC_DYNAMIC_MINOR; + misc_device->name = kstrdup(name, GFP_KERNEL); + if (!misc_device->name) { + err = -ENOMEM; + goto err_ida_remove; + } + + err = misc_register(misc_device); + if (err) { + dev_err(dev, "failed to register device\n"); + goto err_kfree_name; + } INIT_WORK(&vk->wq_work, bcm_vk_wq_handler); @@ -727,7 +742,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!vk->wq_thread) { dev_err(dev, "Fail to create workqueue thread\n"); err = -ENOMEM; - goto err_ida_remove; + goto err_misc_deregister; } /* sync other info */ @@ -749,11 +764,20 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } + dev_dbg(dev, "BCM-VK:%u created\n", id); + return 0; err_destroy_workqueue: destroy_workqueue(vk->wq_thread); +err_misc_deregister: + misc_deregister(misc_device); + +err_kfree_name: + kfree(misc_device->name); + misc_device->name = NULL; + err_ida_remove: ida_simple_remove(&bcm_vk_ida, id); @@ -783,6 +807,7 @@ static void bcm_vk_remove(struct pci_dev *pdev) { int i; struct bcm_vk *vk = pci_get_drvdata(pdev); + struct miscdevice *misc_device = &vk->miscdev; /* * Trigger a reset to card and wait enough time for UCODE to rerun, @@ -797,6 +822,13 @@ static void bcm_vk_remove(struct pci_dev *pdev) dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, vk->tdma_vaddr, vk->tdma_addr); + /* remove if name is set which means misc dev registered */ + if (misc_device->name) { + misc_deregister(misc_device); + kfree(misc_device->name); + ida_simple_remove(&bcm_vk_ida, vk->devid); + } + cancel_work_sync(&vk->wq_work); destroy_workqueue(vk->wq_thread); @@ -805,6 +837,8 @@ static void bcm_vk_remove(struct pci_dev *pdev) pci_iounmap(pdev, vk->bar[i]); } + dev_dbg(&pdev->dev, "BCM-VK:%d released\n", vk->devid); + pci_release_regions(pdev); pci_free_irq_vectors(pdev); pci_disable_device(pdev); From af22527e82d12f9d0b5afb39f25926a91d5fa7e7 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:19 -0800 Subject: [PATCH 111/633] misc: bcm-vk: add triggers when host panic or reboots to notify card Pass down an interrupt to card in case of panic or reboot so that card can take appropriate action to perform a clean reset. Uses kernel notifier block either directly (register on panic list), or implicitly (add shutdown method for PCI device). Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-6-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 2 ++ drivers/misc/bcm-vk/bcm_vk_dev.c | 29 ++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 0a366db693c8..f428ad9a0c3d 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -223,6 +223,8 @@ struct bcm_vk { unsigned long wq_offload[1]; /* various flags on wq requested */ void *tdma_vaddr; /* test dma segment virtual addr */ dma_addr_t tdma_addr; /* test dma segment bus addr */ + + struct notifier_block panic_nb; }; /* wq offload work items bits definitions */ diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 4ecd5b5f80d3..09d99bd36e8a 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -635,6 +635,16 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) return 0; } +static int bcm_vk_on_panic(struct notifier_block *nb, + unsigned long e, void *p) +{ + struct bcm_vk *vk = container_of(nb, struct bcm_vk, panic_nb); + + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD); + + return 0; +} + static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { int err; @@ -748,6 +758,15 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* sync other info */ bcm_vk_sync_card_info(vk); + /* register for panic notifier */ + vk->panic_nb.notifier_call = bcm_vk_on_panic; + err = atomic_notifier_chain_register(&panic_notifier_list, + &vk->panic_nb); + if (err) { + dev_err(dev, "Fail to register panic notifier\n"); + goto err_destroy_workqueue; + } + /* * lets trigger an auto download. We don't want to do it serially here * because at probing time, it is not supposed to block for a long time. @@ -756,7 +775,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (auto_load) { if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { if (bcm_vk_trigger_autoload(vk)) - goto err_destroy_workqueue; + goto err_unregister_panic_notifier; } else { dev_err(dev, "Auto-load skipped - BROM not in proper state (0x%x)\n", @@ -768,6 +787,10 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; +err_unregister_panic_notifier: + atomic_notifier_chain_unregister(&panic_notifier_list, + &vk->panic_nb); + err_destroy_workqueue: destroy_workqueue(vk->wq_thread); @@ -818,6 +841,10 @@ static void bcm_vk_remove(struct pci_dev *pdev) bcm_vk_trigger_reset(vk); usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US); + /* unregister panic notifier */ + atomic_notifier_chain_unregister(&panic_notifier_list, + &vk->panic_nb); + if (vk->tdma_vaddr) dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, vk->tdma_vaddr, vk->tdma_addr); From 22c30607d1e0c604b2450a7aa5bc90a63e24f088 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:20 -0800 Subject: [PATCH 112/633] misc: bcm-vk: add open/release Add open/release to replace private data with context for other methods to use. Reason for the context is because it is allowed for multiple sessions to open sysfs. For each file open, when upper layer queries the response, only those that are tied to a specified open should be returned. Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-7-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/Makefile | 4 +- drivers/misc/bcm-vk/bcm_vk.h | 15 ++++ drivers/misc/bcm-vk/bcm_vk_dev.c | 23 ++++++ drivers/misc/bcm-vk/bcm_vk_msg.c | 127 +++++++++++++++++++++++++++++++ drivers/misc/bcm-vk/bcm_vk_msg.h | 31 ++++++++ 5 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.c create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.h diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile index f8a7ac4c242f..a2ae79858409 100644 --- a/drivers/misc/bcm-vk/Makefile +++ b/drivers/misc/bcm-vk/Makefile @@ -5,4 +5,6 @@ obj-$(CONFIG_BCM_VK) += bcm_vk.o bcm_vk-objs := \ - bcm_vk_dev.o + bcm_vk_dev.o \ + bcm_vk_msg.o + diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index f428ad9a0c3d..5f0fcfdaf265 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -7,9 +7,14 @@ #define BCM_VK_H #include +#include #include +#include #include #include +#include + +#include "bcm_vk_msg.h" #define DRV_MODULE_NAME "bcm-vk" @@ -218,6 +223,13 @@ struct bcm_vk { struct miscdevice miscdev; int devid; /* dev id allocated */ + /* Reference-counting to handle file operations */ + struct kref kref; + + spinlock_t ctx_lock; /* Spinlock for component context */ + struct bcm_vk_ctx ctx[VK_CMPT_CTX_MAX]; + struct bcm_vk_ht_entry pid_ht[VK_PID_HT_SZ]; + struct workqueue_struct *wq_thread; struct work_struct wq_work; /* work queue for deferred job */ unsigned long wq_offload[1]; /* various flags on wq requested */ @@ -278,6 +290,9 @@ static inline bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk) return (rdy_marker == VK_BAR1_MSGQ_RDY_MARKER); } +int bcm_vk_open(struct inode *inode, struct file *p_file); +int bcm_vk_release(struct inode *inode, struct file *p_file); +void bcm_vk_release_data(struct kref *kref); int bcm_vk_auto_load_all_images(struct bcm_vk *vk); #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 09d99bd36e8a..79fffb1e6f84 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -635,6 +636,12 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) return 0; } +static const struct file_operations bcm_vk_fops = { + .owner = THIS_MODULE, + .open = bcm_vk_open, + .release = bcm_vk_release, +}; + static int bcm_vk_on_panic(struct notifier_block *nb, unsigned long e, void *p) { @@ -657,10 +664,13 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct miscdevice *misc_device; u32 boot_status; + /* allocate vk structure which is tied to kref for freeing */ vk = kzalloc(sizeof(*vk), GFP_KERNEL); if (!vk) return -ENOMEM; + kref_init(&vk->kref); + err = pci_enable_device(pdev); if (err) { dev_err(dev, "Cannot enable PCI device\n"); @@ -738,6 +748,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = -ENOMEM; goto err_ida_remove; } + misc_device->fops = &bcm_vk_fops, err = misc_register(misc_device); if (err) { @@ -826,6 +837,16 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return err; } +void bcm_vk_release_data(struct kref *kref) +{ + struct bcm_vk *vk = container_of(kref, struct bcm_vk, kref); + struct pci_dev *pdev = vk->pdev; + + dev_dbg(&pdev->dev, "BCM-VK:%d release data 0x%p\n", vk->devid, vk); + pci_dev_put(pdev); + kfree(vk); +} + static void bcm_vk_remove(struct pci_dev *pdev) { int i; @@ -869,6 +890,8 @@ static void bcm_vk_remove(struct pci_dev *pdev) pci_release_regions(pdev); pci_free_irq_vectors(pdev); pci_disable_device(pdev); + + kref_put(&vk->kref, bcm_vk_release_data); } static void bcm_vk_shutdown(struct pci_dev *pdev) diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c new file mode 100644 index 000000000000..2d9a6b4e5f61 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include "bcm_vk.h" +#include "bcm_vk_msg.h" + +/* + * allocate a ctx per file struct + */ +static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) +{ + u32 i; + struct bcm_vk_ctx *ctx = NULL; + u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT); + + spin_lock(&vk->ctx_lock); + + for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { + if (!vk->ctx[i].in_use) { + vk->ctx[i].in_use = true; + ctx = &vk->ctx[i]; + break; + } + } + + if (!ctx) { + dev_err(&vk->pdev->dev, "All context in use\n"); + + goto all_in_use_exit; + } + + /* set the pid and insert it to hash table */ + ctx->pid = pid; + ctx->hash_idx = hash_idx; + list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head); + + /* increase kref */ + kref_get(&vk->kref); + +all_in_use_exit: + spin_unlock(&vk->ctx_lock); + + return ctx; +} + +static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) +{ + u32 idx; + u32 hash_idx; + pid_t pid; + struct bcm_vk_ctx *entry; + int count = 0; + + if (!ctx) { + dev_err(&vk->pdev->dev, "NULL context detected\n"); + return -EINVAL; + } + idx = ctx->idx; + pid = ctx->pid; + + spin_lock(&vk->ctx_lock); + + if (!vk->ctx[idx].in_use) { + dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx); + } else { + vk->ctx[idx].in_use = false; + vk->ctx[idx].miscdev = NULL; + + /* Remove it from hash list and see if it is the last one. */ + list_del(&ctx->node); + hash_idx = ctx->hash_idx; + list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) { + if (entry->pid == pid) + count++; + } + } + + spin_unlock(&vk->ctx_lock); + + return count; +} + +int bcm_vk_open(struct inode *inode, struct file *p_file) +{ + struct bcm_vk_ctx *ctx; + struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data; + struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + int rc = 0; + + /* get a context and set it up for file */ + ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current)); + if (!ctx) { + dev_err(dev, "Error allocating context\n"); + rc = -ENOMEM; + } else { + /* + * set up context and replace private data with context for + * other methods to use. Reason for the context is because + * it is allowed for multiple sessions to open the sysfs, and + * for each file open, when upper layer query the response, + * only those that are tied to a specific open should be + * returned. The context->idx will be used for such binding + */ + ctx->miscdev = miscdev; + p_file->private_data = ctx; + dev_dbg(dev, "ctx_returned with idx %d, pid %d\n", + ctx->idx, ctx->pid); + } + return rc; +} + +int bcm_vk_release(struct inode *inode, struct file *p_file) +{ + int ret; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + + ret = bcm_vk_free_ctx(vk, ctx); + + kref_put(&vk->kref, bcm_vk_release_data); + + return ret; +} + diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h new file mode 100644 index 000000000000..32516abcaf89 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_msg.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_MSG_H +#define BCM_VK_MSG_H + +/* context per session opening of sysfs */ +struct bcm_vk_ctx { + struct list_head node; /* use for linkage in Hash Table */ + unsigned int idx; + bool in_use; + pid_t pid; + u32 hash_idx; + struct miscdevice *miscdev; +}; + +/* pid hash table entry */ +struct bcm_vk_ht_entry { + struct list_head head; +}; + +/* total number of supported ctx, 32 ctx each for 5 components */ +#define VK_CMPT_CTX_MAX (32 * 5) + +/* hash table defines to store the opened FDs */ +#define VK_PID_HT_SHIFT_BIT 7 /* 128 */ +#define VK_PID_HT_SZ BIT(VK_PID_HT_SHIFT_BIT) + +#endif From 7367e0ad77d21ff851404ffd71bd1f626bc93ef2 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:21 -0800 Subject: [PATCH 113/633] misc: bcm-vk: add ioctl load_image Add ioctl support to issue load_image operation to VK card. Co-developed-by: Desmond Yan Co-developed-by: James Hu Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: James Hu Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-8-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 3 + drivers/misc/bcm-vk/bcm_vk_dev.c | 95 ++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 5f0fcfdaf265..726aab71bb6b 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "bcm_vk_msg.h" @@ -220,6 +221,8 @@ struct bcm_vk { struct bcm_vk_dauth_info dauth_info; + /* mutex to protect the ioctls */ + struct mutex mutex; struct miscdevice miscdev; int devid; /* dev id allocated */ diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 79fffb1e6f84..203a1cf2bae3 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -580,6 +581,71 @@ static void bcm_vk_wq_handler(struct work_struct *work) } } +static long bcm_vk_load_image(struct bcm_vk *vk, + const struct vk_image __user *arg) +{ + struct device *dev = &vk->pdev->dev; + const char *image_name; + struct vk_image image; + u32 next_loadable; + enum soc_idx idx; + int image_idx; + int ret = -EPERM; + + if (copy_from_user(&image, arg, sizeof(image))) + return -EACCES; + + if ((image.type != VK_IMAGE_TYPE_BOOT1) && + (image.type != VK_IMAGE_TYPE_BOOT2)) { + dev_err(dev, "invalid image.type %u\n", image.type); + return ret; + } + + next_loadable = bcm_vk_next_boot_image(vk); + if (next_loadable != image.type) { + dev_err(dev, "Next expected image %u, Loading %u\n", + next_loadable, image.type); + return ret; + } + + /* + * if something is pending download already. This could only happen + * for now when the driver is being loaded, or if someone has issued + * another download command in another shell. + */ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) { + dev_err(dev, "Download operation already pending.\n"); + return ret; + } + + image_name = image.filename; + if (image_name[0] == '\0') { + /* Use default image name if NULL */ + idx = get_soc_idx(vk); + if (idx == VK_IDX_INVALID) + goto err_idx; + + /* Image idx starts with boot1 */ + image_idx = image.type - VK_IMAGE_TYPE_BOOT1; + image_name = get_load_fw_name(vk, &image_tab[idx][image_idx]); + if (!image_name) { + dev_err(dev, "No suitable image found for type %d", + image.type); + ret = -ENOENT; + goto err_idx; + } + } else { + /* Ensure filename is NULL terminated */ + image.filename[sizeof(image.filename) - 1] = '\0'; + } + ret = bcm_vk_load_image_by_type(vk, image.type, image_name); + dev_info(dev, "Load %s, ret %d\n", image_name, ret); +err_idx: + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + + return ret; +} + static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val) { vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE); @@ -636,10 +702,38 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) return 0; } +static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = -EINVAL; + struct bcm_vk_ctx *ctx = file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + void __user *argp = (void __user *)arg; + + dev_dbg(&vk->pdev->dev, + "ioctl, cmd=0x%02x, arg=0x%02lx\n", + cmd, arg); + + mutex_lock(&vk->mutex); + + switch (cmd) { + case VK_IOCTL_LOAD_IMAGE: + ret = bcm_vk_load_image(vk, argp); + break; + + default: + break; + } + + mutex_unlock(&vk->mutex); + + return ret; +} + static const struct file_operations bcm_vk_fops = { .owner = THIS_MODULE, .open = bcm_vk_open, .release = bcm_vk_release, + .unlocked_ioctl = bcm_vk_ioctl, }; static int bcm_vk_on_panic(struct notifier_block *nb, @@ -670,6 +764,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; kref_init(&vk->kref); + mutex_init(&vk->mutex); err = pci_enable_device(pdev); if (err) { From ff428d052b3b6fb22242d17c213c4898e5136323 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:22 -0800 Subject: [PATCH 114/633] misc: bcm-vk: add get_card_info, peerlog_info, and proc_mon_info Add support to get card_info (details about card), peerlog_info (to get details of peerlog on card), and proc_mon_info (process monitoring on card). This info is used for collection of logs via direct read of BAR space and by sysfs access (in a follow on commit). Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-9-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 60 ++++++++++++++++++ drivers/misc/bcm-vk/bcm_vk_dev.c | 105 +++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 726aab71bb6b..50f2a0cd6e13 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -205,6 +205,21 @@ enum pci_barno { #define BCM_VK_NUM_TTY 2 +/* VK device max power state, supports 3, full, reduced and low */ +#define MAX_OPP 3 +#define MAX_CARD_INFO_TAG_SIZE 64 + +struct bcm_vk_card_info { + u32 version; + char os_tag[MAX_CARD_INFO_TAG_SIZE]; + char cmpt_tag[MAX_CARD_INFO_TAG_SIZE]; + u32 cpu_freq_mhz; + u32 cpu_scale[MAX_OPP]; + u32 ddr_freq_mhz; + u32 ddr_size_MB; + u32 video_core_freq_mhz; +}; + /* DAUTH related info */ struct bcm_vk_dauth_key { char store[VK_BAR1_DAUTH_STORE_SIZE]; @@ -215,10 +230,49 @@ struct bcm_vk_dauth_info { struct bcm_vk_dauth_key keys[VK_BAR1_DAUTH_MAX]; }; +/* + * Control structure of logging messages from the card. This + * buffer is for logmsg that comes from vk + */ +struct bcm_vk_peer_log { + u32 rd_idx; + u32 wr_idx; + u32 buf_size; + u32 mask; + char data[0]; +}; + +/* max buf size allowed */ +#define BCM_VK_PEER_LOG_BUF_MAX SZ_16K +/* max size per line of peer log */ +#define BCM_VK_PEER_LOG_LINE_MAX 256 + +/* + * single entry for processing type + utilization + */ +#define BCM_VK_PROC_TYPE_TAG_LEN 8 +struct bcm_vk_proc_mon_entry_t { + char tag[BCM_VK_PROC_TYPE_TAG_LEN]; + u32 used; + u32 max; /**< max capacity */ +}; + +/** + * Structure for run time utilization + */ +#define BCM_VK_PROC_MON_MAX 8 /* max entries supported */ +struct bcm_vk_proc_mon_info { + u32 num; /**< no of entries */ + u32 entry_size; /**< per entry size */ + struct bcm_vk_proc_mon_entry_t entries[BCM_VK_PROC_MON_MAX]; +}; + struct bcm_vk { struct pci_dev *pdev; void __iomem *bar[MAX_BAR]; + struct bcm_vk_card_info card_info; + struct bcm_vk_proc_mon_info proc_mon_info; struct bcm_vk_dauth_info dauth_info; /* mutex to protect the ioctls */ @@ -240,6 +294,12 @@ struct bcm_vk { dma_addr_t tdma_addr; /* test dma segment bus addr */ struct notifier_block panic_nb; + + /* offset of the peer log control in BAR2 */ + u32 peerlog_off; + struct bcm_vk_peer_log peerlog_info; /* record of peer log info */ + /* offset of processing monitoring info in BAR2 */ + u32 proc_mon_off; }; /* wq offload work items bits definitions */ diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 203a1cf2bae3..a63208513c64 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -172,6 +172,104 @@ static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar, return 0; } +static void bcm_vk_get_card_info(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 offset; + int i; + u8 *dst; + struct bcm_vk_card_info *info = &vk->card_info; + + /* first read the offset from spare register */ + offset = vkread32(vk, BAR_0, BAR_CARD_STATIC_INFO); + offset &= (pci_resource_len(vk->pdev, BAR_2 * 2) - 1); + + /* based on the offset, read info to internal card info structure */ + dst = (u8 *)info; + for (i = 0; i < sizeof(*info); i++) + *dst++ = vkread8(vk, BAR_2, offset++); + +#define CARD_INFO_LOG_FMT "version : %x\n" \ + "os_tag : %s\n" \ + "cmpt_tag : %s\n" \ + "cpu_freq : %d MHz\n" \ + "cpu_scale : %d full, %d lowest\n" \ + "ddr_freq : %d MHz\n" \ + "ddr_size : %d MB\n" \ + "video_freq: %d MHz\n" + dev_dbg(dev, CARD_INFO_LOG_FMT, info->version, info->os_tag, + info->cmpt_tag, info->cpu_freq_mhz, info->cpu_scale[0], + info->cpu_scale[MAX_OPP - 1], info->ddr_freq_mhz, + info->ddr_size_MB, info->video_core_freq_mhz); + + /* + * get the peer log pointer, only need the offset, and get record + * of the log buffer information which would be used for checking + * before dump, in case the BAR2 memory has been corrupted. + */ + vk->peerlog_off = offset; + memcpy_fromio(&vk->peerlog_info, vk->bar[BAR_2] + vk->peerlog_off, + sizeof(vk->peerlog_info)); + + /* + * Do a range checking and if out of bound, the record will be zeroed + * which guarantees that nothing would be dumped. In other words, + * peer dump is disabled. + */ + if ((vk->peerlog_info.buf_size > BCM_VK_PEER_LOG_BUF_MAX) || + (vk->peerlog_info.mask != (vk->peerlog_info.buf_size - 1)) || + (vk->peerlog_info.rd_idx > vk->peerlog_info.mask) || + (vk->peerlog_info.wr_idx > vk->peerlog_info.mask)) { + dev_err(dev, "Peer log disabled - range error: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + vk->peerlog_info.buf_size, + vk->peerlog_info.mask, + vk->peerlog_info.rd_idx, + vk->peerlog_info.wr_idx); + memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info)); + } else { + dev_dbg(dev, "Peer log: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + vk->peerlog_info.buf_size, + vk->peerlog_info.mask, + vk->peerlog_info.rd_idx, + vk->peerlog_info.wr_idx); + } +} + +static void bcm_vk_get_proc_mon_info(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + struct bcm_vk_proc_mon_info *mon = &vk->proc_mon_info; + u32 num, entry_size, offset, buf_size; + u8 *dst; + + /* calculate offset which is based on peerlog offset */ + buf_size = vkread32(vk, BAR_2, + vk->peerlog_off + + offsetof(struct bcm_vk_peer_log, buf_size)); + offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log) + + buf_size; + + /* first read the num and entry size */ + num = vkread32(vk, BAR_2, offset); + entry_size = vkread32(vk, BAR_2, offset + sizeof(num)); + + /* check for max allowed */ + if (num > BCM_VK_PROC_MON_MAX) { + dev_err(dev, "Processing monitoring entry %d exceeds max %d\n", + num, BCM_VK_PROC_MON_MAX); + return; + } + mon->num = num; + mon->entry_size = entry_size; + + vk->proc_mon_off = offset; + + /* read it once that will capture those static info */ + dst = (u8 *)&mon->entries[0]; + offset += sizeof(num) + sizeof(entry_size); + memcpy_fromio(dst, vk->bar[BAR_2] + offset, num * entry_size); +} + static int bcm_vk_sync_card_info(struct bcm_vk *vk) { u32 rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); @@ -193,6 +291,13 @@ static int bcm_vk_sync_card_info(struct bcm_vk *vk) vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1, VK_BAR1_SCRATCH_SZ_ADDR); } + + /* get static card info, only need to read once */ + bcm_vk_get_card_info(vk); + + /* get the proc mon info once */ + bcm_vk_get_proc_mon_info(vk); + return 0; } From 111d746bb4767ad476f80fe49067e3df3d9a9375 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:23 -0800 Subject: [PATCH 115/633] misc: bcm-vk: add VK messaging support Add message support in order to be able to communicate to VK card via message queues. This info is used for debug purposes via collection of logs via direct read of BAR space and by sysfs access (in a follow on commit). Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-10-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/Makefile | 3 +- drivers/misc/bcm-vk/bcm_vk.h | 123 ++++ drivers/misc/bcm-vk/bcm_vk_dev.c | 309 +++++++- drivers/misc/bcm-vk/bcm_vk_msg.c | 1187 ++++++++++++++++++++++++++++++ drivers/misc/bcm-vk/bcm_vk_msg.h | 132 ++++ drivers/misc/bcm-vk/bcm_vk_sg.c | 275 +++++++ drivers/misc/bcm-vk/bcm_vk_sg.h | 61 ++ 7 files changed, 2087 insertions(+), 3 deletions(-) create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.c create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.h diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile index a2ae79858409..79b4e365c9e6 100644 --- a/drivers/misc/bcm-vk/Makefile +++ b/drivers/misc/bcm-vk/Makefile @@ -6,5 +6,6 @@ obj-$(CONFIG_BCM_VK) += bcm_vk.o bcm_vk-objs := \ bcm_vk_dev.o \ - bcm_vk_msg.o + bcm_vk_msg.o \ + bcm_vk_sg.o diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 50f2a0cd6e13..d847a512d0ed 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -6,11 +6,13 @@ #ifndef BCM_VK_H #define BCM_VK_H +#include #include #include #include #include #include +#include #include #include #include @@ -93,14 +95,53 @@ #define MAJOR_SOC_REV(_chip_id) (((_chip_id) >> 20) & 0xf) #define BAR_CARD_TEMPERATURE 0x45c +/* defines for all temperature sensor */ +#define BCM_VK_TEMP_FIELD_MASK 0xff +#define BCM_VK_CPU_TEMP_SHIFT 0 +#define BCM_VK_DDR0_TEMP_SHIFT 8 +#define BCM_VK_DDR1_TEMP_SHIFT 16 #define BAR_CARD_VOLTAGE 0x460 +/* defines for voltage rail conversion */ +#define BCM_VK_VOLT_RAIL_MASK 0xffff +#define BCM_VK_3P3_VOLT_REG_SHIFT 16 #define BAR_CARD_ERR_LOG 0x464 +/* Error log register bit definition - register for error alerts */ +#define ERR_LOG_UECC BIT(0) +#define ERR_LOG_SSIM_BUSY BIT(1) +#define ERR_LOG_AFBC_BUSY BIT(2) +#define ERR_LOG_HIGH_TEMP_ERR BIT(3) +#define ERR_LOG_WDOG_TIMEOUT BIT(4) +#define ERR_LOG_SYS_FAULT BIT(5) +#define ERR_LOG_RAMDUMP BIT(6) +#define ERR_LOG_COP_WDOG_TIMEOUT BIT(7) +/* warnings */ +#define ERR_LOG_MEM_ALLOC_FAIL BIT(8) +#define ERR_LOG_LOW_TEMP_WARN BIT(9) +#define ERR_LOG_ECC BIT(10) +#define ERR_LOG_IPC_DWN BIT(11) + +/* Alert bit definitions detectd on host */ +#define ERR_LOG_HOST_INTF_V_FAIL BIT(13) +#define ERR_LOG_HOST_HB_FAIL BIT(14) +#define ERR_LOG_HOST_PCIE_DWN BIT(15) #define BAR_CARD_ERR_MEM 0x468 +/* defines for mem err, all fields have same width */ +#define BCM_VK_MEM_ERR_FIELD_MASK 0xff +#define BCM_VK_ECC_MEM_ERR_SHIFT 0 +#define BCM_VK_UECC_MEM_ERR_SHIFT 8 +/* threshold of event occurrence and logs start to come out */ +#define BCM_VK_ECC_THRESHOLD 10 +#define BCM_VK_UECC_THRESHOLD 1 #define BAR_CARD_PWR_AND_THRE 0x46c +/* defines for power and temp threshold, all fields have same width */ +#define BCM_VK_PWR_AND_THRE_FIELD_MASK 0xff +#define BCM_VK_LOW_TEMP_THRE_SHIFT 0 +#define BCM_VK_HIGH_TEMP_THRE_SHIFT 8 +#define BCM_VK_PWR_STATE_SHIFT 16 #define BAR_CARD_STATIC_INFO 0x470 @@ -143,6 +184,11 @@ #define BAR_FIRMWARE_TAG_SIZE 50 #define FIRMWARE_STATUS_PRE_INIT_DONE 0x1f +/* VK MSG_ID defines */ +#define VK_MSG_ID_BITMAP_SIZE 4096 +#define VK_MSG_ID_BITMAP_MASK (VK_MSG_ID_BITMAP_SIZE - 1) +#define VK_MSG_ID_OVERFLOW 0xffff + /* * BAR1 */ @@ -197,6 +243,10 @@ /* VK device supports a maximum of 3 bars */ #define MAX_BAR 3 +/* default number of msg blk for inband SGL */ +#define BCM_VK_DEF_IB_SGL_BLK_LEN 16 +#define BCM_VK_IB_SGL_BLK_MAX 24 + enum pci_barno { BAR_0 = 0, BAR_1, @@ -267,9 +317,27 @@ struct bcm_vk_proc_mon_info { struct bcm_vk_proc_mon_entry_t entries[BCM_VK_PROC_MON_MAX]; }; +struct bcm_vk_hb_ctrl { + struct timer_list timer; + u32 last_uptime; + u32 lost_cnt; +}; + +struct bcm_vk_alert { + u16 flags; + u16 notfs; +}; + +/* some alert counters that the driver will keep track */ +struct bcm_vk_alert_cnts { + u16 ecc; + u16 uecc; +}; + struct bcm_vk { struct pci_dev *pdev; void __iomem *bar[MAX_BAR]; + int num_irqs; struct bcm_vk_card_info card_info; struct bcm_vk_proc_mon_info proc_mon_info; @@ -283,9 +351,17 @@ struct bcm_vk { /* Reference-counting to handle file operations */ struct kref kref; + spinlock_t msg_id_lock; /* Spinlock for msg_id */ + u16 msg_id; + DECLARE_BITMAP(bmap, VK_MSG_ID_BITMAP_SIZE); spinlock_t ctx_lock; /* Spinlock for component context */ struct bcm_vk_ctx ctx[VK_CMPT_CTX_MAX]; struct bcm_vk_ht_entry pid_ht[VK_PID_HT_SZ]; + pid_t reset_pid; /* process that issue reset */ + + atomic_t msgq_inited; /* indicate if info has been synced with vk */ + struct bcm_vk_msg_chan to_v_msg_chan; + struct bcm_vk_msg_chan to_h_msg_chan; struct workqueue_struct *wq_thread; struct work_struct wq_work; /* work queue for deferred job */ @@ -294,6 +370,15 @@ struct bcm_vk { dma_addr_t tdma_addr; /* test dma segment bus addr */ struct notifier_block panic_nb; + u32 ib_sgl_size; /* size allocated for inband sgl insertion */ + + /* heart beat mechanism control structure */ + struct bcm_vk_hb_ctrl hb_ctrl; + /* house-keeping variable of error logs */ + spinlock_t host_alert_lock; /* protection to access host_alert struct */ + struct bcm_vk_alert host_alert; + struct bcm_vk_alert peer_alert; /* bits set by the card */ + struct bcm_vk_alert_cnts alert_cnts; /* offset of the peer log control in BAR2 */ u32 peerlog_off; @@ -306,8 +391,26 @@ struct bcm_vk { enum bcm_vk_wq_offload_flags { BCM_VK_WQ_DWNLD_PEND = 0, BCM_VK_WQ_DWNLD_AUTO = 1, + BCM_VK_WQ_NOTF_PEND = 2, }; +/* a macro to get an individual field with mask and shift */ +#define BCM_VK_EXTRACT_FIELD(_field, _reg, _mask, _shift) \ + (_field = (((_reg) >> (_shift)) & (_mask))) + +struct bcm_vk_entry { + const u32 mask; + const u32 exp_val; + const char *str; +}; + +/* alerts that could be generated from peer */ +#define BCM_VK_PEER_ERR_NUM 12 +extern struct bcm_vk_entry const bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM]; +/* alerts detected by the host */ +#define BCM_VK_HOST_ERR_NUM 3 +extern struct bcm_vk_entry const bcm_vk_host_err[BCM_VK_HOST_ERR_NUM]; + /* * check if PCIe interface is down on read. Use it when it is * certain that _val should never be all ones. @@ -354,8 +457,28 @@ static inline bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk) } int bcm_vk_open(struct inode *inode, struct file *p_file); +ssize_t bcm_vk_read(struct file *p_file, char __user *buf, size_t count, + loff_t *f_pos); +ssize_t bcm_vk_write(struct file *p_file, const char __user *buf, + size_t count, loff_t *f_pos); +__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait); int bcm_vk_release(struct inode *inode, struct file *p_file); void bcm_vk_release_data(struct kref *kref); +irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id); +irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id); +int bcm_vk_msg_init(struct bcm_vk *vk); +void bcm_vk_msg_remove(struct bcm_vk *vk); +int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync); +void bcm_vk_blk_drv_access(struct bcm_vk *vk); +s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk); +int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, + const pid_t pid, const u32 q_num); +void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val); int bcm_vk_auto_load_all_images(struct bcm_vk *vk); +void bcm_vk_hb_init(struct bcm_vk *vk); +void bcm_vk_hb_deinit(struct bcm_vk *vk); +void bcm_vk_handle_notf(struct bcm_vk *vk); +bool bcm_vk_drv_access_ok(struct bcm_vk *vk); +void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask); #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index a63208513c64..5d82f02c0f27 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,54 @@ static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES; module_param(nr_scratch_pages, uint, 0444); MODULE_PARM_DESC(nr_scratch_pages, "Number of pre allocated DMAable coherent pages.\n"); +static uint nr_ib_sgl_blk = BCM_VK_DEF_IB_SGL_BLK_LEN; +module_param(nr_ib_sgl_blk, uint, 0444); +MODULE_PARM_DESC(nr_ib_sgl_blk, + "Number of in-band msg blks for short SGL.\n"); + +/* + * alerts that could be generated from peer + */ +const struct bcm_vk_entry bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM] = { + {ERR_LOG_UECC, ERR_LOG_UECC, "uecc"}, + {ERR_LOG_SSIM_BUSY, ERR_LOG_SSIM_BUSY, "ssim_busy"}, + {ERR_LOG_AFBC_BUSY, ERR_LOG_AFBC_BUSY, "afbc_busy"}, + {ERR_LOG_HIGH_TEMP_ERR, ERR_LOG_HIGH_TEMP_ERR, "high_temp"}, + {ERR_LOG_WDOG_TIMEOUT, ERR_LOG_WDOG_TIMEOUT, "wdog_timeout"}, + {ERR_LOG_SYS_FAULT, ERR_LOG_SYS_FAULT, "sys_fault"}, + {ERR_LOG_RAMDUMP, ERR_LOG_RAMDUMP, "ramdump"}, + {ERR_LOG_COP_WDOG_TIMEOUT, ERR_LOG_COP_WDOG_TIMEOUT, + "cop_wdog_timeout"}, + {ERR_LOG_MEM_ALLOC_FAIL, ERR_LOG_MEM_ALLOC_FAIL, "malloc_fail warn"}, + {ERR_LOG_LOW_TEMP_WARN, ERR_LOG_LOW_TEMP_WARN, "low_temp warn"}, + {ERR_LOG_ECC, ERR_LOG_ECC, "ecc"}, + {ERR_LOG_IPC_DWN, ERR_LOG_IPC_DWN, "ipc_down"}, +}; + +/* alerts detected by the host */ +const struct bcm_vk_entry bcm_vk_host_err[BCM_VK_HOST_ERR_NUM] = { + {ERR_LOG_HOST_PCIE_DWN, ERR_LOG_HOST_PCIE_DWN, "PCIe_down"}, + {ERR_LOG_HOST_HB_FAIL, ERR_LOG_HOST_HB_FAIL, "hb_fail"}, + {ERR_LOG_HOST_INTF_V_FAIL, ERR_LOG_HOST_INTF_V_FAIL, "intf_ver_fail"}, +}; + +irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + if (!bcm_vk_drv_access_ok(vk)) { + dev_err(&vk->pdev->dev, + "Interrupt %d received when msgq not inited\n", irq); + goto skip_schedule_work; + } + + /* if notification is not pending, set bit and schedule work */ + if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0) + queue_work(vk->wq_thread, &vk->wq_work); + +skip_schedule_work: + return IRQ_HANDLED; +} static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) { @@ -126,6 +175,7 @@ static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) dev_err(dev, "Intf major.minor=%d.%d rejected - drv %d.%d\n", major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_INTF_V_FAIL); ret = -EPFNOSUPPORT; } else { dev_dbg(dev, @@ -135,6 +185,154 @@ static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) return ret; } +static void bcm_vk_log_notf(struct bcm_vk *vk, + struct bcm_vk_alert *alert, + struct bcm_vk_entry const *entry_tab, + const u32 table_size) +{ + u32 i; + u32 masked_val, latched_val; + struct bcm_vk_entry const *entry; + u32 reg; + u16 ecc_mem_err, uecc_mem_err; + struct device *dev = &vk->pdev->dev; + + for (i = 0; i < table_size; i++) { + entry = &entry_tab[i]; + masked_val = entry->mask & alert->notfs; + latched_val = entry->mask & alert->flags; + + if (masked_val == ERR_LOG_UECC) { + /* + * if there is difference between stored cnt and it + * is greater than threshold, log it. + */ + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM); + BCM_VK_EXTRACT_FIELD(uecc_mem_err, reg, + BCM_VK_MEM_ERR_FIELD_MASK, + BCM_VK_UECC_MEM_ERR_SHIFT); + if ((uecc_mem_err != vk->alert_cnts.uecc) && + (uecc_mem_err >= BCM_VK_UECC_THRESHOLD)) + dev_info(dev, + "ALERT! %s.%d uecc RAISED - ErrCnt %d\n", + DRV_MODULE_NAME, vk->devid, + uecc_mem_err); + vk->alert_cnts.uecc = uecc_mem_err; + } else if (masked_val == ERR_LOG_ECC) { + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM); + BCM_VK_EXTRACT_FIELD(ecc_mem_err, reg, + BCM_VK_MEM_ERR_FIELD_MASK, + BCM_VK_ECC_MEM_ERR_SHIFT); + if ((ecc_mem_err != vk->alert_cnts.ecc) && + (ecc_mem_err >= BCM_VK_ECC_THRESHOLD)) + dev_info(dev, "ALERT! %s.%d ecc RAISED - ErrCnt %d\n", + DRV_MODULE_NAME, vk->devid, + ecc_mem_err); + vk->alert_cnts.ecc = ecc_mem_err; + } else if (masked_val != latched_val) { + /* print a log as info */ + dev_info(dev, "ALERT! %s.%d %s %s\n", + DRV_MODULE_NAME, vk->devid, entry->str, + masked_val ? "RAISED" : "CLEARED"); + } + } +} + +static void bcm_vk_dump_peer_log(struct bcm_vk *vk) +{ + struct bcm_vk_peer_log log; + struct bcm_vk_peer_log *log_info = &vk->peerlog_info; + char loc_buf[BCM_VK_PEER_LOG_LINE_MAX]; + int cnt; + struct device *dev = &vk->pdev->dev; + unsigned int data_offset; + + memcpy_fromio(&log, vk->bar[BAR_2] + vk->peerlog_off, sizeof(log)); + + dev_dbg(dev, "Peer PANIC: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", + log.buf_size, log.mask, log.rd_idx, log.wr_idx); + + if (!log_info->buf_size) { + dev_err(dev, "Peer log dump disabled - skipped!\n"); + return; + } + + /* perform range checking for rd/wr idx */ + if ((log.rd_idx > log_info->mask) || + (log.wr_idx > log_info->mask) || + (log.buf_size != log_info->buf_size) || + (log.mask != log_info->mask)) { + dev_err(dev, + "Corrupted Ptrs: Size 0x%x(0x%x) Mask 0x%x(0x%x) [Rd Wr] = [%d %d], skip log dump.\n", + log_info->buf_size, log.buf_size, + log_info->mask, log.mask, + log.rd_idx, log.wr_idx); + return; + } + + cnt = 0; + data_offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log); + loc_buf[BCM_VK_PEER_LOG_LINE_MAX - 1] = '\0'; + while (log.rd_idx != log.wr_idx) { + loc_buf[cnt] = vkread8(vk, BAR_2, data_offset + log.rd_idx); + + if ((loc_buf[cnt] == '\0') || + (cnt == (BCM_VK_PEER_LOG_LINE_MAX - 1))) { + dev_err(dev, "%s", loc_buf); + cnt = 0; + } else { + cnt++; + } + log.rd_idx = (log.rd_idx + 1) & log.mask; + } + /* update rd idx at the end */ + vkwrite32(vk, log.rd_idx, BAR_2, + vk->peerlog_off + offsetof(struct bcm_vk_peer_log, rd_idx)); +} + +void bcm_vk_handle_notf(struct bcm_vk *vk) +{ + u32 reg; + struct bcm_vk_alert alert; + bool intf_down; + unsigned long flags; + + /* handle peer alerts and then locally detected ones */ + reg = vkread32(vk, BAR_0, BAR_CARD_ERR_LOG); + intf_down = BCM_VK_INTF_IS_DOWN(reg); + if (!intf_down) { + vk->peer_alert.notfs = reg; + bcm_vk_log_notf(vk, &vk->peer_alert, bcm_vk_peer_err, + ARRAY_SIZE(bcm_vk_peer_err)); + vk->peer_alert.flags = vk->peer_alert.notfs; + } else { + /* turn off access */ + bcm_vk_blk_drv_access(vk); + } + + /* check and make copy of alert with lock and then free lock */ + spin_lock_irqsave(&vk->host_alert_lock, flags); + if (intf_down) + vk->host_alert.notfs |= ERR_LOG_HOST_PCIE_DWN; + + alert = vk->host_alert; + vk->host_alert.flags = vk->host_alert.notfs; + spin_unlock_irqrestore(&vk->host_alert_lock, flags); + + /* call display with copy */ + bcm_vk_log_notf(vk, &alert, bcm_vk_host_err, + ARRAY_SIZE(bcm_vk_host_err)); + + /* + * If it is a sys fault or heartbeat timeout, we would like extract + * log msg from the card so that we would know what is the last fault + */ + if (!intf_down && + ((vk->host_alert.flags & ERR_LOG_HOST_HB_FAIL) || + (vk->peer_alert.flags & ERR_LOG_SYS_FAULT))) + bcm_vk_dump_peer_log(vk); +} + static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar, u64 offset, u32 mask, u32 value, unsigned long timeout_ms) @@ -301,6 +499,31 @@ static int bcm_vk_sync_card_info(struct bcm_vk *vk) return 0; } +void bcm_vk_blk_drv_access(struct bcm_vk *vk) +{ + int i; + + /* + * kill all the apps + */ + spin_lock(&vk->ctx_lock); + + /* set msgq_inited to 0 so that all rd/wr will be blocked */ + atomic_set(&vk->msgq_inited, 0); + + for (i = 0; i < VK_PID_HT_SZ; i++) { + struct bcm_vk_ctx *ctx; + + list_for_each_entry(ctx, &vk->pid_ht[i].head, node) { + dev_dbg(&vk->pdev->dev, + "Send kill signal to pid %d\n", + ctx->pid); + kill_pid(find_vpid(ctx->pid), SIGKILL, 1); + } + } + spin_unlock(&vk->ctx_lock); +} + static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp, dma_addr_t host_buf_addr, u32 buf_size) { @@ -518,6 +741,17 @@ static int bcm_vk_load_image_by_type(struct bcm_vk *vk, u32 load_type, goto err_firmware_out; } + /* + * Next, initialize Message Q if we are loading boot2. + * Do a force sync + */ + ret = bcm_vk_sync_msgq(vk, true); + if (ret) { + dev_err(dev, "Boot2 Error reading comm msg Q info\n"); + ret = -EIO; + goto err_firmware_out; + } + /* sync & channel other info */ ret = bcm_vk_sync_card_info(vk); if (ret) { @@ -668,12 +902,20 @@ static int bcm_vk_trigger_autoload(struct bcm_vk *vk) } /* - * deferred work queue for auto download. + * deferred work queue for draining and auto download. */ static void bcm_vk_wq_handler(struct work_struct *work) { struct bcm_vk *vk = container_of(work, struct bcm_vk, wq_work); + struct device *dev = &vk->pdev->dev; + s32 ret; + /* check wq offload bit map to perform various operations */ + if (test_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload)) { + /* clear bit right the way for notification */ + clear_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload); + bcm_vk_handle_notf(vk); + } if (test_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload)) { bcm_vk_auto_load_all_images(vk); @@ -684,6 +926,14 @@ static void bcm_vk_wq_handler(struct work_struct *work) clear_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); } + + /* next, try to drain */ + ret = bcm_to_h_msg_dequeue(vk); + + if (ret == 0) + dev_dbg(dev, "Spurious trigger for workqueue\n"); + else if (ret < 0) + bcm_vk_blk_drv_access(vk); } static long bcm_vk_load_image(struct bcm_vk *vk, @@ -837,6 +1087,9 @@ static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) static const struct file_operations bcm_vk_fops = { .owner = THIS_MODULE, .open = bcm_vk_open, + .read = bcm_vk_read, + .write = bcm_vk_write, + .poll = bcm_vk_poll, .release = bcm_vk_release, .unlocked_ioctl = bcm_vk_ioctl, }; @@ -869,6 +1122,12 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; kref_init(&vk->kref); + if (nr_ib_sgl_blk > BCM_VK_IB_SGL_BLK_MAX) { + dev_warn(dev, "Inband SGL blk %d limited to max %d\n", + nr_ib_sgl_blk, BCM_VK_IB_SGL_BLK_MAX); + nr_ib_sgl_blk = BCM_VK_IB_SGL_BLK_MAX; + } + vk->ib_sgl_size = nr_ib_sgl_blk * VK_MSGQ_BLK_SIZE; mutex_init(&vk->mutex); err = pci_enable_device(pdev); @@ -932,11 +1191,34 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } + for (vk->num_irqs = 0; + vk->num_irqs < VK_MSIX_MSGQ_MAX; + vk->num_irqs++) { + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_msgq_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed to request msgq IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + } + /* one irq for notification from VK */ + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_notf_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed to request notf IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + vk->num_irqs++; + id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); if (id < 0) { err = id; dev_err(dev, "unable to get id\n"); - goto err_iounmap; + goto err_irq; } vk->devid = id; @@ -966,6 +1248,12 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_misc_deregister; } + err = bcm_vk_msg_init(vk); + if (err) { + dev_err(dev, "failed to init msg queue info\n"); + goto err_destroy_workqueue; + } + /* sync other info */ bcm_vk_sync_card_info(vk); @@ -994,6 +1282,9 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } + /* enable hb */ + bcm_vk_hb_init(vk); + dev_dbg(dev, "BCM-VK:%u created\n", id); return 0; @@ -1015,6 +1306,13 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_ida_remove: ida_simple_remove(&bcm_vk_ida, id); +err_irq: + for (i = 0; i < vk->num_irqs; i++) + devm_free_irq(dev, pci_irq_vector(pdev, i), vk); + + pci_disable_msix(pdev); + pci_disable_msi(pdev); + err_iounmap: for (i = 0; i < MAX_BAR; i++) { if (vk->bar[i]) @@ -1053,6 +1351,8 @@ static void bcm_vk_remove(struct pci_dev *pdev) struct bcm_vk *vk = pci_get_drvdata(pdev); struct miscdevice *misc_device = &vk->miscdev; + bcm_vk_hb_deinit(vk); + /* * Trigger a reset to card and wait enough time for UCODE to rerun, * which re-initialize the card into its default state. @@ -1076,6 +1376,11 @@ static void bcm_vk_remove(struct pci_dev *pdev) kfree(misc_device->name); ida_simple_remove(&bcm_vk_ida, vk->devid); } + for (i = 0; i < vk->num_irqs; i++) + devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), vk); + + pci_disable_msix(pdev); + pci_disable_msi(pdev); cancel_work_sync(&vk->wq_work); destroy_workqueue(vk->wq_thread); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 2d9a6b4e5f61..b05e20a72a8b 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -3,8 +3,200 @@ * Copyright 2018-2020 Broadcom. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include "bcm_vk.h" #include "bcm_vk_msg.h" +#include "bcm_vk_sg.h" + +/* functions to manipulate the transport id in msg block */ +#define BCM_VK_MSG_Q_SHIFT 4 +#define BCM_VK_MSG_Q_MASK 0xF +#define BCM_VK_MSG_ID_MASK 0xFFF + +#define BCM_VK_DMA_DRAIN_MAX_MS 2000 + +/* number x q_size will be the max number of msg processed per loop */ +#define BCM_VK_MSG_PROC_MAX_LOOP 2 + +/* module parameter */ +static bool hb_mon = true; +module_param(hb_mon, bool, 0444); +MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n"); +static int batch_log = 1; +module_param(batch_log, int, 0444); +MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n"); + +static bool hb_mon_is_on(void) +{ + return hb_mon; +} + +static u32 get_q_num(const struct vk_msg_blk *msg) +{ + u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK; + + if (q_num >= VK_MSGQ_PER_CHAN_MAX) + q_num = VK_MSGQ_NUM_DEFAULT; + return q_num; +} + +static void set_q_num(struct vk_msg_blk *msg, u32 q_num) +{ + msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | + (q_num >= VK_MSGQ_PER_CHAN_MAX) ? VK_MSGQ_NUM_DEFAULT : q_num; +} + +static u32 get_msg_id(const struct vk_msg_blk *msg) +{ + return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK); +} + +static void set_msg_id(struct vk_msg_blk *msg, u32 val) +{ + msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg); +} + +static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc) +{ + return ((idx + inc) & qinfo->q_mask); +} + +static +struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo, + u32 idx) +{ + return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx); +} + +static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq, + const struct bcm_vk_sync_qinfo *qinfo) +{ + u32 wr_idx, rd_idx; + + wr_idx = readl_relaxed(&msgq->wr_idx); + rd_idx = readl_relaxed(&msgq->rd_idx); + + return ((wr_idx - rd_idx) & qinfo->q_mask); +} + +static +u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, + const struct bcm_vk_sync_qinfo *qinfo) +{ + return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1); +} + +/* number of retries when enqueue message fails before returning EAGAIN */ +#define BCM_VK_H2VK_ENQ_RETRY 10 +#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50 + +bool bcm_vk_drv_access_ok(struct bcm_vk *vk) +{ + return (!!atomic_read(&vk->msgq_inited)); +} + +void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) +{ + struct bcm_vk_alert *alert = &vk->host_alert; + unsigned long flags; + + /* use irqsave version as this maybe called inside timer interrupt */ + spin_lock_irqsave(&vk->host_alert_lock, flags); + alert->notfs |= bit_mask; + spin_unlock_irqrestore(&vk->host_alert_lock, flags); + + if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0) + queue_work(vk->wq_thread, &vk->wq_work); +} + +/* + * Heartbeat related defines + * The heartbeat from host is a last resort. If stuck condition happens + * on the card, firmware is supposed to detect it. Therefore, the heartbeat + * values used will be more relaxed on the driver, which need to be bigger + * than the watchdog timeout on the card. The watchdog timeout on the card + * is 20s, with a jitter of 2s => 22s. We use a value of 27s here. + */ +#define BCM_VK_HB_TIMER_S 3 +#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) +#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) + +static void bcm_vk_hb_poll(struct timer_list *t) +{ + u32 uptime_s; + struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl, + timer); + struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl); + + if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) { + /* read uptime from register and compare */ + uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME); + + if (uptime_s == hb->last_uptime) + hb->lost_cnt++; + else /* reset to avoid accumulation */ + hb->lost_cnt = 0; + + dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n", + hb->last_uptime, uptime_s, hb->lost_cnt); + + /* + * if the interface goes down without any activity, a value + * of 0xFFFFFFFF will be continuously read, and the detection + * will be happened eventually. + */ + hb->last_uptime = uptime_s; + } else { + /* reset heart beat lost cnt */ + hb->lost_cnt = 0; + } + + /* next, check if heartbeat exceeds limit */ + if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) { + dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n", + BCM_VK_HB_LOST_MAX, + BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S); + + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); + } + /* re-arm timer */ + mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); +} + +void bcm_vk_hb_init(struct bcm_vk *vk) +{ + struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; + + timer_setup(&hb->timer, bcm_vk_hb_poll, 0); + mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE); +} + +void bcm_vk_hb_deinit(struct bcm_vk *vk) +{ + struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; + + del_timer(&hb->timer); +} + +static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk, + unsigned int start, + unsigned int nbits) +{ + spin_lock(&vk->msg_id_lock); + bitmap_clear(vk->bmap, start, nbits); + spin_unlock(&vk->msg_id_lock); +} /* * allocate a ctx per file struct @@ -39,12 +231,47 @@ static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) /* increase kref */ kref_get(&vk->kref); + /* clear counter */ + atomic_set(&ctx->pend_cnt, 0); + atomic_set(&ctx->dma_cnt, 0); + init_waitqueue_head(&ctx->rd_wq); + all_in_use_exit: spin_unlock(&vk->ctx_lock); return ctx; } +static u16 bcm_vk_get_msg_id(struct bcm_vk *vk) +{ + u16 rc = VK_MSG_ID_OVERFLOW; + u16 test_bit_count = 0; + + spin_lock(&vk->msg_id_lock); + while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) { + /* + * first time come in this loop, msg_id will be 0 + * and the first one tested will be 1. We skip + * VK_SIMPLEX_MSG_ID (0) for one way host2vk + * communication + */ + vk->msg_id++; + if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE) + vk->msg_id = 1; + + if (test_bit(vk->msg_id, vk->bmap)) { + test_bit_count++; + continue; + } + rc = vk->msg_id; + bitmap_set(vk->bmap, vk->msg_id, 1); + break; + } + spin_unlock(&vk->msg_id_lock); + + return rc; +} + static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) { u32 idx; @@ -82,6 +309,636 @@ static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) return count; } +static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry) +{ + int proc_cnt; + + bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt); + if (proc_cnt) + atomic_dec(&entry->ctx->dma_cnt); + + kfree(entry->to_h_msg); + kfree(entry); +} + +static void bcm_vk_drain_all_pend(struct device *dev, + struct bcm_vk_msg_chan *chan, + struct bcm_vk_ctx *ctx) +{ + u32 num; + struct bcm_vk_wkent *entry, *tmp; + struct bcm_vk *vk; + struct list_head del_q; + + if (ctx) + vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + + INIT_LIST_HEAD(&del_q); + spin_lock(&chan->pendq_lock); + for (num = 0; num < chan->q_nr; num++) { + list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { + if ((!ctx) || (entry->ctx->idx == ctx->idx)) { + list_del(&entry->node); + list_add_tail(&entry->node, &del_q); + } + } + } + spin_unlock(&chan->pendq_lock); + + /* batch clean up */ + num = 0; + list_for_each_entry_safe(entry, tmp, &del_q, node) { + list_del(&entry->node); + num++; + if (ctx) { + struct vk_msg_blk *msg; + int bit_set; + bool responded; + u32 msg_id; + + /* if it is specific ctx, log for any stuck */ + msg = entry->to_v_msg; + msg_id = get_msg_id(msg); + bit_set = test_bit(msg_id, vk->bmap); + responded = entry->to_h_msg ? true : false; + if (num <= batch_log) + dev_info(dev, + "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n", + msg->function_id, msg->size, + msg_id, entry->seq_num, + msg->context_id, entry->ctx->idx, + msg->cmd, msg->arg, + responded ? "T" : "F", bit_set); + if (responded) + atomic_dec(&ctx->pend_cnt); + else if (bit_set) + bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); + } + bcm_vk_free_wkent(dev, entry); + } + if (num && ctx) + dev_info(dev, "Total drained items %d [fd-%d]\n", + num, ctx->idx); +} + +/* + * Function to sync up the messages queue info that is provided by BAR1 + */ +int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) +{ + struct bcm_vk_msgq __iomem *msgq; + struct device *dev = &vk->pdev->dev; + u32 msgq_off; + u32 num_q; + struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan, + &vk->to_h_msg_chan}; + struct bcm_vk_msg_chan *chan; + int i, j; + int ret = 0; + + /* + * If the driver is loaded at startup where vk OS is not up yet, + * the msgq-info may not be available until a later time. In + * this case, we skip and the sync function is supposed to be + * called again. + */ + if (!bcm_vk_msgq_marker_valid(vk)) { + dev_info(dev, "BAR1 msgq marker not initialized.\n"); + return -EAGAIN; + } + + msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF); + + /* each side is always half the total */ + num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2; + if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) { + dev_err(dev, + "Advertised msgq %d error - max %d allowed\n", + num_q, VK_MSGQ_PER_CHAN_MAX); + return -EINVAL; + } + + vk->to_v_msg_chan.q_nr = num_q; + vk->to_h_msg_chan.q_nr = num_q; + + /* first msgq location */ + msgq = vk->bar[BAR_1] + msgq_off; + + /* + * if this function is called when it is already inited, + * something is wrong + */ + if (bcm_vk_drv_access_ok(vk) && !force_sync) { + dev_err(dev, "Msgq info already in sync\n"); + return -EPERM; + } + + for (i = 0; i < ARRAY_SIZE(chan_list); i++) { + chan = chan_list[i]; + memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo)); + + for (j = 0; j < num_q; j++) { + struct bcm_vk_sync_qinfo *qinfo; + u32 msgq_start; + u32 msgq_size; + u32 msgq_nxt; + u32 msgq_db_offset, q_db_offset; + + chan->msgq[j] = msgq; + msgq_start = readl_relaxed(&msgq->start); + msgq_size = readl_relaxed(&msgq->size); + msgq_nxt = readl_relaxed(&msgq->nxt); + msgq_db_offset = readl_relaxed(&msgq->db_offset); + q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1)); + if (q_db_offset == (~msgq_db_offset >> DB_SHIFT)) + msgq_db_offset = q_db_offset; + else + /* fall back to default */ + msgq_db_offset = VK_BAR0_Q_DB_BASE(j); + + dev_info(dev, + "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n", + j, + readw_relaxed(&msgq->type), + readw_relaxed(&msgq->num), + msgq_start, + msgq_db_offset, + readl_relaxed(&msgq->rd_idx), + readl_relaxed(&msgq->wr_idx), + msgq_size, + msgq_nxt); + + qinfo = &chan->sync_qinfo[j]; + /* formulate and record static info */ + qinfo->q_start = vk->bar[BAR_1] + msgq_start; + qinfo->q_size = msgq_size; + /* set low threshold as 50% or 1/2 */ + qinfo->q_low = qinfo->q_size >> 1; + qinfo->q_mask = qinfo->q_size - 1; + qinfo->q_db_offset = msgq_db_offset; + + msgq++; + } + } + atomic_set(&vk->msgq_inited, 1); + + return ret; +} + +static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan) +{ + u32 i; + + mutex_init(&chan->msgq_mutex); + spin_lock_init(&chan->pendq_lock); + for (i = 0; i < VK_MSGQ_MAX_NR; i++) + INIT_LIST_HEAD(&chan->pendq[i]); + + return 0; +} + +static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num, + struct bcm_vk_wkent *entry) +{ + struct bcm_vk_ctx *ctx; + + spin_lock(&chan->pendq_lock); + list_add_tail(&entry->node, &chan->pendq[q_num]); + if (entry->to_h_msg) { + ctx = entry->ctx; + atomic_inc(&ctx->pend_cnt); + wake_up_interruptible(&ctx->rd_wq); + } + spin_unlock(&chan->pendq_lock); +} + +static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk, + struct bcm_vk_wkent *entry, + struct _vk_data *data, + unsigned int num_planes) +{ + unsigned int i; + unsigned int item_cnt = 0; + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct vk_msg_blk *msg = &entry->to_v_msg[0]; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + u32 ib_sgl_size = 0; + u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks]; + u32 avail; + u32 q_num; + + /* check if high watermark is hit, and if so, skip */ + q_num = get_q_num(msg); + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + avail = msgq_avail_space(msgq, qinfo); + if (avail < qinfo->q_low) { + dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n", + avail, qinfo->q_size); + return 0; + } + + for (i = 0; i < num_planes; i++) { + if (data[i].address && + (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) { + item_cnt++; + memcpy(buf, entry->dma[i].sglist, data[i].size); + ib_sgl_size += data[i].size; + buf += data[i].size; + } + } + + dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n", + item_cnt, ib_sgl_size, vk->ib_sgl_size); + + /* round up size */ + ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1) + >> VK_MSGQ_BLK_SZ_SHIFT; + + return ib_sgl_size; +} + +void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val) +{ + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num]; + + vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset); +} + +static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry) +{ + static u32 seq_num; + struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; + struct device *dev = &vk->pdev->dev; + struct vk_msg_blk *src = &entry->to_v_msg[0]; + + struct vk_msg_blk __iomem *dst; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + u32 q_num = get_q_num(src); + u32 wr_idx; /* local copy */ + u32 i; + u32 avail; + u32 retry; + + if (entry->to_v_blks != src->size + 1) { + dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n", + entry->to_v_blks, + src->size + 1, + get_msg_id(src), + src->function_id, + src->context_id); + return -EMSGSIZE; + } + + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + + mutex_lock(&chan->msgq_mutex); + + avail = msgq_avail_space(msgq, qinfo); + + /* if not enough space, return EAGAIN and let app handles it */ + retry = 0; + while ((avail < entry->to_v_blks) && + (retry++ < BCM_VK_H2VK_ENQ_RETRY)) { + mutex_unlock(&chan->msgq_mutex); + + msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS); + mutex_lock(&chan->msgq_mutex); + avail = msgq_avail_space(msgq, qinfo); + } + if (retry > BCM_VK_H2VK_ENQ_RETRY) { + mutex_unlock(&chan->msgq_mutex); + return -EAGAIN; + } + + /* at this point, mutex is taken and there is enough space */ + entry->seq_num = seq_num++; /* update debug seq number */ + wr_idx = readl_relaxed(&msgq->wr_idx); + + if (wr_idx >= qinfo->q_size) { + dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!", + wr_idx, qinfo->q_size); + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN); + goto idx_err; + } + + dst = msgq_blk_addr(qinfo, wr_idx); + for (i = 0; i < entry->to_v_blks; i++) { + memcpy_toio(dst, src, sizeof(*dst)); + + src++; + wr_idx = msgq_inc(qinfo, wr_idx, 1); + dst = msgq_blk_addr(qinfo, wr_idx); + } + + /* flush the write pointer */ + writel(wr_idx, &msgq->wr_idx); + + /* log new info for debugging */ + dev_dbg(dev, + "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n", + readl_relaxed(&msgq->num), + readl_relaxed(&msgq->rd_idx), + wr_idx, + entry->to_v_blks, + msgq_occupied(msgq, qinfo), + msgq_avail_space(msgq, qinfo), + readl_relaxed(&msgq->size)); + /* + * press door bell based on queue number. 1 is added to the wr_idx + * to avoid the value of 0 appearing on the VK side to distinguish + * from initial value. + */ + bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1); +idx_err: + mutex_unlock(&chan->msgq_mutex); + return 0; +} + +int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, + const pid_t pid, const u32 q_num) +{ + int rc = 0; + struct bcm_vk_wkent *entry; + struct device *dev = &vk->pdev->dev; + + /* + * check if the marker is still good. Sometimes, the PCIe interface may + * have gone done, and if so and we ship down thing based on broken + * values, kernel may panic. + */ + if (!bcm_vk_msgq_marker_valid(vk)) { + dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n", + vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY)); + return -EINVAL; + } + + entry = kzalloc(sizeof(*entry) + + sizeof(struct vk_msg_blk), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + /* fill up necessary data */ + entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN; + set_q_num(&entry->to_v_msg[0], q_num); + set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID); + entry->to_v_blks = 1; /* always 1 block */ + + entry->to_v_msg[0].cmd = shut_type; + entry->to_v_msg[0].arg = pid; + + rc = bcm_to_v_msg_enqueue(vk, entry); + if (rc) + dev_err(dev, + "Sending shutdown message to q %d for pid %d fails.\n", + get_q_num(&entry->to_v_msg[0]), pid); + + kfree(entry); + + return rc; +} + +static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid, + const u32 q_num) +{ + int rc = 0; + struct device *dev = &vk->pdev->dev; + + /* + * don't send down or do anything if message queue is not initialized + */ + if (!bcm_vk_drv_access_ok(vk)) + return -EPERM; + + dev_dbg(dev, "No more sessions, shut down pid %d\n", pid); + + rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); + + return rc; +} + +static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk, + struct bcm_vk_msg_chan *chan, + u16 q_num, + u16 msg_id) +{ + bool found = false; + struct bcm_vk_wkent *entry; + + spin_lock(&chan->pendq_lock); + list_for_each_entry(entry, &chan->pendq[q_num], node) { + if (get_msg_id(&entry->to_v_msg[0]) == msg_id) { + list_del(&entry->node); + found = true; + bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); + break; + } + } + spin_unlock(&chan->pendq_lock); + return ((found) ? entry : NULL); +} + +s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; + struct vk_msg_blk *data; + struct vk_msg_blk __iomem *src; + struct vk_msg_blk *dst; + struct bcm_vk_msgq __iomem *msgq; + struct bcm_vk_sync_qinfo *qinfo; + struct bcm_vk_wkent *entry; + u32 rd_idx, wr_idx; + u32 q_num, msg_id, j; + u32 num_blks; + s32 total = 0; + int cnt = 0; + int msg_processed = 0; + int max_msg_to_process; + bool exit_loop; + + /* + * drain all the messages from the queues, and find its pending + * entry in the to_v queue, based on msg_id & q_num, and move the + * entry to the to_h pending queue, waiting for user space + * program to extract + */ + mutex_lock(&chan->msgq_mutex); + + for (q_num = 0; q_num < chan->q_nr; q_num++) { + msgq = chan->msgq[q_num]; + qinfo = &chan->sync_qinfo[q_num]; + max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size; + + rd_idx = readl_relaxed(&msgq->rd_idx); + wr_idx = readl_relaxed(&msgq->wr_idx); + msg_processed = 0; + exit_loop = false; + while ((rd_idx != wr_idx) && !exit_loop) { + u8 src_size; + + /* + * Make a local copy and get pointer to src blk + * The rd_idx is masked before getting the pointer to + * avoid out of bound access in case the interface goes + * down. It will end up pointing to the last block in + * the buffer, but subsequent src->size check would be + * able to catch this. + */ + src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask); + src_size = readb(&src->size); + + if ((rd_idx >= qinfo->q_size) || + (src_size > (qinfo->q_size - 1))) { + dev_crit(dev, + "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!", + rd_idx, src_size, qinfo->q_size); + bcm_vk_blk_drv_access(vk); + bcm_vk_set_host_alert(vk, + ERR_LOG_HOST_PCIE_DWN); + goto idx_err; + } + + num_blks = src_size + 1; + data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL); + if (data) { + /* copy messages and linearize it */ + dst = data; + for (j = 0; j < num_blks; j++) { + memcpy_fromio(dst, src, sizeof(*dst)); + + dst++; + rd_idx = msgq_inc(qinfo, rd_idx, 1); + src = msgq_blk_addr(qinfo, rd_idx); + } + total++; + } else { + /* + * if we could not allocate memory in kernel, + * that is fatal. + */ + dev_crit(dev, "Kernel mem allocation failure.\n"); + return -ENOMEM; + } + + /* flush rd pointer after a message is dequeued */ + writel(rd_idx, &msgq->rd_idx); + + /* log new info for debugging */ + dev_dbg(dev, + "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n", + readl_relaxed(&msgq->num), + rd_idx, + wr_idx, + num_blks, + msgq_occupied(msgq, qinfo), + msgq_avail_space(msgq, qinfo), + readl_relaxed(&msgq->size)); + + /* + * No need to search if it is an autonomous one-way + * message from driver, as these messages do not bear + * a to_v pending item. Currently, only the shutdown + * message falls into this category. + */ + if (data->function_id == VK_FID_SHUTDOWN) { + kfree(data); + continue; + } + + msg_id = get_msg_id(data); + /* lookup original message in to_v direction */ + entry = bcm_vk_dequeue_pending(vk, + &vk->to_v_msg_chan, + q_num, + msg_id); + + /* + * if there is message to does not have prior send, + * this is the location to add here + */ + if (entry) { + entry->to_h_blks = num_blks; + entry->to_h_msg = data; + bcm_vk_append_pendq(&vk->to_h_msg_chan, + q_num, entry); + + } else { + if (cnt++ < batch_log) + dev_info(dev, + "Could not find MsgId[0x%x] for resp func %d bmap %d\n", + msg_id, data->function_id, + test_bit(msg_id, vk->bmap)); + kfree(data); + } + /* Fetch wr_idx to handle more back-to-back events */ + wr_idx = readl(&msgq->wr_idx); + + /* + * cap the max so that even we try to handle more back-to-back events, + * so that it won't hold CPU too long or in case rd/wr idexes are + * corrupted which triggers infinite looping. + */ + if (++msg_processed >= max_msg_to_process) { + dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n", + q_num, max_msg_to_process); + exit_loop = true; + } + } + } +idx_err: + mutex_unlock(&chan->msgq_mutex); + dev_dbg(dev, "total %d drained from queues\n", total); + + return total; +} + +/* + * init routine for all required data structures + */ +static int bcm_vk_data_init(struct bcm_vk *vk) +{ + int i; + + spin_lock_init(&vk->ctx_lock); + for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { + vk->ctx[i].in_use = false; + vk->ctx[i].idx = i; /* self identity */ + vk->ctx[i].miscdev = NULL; + } + spin_lock_init(&vk->msg_id_lock); + spin_lock_init(&vk->host_alert_lock); + vk->msg_id = 0; + + /* initialize hash table */ + for (i = 0; i < VK_PID_HT_SZ; i++) + INIT_LIST_HEAD(&vk->pid_ht[i].head); + + return 0; +} + +irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + if (!bcm_vk_drv_access_ok(vk)) { + dev_err(&vk->pdev->dev, + "Interrupt %d received when msgq not inited\n", irq); + goto skip_schedule_work; + } + + queue_work(vk->wq_thread, &vk->wq_work); + +skip_schedule_work: + return IRQ_HANDLED; +} + int bcm_vk_open(struct inode *inode, struct file *p_file) { struct bcm_vk_ctx *ctx; @@ -112,16 +969,346 @@ int bcm_vk_open(struct inode *inode, struct file *p_file) return rc; } +ssize_t bcm_vk_read(struct file *p_file, + char __user *buf, + size_t count, + loff_t *f_pos) +{ + ssize_t rc = -ENOMSG; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, + miscdev); + struct device *dev = &vk->pdev->dev; + struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; + struct bcm_vk_wkent *entry = NULL; + u32 q_num; + u32 rsp_length; + bool found = false; + + if (!bcm_vk_drv_access_ok(vk)) + return -EPERM; + + dev_dbg(dev, "Buf count %zu\n", count); + found = false; + + /* + * search through the pendq on the to_h chan, and return only those + * that belongs to the same context. Search is always from the high to + * the low priority queues + */ + spin_lock(&chan->pendq_lock); + for (q_num = 0; q_num < chan->q_nr; q_num++) { + list_for_each_entry(entry, &chan->pendq[q_num], node) { + if (entry->ctx->idx == ctx->idx) { + if (count >= + (entry->to_h_blks * VK_MSGQ_BLK_SIZE)) { + list_del(&entry->node); + atomic_dec(&ctx->pend_cnt); + found = true; + } else { + /* buffer not big enough */ + rc = -EMSGSIZE; + } + goto read_loop_exit; + } + } + } +read_loop_exit: + spin_unlock(&chan->pendq_lock); + + if (found) { + /* retrieve the passed down msg_id */ + set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id); + rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE; + if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0) + rc = rsp_length; + + bcm_vk_free_wkent(dev, entry); + } else if (rc == -EMSGSIZE) { + struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; + + /* + * in this case, return just the first block, so + * that app knows what size it is looking for. + */ + set_msg_id(&tmp_msg, entry->usr_msg_id); + tmp_msg.size = entry->to_h_blks - 1; + if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { + dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); + rc = -EFAULT; + } + } + return rc; +} + +ssize_t bcm_vk_write(struct file *p_file, + const char __user *buf, + size_t count, + loff_t *f_pos) +{ + ssize_t rc; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, + miscdev); + struct bcm_vk_msgq __iomem *msgq; + struct device *dev = &vk->pdev->dev; + struct bcm_vk_wkent *entry; + u32 sgl_extra_blks; + u32 q_num; + u32 msg_size; + u32 msgq_size; + + if (!bcm_vk_drv_access_ok(vk)) + return -EPERM; + + dev_dbg(dev, "Msg count %zu\n", count); + + /* first, do sanity check where count should be multiple of basic blk */ + if (count & (VK_MSGQ_BLK_SIZE - 1)) { + dev_err(dev, "Failure with size %zu not multiple of %zu\n", + count, VK_MSGQ_BLK_SIZE); + rc = -EINVAL; + goto write_err; + } + + /* allocate the work entry + buffer for size count and inband sgl */ + entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size, + GFP_KERNEL); + if (!entry) { + rc = -ENOMEM; + goto write_err; + } + + /* now copy msg from user space, and then formulate the work entry */ + if (copy_from_user(&entry->to_v_msg[0], buf, count)) { + rc = -EFAULT; + goto write_free_ent; + } + + entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT; + entry->ctx = ctx; + + /* do a check on the blk size which could not exceed queue space */ + q_num = get_q_num(&entry->to_v_msg[0]); + msgq = vk->to_v_msg_chan.msgq[q_num]; + msgq_size = readl_relaxed(&msgq->size); + if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT) + > (msgq_size - 1)) { + dev_err(dev, "Blk size %d exceed max queue size allowed %d\n", + entry->to_v_blks, msgq_size - 1); + rc = -EINVAL; + goto write_free_ent; + } + + /* Use internal message id */ + entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]); + rc = bcm_vk_get_msg_id(vk); + if (rc == VK_MSG_ID_OVERFLOW) { + dev_err(dev, "msg_id overflow\n"); + rc = -EOVERFLOW; + goto write_free_ent; + } + set_msg_id(&entry->to_v_msg[0], rc); + ctx->q_num = q_num; + + dev_dbg(dev, + "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n", + ctx->q_num, ctx->idx, entry->usr_msg_id, + get_msg_id(&entry->to_v_msg[0])); + + if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) { + /* Convert any pointers to sg list */ + unsigned int num_planes; + int dir; + struct _vk_data *data; + + num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK; + if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD) + dir = DMA_FROM_DEVICE; + else + dir = DMA_TO_DEVICE; + + /* Calculate vk_data location */ + /* Go to end of the message */ + msg_size = entry->to_v_msg[0].size; + if (msg_size > entry->to_v_blks) { + rc = -EMSGSIZE; + goto write_free_msgid; + } + + data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1]; + + /* Now back up to the start of the pointers */ + data -= num_planes; + + /* Convert user addresses to DMA SG List */ + rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes); + if (rc) + goto write_free_msgid; + + atomic_inc(&ctx->dma_cnt); + /* try to embed inband sgl */ + sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data, + num_planes); + entry->to_v_blks += sgl_extra_blks; + entry->to_v_msg[0].size += sgl_extra_blks; + } else if (entry->to_v_msg[0].function_id == VK_FID_INIT && + entry->to_v_msg[0].context_id == VK_NEW_CTX) { + /* + * Init happens in 2 stages, only the first stage contains the + * pid that needs translating. + */ + pid_t org_pid, pid; + + /* + * translate the pid into the unique host space as user + * may run sessions inside containers or process + * namespaces. + */ +#define VK_MSG_PID_MASK 0xffffff00 +#define VK_MSG_PID_SH 8 + org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK) + >> VK_MSG_PID_SH; + + pid = task_tgid_nr(current); + entry->to_v_msg[0].arg = + (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) | + (pid << VK_MSG_PID_SH); + if (org_pid != pid) + dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n", + org_pid, org_pid, pid, pid); + } + + /* + * store work entry to pending queue until a response is received. + * This needs to be done before enqueuing the message + */ + bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry); + + rc = bcm_to_v_msg_enqueue(vk, entry); + if (rc) { + dev_err(dev, "Fail to enqueue msg to to_v queue\n"); + + /* remove message from pending list */ + entry = bcm_vk_dequeue_pending + (vk, + &vk->to_v_msg_chan, + q_num, + get_msg_id(&entry->to_v_msg[0])); + goto write_free_ent; + } + + return count; + +write_free_msgid: + bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1); +write_free_ent: + kfree(entry); +write_err: + return rc; +} + +__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait) +{ + __poll_t ret = 0; + int cnt; + struct bcm_vk_ctx *ctx = p_file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + + poll_wait(p_file, &ctx->rd_wq, wait); + + cnt = atomic_read(&ctx->pend_cnt); + if (cnt) { + ret = (__force __poll_t)(POLLIN | POLLRDNORM); + if (cnt < 0) { + dev_err(dev, "Error cnt %d, setting back to 0", cnt); + atomic_set(&ctx->pend_cnt, 0); + } + } + + return ret; +} + int bcm_vk_release(struct inode *inode, struct file *p_file) { int ret; struct bcm_vk_ctx *ctx = p_file->private_data; struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + struct device *dev = &vk->pdev->dev; + pid_t pid = ctx->pid; + int dma_cnt; + unsigned long timeout, start_time; + + /* + * if there are outstanding DMA transactions, need to delay long enough + * to ensure that the card side would have stopped touching the host buffer + * and its SGL list. A race condition could happen if the host app is killed + * abruptly, eg kill -9, while some DMA transfer orders are still inflight. + * Nothing could be done except for a delay as host side is running in a + * completely async fashion. + */ + start_time = jiffies; + timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS); + do { + if (time_after(jiffies, timeout)) { + dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n", + dma_cnt, ctx->idx, pid); + break; + } + dma_cnt = atomic_read(&ctx->dma_cnt); + cpu_relax(); + cond_resched(); + } while (dma_cnt); + dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n", + ctx->idx, pid, jiffies_to_msecs(jiffies - start_time)); + + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx); ret = bcm_vk_free_ctx(vk, ctx); + if (ret == 0) + ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num); + else + ret = 0; kref_put(&vk->kref, bcm_vk_release_data); return ret; } +int bcm_vk_msg_init(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + int ret; + + if (bcm_vk_data_init(vk)) { + dev_err(dev, "Error initializing internal data structures\n"); + return -EINVAL; + } + + if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) || + bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) { + dev_err(dev, "Error initializing communication channel\n"); + return -EIO; + } + + /* read msgq info if ready */ + ret = bcm_vk_sync_msgq(vk, false); + if (ret && (ret != -EAGAIN)) { + dev_err(dev, "Error reading comm msg Q info\n"); + return -EIO; + } + + return 0; +} + +void bcm_vk_msg_remove(struct bcm_vk *vk) +{ + bcm_vk_blk_drv_access(vk); + + /* drain all pending items */ + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); +} + diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h index 32516abcaf89..4eaad84825d6 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.h +++ b/drivers/misc/bcm-vk/bcm_vk_msg.h @@ -6,6 +6,78 @@ #ifndef BCM_VK_MSG_H #define BCM_VK_MSG_H +#include +#include "bcm_vk_sg.h" + +/* Single message queue control structure */ +struct bcm_vk_msgq { + u16 type; /* queue type */ + u16 num; /* queue number */ + u32 start; /* offset in BAR1 where the queue memory starts */ + + u32 rd_idx; /* read idx */ + u32 wr_idx; /* write idx */ + + u32 size; /* + * size, which is in number of 16byte blocks, + * to align with the message data structure. + */ + u32 nxt; /* + * nxt offset to the next msg queue struct. + * This is to provide flexibity for alignment purposes. + */ + +/* Least significant 16 bits in below field hold doorbell register offset */ +#define DB_SHIFT 16 + + u32 db_offset; /* queue doorbell register offset in BAR0 */ + + u32 rsvd; +}; + +/* + * Structure to record static info from the msgq sync. We keep local copy + * for some of these variables for both performance + checking purpose. + */ +struct bcm_vk_sync_qinfo { + void __iomem *q_start; + u32 q_size; + u32 q_mask; + u32 q_low; + u32 q_db_offset; +}; + +#define VK_MSGQ_MAX_NR 4 /* Maximum number of message queues */ + +/* + * message block - basic unit in the message where a message's size is always + * N x sizeof(basic_block) + */ +struct vk_msg_blk { + u8 function_id; +#define VK_FID_TRANS_BUF 5 +#define VK_FID_SHUTDOWN 8 +#define VK_FID_INIT 9 + u8 size; /* size of the message in number of vk_msg_blk's */ + u16 trans_id; /* transport id, queue & msg_id */ + u32 context_id; +#define VK_NEW_CTX 0 + u32 cmd; +#define VK_CMD_PLANES_MASK 0x000f /* number of planes to up/download */ +#define VK_CMD_UPLOAD 0x0400 /* memory transfer to vk */ +#define VK_CMD_DOWNLOAD 0x0500 /* memory transfer from vk */ +#define VK_CMD_MASK 0x0f00 /* command mask */ + u32 arg; +}; + +/* vk_msg_blk is 16 bytes fixed */ +#define VK_MSGQ_BLK_SIZE (sizeof(struct vk_msg_blk)) +/* shift for fast division of basic msg blk size */ +#define VK_MSGQ_BLK_SZ_SHIFT 4 + +/* use msg_id 0 for any simplex host2vk communication */ +#define VK_SIMPLEX_MSG_ID 0 + /* context per session opening of sysfs */ struct bcm_vk_ctx { struct list_head node; /* use for linkage in Hash Table */ @@ -13,7 +85,11 @@ struct bcm_vk_ctx { bool in_use; pid_t pid; u32 hash_idx; + u32 q_num; /* queue number used by the stream */ struct miscdevice *miscdev; + atomic_t pend_cnt; /* number of items pending to be read from host */ + atomic_t dma_cnt; /* any dma transaction outstanding */ + wait_queue_head_t rd_wq; }; /* pid hash table entry */ @@ -21,6 +97,55 @@ struct bcm_vk_ht_entry { struct list_head head; }; +#define VK_DMA_MAX_ADDRS 4 /* Max 4 DMA Addresses */ +/* structure for house keeping a single work entry */ +struct bcm_vk_wkent { + struct list_head node; /* for linking purpose */ + struct bcm_vk_ctx *ctx; + + /* Store up to 4 dma pointers */ + struct bcm_vk_dma dma[VK_DMA_MAX_ADDRS]; + + u32 to_h_blks; /* response */ + struct vk_msg_blk *to_h_msg; + + /* + * put the to_v_msg at the end so that we could simply append to_v msg + * to the end of the allocated block + */ + u32 usr_msg_id; + u32 to_v_blks; + u32 seq_num; + struct vk_msg_blk to_v_msg[0]; +}; + +/* queue stats counters */ +struct bcm_vk_qs_cnts { + u32 cnt; /* general counter, used to limit output */ + u32 acc_sum; + u32 max_occ; /* max during a sampling period */ + u32 max_abs; /* the abs max since reset */ +}; + +/* control channel structure for either to_v or to_h communication */ +struct bcm_vk_msg_chan { + u32 q_nr; + /* Mutex to access msgq */ + struct mutex msgq_mutex; + /* pointing to BAR locations */ + struct bcm_vk_msgq __iomem *msgq[VK_MSGQ_MAX_NR]; + /* Spinlock to access pending queue */ + spinlock_t pendq_lock; + /* for temporary storing pending items, one for each queue */ + struct list_head pendq[VK_MSGQ_MAX_NR]; + /* static queue info from the sync */ + struct bcm_vk_sync_qinfo sync_qinfo[VK_MSGQ_MAX_NR]; +}; + +/* totol number of message q allowed by the driver */ +#define VK_MSGQ_PER_CHAN_MAX 3 +#define VK_MSGQ_NUM_DEFAULT (VK_MSGQ_PER_CHAN_MAX - 1) + /* total number of supported ctx, 32 ctx each for 5 components */ #define VK_CMPT_CTX_MAX (32 * 5) @@ -28,4 +153,11 @@ struct bcm_vk_ht_entry { #define VK_PID_HT_SHIFT_BIT 7 /* 128 */ #define VK_PID_HT_SZ BIT(VK_PID_HT_SHIFT_BIT) +/* The following are offsets of DDR info provided by the vk card */ +#define VK_BAR0_SEG_SIZE (4 * SZ_1K) /* segment size for BAR0 */ + +/* shutdown types supported */ +#define VK_SHUTDOWN_PID 1 +#define VK_SHUTDOWN_GRACEFUL 2 + #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.c b/drivers/misc/bcm-vk/bcm_vk_sg.c new file mode 100644 index 000000000000..2e9daaf3e492 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_sg.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "bcm_vk.h" +#include "bcm_vk_msg.h" +#include "bcm_vk_sg.h" + +/* + * Valkyrie has a hardware limitation of 16M transfer size. + * So limit the SGL chunks to 16M. + */ +#define BCM_VK_MAX_SGL_CHUNK SZ_16M + +static int bcm_vk_dma_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata); +static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma); + +/* Uncomment to dump SGLIST */ +/* #define BCM_VK_DUMP_SGLIST */ + +static int bcm_vk_dma_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int direction, + struct _vk_data *vkdata) +{ + dma_addr_t addr, sg_addr; + int err; + int i; + int offset; + u32 size; + u32 remaining_size; + u32 transfer_size; + u64 data; + unsigned long first, last; + struct _vk_data *sgdata; + + /* Get 64-bit user address */ + data = get_unaligned(&vkdata->address); + + /* offset into first page */ + offset = offset_in_page(data); + + /* Calculate number of pages */ + first = (data & PAGE_MASK) >> PAGE_SHIFT; + last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT; + dma->nr_pages = last - first + 1; + + /* Allocate DMA pages */ + dma->pages = kmalloc_array(dma->nr_pages, + sizeof(struct page *), + GFP_KERNEL); + if (!dma->pages) + return -ENOMEM; + + dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n", + data, vkdata->size, dma->nr_pages); + + dma->direction = direction; + + /* Get user pages into memory */ + err = get_user_pages_fast(data & PAGE_MASK, + dma->nr_pages, + direction == DMA_FROM_DEVICE, + dma->pages); + if (err != dma->nr_pages) { + dma->nr_pages = (err >= 0) ? err : 0; + dev_err(dev, "get_user_pages_fast, err=%d [%d]\n", + err, dma->nr_pages); + return err < 0 ? err : -EINVAL; + } + + /* Max size of sg list is 1 per mapped page + fields at start */ + dma->sglen = (dma->nr_pages * sizeof(*sgdata)) + + (sizeof(u32) * SGLIST_VKDATA_START); + + /* Allocate sglist */ + dma->sglist = dma_alloc_coherent(dev, + dma->sglen, + &dma->handle, + GFP_KERNEL); + if (!dma->sglist) + return -ENOMEM; + + dma->sglist[SGLIST_NUM_SG] = 0; + dma->sglist[SGLIST_TOTALSIZE] = vkdata->size; + remaining_size = vkdata->size; + sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; + + /* Map all pages into DMA */ + size = min_t(size_t, PAGE_SIZE - offset, remaining_size); + remaining_size -= size; + sg_addr = dma_map_page(dev, + dma->pages[0], + offset, + size, + dma->direction); + transfer_size = size; + if (unlikely(dma_mapping_error(dev, sg_addr))) { + __free_page(dma->pages[0]); + return -EIO; + } + + for (i = 1; i < dma->nr_pages; i++) { + size = min_t(size_t, PAGE_SIZE, remaining_size); + remaining_size -= size; + addr = dma_map_page(dev, + dma->pages[i], + 0, + size, + dma->direction); + if (unlikely(dma_mapping_error(dev, addr))) { + __free_page(dma->pages[i]); + return -EIO; + } + + /* + * Compress SG list entry when pages are contiguous + * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK + */ + if ((addr == (sg_addr + transfer_size)) && + ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) { + /* pages are contiguous, add to same sg entry */ + transfer_size += size; + } else { + /* pages are not contiguous, write sg entry */ + sgdata->size = transfer_size; + put_unaligned(sg_addr, (u64 *)&sgdata->address); + dma->sglist[SGLIST_NUM_SG]++; + + /* start new sg entry */ + sgdata++; + sg_addr = addr; + transfer_size = size; + } + } + /* Write last sg list entry */ + sgdata->size = transfer_size; + put_unaligned(sg_addr, (u64 *)&sgdata->address); + dma->sglist[SGLIST_NUM_SG]++; + + /* Update pointers and size field to point to sglist */ + put_unaligned((u64)dma->handle, &vkdata->address); + vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) + + (sizeof(u32) * SGLIST_VKDATA_START); + +#ifdef BCM_VK_DUMP_SGLIST + dev_dbg(dev, + "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n", + (u64)dma->sglist, + dma->handle, + dma->sglen, + vkdata->size); + for (i = 0; i < vkdata->size / sizeof(u32); i++) + dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]); +#endif + + return 0; +} + +int bcm_vk_sg_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata, + int num) +{ + int i; + int rc = -EINVAL; + + /* Convert user addresses to DMA SG List */ + for (i = 0; i < num; i++) { + if (vkdata[i].size && vkdata[i].address) { + /* + * If both size and address are non-zero + * then DMA alloc. + */ + rc = bcm_vk_dma_alloc(dev, + &dma[i], + dir, + &vkdata[i]); + } else if (vkdata[i].size || + vkdata[i].address) { + /* + * If one of size and address are zero + * there is a problem. + */ + dev_err(dev, + "Invalid vkdata %x 0x%x 0x%llx\n", + i, vkdata[i].size, vkdata[i].address); + rc = -EINVAL; + } else { + /* + * If size and address are both zero + * don't convert, but return success. + */ + rc = 0; + } + + if (rc) + goto fail_alloc; + } + return rc; + +fail_alloc: + while (i > 0) { + i--; + if (dma[i].sglist) + bcm_vk_dma_free(dev, &dma[i]); + } + return rc; +} + +static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma) +{ + dma_addr_t addr; + int i; + int num_sg; + u32 size; + struct _vk_data *vkdata; + + dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen); + + /* Unmap all pages in the sglist */ + num_sg = dma->sglist[SGLIST_NUM_SG]; + vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START]; + for (i = 0; i < num_sg; i++) { + size = vkdata[i].size; + addr = get_unaligned(&vkdata[i].address); + + dma_unmap_page(dev, addr, size, dma->direction); + } + + /* Free allocated sglist */ + dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle); + + /* Release lock on all pages */ + for (i = 0; i < dma->nr_pages; i++) + put_page(dma->pages[i]); + + /* Free allocated dma pages */ + kfree(dma->pages); + dma->sglist = NULL; + + return 0; +} + +int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, + int *proc_cnt) +{ + int i; + + *proc_cnt = 0; + /* Unmap and free all pages and sglists */ + for (i = 0; i < num; i++) { + if (dma[i].sglist) { + bcm_vk_dma_free(dev, &dma[i]); + *proc_cnt += 1; + } + } + + return 0; +} diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.h b/drivers/misc/bcm-vk/bcm_vk_sg.h new file mode 100644 index 000000000000..81b3d0976ddb --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_sg.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2018-2020 Broadcom. + */ + +#ifndef BCM_VK_SG_H +#define BCM_VK_SG_H + +#include + +struct bcm_vk_dma { + /* for userland buffer */ + struct page **pages; + int nr_pages; + + /* common */ + dma_addr_t handle; + /* + * sglist is of the following LE format + * [U32] num_sg = number of sg addresses (N) + * [U32] totalsize = totalsize of data being transferred in sglist + * [U32] size[0] = size of data in address0 + * [U32] addr_l[0] = lower 32-bits of address0 + * [U32] addr_h[0] = higher 32-bits of address0 + * .. + * [U32] size[N-1] = size of data in addressN-1 + * [U32] addr_l[N-1] = lower 32-bits of addressN-1 + * [U32] addr_h[N-1] = higher 32-bits of addressN-1 + */ + u32 *sglist; +#define SGLIST_NUM_SG 0 +#define SGLIST_TOTALSIZE 1 +#define SGLIST_VKDATA_START 2 + + int sglen; /* Length (bytes) of sglist */ + int direction; +}; + +struct _vk_data { + u32 size; /* data size in bytes */ + u64 address; /* Pointer to data */ +} __packed; + +/* + * Scatter-gather DMA buffer API. + * + * These functions provide a simple way to create a page list and a + * scatter-gather list from userspace address and map the memory + * for DMA operation. + */ +int bcm_vk_sg_alloc(struct device *dev, + struct bcm_vk_dma *dma, + int dir, + struct _vk_data *vkdata, + int num); + +int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num, + int *proc_cnt); + +#endif + From d63d658f74727749088bfe436230b810f3880b0a Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:24 -0800 Subject: [PATCH 116/633] misc: bcm-vk: reset_pid support Add reset support via ioctl. Kill user processes that are open when VK card is reset. If a particular PID has issued the reset request do not kill that process as it issued the ioctl. Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-11-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk.h | 1 + drivers/misc/bcm-vk/bcm_vk_dev.c | 158 +++++++++++++++++++++++++++++-- drivers/misc/bcm-vk/bcm_vk_msg.c | 40 +++++++- 3 files changed, 191 insertions(+), 8 deletions(-) diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index d847a512d0ed..a1d0bf6e694c 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -468,6 +468,7 @@ irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id); irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id); int bcm_vk_msg_init(struct bcm_vk *vk); void bcm_vk_msg_remove(struct bcm_vk *vk); +void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk); int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync); void bcm_vk_blk_drv_access(struct bcm_vk *vk); s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk); diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index 5d82f02c0f27..e572a7b18fab 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -504,7 +504,9 @@ void bcm_vk_blk_drv_access(struct bcm_vk *vk) int i; /* - * kill all the apps + * kill all the apps except for the process that is resetting. + * If not called during reset, reset_pid will be 0, and all will be + * killed. */ spin_lock(&vk->ctx_lock); @@ -515,10 +517,12 @@ void bcm_vk_blk_drv_access(struct bcm_vk *vk) struct bcm_vk_ctx *ctx; list_for_each_entry(ctx, &vk->pid_ht[i].head, node) { - dev_dbg(&vk->pdev->dev, - "Send kill signal to pid %d\n", - ctx->pid); - kill_pid(find_vpid(ctx->pid), SIGKILL, 1); + if (ctx->pid != vk->reset_pid) { + dev_dbg(&vk->pdev->dev, + "Send kill signal to pid %d\n", + ctx->pid); + kill_pid(find_vpid(ctx->pid), SIGKILL, 1); + } } } spin_unlock(&vk->ctx_lock); @@ -1001,6 +1005,49 @@ static long bcm_vk_load_image(struct bcm_vk *vk, return ret; } +static int bcm_vk_reset_successful(struct bcm_vk *vk) +{ + struct device *dev = &vk->pdev->dev; + u32 fw_status, reset_reason; + int ret = -EAGAIN; + + /* + * Reset could be triggered when the card in several state: + * i) in bootROM + * ii) after boot1 + * iii) boot2 running + * + * i) & ii) - no status bits will be updated. If vkboot1 + * runs automatically after reset, it will update the reason + * to be unknown reason + * iii) - reboot reason match + deinit done. + */ + fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); + /* immediate exit if interface goes down */ + if (BCM_VK_INTF_IS_DOWN(fw_status)) { + dev_err(dev, "PCIe Intf Down!\n"); + goto reset_exit; + } + + reset_reason = (fw_status & VK_FWSTS_RESET_REASON_MASK); + if ((reset_reason == VK_FWSTS_RESET_MBOX_DB) || + (reset_reason == VK_FWSTS_RESET_UNKNOWN)) + ret = 0; + + /* + * if some of the deinit bits are set, but done + * bit is not, this is a failure if triggered while boot2 is running + */ + if ((fw_status & VK_FWSTS_DEINIT_TRIGGERED) && + !(fw_status & VK_FWSTS_RESET_DONE)) + ret = -EAGAIN; + +reset_exit: + dev_dbg(dev, "FW status = 0x%x ret %d\n", fw_status, ret); + + return ret; +} + static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val) { vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE); @@ -1010,12 +1057,16 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) { u32 i; u32 value, boot_status; + bool is_stdalone, is_boot2; static const u32 bar0_reg_clr_list[] = { BAR_OS_UPTIME, BAR_INTF_VER, BAR_CARD_VOLTAGE, BAR_CARD_TEMPERATURE, BAR_CARD_PWR_AND_THRE }; + /* clean up before pressing the door bell */ + bcm_vk_drain_msg_on_reset(vk); + vkwrite32(vk, 0, BAR_1, VK_BAR1_MSGQ_DEF_RDY); /* make tag '\0' terminated */ vkwrite32(vk, 0, BAR_1, VK_BAR1_BOOT1_VER_TAG); @@ -1026,6 +1077,11 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) for (i = 0; i < VK_BAR1_SOTP_REVID_MAX; i++) vkwrite32(vk, 0, BAR_1, VK_BAR1_SOTP_REVID_ADDR(i)); + memset(&vk->card_info, 0, sizeof(vk->card_info)); + memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info)); + memset(&vk->proc_mon_info, 0, sizeof(vk->proc_mon_info)); + memset(&vk->alert_cnts, 0, sizeof(vk->alert_cnts)); + /* * When boot request fails, the CODE_PUSH_OFFSET stays persistent. * Allowing us to debug the failure. When we call reset, @@ -1046,17 +1102,103 @@ static int bcm_vk_trigger_reset(struct bcm_vk *vk) } vkwrite32(vk, value, BAR_0, BAR_CODEPUSH_SBL); + /* special reset handling */ + is_stdalone = boot_status & BOOT_STDALONE_RUNNING; + is_boot2 = (boot_status & BOOT_STATE_MASK) == BOOT2_RUNNING; + if (vk->peer_alert.flags & ERR_LOG_RAMDUMP) { + /* + * if card is in ramdump mode, it is hitting an error. Don't + * reset the reboot reason as it will contain valid info that + * is important - simply use special reset + */ + vkwrite32(vk, VK_BAR0_RESET_RAMPDUMP, BAR_0, VK_BAR_FWSTS); + return VK_BAR0_RESET_RAMPDUMP; + } else if (is_stdalone && !is_boot2) { + dev_info(&vk->pdev->dev, "Hard reset on Standalone mode"); + bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD); + return VK_BAR0_RESET_DB_HARD; + } + /* reset fw_status with proper reason, and press db */ vkwrite32(vk, VK_FWSTS_RESET_MBOX_DB, BAR_0, VK_BAR_FWSTS); bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_SOFT); - /* clear other necessary registers records */ + /* clear other necessary registers and alert records */ for (i = 0; i < ARRAY_SIZE(bar0_reg_clr_list); i++) vkwrite32(vk, 0, BAR_0, bar0_reg_clr_list[i]); + memset(&vk->host_alert, 0, sizeof(vk->host_alert)); + memset(&vk->peer_alert, 0, sizeof(vk->peer_alert)); + /* clear 4096 bits of bitmap */ + bitmap_clear(vk->bmap, 0, VK_MSG_ID_BITMAP_SIZE); return 0; } +static long bcm_vk_reset(struct bcm_vk *vk, struct vk_reset __user *arg) +{ + struct device *dev = &vk->pdev->dev; + struct vk_reset reset; + int ret = 0; + u32 ramdump_reset; + int special_reset; + + if (copy_from_user(&reset, arg, sizeof(struct vk_reset))) + return -EFAULT; + + /* check if any download is in-progress, if so return error */ + if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) { + dev_err(dev, "Download operation pending - skip reset.\n"); + return -EPERM; + } + + ramdump_reset = vk->peer_alert.flags & ERR_LOG_RAMDUMP; + dev_info(dev, "Issue Reset %s\n", + ramdump_reset ? "in ramdump mode" : ""); + + /* + * The following is the sequence of reset: + * - send card level graceful shut down + * - wait enough time for VK to handle its business, stopping DMA etc + * - kill host apps + * - Trigger interrupt with DB + */ + bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_GRACEFUL, 0, 0); + + spin_lock(&vk->ctx_lock); + if (!vk->reset_pid) { + vk->reset_pid = task_pid_nr(current); + } else { + dev_err(dev, "Reset already launched by process pid %d\n", + vk->reset_pid); + ret = -EACCES; + } + spin_unlock(&vk->ctx_lock); + if (ret) + goto err_exit; + + bcm_vk_blk_drv_access(vk); + special_reset = bcm_vk_trigger_reset(vk); + + /* + * Wait enough time for card os to deinit + * and populate the reset reason. + */ + msleep(BCM_VK_DEINIT_TIME_MS); + + if (special_reset) { + /* if it is special ramdump reset, return the type to user */ + reset.arg2 = special_reset; + if (copy_to_user(arg, &reset, sizeof(reset))) + ret = -EFAULT; + } else { + ret = bcm_vk_reset_successful(vk); + } + +err_exit: + clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); + return ret; +} + static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long ret = -EINVAL; @@ -1075,6 +1217,10 @@ static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) ret = bcm_vk_load_image(vk, argp); break; + case VK_IOCTL_RESET: + ret = bcm_vk_reset(vk, argp); + break; + default: break; } diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index b05e20a72a8b..eec90494777d 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -209,6 +209,15 @@ static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) spin_lock(&vk->ctx_lock); + /* check if it is in reset, if so, don't allow */ + if (vk->reset_pid) { + dev_err(&vk->pdev->dev, + "No context allowed during reset by pid %d\n", + vk->reset_pid); + + goto in_reset_exit; + } + for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { if (!vk->ctx[i].in_use) { vk->ctx[i].in_use = true; @@ -237,6 +246,7 @@ static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) init_waitqueue_head(&ctx->rd_wq); all_in_use_exit: +in_reset_exit: spin_unlock(&vk->ctx_lock); return ctx; @@ -381,6 +391,12 @@ static void bcm_vk_drain_all_pend(struct device *dev, num, ctx->idx); } +void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk) +{ + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); + bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); +} + /* * Function to sync up the messages queue info that is provided by BAR1 */ @@ -712,13 +728,22 @@ static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid, /* * don't send down or do anything if message queue is not initialized + * and if it is the reset session, clear it. */ - if (!bcm_vk_drv_access_ok(vk)) + if (!bcm_vk_drv_access_ok(vk)) { + if (vk->reset_pid == pid) + vk->reset_pid = 0; return -EPERM; + } dev_dbg(dev, "No more sessions, shut down pid %d\n", pid); - rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); + /* only need to do it if it is not the reset process */ + if (vk->reset_pid != pid) + rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); + else + /* put reset_pid to 0 if it is exiting last session */ + vk->reset_pid = 0; return rc; } @@ -1122,6 +1147,17 @@ ssize_t bcm_vk_write(struct file *p_file, int dir; struct _vk_data *data; + /* + * check if we are in reset, if so, no buffer transfer is + * allowed and return error. + */ + if (vk->reset_pid) { + dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n", + ctx->pid); + rc = -EACCES; + goto write_free_msgid; + } + num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK; if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD) dir = DMA_FROM_DEVICE; From 483050c04738c685e3982c1baec202d0cc1beb0d Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:25 -0800 Subject: [PATCH 117/633] misc: bcm-vk: add mmap function for exposing BAR2 Add mmap function that allows host application to open up BAR2 memory for remote spooling out messages from the VK logger. Co-developed-by: Desmond Yan Acked-by: Olof Johansson Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-12-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk_dev.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index e572a7b18fab..cac07419f041 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -1199,6 +1199,29 @@ static long bcm_vk_reset(struct bcm_vk *vk, struct vk_reset __user *arg) return ret; } +static int bcm_vk_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct bcm_vk_ctx *ctx = file->private_data; + struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); + unsigned long pg_size; + + /* only BAR2 is mmap possible, which is bar num 4 due to 64bit */ +#define VK_MMAPABLE_BAR 4 + + pg_size = ((pci_resource_len(vk->pdev, VK_MMAPABLE_BAR) - 1) + >> PAGE_SHIFT) + 1; + if (vma->vm_pgoff + vma_pages(vma) > pg_size) + return -EINVAL; + + vma->vm_pgoff += (pci_resource_start(vk->pdev, VK_MMAPABLE_BAR) + >> PAGE_SHIFT); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long ret = -EINVAL; @@ -1237,6 +1260,7 @@ static const struct file_operations bcm_vk_fops = { .write = bcm_vk_write, .poll = bcm_vk_poll, .release = bcm_vk_release, + .mmap = bcm_vk_mmap, .unlocked_ioctl = bcm_vk_ioctl, }; From 68f1fae62c37ff739c38ac812165cbbab85b65d4 Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:26 -0800 Subject: [PATCH 118/633] MAINTAINERS: bcm-vk: add maintainer for Broadcom VK Driver Add maintainer entry for new Broadcom VK Driver Acked-by: Olof Johansson Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-13-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 93d182e4808d..23273a59da6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3744,6 +3744,13 @@ L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/tg3.* +BROADCOM VK DRIVER +M: Scott Branden +L: bcm-kernel-feedback-list@broadcom.com +S: Supported +F: drivers/misc/bcm-vk/ +F: include/uapi/linux/misc/bcm_vk.h + BROCADE BFA FC SCSI DRIVER M: Anil Gurumurthy M: Sudarsana Kalluru From 91ca10d6fa0720e35596c720e494d9c18624418a Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 20 Jan 2021 09:58:27 -0800 Subject: [PATCH 119/633] misc: bcm-vk: add ttyVK support Add ttyVK support to driver to allow console access to VK card from host. Device node will be in the follow form /dev/bcm-vk.x_ttyVKy where: x is the instance of the VK card y is the tty device number on the VK card Acked-by: Olof Johansson Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210120175827.14820-14-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/Makefile | 3 +- drivers/misc/bcm-vk/bcm_vk.h | 28 +++ drivers/misc/bcm-vk/bcm_vk_dev.c | 30 ++- drivers/misc/bcm-vk/bcm_vk_tty.c | 333 +++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 2 deletions(-) create mode 100644 drivers/misc/bcm-vk/bcm_vk_tty.c diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile index 79b4e365c9e6..e4a1486f7209 100644 --- a/drivers/misc/bcm-vk/Makefile +++ b/drivers/misc/bcm-vk/Makefile @@ -7,5 +7,6 @@ obj-$(CONFIG_BCM_VK) += bcm_vk.o bcm_vk-objs := \ bcm_vk_dev.o \ bcm_vk_msg.o \ - bcm_vk_sg.o + bcm_vk_sg.o \ + bcm_vk_tty.o diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index a1d0bf6e694c..3f37c640a814 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -8,12 +8,14 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -84,6 +86,9 @@ #define CODEPUSH_BOOT2_ENTRY 0x60000000 #define BAR_CARD_STATUS 0x410 +/* CARD_STATUS definitions */ +#define CARD_STATUS_TTYVK0_READY BIT(0) +#define CARD_STATUS_TTYVK1_READY BIT(1) #define BAR_BOOT1_STDALONE_PROGRESS 0x420 #define BOOT1_STDALONE_SUCCESS (BIT(13) | BIT(14)) @@ -255,6 +260,19 @@ enum pci_barno { #define BCM_VK_NUM_TTY 2 +struct bcm_vk_tty { + struct tty_port port; + u32 to_offset; /* bar offset to use */ + u32 to_size; /* to VK buffer size */ + u32 wr; /* write offset shadow */ + u32 from_offset; /* bar offset to use */ + u32 from_size; /* from VK buffer size */ + u32 rd; /* read offset shadow */ + pid_t pid; + bool irq_enabled; + bool is_opened; /* tracks tty open/close */ +}; + /* VK device max power state, supports 3, full, reduced and low */ #define MAX_OPP 3 #define MAX_CARD_INFO_TAG_SIZE 64 @@ -348,6 +366,12 @@ struct bcm_vk { struct miscdevice miscdev; int devid; /* dev id allocated */ + struct tty_driver *tty_drv; + struct timer_list serial_timer; + struct bcm_vk_tty tty[BCM_VK_NUM_TTY]; + struct workqueue_struct *tty_wq_thread; + struct work_struct tty_wq_work; + /* Reference-counting to handle file operations */ struct kref kref; @@ -466,6 +490,7 @@ int bcm_vk_release(struct inode *inode, struct file *p_file); void bcm_vk_release_data(struct kref *kref); irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id); irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id); +irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id); int bcm_vk_msg_init(struct bcm_vk *vk); void bcm_vk_msg_remove(struct bcm_vk *vk); void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk); @@ -476,6 +501,9 @@ int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, const pid_t pid, const u32 q_num); void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val); int bcm_vk_auto_load_all_images(struct bcm_vk *vk); +int bcm_vk_tty_init(struct bcm_vk *vk, char *name); +void bcm_vk_tty_exit(struct bcm_vk *vk); +void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk); void bcm_vk_hb_init(struct bcm_vk *vk); void bcm_vk_hb_deinit(struct bcm_vk *vk); void bcm_vk_handle_notf(struct bcm_vk *vk); diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index cac07419f041..c3d2bba68ef1 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -525,6 +525,7 @@ void bcm_vk_blk_drv_access(struct bcm_vk *vk) } } } + bcm_vk_tty_terminate_tty_user(vk); spin_unlock(&vk->ctx_lock); } @@ -1384,6 +1385,20 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } vk->num_irqs++; + for (i = 0; + (i < VK_MSIX_TTY_MAX) && (vk->num_irqs < irq); + i++, vk->num_irqs++) { + err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs), + bcm_vk_tty_irqhandler, + IRQF_SHARED, DRV_MODULE_NAME, vk); + if (err) { + dev_err(dev, "failed request tty IRQ %d for MSIX %d\n", + pdev->irq + vk->num_irqs, vk->num_irqs + 1); + goto err_irq; + } + vk->tty[i].irq_enabled = true; + } + id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); if (id < 0) { err = id; @@ -1436,6 +1451,11 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_destroy_workqueue; } + snprintf(name, sizeof(name), KBUILD_MODNAME ".%d_ttyVK", id); + err = bcm_vk_tty_init(vk, name); + if (err) + goto err_unregister_panic_notifier; + /* * lets trigger an auto download. We don't want to do it serially here * because at probing time, it is not supposed to block for a long time. @@ -1444,7 +1464,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (auto_load) { if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { if (bcm_vk_trigger_autoload(vk)) - goto err_unregister_panic_notifier; + goto err_bcm_vk_tty_exit; } else { dev_err(dev, "Auto-load skipped - BROM not in proper state (0x%x)\n", @@ -1459,6 +1479,9 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; +err_bcm_vk_tty_exit: + bcm_vk_tty_exit(vk); + err_unregister_panic_notifier: atomic_notifier_chain_unregister(&panic_notifier_list, &vk->panic_nb); @@ -1536,6 +1559,9 @@ static void bcm_vk_remove(struct pci_dev *pdev) atomic_notifier_chain_unregister(&panic_notifier_list, &vk->panic_nb); + bcm_vk_msg_remove(vk); + bcm_vk_tty_exit(vk); + if (vk->tdma_vaddr) dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, vk->tdma_vaddr, vk->tdma_addr); @@ -1554,6 +1580,8 @@ static void bcm_vk_remove(struct pci_dev *pdev) cancel_work_sync(&vk->wq_work); destroy_workqueue(vk->wq_thread); + cancel_work_sync(&vk->tty_wq_work); + destroy_workqueue(vk->tty_wq_thread); for (i = 0; i < MAX_BAR; i++) { if (vk->bar[i]) diff --git a/drivers/misc/bcm-vk/bcm_vk_tty.c b/drivers/misc/bcm-vk/bcm_vk_tty.c new file mode 100644 index 000000000000..be3964949b63 --- /dev/null +++ b/drivers/misc/bcm-vk/bcm_vk_tty.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018-2020 Broadcom. + */ + +#include +#include +#include + +#include "bcm_vk.h" + +/* TTYVK base offset is 0x30000 into BAR1 */ +#define BAR1_TTYVK_BASE_OFFSET 0x300000 +/* Each TTYVK channel (TO or FROM) is 0x10000 */ +#define BAR1_TTYVK_CHAN_OFFSET 0x100000 +/* Each TTYVK channel has TO and FROM, hence the * 2 */ +#define BAR1_TTYVK_BASE(index) (BAR1_TTYVK_BASE_OFFSET + \ + ((index) * BAR1_TTYVK_CHAN_OFFSET * 2)) +/* TO TTYVK channel base comes before FROM for each index */ +#define TO_TTYK_BASE(index) BAR1_TTYVK_BASE(index) +#define FROM_TTYK_BASE(index) (BAR1_TTYVK_BASE(index) + \ + BAR1_TTYVK_CHAN_OFFSET) + +struct bcm_vk_tty_chan { + u32 reserved; + u32 size; + u32 wr; + u32 rd; + u32 *data; +}; + +#define VK_BAR_CHAN(v, DIR, e) ((v)->DIR##_offset \ + + offsetof(struct bcm_vk_tty_chan, e)) +#define VK_BAR_CHAN_SIZE(v, DIR) VK_BAR_CHAN(v, DIR, size) +#define VK_BAR_CHAN_WR(v, DIR) VK_BAR_CHAN(v, DIR, wr) +#define VK_BAR_CHAN_RD(v, DIR) VK_BAR_CHAN(v, DIR, rd) +#define VK_BAR_CHAN_DATA(v, DIR, off) (VK_BAR_CHAN(v, DIR, data) + (off)) + +#define VK_BAR0_REGSEG_TTY_DB_OFFSET 0x86c + +/* Poll every 1/10 of second - temp hack till we use MSI interrupt */ +#define SERIAL_TIMER_VALUE (HZ / 10) + +static void bcm_vk_tty_poll(struct timer_list *t) +{ + struct bcm_vk *vk = from_timer(vk, t, serial_timer); + + queue_work(vk->tty_wq_thread, &vk->tty_wq_work); + mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE); +} + +irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id) +{ + struct bcm_vk *vk = dev_id; + + queue_work(vk->tty_wq_thread, &vk->tty_wq_work); + + return IRQ_HANDLED; +} + +static void bcm_vk_tty_wq_handler(struct work_struct *work) +{ + struct bcm_vk *vk = container_of(work, struct bcm_vk, tty_wq_work); + struct bcm_vk_tty *vktty; + int card_status; + int count; + unsigned char c; + int i; + int wr; + + card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS); + if (BCM_VK_INTF_IS_DOWN(card_status)) + return; + + for (i = 0; i < BCM_VK_NUM_TTY; i++) { + count = 0; + /* Check the card status that the tty channel is ready */ + if ((card_status & BIT(i)) == 0) + continue; + + vktty = &vk->tty[i]; + + /* Don't increment read index if tty app is closed */ + if (!vktty->is_opened) + continue; + + /* Fetch the wr offset in buffer from VK */ + wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, from)); + + /* safe to ignore until bar read gives proper size */ + if (vktty->from_size == 0) + continue; + + if (wr >= vktty->from_size) { + dev_err(&vk->pdev->dev, + "ERROR: wq handler ttyVK%d wr:0x%x > 0x%x\n", + i, wr, vktty->from_size); + /* Need to signal and close device in this case */ + continue; + } + + /* + * Simple read of circular buffer and + * insert into tty flip buffer + */ + while (vk->tty[i].rd != wr) { + c = vkread8(vk, BAR_1, + VK_BAR_CHAN_DATA(vktty, from, vktty->rd)); + vktty->rd++; + if (vktty->rd >= vktty->from_size) + vktty->rd = 0; + tty_insert_flip_char(&vktty->port, c, TTY_NORMAL); + count++; + } + + if (count) { + tty_flip_buffer_push(&vktty->port); + + /* Update read offset from shadow register to card */ + vkwrite32(vk, vktty->rd, BAR_1, + VK_BAR_CHAN_RD(vktty, from)); + } + } +} + +static int bcm_vk_tty_open(struct tty_struct *tty, struct file *file) +{ + int card_status; + struct bcm_vk *vk; + struct bcm_vk_tty *vktty; + int index; + + /* initialize the pointer in case something fails */ + tty->driver_data = NULL; + + vk = (struct bcm_vk *)dev_get_drvdata(tty->dev); + index = tty->index; + + if (index >= BCM_VK_NUM_TTY) + return -EINVAL; + + vktty = &vk->tty[index]; + + vktty->pid = task_pid_nr(current); + vktty->to_offset = TO_TTYK_BASE(index); + vktty->from_offset = FROM_TTYK_BASE(index); + + /* Do not allow tty device to be opened if tty on card not ready */ + card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS); + if (BCM_VK_INTF_IS_DOWN(card_status) || ((card_status & BIT(index)) == 0)) + return -EBUSY; + + /* + * Get shadow registers of the buffer sizes and the "to" write offset + * and "from" read offset + */ + vktty->to_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, to)); + vktty->wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, to)); + vktty->from_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, from)); + vktty->rd = vkread32(vk, BAR_1, VK_BAR_CHAN_RD(vktty, from)); + vktty->is_opened = true; + + if (tty->count == 1 && !vktty->irq_enabled) { + timer_setup(&vk->serial_timer, bcm_vk_tty_poll, 0); + mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE); + } + return 0; +} + +static void bcm_vk_tty_close(struct tty_struct *tty, struct file *file) +{ + struct bcm_vk *vk = dev_get_drvdata(tty->dev); + + if (tty->index >= BCM_VK_NUM_TTY) + return; + + vk->tty[tty->index].is_opened = false; + + if (tty->count == 1) + del_timer_sync(&vk->serial_timer); +} + +static void bcm_vk_tty_doorbell(struct bcm_vk *vk, u32 db_val) +{ + vkwrite32(vk, db_val, BAR_0, + VK_BAR0_REGSEG_DB_BASE + VK_BAR0_REGSEG_TTY_DB_OFFSET); +} + +static int bcm_vk_tty_write(struct tty_struct *tty, + const unsigned char *buffer, + int count) +{ + int index; + struct bcm_vk *vk; + struct bcm_vk_tty *vktty; + int i; + + index = tty->index; + vk = dev_get_drvdata(tty->dev); + vktty = &vk->tty[index]; + + /* Simple write each byte to circular buffer */ + for (i = 0; i < count; i++) { + vkwrite8(vk, buffer[i], BAR_1, + VK_BAR_CHAN_DATA(vktty, to, vktty->wr)); + vktty->wr++; + if (vktty->wr >= vktty->to_size) + vktty->wr = 0; + } + /* Update write offset from shadow register to card */ + vkwrite32(vk, vktty->wr, BAR_1, VK_BAR_CHAN_WR(vktty, to)); + bcm_vk_tty_doorbell(vk, 0); + + return count; +} + +static int bcm_vk_tty_write_room(struct tty_struct *tty) +{ + struct bcm_vk *vk = dev_get_drvdata(tty->dev); + + return vk->tty[tty->index].to_size - 1; +} + +static const struct tty_operations serial_ops = { + .open = bcm_vk_tty_open, + .close = bcm_vk_tty_close, + .write = bcm_vk_tty_write, + .write_room = bcm_vk_tty_write_room, +}; + +int bcm_vk_tty_init(struct bcm_vk *vk, char *name) +{ + int i; + int err; + struct tty_driver *tty_drv; + struct device *dev = &vk->pdev->dev; + + tty_drv = tty_alloc_driver + (BCM_VK_NUM_TTY, + TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); + if (IS_ERR(tty_drv)) + return PTR_ERR(tty_drv); + + /* Save struct tty_driver for uninstalling the device */ + vk->tty_drv = tty_drv; + + /* initialize the tty driver */ + tty_drv->driver_name = KBUILD_MODNAME; + tty_drv->name = kstrdup(name, GFP_KERNEL); + if (!tty_drv->name) { + err = -ENOMEM; + goto err_put_tty_driver; + } + tty_drv->type = TTY_DRIVER_TYPE_SERIAL; + tty_drv->subtype = SERIAL_TYPE_NORMAL; + tty_drv->init_termios = tty_std_termios; + tty_set_operations(tty_drv, &serial_ops); + + /* register the tty driver */ + err = tty_register_driver(tty_drv); + if (err) { + dev_err(dev, "tty_register_driver failed\n"); + goto err_kfree_tty_name; + } + + for (i = 0; i < BCM_VK_NUM_TTY; i++) { + struct device *tty_dev; + + tty_port_init(&vk->tty[i].port); + tty_dev = tty_port_register_device(&vk->tty[i].port, tty_drv, + i, dev); + if (IS_ERR(tty_dev)) { + err = PTR_ERR(tty_dev); + goto unwind; + } + dev_set_drvdata(tty_dev, vk); + vk->tty[i].is_opened = false; + } + + INIT_WORK(&vk->tty_wq_work, bcm_vk_tty_wq_handler); + vk->tty_wq_thread = create_singlethread_workqueue("tty"); + if (!vk->tty_wq_thread) { + dev_err(dev, "Fail to create tty workqueue thread\n"); + err = -ENOMEM; + goto unwind; + } + return 0; + +unwind: + while (--i >= 0) + tty_port_unregister_device(&vk->tty[i].port, tty_drv, i); + tty_unregister_driver(tty_drv); + +err_kfree_tty_name: + kfree(tty_drv->name); + tty_drv->name = NULL; + +err_put_tty_driver: + put_tty_driver(tty_drv); + + return err; +} + +void bcm_vk_tty_exit(struct bcm_vk *vk) +{ + int i; + + del_timer_sync(&vk->serial_timer); + for (i = 0; i < BCM_VK_NUM_TTY; ++i) { + tty_port_unregister_device(&vk->tty[i].port, + vk->tty_drv, + i); + tty_port_destroy(&vk->tty[i].port); + } + tty_unregister_driver(vk->tty_drv); + + kfree(vk->tty_drv->name); + vk->tty_drv->name = NULL; + + put_tty_driver(vk->tty_drv); +} + +void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk) +{ + struct bcm_vk_tty *vktty; + int i; + + for (i = 0; i < BCM_VK_NUM_TTY; ++i) { + vktty = &vk->tty[i]; + if (vktty->pid) + kill_pid(find_vpid(vktty->pid), SIGKILL, 1); + } +} From fc716ffb9afe789dcf8a5d792e3980f817e0ad1b Mon Sep 17 00:00:00 2001 From: mateng Date: Tue, 26 Jan 2021 16:40:10 +0800 Subject: [PATCH 120/633] misc/vmw_vmci: fix typo change 'addres' to 'address' Signed-off-by: mateng Link: https://lore.kernel.org/r/20210126084010.1941-1-ayowoe@163.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.h b/drivers/misc/vmw_vmci/vmci_queue_pair.h index 00017fc29a52..c4e6e924d9be 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.h +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.h @@ -104,7 +104,7 @@ struct vmci_qp_dtch_info { struct vmci_qp_page_store { /* Reference to pages backing the queue pair. */ u64 pages; - /* Length of pageList/virtual addres range (in pages). */ + /* Length of pageList/virtual address range (in pages). */ u32 len; }; From e8266c4c3307d2a64f8ebcb22323347be8854742 Mon Sep 17 00:00:00 2001 From: Jorgen Hansen Date: Wed, 20 Jan 2021 08:32:20 -0800 Subject: [PATCH 121/633] VMCI: Stop log spew when qp allocation isn't possible VMCI queue pair allocation is disabled, if a VM is in FT mode. In these cases, VMware Tools may still once in a while attempt to create a vSocket stream connection, resulting in multiple warnings in the kernel logs. Therefore downgrade the error log to a debug log. Reviewed-by: Vishnu Dasa Signed-off-by: Jorgen Hansen Link: https://lore.kernel.org/r/1611160340-30158-1-git-send-email-jhansen@vmware.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index c49065887e8f..a3691c16d45a 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -1207,7 +1207,7 @@ static int qp_alloc_guest_work(struct vmci_handle *handle, } else { result = qp_alloc_hypercall(queue_pair_entry); if (result < VMCI_SUCCESS) { - pr_warn("qp_alloc_hypercall result = %d\n", result); + pr_devel("qp_alloc_hypercall result = %d\n", result); goto error; } } From 5a16c535409f8dcb7568e20737309e3027ae3e49 Mon Sep 17 00:00:00 2001 From: Jorgen Hansen Date: Wed, 20 Jan 2021 08:32:40 -0800 Subject: [PATCH 122/633] VMCI: Use set_page_dirty_lock() when unregistering guest memory When the VMCI host support releases guest memory in the case where the VM was killed, the pinned guest pages aren't locked. Use set_page_dirty_lock() instead of set_page_dirty(). Testing done: Killed VM while having an active VMCI based vSocket connection and observed warning from ext4. With this fix, no warning was observed. Ran various vSocket tests without issues. Fixes: 06164d2b72aa ("VMCI: queue pairs implementation.") Reviewed-by: Vishnu Dasa Signed-off-by: Jorgen Hansen Link: https://lore.kernel.org/r/1611160360-30299-1-git-send-email-jhansen@vmware.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index a3691c16d45a..525ef96d3a07 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -630,7 +630,7 @@ static void qp_release_pages(struct page **pages, for (i = 0; i < num_pages; i++) { if (dirty) - set_page_dirty(pages[i]); + set_page_dirty_lock(pages[i]); put_page(pages[i]); pages[i] = NULL; From 7eecea89e44f64e60f1410483e79a0cab18ad580 Mon Sep 17 00:00:00 2001 From: Jorgen Hansen Date: Wed, 20 Jan 2021 08:33:40 -0800 Subject: [PATCH 123/633] VMCI: Enforce queuepair max size for IOCTL_VMCI_QUEUEPAIR_ALLOC When create the VMCI queue pair tracking data structures on the host side, the IOCTL for creating the VMCI queue pair didn't validate the queue pair size parameters. This change adds checks for this. This avoids a memory allocation issue in qp_host_alloc_queue, as reported by nslusarek@gmx.net. The check in qp_host_alloc_queue has also been updated to enforce the maximum queue pair size as defined by VMCI_MAX_GUEST_QP_MEMORY. The fix has been verified using sample code supplied by nslusarek@gmx.net. Reported-by: nslusarek@gmx.net Reviewed-by: Vishnu Dasa Signed-off-by: Jorgen Hansen Link: https://lore.kernel.org/r/1611160420-30573-1-git-send-email-jhansen@vmware.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 12 ++++++++---- include/linux/vmw_vmci_defs.h | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index 525ef96d3a07..d787ddecee77 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -237,7 +237,9 @@ static struct qp_list qp_guest_endpoints = { #define QPE_NUM_PAGES(_QPE) ((u32) \ (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \ DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2)) - +#define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \ + ((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \ + (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY) /* * Frees kernel VA space for a given queue and its queue header, and @@ -528,7 +530,7 @@ static struct vmci_queue *qp_host_alloc_queue(u64 size) u64 num_pages; const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if)); - if (size > SIZE_MAX - PAGE_SIZE) + if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE)) return NULL; num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1; if (num_pages > (SIZE_MAX - queue_size) / @@ -1929,6 +1931,9 @@ int vmci_qp_broker_alloc(struct vmci_handle handle, struct vmci_qp_page_store *page_store, struct vmci_ctx *context) { + if (!QP_SIZES_ARE_VALID(produce_size, consume_size)) + return VMCI_ERROR_NO_RESOURCES; + return qp_broker_alloc(handle, peer, flags, priv_flags, produce_size, consume_size, page_store, context, NULL, NULL, NULL, NULL); @@ -2685,8 +2690,7 @@ int vmci_qpair_alloc(struct vmci_qp **qpair, * used by the device is NO_RESOURCES, so use that here too. */ - if (produce_qsize + consume_qsize < max(produce_qsize, consume_qsize) || - produce_qsize + consume_qsize > VMCI_MAX_GUEST_QP_MEMORY) + if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize)) return VMCI_ERROR_NO_RESOURCES; retval = vmci_route(&src, &dst, false, &route); diff --git a/include/linux/vmw_vmci_defs.h b/include/linux/vmw_vmci_defs.h index be0afe6f379b..e36cb114c188 100644 --- a/include/linux/vmw_vmci_defs.h +++ b/include/linux/vmw_vmci_defs.h @@ -66,7 +66,7 @@ enum { * consists of at least two pages, the memory limit also dictates the * number of queue pairs a guest can create. */ -#define VMCI_MAX_GUEST_QP_MEMORY (128 * 1024 * 1024) +#define VMCI_MAX_GUEST_QP_MEMORY ((size_t)(128 * 1024 * 1024)) #define VMCI_MAX_GUEST_QP_COUNT (VMCI_MAX_GUEST_QP_MEMORY / PAGE_SIZE / 2) /* @@ -80,7 +80,7 @@ enum { * too much kernel memory (especially on vmkernel). We limit a queuepair to * 32 KB, or 16 KB per queue for symmetrical pairs. */ -#define VMCI_MAX_PINNED_QP_MEMORY (32 * 1024) +#define VMCI_MAX_PINNED_QP_MEMORY ((size_t)(32 * 1024)) /* * We have a fixed set of resource IDs available in the VMX. From e4240253ac853886e7066e0bb765e2d96e3b1eb6 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 18 Jan 2021 15:46:29 +0100 Subject: [PATCH 124/633] greybus: es2: drop short control-transfer checks There's no need to check for short USB control transfers when sending data using so remove the redundant sanity checks. Reviewed-by: Viresh Kumar Signed-off-by: Johan Hovold Link: https://lore.kernel.org/r/20210118144629.25533-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/greybus/es2.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/drivers/greybus/es2.c b/drivers/greybus/es2.c index 1df6ab5d339d..48ad154df3a7 100644 --- a/drivers/greybus/es2.c +++ b/drivers/greybus/es2.c @@ -567,12 +567,9 @@ static int cport_enable(struct gb_host_device *hd, u16 cport_id, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, cport_id, 0, req, sizeof(*req), ES2_USB_CTRL_TIMEOUT); - if (ret != sizeof(*req)) { + if (ret < 0) { dev_err(&udev->dev, "failed to set cport flags for port %d\n", cport_id); - if (ret >= 0) - ret = -EIO; - goto out; } @@ -961,12 +958,10 @@ static int arpc_send(struct es2_ap_dev *es2, struct arpc *rpc, int timeout) 0, 0, rpc->req, le16_to_cpu(rpc->req->size), ES2_USB_CTRL_TIMEOUT); - if (retval != le16_to_cpu(rpc->req->size)) { + if (retval < 0) { dev_err(&udev->dev, "failed to send ARPC request %d: %d\n", rpc->req->type, retval); - if (retval > 0) - retval = -EIO; return retval; } From 26c2e922614074da5146aa538c5afaf19112bae1 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Fri, 8 Jan 2021 23:15:12 +0100 Subject: [PATCH 125/633] most: core: Constify static attribute_group structs The only usage of these is to put their addresses in arrays of pointers to const attribute_groups. Make them const to allow the compiler to put them in read-only memory. Signed-off-by: Rikard Falkeborn Link: https://lore.kernel.org/r/20210108221512.18811-1-rikard.falkeborn@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/most/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/most/core.c b/drivers/most/core.c index 353ab277cbc6..e4412c7d25b0 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -379,7 +379,7 @@ static struct attribute *channel_attrs[] = { NULL, }; -static struct attribute_group channel_attr_group = { +static const struct attribute_group channel_attr_group = { .attrs = channel_attrs, .is_visible = channel_attr_is_visible, }; @@ -436,7 +436,7 @@ static struct attribute *interface_attrs[] = { NULL, }; -static struct attribute_group interface_attr_group = { +static const struct attribute_group interface_attr_group = { .attrs = interface_attrs, }; @@ -718,7 +718,7 @@ static struct attribute *mc_attrs[] = { NULL, }; -static struct attribute_group mc_attr_group = { +static const struct attribute_group mc_attr_group = { .attrs = mc_attrs, }; From c41e21dca8dc6899611c6df1fa4e6b55097ceada Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Sat, 23 Jan 2021 08:00:03 -0800 Subject: [PATCH 126/633] sgi-xp: remove h from printk format specifier This change fixes the checkpatch warning described in this commit commit cbacb5ab0aa0 ("docs: printk-formats: Stop encouraging use of unnecessary %h[xudi] and %hh[xudi]") Standard integer promotion is already done and %hx and %hhx is useless so do not encourage the use of %hh[xudi] or %h[xudi]. Reviewed-By: Steve Wahl Signed-off-by: Tom Rix Link: https://lore.kernel.org/r/20210123160003.1777766-1-trix@redhat.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/sgi-xp/xpnet.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/misc/sgi-xp/xpnet.c b/drivers/misc/sgi-xp/xpnet.c index 23837d0d6f4a..2508f83bdc3f 100644 --- a/drivers/misc/sgi-xp/xpnet.c +++ b/drivers/misc/sgi-xp/xpnet.c @@ -208,7 +208,7 @@ xpnet_receive(short partid, int channel, struct xpnet_message *msg) } else { dst = (void *)((u64)skb->data & ~(L1_CACHE_BYTES - 1)); dev_dbg(xpnet, "transferring buffer to the skb->data area;\n\t" - "xp_remote_memcpy(0x%p, 0x%p, %hu)\n", dst, + "xp_remote_memcpy(0x%p, 0x%p, %u)\n", dst, (void *)msg->buf_pa, msg->size); ret = xp_remote_memcpy(xp_pa(dst), msg->buf_pa, msg->size); @@ -218,7 +218,7 @@ xpnet_receive(short partid, int channel, struct xpnet_message *msg) * !!! appears in_use and we can't just call * !!! dev_kfree_skb. */ - dev_err(xpnet, "xp_remote_memcpy(0x%p, 0x%p, 0x%hx) " + dev_err(xpnet, "xp_remote_memcpy(0x%p, 0x%p, 0x%x) " "returned error=0x%x\n", dst, (void *)msg->buf_pa, msg->size, ret); From 0fc99422bc034de018607ef6b70f92d4bc4a236d Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Mon, 18 Jan 2021 09:48:56 +0100 Subject: [PATCH 127/633] firmware: xilinx: Remove PM_API_MAX value There is no reason to keep PM_API_MAX around. The commit acfdd18591ea ("firmware: xilinx: Use hash-table for api feature check") removed its usage that's why it is not used anywhere now. Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/86e94593a29a1b5b1958c539a1bfabdd08c0948e.1610959734.git.michal.simek@xilinx.com Signed-off-by: Greg Kroah-Hartman --- include/linux/firmware/xlnx-zynqmp.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 2a0da841c942..0a483249ff30 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -85,7 +85,6 @@ enum pm_api_id { PM_CLOCK_GETPARENT, PM_SECURE_AES = 47, PM_FEATURE_CHECK = 63, - PM_API_MAX, }; /* PMU-FW return status codes */ From acda36189cb8df27ef073d391ebd67c64ac36cf9 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 27 Jan 2021 13:11:33 +0200 Subject: [PATCH 128/633] dt-bindings: interconnect: Add Qualcomm SDX55 DT bindings The Qualcomm SDX55 platform has several bus fabrics that could be controlled and tuned dynamically over RPMh according to the bandwidth demand. Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20210121053254.8355-2-manivannan.sadhasivam@linaro.org Signed-off-by: Georgi Djakov --- .../bindings/interconnect/qcom,rpmh.yaml | 4 + include/dt-bindings/interconnect/qcom,sdx55.h | 76 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 include/dt-bindings/interconnect/qcom,sdx55.h diff --git a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml index 30c2a092d2d3..f9b150b817d8 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml @@ -45,6 +45,10 @@ properties: - qcom,sdm845-mem-noc - qcom,sdm845-mmss-noc - qcom,sdm845-system-noc + - qcom,sdx55-ipa-virt + - qcom,sdx55-mc-virt + - qcom,sdx55-mem-noc + - qcom,sdx55-system-noc - qcom,sm8150-aggre1-noc - qcom,sm8150-aggre2-noc - qcom,sm8150-camnoc-noc diff --git a/include/dt-bindings/interconnect/qcom,sdx55.h b/include/dt-bindings/interconnect/qcom,sdx55.h new file mode 100644 index 000000000000..bfb6524a2d90 --- /dev/null +++ b/include/dt-bindings/interconnect/qcom,sdx55.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Qualcomm SDX55 interconnect IDs + * + * Copyright (c) 2021, Linaro Ltd. + * Author: Manivannan Sadhasivam + */ + +#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SDX55_H +#define __DT_BINDINGS_INTERCONNECT_QCOM_SDX55_H + +#define MASTER_LLCC 0 +#define SLAVE_EBI_CH0 1 + +#define MASTER_TCU_0 0 +#define MASTER_SNOC_GC_MEM_NOC 1 +#define MASTER_AMPSS_M0 2 +#define SLAVE_LLCC 3 +#define SLAVE_MEM_NOC_SNOC 4 +#define SLAVE_MEM_NOC_PCIE_SNOC 5 + +#define MASTER_AUDIO 0 +#define MASTER_BLSP_1 1 +#define MASTER_QDSS_BAM 2 +#define MASTER_QPIC 3 +#define MASTER_SNOC_CFG 4 +#define MASTER_SPMI_FETCHER 5 +#define MASTER_ANOC_SNOC 6 +#define MASTER_IPA 7 +#define MASTER_MEM_NOC_SNOC 8 +#define MASTER_MEM_NOC_PCIE_SNOC 9 +#define MASTER_CRYPTO_CORE_0 10 +#define MASTER_EMAC 11 +#define MASTER_IPA_PCIE 12 +#define MASTER_PCIE 13 +#define MASTER_QDSS_ETR 14 +#define MASTER_SDCC_1 15 +#define MASTER_USB3 16 +#define SLAVE_AOP 17 +#define SLAVE_AOSS 18 +#define SLAVE_APPSS 19 +#define SLAVE_AUDIO 20 +#define SLAVE_BLSP_1 21 +#define SLAVE_CLK_CTL 22 +#define SLAVE_CRYPTO_0_CFG 23 +#define SLAVE_CNOC_DDRSS 24 +#define SLAVE_ECC_CFG 25 +#define SLAVE_EMAC_CFG 26 +#define SLAVE_IMEM_CFG 27 +#define SLAVE_IPA_CFG 28 +#define SLAVE_CNOC_MSS 29 +#define SLAVE_PCIE_PARF 30 +#define SLAVE_PDM 31 +#define SLAVE_PRNG 32 +#define SLAVE_QDSS_CFG 33 +#define SLAVE_QPIC 34 +#define SLAVE_SDCC_1 35 +#define SLAVE_SNOC_CFG 36 +#define SLAVE_SPMI_FETCHER 37 +#define SLAVE_SPMI_VGI_COEX 38 +#define SLAVE_TCSR 39 +#define SLAVE_TLMM 40 +#define SLAVE_USB3 41 +#define SLAVE_USB3_PHY_CFG 42 +#define SLAVE_ANOC_SNOC 43 +#define SLAVE_SNOC_MEM_NOC_GC 44 +#define SLAVE_OCIMEM 45 +#define SLAVE_SERVICE_SNOC 46 +#define SLAVE_PCIE_0 47 +#define SLAVE_QDSS_STM 48 +#define SLAVE_TCU 49 + +#define MASTER_IPA_CORE 0 +#define SLAVE_IPA_CORE 1 + +#endif From cbb382c5fb371c079b4e984da03cdb5d8940518b Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Wed, 27 Jan 2021 13:11:33 +0200 Subject: [PATCH 129/633] interconnect: qcom: Add SDX55 interconnect provider driver Add driver for the Qualcomm interconnect buses found in SDX55 based platforms. The topology consists of several NoCs that are controlled by a remote processor that collects the aggregated bandwidth for each master-slave pairs. Based on SM8250 driver and generated from downstream dts. Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20210121053254.8355-3-manivannan.sadhasivam@linaro.org Signed-off-by: Georgi Djakov --- drivers/interconnect/qcom/Kconfig | 9 + drivers/interconnect/qcom/Makefile | 2 + drivers/interconnect/qcom/sdx55.c | 356 +++++++++++++++++++++++++++++ drivers/interconnect/qcom/sdx55.h | 70 ++++++ 4 files changed, 437 insertions(+) create mode 100644 drivers/interconnect/qcom/sdx55.c create mode 100644 drivers/interconnect/qcom/sdx55.h diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig index b3fb5b02bcf1..165e65174ab2 100644 --- a/drivers/interconnect/qcom/Kconfig +++ b/drivers/interconnect/qcom/Kconfig @@ -74,6 +74,15 @@ config INTERCONNECT_QCOM_SDM845 This is a driver for the Qualcomm Network-on-Chip on sdm845-based platforms. +config INTERCONNECT_QCOM_SDX55 + tristate "Qualcomm SDX55 interconnect driver" + depends on INTERCONNECT_QCOM_RPMH_POSSIBLE + select INTERCONNECT_QCOM_RPMH + select INTERCONNECT_QCOM_BCM_VOTER + help + This is a driver for the Qualcomm Network-on-Chip on sdx55-based + platforms. + config INTERCONNECT_QCOM_SM8150 tristate "Qualcomm SM8150 interconnect driver" depends on INTERCONNECT_QCOM_RPMH_POSSIBLE diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile index cf628f7990cd..981a5ea45af9 100644 --- a/drivers/interconnect/qcom/Makefile +++ b/drivers/interconnect/qcom/Makefile @@ -8,6 +8,7 @@ qnoc-qcs404-objs := qcs404.o icc-rpmh-obj := icc-rpmh.o qnoc-sc7180-objs := sc7180.o qnoc-sdm845-objs := sdm845.o +qnoc-sdx55-objs := sdx55.o qnoc-sm8150-objs := sm8150.o qnoc-sm8250-objs := sm8250.o icc-smd-rpm-objs := smd-rpm.o @@ -20,6 +21,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM_QCS404) += qnoc-qcs404.o obj-$(CONFIG_INTERCONNECT_QCOM_RPMH) += icc-rpmh.o obj-$(CONFIG_INTERCONNECT_QCOM_SC7180) += qnoc-sc7180.o obj-$(CONFIG_INTERCONNECT_QCOM_SDM845) += qnoc-sdm845.o +obj-$(CONFIG_INTERCONNECT_QCOM_SDX55) += qnoc-sdx55.o obj-$(CONFIG_INTERCONNECT_QCOM_SM8150) += qnoc-sm8150.o obj-$(CONFIG_INTERCONNECT_QCOM_SM8250) += qnoc-sm8250.o obj-$(CONFIG_INTERCONNECT_QCOM_SMD_RPM) += icc-smd-rpm.o diff --git a/drivers/interconnect/qcom/sdx55.c b/drivers/interconnect/qcom/sdx55.c new file mode 100644 index 000000000000..a5a122ee3d21 --- /dev/null +++ b/drivers/interconnect/qcom/sdx55.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm SDX55 interconnect driver + * Author: Manivannan Sadhasivam + * + * Copyright (c) 2021, Linaro Ltd. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "bcm-voter.h" +#include "icc-rpmh.h" +#include "sdx55.h" + +DEFINE_QNODE(ipa_core_master, SDX55_MASTER_IPA_CORE, 1, 8, SDX55_SLAVE_IPA_CORE); +DEFINE_QNODE(llcc_mc, SDX55_MASTER_LLCC, 4, 4, SDX55_SLAVE_EBI_CH0); +DEFINE_QNODE(acm_tcu, SDX55_MASTER_TCU_0, 1, 8, SDX55_SLAVE_LLCC, SDX55_SLAVE_MEM_NOC_SNOC, SDX55_SLAVE_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qnm_snoc_gc, SDX55_MASTER_SNOC_GC_MEM_NOC, 1, 8, SDX55_SLAVE_LLCC); +DEFINE_QNODE(xm_apps_rdwr, SDX55_MASTER_AMPSS_M0, 1, 16, SDX55_SLAVE_LLCC, SDX55_SLAVE_MEM_NOC_SNOC, SDX55_SLAVE_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qhm_audio, SDX55_MASTER_AUDIO, 1, 4, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(qhm_blsp1, SDX55_MASTER_BLSP_1, 1, 4, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(qhm_qdss_bam, SDX55_MASTER_QDSS_BAM, 1, 4, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qhm_qpic, SDX55_MASTER_QPIC, 1, 4, SDX55_SLAVE_AOSS, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP, SDX55_SLAVE_AUDIO); +DEFINE_QNODE(qhm_snoc_cfg, SDX55_MASTER_SNOC_CFG, 1, 4, SDX55_SLAVE_SERVICE_SNOC); +DEFINE_QNODE(qhm_spmi_fetcher1, SDX55_MASTER_SPMI_FETCHER, 1, 4, SDX55_SLAVE_AOSS, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP); +DEFINE_QNODE(qnm_aggre_noc, SDX55_MASTER_ANOC_SNOC, 1, 8, SDX55_SLAVE_PCIE_0, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_USB3, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_APPSS, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_ipa, SDX55_MASTER_IPA, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_AOSS, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_TLMM, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_memnoc, SDX55_MASTER_MEM_NOC_SNOC, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_TLMM, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QDSS_STM, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_APPSS, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(qnm_memnoc_pcie, SDX55_MASTER_MEM_NOC_PCIE_SNOC, 1, 8, SDX55_SLAVE_PCIE_0); +DEFINE_QNODE(qxm_crypto, SDX55_MASTER_CRYPTO_CORE_0, 1, 8, SDX55_SLAVE_AOSS, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP); +DEFINE_QNODE(xm_emac, SDX55_MASTER_EMAC, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(xm_ipa2pcie_slv, SDX55_MASTER_IPA_PCIE, 1, 8, SDX55_SLAVE_PCIE_0); +DEFINE_QNODE(xm_pcie, SDX55_MASTER_PCIE, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(xm_qdss_etr, SDX55_MASTER_QDSS_ETR, 1, 8, SDX55_SLAVE_SNOC_CFG, SDX55_SLAVE_EMAC_CFG, SDX55_SLAVE_USB3, SDX55_SLAVE_AOSS, SDX55_SLAVE_SPMI_FETCHER, SDX55_SLAVE_QDSS_CFG, SDX55_SLAVE_PDM, SDX55_SLAVE_SNOC_MEM_NOC_GC, SDX55_SLAVE_TCSR, SDX55_SLAVE_CNOC_DDRSS, SDX55_SLAVE_SPMI_VGI_COEX, SDX55_SLAVE_QPIC, SDX55_SLAVE_OCIMEM, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_USB3_PHY_CFG, SDX55_SLAVE_AOP, SDX55_SLAVE_BLSP_1, SDX55_SLAVE_SDCC_1, SDX55_SLAVE_CNOC_MSS, SDX55_SLAVE_PCIE_PARF, SDX55_SLAVE_ECC_CFG, SDX55_SLAVE_AUDIO, SDX55_SLAVE_AOSS, SDX55_SLAVE_PRNG, SDX55_SLAVE_CRYPTO_0_CFG, SDX55_SLAVE_TCU, SDX55_SLAVE_CLK_CTL, SDX55_SLAVE_IMEM_CFG); +DEFINE_QNODE(xm_sdc1, SDX55_MASTER_SDCC_1, 1, 8, SDX55_SLAVE_AOSS, SDX55_SLAVE_IPA_CFG, SDX55_SLAVE_ANOC_SNOC, SDX55_SLAVE_AOP, SDX55_SLAVE_AUDIO); +DEFINE_QNODE(xm_usb3, SDX55_MASTER_USB3, 1, 8, SDX55_SLAVE_ANOC_SNOC); +DEFINE_QNODE(ipa_core_slave, SDX55_SLAVE_IPA_CORE, 1, 8); +DEFINE_QNODE(ebi, SDX55_SLAVE_EBI_CH0, 1, 4); +DEFINE_QNODE(qns_llcc, SDX55_SLAVE_LLCC, 1, 16, SDX55_SLAVE_EBI_CH0); +DEFINE_QNODE(qns_memnoc_snoc, SDX55_SLAVE_MEM_NOC_SNOC, 1, 8, SDX55_MASTER_MEM_NOC_SNOC); +DEFINE_QNODE(qns_sys_pcie, SDX55_SLAVE_MEM_NOC_PCIE_SNOC, 1, 8, SDX55_MASTER_MEM_NOC_PCIE_SNOC); +DEFINE_QNODE(qhs_aop, SDX55_SLAVE_AOP, 1, 4); +DEFINE_QNODE(qhs_aoss, SDX55_SLAVE_AOSS, 1, 4); +DEFINE_QNODE(qhs_apss, SDX55_SLAVE_APPSS, 1, 4); +DEFINE_QNODE(qhs_audio, SDX55_SLAVE_AUDIO, 1, 4); +DEFINE_QNODE(qhs_blsp1, SDX55_SLAVE_BLSP_1, 1, 4); +DEFINE_QNODE(qhs_clk_ctl, SDX55_SLAVE_CLK_CTL, 1, 4); +DEFINE_QNODE(qhs_crypto0_cfg, SDX55_SLAVE_CRYPTO_0_CFG, 1, 4); +DEFINE_QNODE(qhs_ddrss_cfg, SDX55_SLAVE_CNOC_DDRSS, 1, 4); +DEFINE_QNODE(qhs_ecc_cfg, SDX55_SLAVE_ECC_CFG, 1, 4); +DEFINE_QNODE(qhs_emac_cfg, SDX55_SLAVE_EMAC_CFG, 1, 4); +DEFINE_QNODE(qhs_imem_cfg, SDX55_SLAVE_IMEM_CFG, 1, 4); +DEFINE_QNODE(qhs_ipa, SDX55_SLAVE_IPA_CFG, 1, 4); +DEFINE_QNODE(qhs_mss_cfg, SDX55_SLAVE_CNOC_MSS, 1, 4); +DEFINE_QNODE(qhs_pcie_parf, SDX55_SLAVE_PCIE_PARF, 1, 4); +DEFINE_QNODE(qhs_pdm, SDX55_SLAVE_PDM, 1, 4); +DEFINE_QNODE(qhs_prng, SDX55_SLAVE_PRNG, 1, 4); +DEFINE_QNODE(qhs_qdss_cfg, SDX55_SLAVE_QDSS_CFG, 1, 4); +DEFINE_QNODE(qhs_qpic, SDX55_SLAVE_QPIC, 1, 4); +DEFINE_QNODE(qhs_sdc1, SDX55_SLAVE_SDCC_1, 1, 4); +DEFINE_QNODE(qhs_snoc_cfg, SDX55_SLAVE_SNOC_CFG, 1, 4, SDX55_MASTER_SNOC_CFG); +DEFINE_QNODE(qhs_spmi_fetcher, SDX55_SLAVE_SPMI_FETCHER, 1, 4); +DEFINE_QNODE(qhs_spmi_vgi_coex, SDX55_SLAVE_SPMI_VGI_COEX, 1, 4); +DEFINE_QNODE(qhs_tcsr, SDX55_SLAVE_TCSR, 1, 4); +DEFINE_QNODE(qhs_tlmm, SDX55_SLAVE_TLMM, 1, 4); +DEFINE_QNODE(qhs_usb3, SDX55_SLAVE_USB3, 1, 4); +DEFINE_QNODE(qhs_usb3_phy, SDX55_SLAVE_USB3_PHY_CFG, 1, 4); +DEFINE_QNODE(qns_aggre_noc, SDX55_SLAVE_ANOC_SNOC, 1, 8, SDX55_MASTER_ANOC_SNOC); +DEFINE_QNODE(qns_snoc_memnoc, SDX55_SLAVE_SNOC_MEM_NOC_GC, 1, 8, SDX55_MASTER_SNOC_GC_MEM_NOC); +DEFINE_QNODE(qxs_imem, SDX55_SLAVE_OCIMEM, 1, 8); +DEFINE_QNODE(srvc_snoc, SDX55_SLAVE_SERVICE_SNOC, 1, 4); +DEFINE_QNODE(xs_pcie, SDX55_SLAVE_PCIE_0, 1, 8); +DEFINE_QNODE(xs_qdss_stm, SDX55_SLAVE_QDSS_STM, 1, 4); +DEFINE_QNODE(xs_sys_tcu_cfg, SDX55_SLAVE_TCU, 1, 8); + +DEFINE_QBCM(bcm_mc0, "MC0", true, &ebi); +DEFINE_QBCM(bcm_sh0, "SH0", true, &qns_llcc); +DEFINE_QBCM(bcm_ce0, "CE0", false, &qxm_crypto); +DEFINE_QBCM(bcm_ip0, "IP0", false, &ipa_core_slave); +DEFINE_QBCM(bcm_pn0, "PN0", false, &qhm_snoc_cfg); +DEFINE_QBCM(bcm_sh3, "SH3", false, &xm_apps_rdwr); +DEFINE_QBCM(bcm_sh4, "SH4", false, &qns_memnoc_snoc, &qns_sys_pcie); +DEFINE_QBCM(bcm_sn0, "SN0", true, &qns_snoc_memnoc); +DEFINE_QBCM(bcm_sn1, "SN1", false, &qxs_imem); +DEFINE_QBCM(bcm_pn1, "PN1", false, &xm_sdc1); +DEFINE_QBCM(bcm_pn2, "PN2", false, &qhm_audio, &qhm_spmi_fetcher1); +DEFINE_QBCM(bcm_sn3, "SN3", false, &xs_qdss_stm); +DEFINE_QBCM(bcm_pn3, "PN3", false, &qhm_blsp1, &qhm_qpic); +DEFINE_QBCM(bcm_sn4, "SN4", false, &xs_sys_tcu_cfg); +DEFINE_QBCM(bcm_pn5, "PN5", false, &qxm_crypto); +DEFINE_QBCM(bcm_sn6, "SN6", false, &xs_pcie); +DEFINE_QBCM(bcm_sn7, "SN7", false, &qnm_aggre_noc, &xm_emac, &xm_emac, &xm_usb3, + &qns_aggre_noc); +DEFINE_QBCM(bcm_sn8, "SN8", false, &qhm_qdss_bam, &xm_qdss_etr); +DEFINE_QBCM(bcm_sn9, "SN9", false, &qnm_memnoc); +DEFINE_QBCM(bcm_sn10, "SN10", false, &qnm_memnoc_pcie); +DEFINE_QBCM(bcm_sn11, "SN11", false, &qnm_ipa, &xm_ipa2pcie_slv); + +static struct qcom_icc_bcm *mc_virt_bcms[] = { + &bcm_mc0, +}; + +static struct qcom_icc_node *mc_virt_nodes[] = { + [MASTER_LLCC] = &llcc_mc, + [SLAVE_EBI_CH0] = &ebi, +}; + +static const struct qcom_icc_desc sdx55_mc_virt = { + .nodes = mc_virt_nodes, + .num_nodes = ARRAY_SIZE(mc_virt_nodes), + .bcms = mc_virt_bcms, + .num_bcms = ARRAY_SIZE(mc_virt_bcms), +}; + +static struct qcom_icc_bcm *mem_noc_bcms[] = { + &bcm_sh0, + &bcm_sh3, + &bcm_sh4, +}; + +static struct qcom_icc_node *mem_noc_nodes[] = { + [MASTER_TCU_0] = &acm_tcu, + [MASTER_SNOC_GC_MEM_NOC] = &qnm_snoc_gc, + [MASTER_AMPSS_M0] = &xm_apps_rdwr, + [SLAVE_LLCC] = &qns_llcc, + [SLAVE_MEM_NOC_SNOC] = &qns_memnoc_snoc, + [SLAVE_MEM_NOC_PCIE_SNOC] = &qns_sys_pcie, +}; + +static const struct qcom_icc_desc sdx55_mem_noc = { + .nodes = mem_noc_nodes, + .num_nodes = ARRAY_SIZE(mem_noc_nodes), + .bcms = mem_noc_bcms, + .num_bcms = ARRAY_SIZE(mem_noc_bcms), +}; + +static struct qcom_icc_bcm *system_noc_bcms[] = { + &bcm_ce0, + &bcm_pn0, + &bcm_pn1, + &bcm_pn2, + &bcm_pn3, + &bcm_pn5, + &bcm_sn0, + &bcm_sn1, + &bcm_sn3, + &bcm_sn4, + &bcm_sn6, + &bcm_sn7, + &bcm_sn8, + &bcm_sn9, + &bcm_sn10, + &bcm_sn11, +}; + +static struct qcom_icc_node *system_noc_nodes[] = { + [MASTER_AUDIO] = &qhm_audio, + [MASTER_BLSP_1] = &qhm_blsp1, + [MASTER_QDSS_BAM] = &qhm_qdss_bam, + [MASTER_QPIC] = &qhm_qpic, + [MASTER_SNOC_CFG] = &qhm_snoc_cfg, + [MASTER_SPMI_FETCHER] = &qhm_spmi_fetcher1, + [MASTER_ANOC_SNOC] = &qnm_aggre_noc, + [MASTER_IPA] = &qnm_ipa, + [MASTER_MEM_NOC_SNOC] = &qnm_memnoc, + [MASTER_MEM_NOC_PCIE_SNOC] = &qnm_memnoc_pcie, + [MASTER_CRYPTO_CORE_0] = &qxm_crypto, + [MASTER_EMAC] = &xm_emac, + [MASTER_IPA_PCIE] = &xm_ipa2pcie_slv, + [MASTER_PCIE] = &xm_pcie, + [MASTER_QDSS_ETR] = &xm_qdss_etr, + [MASTER_SDCC_1] = &xm_sdc1, + [MASTER_USB3] = &xm_usb3, + [SLAVE_AOP] = &qhs_aop, + [SLAVE_AOSS] = &qhs_aoss, + [SLAVE_APPSS] = &qhs_apss, + [SLAVE_AUDIO] = &qhs_audio, + [SLAVE_BLSP_1] = &qhs_blsp1, + [SLAVE_CLK_CTL] = &qhs_clk_ctl, + [SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg, + [SLAVE_CNOC_DDRSS] = &qhs_ddrss_cfg, + [SLAVE_ECC_CFG] = &qhs_ecc_cfg, + [SLAVE_EMAC_CFG] = &qhs_emac_cfg, + [SLAVE_IMEM_CFG] = &qhs_imem_cfg, + [SLAVE_IPA_CFG] = &qhs_ipa, + [SLAVE_CNOC_MSS] = &qhs_mss_cfg, + [SLAVE_PCIE_PARF] = &qhs_pcie_parf, + [SLAVE_PDM] = &qhs_pdm, + [SLAVE_PRNG] = &qhs_prng, + [SLAVE_QDSS_CFG] = &qhs_qdss_cfg, + [SLAVE_QPIC] = &qhs_qpic, + [SLAVE_SDCC_1] = &qhs_sdc1, + [SLAVE_SNOC_CFG] = &qhs_snoc_cfg, + [SLAVE_SPMI_FETCHER] = &qhs_spmi_fetcher, + [SLAVE_SPMI_VGI_COEX] = &qhs_spmi_vgi_coex, + [SLAVE_TCSR] = &qhs_tcsr, + [SLAVE_TLMM] = &qhs_tlmm, + [SLAVE_USB3] = &qhs_usb3, + [SLAVE_USB3_PHY_CFG] = &qhs_usb3_phy, + [SLAVE_ANOC_SNOC] = &qns_aggre_noc, + [SLAVE_SNOC_MEM_NOC_GC] = &qns_snoc_memnoc, + [SLAVE_OCIMEM] = &qxs_imem, + [SLAVE_SERVICE_SNOC] = &srvc_snoc, + [SLAVE_PCIE_0] = &xs_pcie, + [SLAVE_QDSS_STM] = &xs_qdss_stm, + [SLAVE_TCU] = &xs_sys_tcu_cfg, +}; + +static const struct qcom_icc_desc sdx55_system_noc = { + .nodes = system_noc_nodes, + .num_nodes = ARRAY_SIZE(system_noc_nodes), + .bcms = system_noc_bcms, + .num_bcms = ARRAY_SIZE(system_noc_bcms), +}; + +static struct qcom_icc_bcm *ipa_virt_bcms[] = { + &bcm_ip0, +}; + +static struct qcom_icc_node *ipa_virt_nodes[] = { + [MASTER_IPA_CORE] = &ipa_core_master, + [SLAVE_IPA_CORE] = &ipa_core_slave, +}; + +static const struct qcom_icc_desc sdx55_ipa_virt = { + .nodes = ipa_virt_nodes, + .num_nodes = ARRAY_SIZE(ipa_virt_nodes), + .bcms = ipa_virt_bcms, + .num_bcms = ARRAY_SIZE(ipa_virt_bcms), +}; + +static int qnoc_probe(struct platform_device *pdev) +{ + const struct qcom_icc_desc *desc; + struct icc_onecell_data *data; + struct icc_provider *provider; + struct qcom_icc_node **qnodes; + struct qcom_icc_provider *qp; + struct icc_node *node; + size_t num_nodes, i; + int ret; + + desc = device_get_match_data(&pdev->dev); + if (!desc) + return -EINVAL; + + qnodes = desc->nodes; + num_nodes = desc->num_nodes; + + qp = devm_kzalloc(&pdev->dev, sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + data = devm_kcalloc(&pdev->dev, num_nodes, sizeof(*node), GFP_KERNEL); + if (!data) + return -ENOMEM; + + provider = &qp->provider; + provider->dev = &pdev->dev; + provider->set = qcom_icc_set; + provider->pre_aggregate = qcom_icc_pre_aggregate; + provider->aggregate = qcom_icc_aggregate; + provider->xlate = of_icc_xlate_onecell; + INIT_LIST_HEAD(&provider->nodes); + provider->data = data; + + qp->dev = &pdev->dev; + qp->bcms = desc->bcms; + qp->num_bcms = desc->num_bcms; + + qp->voter = of_bcm_voter_get(qp->dev, NULL); + if (IS_ERR(qp->voter)) + return PTR_ERR(qp->voter); + + ret = icc_provider_add(provider); + if (ret) { + dev_err(&pdev->dev, "error adding interconnect provider\n"); + return ret; + } + + for (i = 0; i < qp->num_bcms; i++) + qcom_icc_bcm_init(qp->bcms[i], &pdev->dev); + + for (i = 0; i < num_nodes; i++) { + size_t j; + + if (!qnodes[i]) + continue; + + node = icc_node_create(qnodes[i]->id); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto err; + } + + node->name = qnodes[i]->name; + node->data = qnodes[i]; + icc_node_add(node, provider); + + for (j = 0; j < qnodes[i]->num_links; j++) + icc_link_create(node, qnodes[i]->links[j]); + + data->nodes[i] = node; + } + data->num_nodes = num_nodes; + + platform_set_drvdata(pdev, qp); + + return 0; +err: + icc_nodes_remove(provider); + icc_provider_del(provider); + return ret; +} + +static int qnoc_remove(struct platform_device *pdev) +{ + struct qcom_icc_provider *qp = platform_get_drvdata(pdev); + + icc_nodes_remove(&qp->provider); + return icc_provider_del(&qp->provider); +} + +static const struct of_device_id qnoc_of_match[] = { + { .compatible = "qcom,sdx55-mc-virt", + .data = &sdx55_mc_virt}, + { .compatible = "qcom,sdx55-mem-noc", + .data = &sdx55_mem_noc}, + { .compatible = "qcom,sdx55-system-noc", + .data = &sdx55_system_noc}, + { .compatible = "qcom,sdx55-ipa-virt", + .data = &sdx55_ipa_virt}, + { } +}; +MODULE_DEVICE_TABLE(of, qnoc_of_match); + +static struct platform_driver qnoc_driver = { + .probe = qnoc_probe, + .remove = qnoc_remove, + .driver = { + .name = "qnoc-sdx55", + .of_match_table = qnoc_of_match, + .sync_state = icc_sync_state, + }, +}; +module_platform_driver(qnoc_driver); + +MODULE_DESCRIPTION("Qualcomm SDX55 NoC driver"); +MODULE_AUTHOR("Manivannan Sadhasivam "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/interconnect/qcom/sdx55.h b/drivers/interconnect/qcom/sdx55.h new file mode 100644 index 000000000000..deff8afe0631 --- /dev/null +++ b/drivers/interconnect/qcom/sdx55.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, Linaro Ltd. + */ + +#ifndef __DRIVERS_INTERCONNECT_QCOM_SDX55_H +#define __DRIVERS_INTERCONNECT_QCOM_SDX55_H + +#define SDX55_MASTER_IPA_CORE 0 +#define SDX55_MASTER_LLCC 1 +#define SDX55_MASTER_TCU_0 2 +#define SDX55_MASTER_SNOC_GC_MEM_NOC 3 +#define SDX55_MASTER_AMPSS_M0 4 +#define SDX55_MASTER_AUDIO 5 +#define SDX55_MASTER_BLSP_1 6 +#define SDX55_MASTER_QDSS_BAM 7 +#define SDX55_MASTER_QPIC 8 +#define SDX55_MASTER_SNOC_CFG 9 +#define SDX55_MASTER_SPMI_FETCHER 10 +#define SDX55_MASTER_ANOC_SNOC 11 +#define SDX55_MASTER_IPA 12 +#define SDX55_MASTER_MEM_NOC_SNOC 13 +#define SDX55_MASTER_MEM_NOC_PCIE_SNOC 14 +#define SDX55_MASTER_CRYPTO_CORE_0 15 +#define SDX55_MASTER_EMAC 16 +#define SDX55_MASTER_IPA_PCIE 17 +#define SDX55_MASTER_PCIE 18 +#define SDX55_MASTER_QDSS_ETR 19 +#define SDX55_MASTER_SDCC_1 20 +#define SDX55_MASTER_USB3 21 +#define SDX55_SLAVE_IPA_CORE 22 +#define SDX55_SLAVE_EBI_CH0 23 +#define SDX55_SLAVE_LLCC 24 +#define SDX55_SLAVE_MEM_NOC_SNOC 25 +#define SDX55_SLAVE_MEM_NOC_PCIE_SNOC 26 +#define SDX55_SLAVE_ANOC_SNOC 27 +#define SDX55_SLAVE_SNOC_CFG 28 +#define SDX55_SLAVE_EMAC_CFG 29 +#define SDX55_SLAVE_USB3 30 +#define SDX55_SLAVE_TLMM 31 +#define SDX55_SLAVE_SPMI_FETCHER 32 +#define SDX55_SLAVE_QDSS_CFG 33 +#define SDX55_SLAVE_PDM 34 +#define SDX55_SLAVE_SNOC_MEM_NOC_GC 35 +#define SDX55_SLAVE_TCSR 36 +#define SDX55_SLAVE_CNOC_DDRSS 37 +#define SDX55_SLAVE_SPMI_VGI_COEX 38 +#define SDX55_SLAVE_QPIC 39 +#define SDX55_SLAVE_OCIMEM 40 +#define SDX55_SLAVE_IPA_CFG 41 +#define SDX55_SLAVE_USB3_PHY_CFG 42 +#define SDX55_SLAVE_AOP 43 +#define SDX55_SLAVE_BLSP_1 44 +#define SDX55_SLAVE_SDCC_1 45 +#define SDX55_SLAVE_CNOC_MSS 46 +#define SDX55_SLAVE_PCIE_PARF 47 +#define SDX55_SLAVE_ECC_CFG 48 +#define SDX55_SLAVE_AUDIO 49 +#define SDX55_SLAVE_AOSS 51 +#define SDX55_SLAVE_PRNG 52 +#define SDX55_SLAVE_CRYPTO_0_CFG 53 +#define SDX55_SLAVE_TCU 54 +#define SDX55_SLAVE_CLK_CTL 55 +#define SDX55_SLAVE_IMEM_CFG 56 +#define SDX55_SLAVE_SERVICE_SNOC 57 +#define SDX55_SLAVE_PCIE_0 58 +#define SDX55_SLAVE_QDSS_STM 59 +#define SDX55_SLAVE_APPSS 60 + +#endif From 1941ab1d25e098e99df18b9041667e99858fd449 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Tue, 26 Jan 2021 23:21:44 +0100 Subject: [PATCH 130/633] speakup: add the missing synth parameter to all io functions So that we can avoid the spk_ttyio_synth global variable in the next commit. Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210126222147.3848175-2-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/serialio.c | 22 ++++++------ .../accessibility/speakup/speakup_acntpc.c | 4 +-- .../accessibility/speakup/speakup_apollo.c | 4 +-- .../accessibility/speakup/speakup_audptr.c | 8 ++--- .../accessibility/speakup/speakup_decext.c | 2 +- drivers/accessibility/speakup/speakup_decpc.c | 4 +-- .../accessibility/speakup/speakup_dectlk.c | 2 +- drivers/accessibility/speakup/speakup_dtlk.c | 4 +-- drivers/accessibility/speakup/speakup_keypc.c | 4 +-- drivers/accessibility/speakup/speakup_ltlk.c | 2 +- drivers/accessibility/speakup/speakup_soft.c | 4 +-- .../accessibility/speakup/speakup_spkout.c | 4 +-- drivers/accessibility/speakup/spk_priv.h | 4 +-- drivers/accessibility/speakup/spk_ttyio.c | 34 +++++++++---------- drivers/accessibility/speakup/spk_types.h | 12 +++---- drivers/accessibility/speakup/synth.c | 6 ++-- 16 files changed, 60 insertions(+), 60 deletions(-) diff --git a/drivers/accessibility/speakup/serialio.c b/drivers/accessibility/speakup/serialio.c index 403b01d66367..53580bdc5baa 100644 --- a/drivers/accessibility/speakup/serialio.c +++ b/drivers/accessibility/speakup/serialio.c @@ -27,11 +27,11 @@ static const struct old_serial_port *serstate; static int timeouts; static int spk_serial_out(struct spk_synth *in_synth, const char ch); -static void spk_serial_send_xchar(char ch); -static void spk_serial_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_serial_in(void); -static unsigned char spk_serial_in_nowait(void); -static void spk_serial_flush_buffer(void); +static void spk_serial_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_serial_in(struct spk_synth *in_synth); +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth); +static void spk_serial_flush_buffer(struct spk_synth *in_synth); static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_serial_io_ops = { @@ -150,7 +150,7 @@ static void start_serial_interrupt(int irq) outb(1, speakup_info.port_tts + UART_FCR); /* Turn FIFO On */ } -static void spk_serial_send_xchar(char ch) +static void spk_serial_send_xchar(struct spk_synth *synth, char ch) { int timeout = SPK_XMITR_TIMEOUT; @@ -162,7 +162,7 @@ static void spk_serial_send_xchar(char ch) outb(ch, speakup_info.port_tts); } -static void spk_serial_tiocmset(unsigned int set, unsigned int clear) +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { int old = inb(speakup_info.port_tts + UART_MCR); @@ -251,7 +251,7 @@ static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char spk_serial_in(void) +static unsigned char spk_serial_in(struct spk_synth *in_synth) { int tmout = SPK_SERIAL_TIMEOUT; @@ -265,7 +265,7 @@ static unsigned char spk_serial_in(void) return inb_p(speakup_info.port_tts + UART_RX); } -static unsigned char spk_serial_in_nowait(void) +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth) { unsigned char lsr; @@ -275,7 +275,7 @@ static unsigned char spk_serial_in_nowait(void) return inb_p(speakup_info.port_tts + UART_RX); } -static void spk_serial_flush_buffer(void) +static void spk_serial_flush_buffer(struct spk_synth *in_synth) { /* TODO: flush the UART 16550 buffer */ } @@ -307,7 +307,7 @@ const char *spk_serial_synth_immediate(struct spk_synth *synth, } EXPORT_SYMBOL_GPL(spk_serial_synth_immediate); -void spk_serial_release(void) +void spk_serial_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts == 0) diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c index c94328a5bd4a..c1ec087dca13 100644 --- a/drivers/accessibility/speakup/speakup_acntpc.c +++ b/drivers/accessibility/speakup/speakup_acntpc.c @@ -25,7 +25,7 @@ #define PROCSPEECH '\r' static int synth_probe(struct spk_synth *synth); -static void accent_release(void); +static void accent_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -294,7 +294,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void accent_release(void) +static void accent_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c index 0877b4044c28..cd63581b2e99 100644 --- a/drivers/accessibility/speakup/speakup_apollo.c +++ b/drivers/accessibility/speakup/speakup_apollo.c @@ -163,8 +163,8 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (!synth->io_ops->synth_out(synth, ch)) { - synth->io_ops->tiocmset(0, UART_MCR_RTS); - synth->io_ops->tiocmset(UART_MCR_RTS, 0); + synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS); + synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0); schedule_timeout(msecs_to_jiffies(full_time_val)); continue; } diff --git a/drivers/accessibility/speakup/speakup_audptr.c b/drivers/accessibility/speakup/speakup_audptr.c index e6a6a9665d8f..e89fd72579e6 100644 --- a/drivers/accessibility/speakup/speakup_audptr.c +++ b/drivers/accessibility/speakup/speakup_audptr.c @@ -119,8 +119,8 @@ static struct spk_synth synth_audptr = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); synth->io_ops->synth_out(synth, PROCSPEECH); } @@ -130,11 +130,11 @@ static void synth_version(struct spk_synth *synth) char synth_id[40] = ""; synth->synth_immediate(synth, "\x05[Q]"); - synth_id[test] = synth->io_ops->synth_in(); + synth_id[test] = synth->io_ops->synth_in(synth); if (synth_id[test] == 'A') { do { /* read version string from synth */ - synth_id[++test] = synth->io_ops->synth_in(); + synth_id[++test] = synth->io_ops->synth_in(synth); } while (synth_id[test] != '\n' && test < 32); synth_id[++test] = 0x00; } diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c index 7408eb29cf38..092cfd08a9e1 100644 --- a/drivers/accessibility/speakup/speakup_decext.c +++ b/drivers/accessibility/speakup/speakup_decext.c @@ -218,7 +218,7 @@ static void do_catch_up(struct spk_synth *synth) static void synth_flush(struct spk_synth *synth) { in_escape = 0; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->synth_immediate(synth, "\033P;10z\033\\"); } diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c index 96f24c848cc5..dec314dee214 100644 --- a/drivers/accessibility/speakup/speakup_decpc.c +++ b/drivers/accessibility/speakup/speakup_decpc.c @@ -125,7 +125,7 @@ enum { PRIMARY_DIC = 0, USER_DIC, COMMAND_DIC, ABBREV_DIC }; #define SYNTH_IO_EXTENT 8 static int synth_probe(struct spk_synth *synth); -static void dtpc_release(void); +static void dtpc_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -474,7 +474,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtpc_release(void) +static void dtpc_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index ab6d61e80b1c..d75de36e96c3 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -289,7 +289,7 @@ static void synth_flush(struct spk_synth *synth) synth->io_ops->synth_out(synth, ']'); in_escape = 0; is_flushing = 1; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, SYNTH_CLEAR); } diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c index dbebed0eeeec..92838d3ae9eb 100644 --- a/drivers/accessibility/speakup/speakup_dtlk.c +++ b/drivers/accessibility/speakup/speakup_dtlk.c @@ -24,7 +24,7 @@ #define PROCSPEECH 0x00 static int synth_probe(struct spk_synth *synth); -static void dtlk_release(void); +static void dtlk_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -365,7 +365,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtlk_release(void) +static void dtlk_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c index 414827e888fc..311f4aa0be22 100644 --- a/drivers/accessibility/speakup/speakup_keypc.c +++ b/drivers/accessibility/speakup/speakup_keypc.c @@ -24,7 +24,7 @@ #define SYNTH_CLEAR 0x03 static int synth_probe(struct spk_synth *synth); -static void keynote_release(void); +static void keynote_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -295,7 +295,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void keynote_release(void) +static void keynote_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (synth_port) diff --git a/drivers/accessibility/speakup/speakup_ltlk.c b/drivers/accessibility/speakup/speakup_ltlk.c index 3c59519a871f..3e59b387d0c4 100644 --- a/drivers/accessibility/speakup/speakup_ltlk.c +++ b/drivers/accessibility/speakup/speakup_ltlk.c @@ -132,7 +132,7 @@ static void synth_interrogate(struct spk_synth *synth) synth->synth_immediate(synth, "\x18\x01?"); for (i = 0; i < 50; i++) { - buf[i] = synth->io_ops->synth_in(); + buf[i] = synth->io_ops->synth_in(synth); if (i > 2 && buf[i] == 0x7f) break; } diff --git a/drivers/accessibility/speakup/speakup_soft.c b/drivers/accessibility/speakup/speakup_soft.c index 9a7029539f35..c3f97c572fb6 100644 --- a/drivers/accessibility/speakup/speakup_soft.c +++ b/drivers/accessibility/speakup/speakup_soft.c @@ -24,7 +24,7 @@ #define CLEAR_SYNTH 0x18 static int softsynth_probe(struct spk_synth *synth); -static void softsynth_release(void); +static void softsynth_release(struct spk_synth *synth); static int softsynth_is_alive(struct spk_synth *synth); static unsigned char get_index(struct spk_synth *synth); @@ -402,7 +402,7 @@ static int softsynth_probe(struct spk_synth *synth) return 0; } -static void softsynth_release(void) +static void softsynth_release(struct spk_synth *synth) { misc_deregister(&synth_device); misc_deregister(&synthu_device); diff --git a/drivers/accessibility/speakup/speakup_spkout.c b/drivers/accessibility/speakup/speakup_spkout.c index 6e933bf1de2e..bd3d8dc300ff 100644 --- a/drivers/accessibility/speakup/speakup_spkout.c +++ b/drivers/accessibility/speakup/speakup_spkout.c @@ -117,8 +117,8 @@ static struct spk_synth synth_spkout = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); } module_param_named(ser, synth_spkout.ser, int, 0444); diff --git a/drivers/accessibility/speakup/spk_priv.h b/drivers/accessibility/speakup/spk_priv.h index 0f4bcbe5ddb9..9da57ead17cb 100644 --- a/drivers/accessibility/speakup/spk_priv.h +++ b/drivers/accessibility/speakup/spk_priv.h @@ -34,8 +34,8 @@ const struct old_serial_port *spk_serial_init(int index); void spk_stop_serial_interrupt(void); -void spk_serial_release(void); -void spk_ttyio_release(void); +void spk_serial_release(struct spk_synth *synth); +void spk_ttyio_release(struct spk_synth *synth); void spk_ttyio_register_ldisc(void); void spk_ttyio_unregister_ldisc(void); diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index 6284aff434a1..d62fb74a5add 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -114,11 +114,11 @@ static struct tty_ldisc_ops spk_ttyio_ldisc_ops = { static int spk_ttyio_out(struct spk_synth *in_synth, const char ch); static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch); -static void spk_ttyio_send_xchar(char ch); -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_ttyio_in(void); -static unsigned char spk_ttyio_in_nowait(void); -static void spk_ttyio_flush_buffer(void); +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_ttyio_in(struct spk_synth *in_synth); +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth); +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth); static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_ttyio_ops = { @@ -281,7 +281,7 @@ static int check_tty(struct tty_struct *tty) return 0; } -static void spk_ttyio_send_xchar(char ch) +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch) { mutex_lock(&speakup_tty_mutex); if (check_tty(speakup_tty)) { @@ -294,7 +294,7 @@ static void spk_ttyio_send_xchar(char ch) mutex_unlock(&speakup_tty_mutex); } -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear) +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { mutex_lock(&speakup_tty_mutex); if (check_tty(speakup_tty)) { @@ -312,7 +312,7 @@ static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char ttyio_in(int timeout) +static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) { struct spk_ldisc_data *ldisc_data = speakup_tty->disc_data; char rv; @@ -339,19 +339,19 @@ static unsigned char ttyio_in(int timeout) return rv; } -static unsigned char spk_ttyio_in(void) +static unsigned char spk_ttyio_in(struct spk_synth *in_synth) { - return ttyio_in(SPK_SYNTH_TIMEOUT); + return ttyio_in(in_synth, SPK_SYNTH_TIMEOUT); } -static unsigned char spk_ttyio_in_nowait(void) +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth) { - u8 rv = ttyio_in(0); + u8 rv = ttyio_in(in_synth, 0); return (rv == 0xff) ? 0 : rv; } -static void spk_ttyio_flush_buffer(void) +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth) { mutex_lock(&speakup_tty_mutex); if (check_tty(speakup_tty)) { @@ -379,7 +379,7 @@ int spk_ttyio_synth_probe(struct spk_synth *synth) } EXPORT_SYMBOL_GPL(spk_ttyio_synth_probe); -void spk_ttyio_release(void) +void spk_ttyio_release(struct spk_synth *in_synth) { if (!speakup_tty) return; @@ -395,15 +395,15 @@ void spk_ttyio_release(void) } EXPORT_SYMBOL_GPL(spk_ttyio_release); -const char *spk_ttyio_synth_immediate(struct spk_synth *synth, const char *buff) +const char *spk_ttyio_synth_immediate(struct spk_synth *in_synth, const char *buff) { u_char ch; while ((ch = *buff)) { if (ch == '\n') - ch = synth->procspeech; + ch = in_synth->procspeech; if (tty_write_room(speakup_tty) < 1 || - !synth->io_ops->synth_out(synth, ch)) + !in_synth->io_ops->synth_out(in_synth, ch)) return buff; buff++; } diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 91fca3033a45..2dc17c2e6749 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -157,11 +157,11 @@ struct spk_synth; struct spk_io_ops { int (*synth_out)(struct spk_synth *synth, const char ch); int (*synth_out_unicode)(struct spk_synth *synth, u16 ch); - void (*send_xchar)(char ch); - void (*tiocmset)(unsigned int set, unsigned int clear); - unsigned char (*synth_in)(void); - unsigned char (*synth_in_nowait)(void); - void (*flush_buffer)(void); + void (*send_xchar)(struct spk_synth *synth, char ch); + void (*tiocmset)(struct spk_synth *synth, unsigned int set, unsigned int clear); + unsigned char (*synth_in)(struct spk_synth *synth); + unsigned char (*synth_in_nowait)(struct spk_synth *synth); + void (*flush_buffer)(struct spk_synth *synth); int (*wait_for_xmitr)(struct spk_synth *synth); }; @@ -188,7 +188,7 @@ struct spk_synth { int *default_vol; struct spk_io_ops *io_ops; int (*probe)(struct spk_synth *synth); - void (*release)(void); + void (*release)(struct spk_synth *synth); const char *(*synth_immediate)(struct spk_synth *synth, const char *buff); void (*catch_up)(struct spk_synth *synth); diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index ac47dbac7207..6c14b682da13 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -137,14 +137,14 @@ EXPORT_SYMBOL_GPL(spk_do_catch_up_unicode); void spk_synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, synth->clear); } EXPORT_SYMBOL_GPL(spk_synth_flush); unsigned char spk_synth_get_index(struct spk_synth *synth) { - return synth->io_ops->synth_in_nowait(); + return synth->io_ops->synth_in_nowait(synth); } EXPORT_SYMBOL_GPL(spk_synth_get_index); @@ -440,7 +440,7 @@ void synth_release(void) sysfs_remove_group(speakup_kobj, &synth->attributes); for (var = synth->vars; var->var_id != MAXVARS; var++) speakup_unregister_var(var->var_id); - synth->release(); + synth->release(synth); synth = NULL; } From 4f2a81f3a88217e7340b2cab5c0a5ebd0112514c Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Tue, 26 Jan 2021 23:21:45 +0100 Subject: [PATCH 131/633] speakup: Reference synth from tty and tty from synth We do not actually need speakup_tty and spk_ttyio_synth global variables, the synth can store the pointer to the tty, and the tty ldisc_data can store the pointer to the synth. Along the way, we can clench the initialization of the synth and the creation of the tty, so that tty is never NULL. Even if the device disappears (e.g. USB unplug), the tty structure will still be there, and we automatically stop speakup in the spk_ttyio_out error handler but keep tty until the user cleans things up. As a result, this simplifies locking a lot. Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210126222147.3848175-3-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/spk_ttyio.c | 117 ++++++++-------------- drivers/accessibility/speakup/spk_types.h | 2 + 2 files changed, 46 insertions(+), 73 deletions(-) diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index d62fb74a5add..adc1cdbae2be 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -12,14 +12,15 @@ struct spk_ldisc_data { char buf; struct completion completion; bool buf_free; + struct spk_synth *synth; }; -static struct spk_synth *spk_ttyio_synth; -static struct tty_struct *speakup_tty; -/* mutex to protect against speakup_tty disappearing from underneath us while - * we are using it. this can happen when the device physically unplugged, - * while in use. it also serialises access to speakup_tty. +/* + * This allows to catch within spk_ttyio_ldisc_open whether it is getting set + * on for a speakup-driven device. */ +static struct tty_struct *speakup_tty; +/* This mutex serializes the use of such global speakup_tty variable */ static DEFINE_MUTEX(speakup_tty_mutex); static int ser_to_dev(int ser, dev_t *dev_no) @@ -67,22 +68,20 @@ static int spk_ttyio_ldisc_open(struct tty_struct *tty) static void spk_ttyio_ldisc_close(struct tty_struct *tty) { - mutex_lock(&speakup_tty_mutex); - kfree(speakup_tty->disc_data); - speakup_tty = NULL; - mutex_unlock(&speakup_tty_mutex); + kfree(tty->disc_data); } static int spk_ttyio_receive_buf2(struct tty_struct *tty, const unsigned char *cp, char *fp, int count) { struct spk_ldisc_data *ldisc_data = tty->disc_data; + struct spk_synth *synth = ldisc_data->synth; - if (spk_ttyio_synth->read_buff_add) { + if (synth->read_buff_add) { int i; for (i = 0; i < count; i++) - spk_ttyio_synth->read_buff_add(cp[i]); + synth->read_buff_add(cp[i]); return count; } @@ -187,13 +186,17 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth) mutex_lock(&speakup_tty_mutex); speakup_tty = tty; ret = tty_set_ldisc(tty, N_SPEAKUP); - if (ret) - speakup_tty = NULL; + speakup_tty = NULL; mutex_unlock(&speakup_tty_mutex); - if (!ret) + if (!ret) { /* Success */ + struct spk_ldisc_data *ldisc_data = tty->disc_data; + + ldisc_data->synth = synth; + synth->dev = tty; return 0; + } pr_err("speakup: Failed to set N_SPEAKUP on tty\n"); @@ -221,11 +224,11 @@ void spk_ttyio_unregister_ldisc(void) static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) { - mutex_lock(&speakup_tty_mutex); - if (in_synth->alive && speakup_tty && speakup_tty->ops->write) { - int ret = speakup_tty->ops->write(speakup_tty, &ch, 1); + struct tty_struct *tty = in_synth->dev; + + if (in_synth->alive && tty->ops->write) { + int ret = tty->ops->write(tty, &ch, 1); - mutex_unlock(&speakup_tty_mutex); if (ret == 0) /* No room */ return 0; @@ -243,7 +246,6 @@ static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) return 1; } - mutex_unlock(&speakup_tty_mutex); return 0; } @@ -264,47 +266,20 @@ static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch) return ret; } -static int check_tty(struct tty_struct *tty) -{ - if (!tty) { - pr_warn("%s: I/O error, deactivating speakup\n", - spk_ttyio_synth->long_name); - /* No synth any more, so nobody will restart TTYs, and we thus - * need to do it ourselves. Now that there is no synth we can - * let application flood anyway - */ - spk_ttyio_synth->alive = 0; - speakup_start_ttys(); - return 1; - } - - return 0; -} - static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->send_xchar) - speakup_tty->ops->send_xchar(speakup_tty, ch); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->send_xchar) + tty->ops->send_xchar(tty, ch); } static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->tiocmset) - speakup_tty->ops->tiocmset(speakup_tty, set, clear); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->tiocmset) + tty->ops->tiocmset(tty, set, clear); } static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) @@ -314,7 +289,8 @@ static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) { - struct spk_ldisc_data *ldisc_data = speakup_tty->disc_data; + struct tty_struct *tty = in_synth->dev; + struct spk_ldisc_data *ldisc_data = tty->disc_data; char rv; if (!timeout) { @@ -334,7 +310,7 @@ static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) mb(); ldisc_data->buf_free = true; /* Let TTY push more characters */ - tty_schedule_flip(speakup_tty->port); + tty_schedule_flip(tty->port); return rv; } @@ -353,16 +329,10 @@ static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth) static void spk_ttyio_flush_buffer(struct spk_synth *in_synth) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->flush_buffer) - speakup_tty->ops->flush_buffer(speakup_tty); - - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->flush_buffer) + tty->ops->flush_buffer(tty); } int spk_ttyio_synth_probe(struct spk_synth *synth) @@ -373,7 +343,6 @@ int spk_ttyio_synth_probe(struct spk_synth *synth) return rv; synth->alive = 1; - spk_ttyio_synth = synth; return 0; } @@ -381,28 +350,30 @@ EXPORT_SYMBOL_GPL(spk_ttyio_synth_probe); void spk_ttyio_release(struct spk_synth *in_synth) { - if (!speakup_tty) - return; + struct tty_struct *tty = in_synth->dev; - tty_lock(speakup_tty); + tty_lock(tty); - if (speakup_tty->ops->close) - speakup_tty->ops->close(speakup_tty, NULL); + if (tty->ops->close) + tty->ops->close(tty, NULL); - tty_ldisc_flush(speakup_tty); - tty_unlock(speakup_tty); - tty_kclose(speakup_tty); + tty_ldisc_flush(tty); + tty_unlock(tty); + tty_kclose(tty); + + in_synth->dev = NULL; } EXPORT_SYMBOL_GPL(spk_ttyio_release); const char *spk_ttyio_synth_immediate(struct spk_synth *in_synth, const char *buff) { + struct tty_struct *tty = in_synth->dev; u_char ch; while ((ch = *buff)) { if (ch == '\n') ch = in_synth->procspeech; - if (tty_write_room(speakup_tty) < 1 || + if (tty_write_room(tty) < 1 || !in_synth->io_ops->synth_out(in_synth, ch)) return buff; buff++; diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 2dc17c2e6749..7789f5dca62a 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -200,6 +200,8 @@ struct spk_synth { struct synth_indexing indexing; int alive; struct attribute_group attributes; + + void *dev; }; /* From 117422521e6c212d32ed7b5d3561cc1e936f8669 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Tue, 26 Jan 2021 23:21:46 +0100 Subject: [PATCH 132/633] speakup: Simplify spk_ttyio_out error handling. This avoids most code indentation Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210126222147.3848175-4-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/spk_ttyio.c | 36 ++++++++++++----------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index adc1cdbae2be..4dbd24c506d6 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -225,27 +225,29 @@ void spk_ttyio_unregister_ldisc(void) static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) { struct tty_struct *tty = in_synth->dev; + int ret; - if (in_synth->alive && tty->ops->write) { - int ret = tty->ops->write(tty, &ch, 1); + if (!in_synth->alive || !tty->ops->write) + return 0; - if (ret == 0) - /* No room */ - return 0; - if (ret < 0) { - pr_warn("%s: I/O error, deactivating speakup\n", - in_synth->long_name); - /* No synth any more, so nobody will restart TTYs, - * and we thus need to do it ourselves. Now that there - * is no synth we can let application flood anyway - */ - in_synth->alive = 0; - speakup_start_ttys(); - return 0; - } + ret = tty->ops->write(tty, &ch, 1); + + if (ret == 0) + /* No room */ + return 0; + + if (ret > 0) + /* Success */ return 1; - } + pr_warn("%s: I/O error, deactivating speakup\n", + in_synth->long_name); + /* No synth any more, so nobody will restart TTYs, + * and we thus need to do it ourselves. Now that there + * is no synth we can let application flood anyway + */ + in_synth->alive = 0; + speakup_start_ttys(); return 0; } From cae2181b498fe52885022772465a7610fd7701f4 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Tue, 26 Jan 2021 23:21:47 +0100 Subject: [PATCH 133/633] speakup: Add documentation on changing the speakup messages language This documents how to use speakup_setlocale to set the speakup messages language. Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210126222147.3848175-5-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/spkguide.txt | 48 ++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/spkguide.txt b/Documentation/admin-guide/spkguide.txt index 5ff6a0fe87d1..977ab3f5a0a8 100644 --- a/Documentation/admin-guide/spkguide.txt +++ b/Documentation/admin-guide/spkguide.txt @@ -1033,7 +1033,9 @@ speakup + keypad 3, you would hear: The speakup key is depressed, so the name of the key state is speakup. This part of the message comes from the states collection. -14.2. Loading Your Own Messages +14.2. Changing language + +14.2.1. Loading Your Own Messages The files under the i18n subdirectory all follow the same format. They consist of lines, with one message per line. @@ -1066,8 +1068,50 @@ echo '1 azul' > /speakup/i18n/colors The next time that Speakup says message 1 from the colors group, it will say "azul", rather than "blue." +14.2.2. Choose a language + In the future, translations into various languages will be made available, -and most users will just load the files necessary for their language. +and most users will just load the files necessary for their language. So far, +only French language is available beyond native Canadian English language. + +French is only available after you are logged in. + +Canadian English is the default language. To toggle another language, +download the source of Speakup and untar it in your home directory. The +following command should let you do this: + +tar xvjf speakup-.tar.bz2 + +where is the version number of the application. + +Next, change to the newly created directory, then into the tools/ directory, and +run the script speakup_setlocale. You are asked the language that you want to +use. Type the number associated to your language (e.g. fr for French) then press +Enter. Needed files are copied in the i18n directory. + +Note: the speakupconf must be installed on your system so that settings are saved. +Otherwise, you will have an error: your language will be loaded but you will +have to run the script again every time Speakup restarts. +See section 16.1. for information about speakupconf. + +You will have to repeat these steps for any change of locale, i.e. if you wish +change the speakup's language or charset (iso-8859-15 ou UTF-8). + +If you wish store the settings, note that at your next login, you will need to +do: + +speakup load + +Alternatively, you can add the above line to your file +~/.bashrc or ~/.bash_profile. + +If your system administrator ran himself the script, all the users will be able +to change from English to the language choosed by root and do directly +speakupconf load (or add this to the ~/.bashrc or +~/.bash_profile file). If there are several languages to handle, the +administrator (or every user) will have to run the first steps until speakupconf +save, choosing the appropriate language, in every user's home directory. Every +user will then be able to do speakupconf load, Speakup will load his own settings. 14.3. No Support for Non-Western-European Languages From 8ba59e9dee31246fc34b4d4bec032093e9c06510 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 22 Jan 2021 13:43:58 +0200 Subject: [PATCH 134/633] misc: pti: Remove driver for deprecated platform Intel Moorestown and Medfield are quite old Intel Atom based 32-bit platforms, which were in limited use in some Android phones, tablets and consumer electronics more than eight years ago. There are no bugs or problems ever reported outside from Intel for breaking any of that platforms for years. It seems no real users exists who run more or less fresh kernel on it. The commit 05f4434bc130 ("ASoC: Intel: remove mfld_machine") also in align with this theory. Due to above and to reduce a burden of supporting outdated drivers we remove the support of outdated platforms completely. Cc: Alexander Shishkin Acked-by: Linus Walleij Acked-by: Arnd Bergmann Acked-by: Alexander Shishkin Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20210122114358.39299-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/pti_intel_mid.rst | 108 --- drivers/misc/Kconfig | 13 - drivers/misc/Makefile | 1 - drivers/misc/pti.c | 978 --------------------- drivers/tty/Makefile | 2 - drivers/tty/n_tracerouter.c | 233 ----- drivers/tty/n_tracesink.c | 228 ----- drivers/tty/n_tracesink.h | 26 - include/linux/intel-pti.h | 35 - 9 files changed, 1624 deletions(-) delete mode 100644 Documentation/driver-api/pti_intel_mid.rst delete mode 100644 drivers/misc/pti.c delete mode 100644 drivers/tty/n_tracerouter.c delete mode 100644 drivers/tty/n_tracesink.c delete mode 100644 drivers/tty/n_tracesink.h delete mode 100644 include/linux/intel-pti.h diff --git a/Documentation/driver-api/pti_intel_mid.rst b/Documentation/driver-api/pti_intel_mid.rst deleted file mode 100644 index bacc2a4ee89f..000000000000 --- a/Documentation/driver-api/pti_intel_mid.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -============= -Intel MID PTI -============= - -The Intel MID PTI project is HW implemented in Intel Atom -system-on-a-chip designs based on the Parallel Trace -Interface for MIPI P1149.7 cJTAG standard. The kernel solution -for this platform involves the following files:: - - ./include/linux/pti.h - ./drivers/.../n_tracesink.h - ./drivers/.../n_tracerouter.c - ./drivers/.../n_tracesink.c - ./drivers/.../pti.c - -pti.c is the driver that enables various debugging features -popular on platforms from certain mobile manufacturers. -n_tracerouter.c and n_tracesink.c allow extra system information to -be collected and routed to the pti driver, such as trace -debugging data from a modem. Although n_tracerouter -and n_tracesink are a part of the complete PTI solution, -these two line disciplines can work separately from -pti.c and route any data stream from one /dev/tty node -to another /dev/tty node via kernel-space. This provides -a stable, reliable connection that will not break unless -the user-space application shuts down (plus avoids -kernel->user->kernel context switch overheads of routing -data). - -An example debugging usage for this driver system: - - * Hook /dev/ttyPTI0 to syslogd. Opening this port will also start - a console device to further capture debugging messages to PTI. - * Hook /dev/ttyPTI1 to modem debugging data to write to PTI HW. - This is where n_tracerouter and n_tracesink are used. - * Hook /dev/pti to a user-level debugging application for writing - to PTI HW. - * `Use mipi_` Kernel Driver API in other device drivers for - debugging to PTI by first requesting a PTI write address via - mipi_request_masterchannel(1). - -Below is example pseudo-code on how a 'privileged' application -can hook up n_tracerouter and n_tracesink to any tty on -a system. 'Privileged' means the application has enough -privileges to successfully manipulate the ldisc drivers -but is not just blindly executing as 'root'. Keep in mind -the use of ioctl(,TIOCSETD,) is not specific to the n_tracerouter -and n_tracesink line discpline drivers but is a generic -operation for a program to use a line discpline driver -on a tty port other than the default n_tty: - -.. code-block:: c - - /////////// To hook up n_tracerouter and n_tracesink ///////// - - // Note that n_tracerouter depends on n_tracesink. - #include - #define ONE_TTY "/dev/ttyOne" - #define TWO_TTY "/dev/ttyTwo" - - // needed global to hand onto ldisc connection - static int g_fd_source = -1; - static int g_fd_sink = -1; - - // these two vars used to grab LDISC values from loaded ldisc drivers - // in OS. Look at /proc/tty/ldiscs to get the right numbers from - // the ldiscs loaded in the system. - int source_ldisc_num, sink_ldisc_num = -1; - int retval; - - g_fd_source = open(ONE_TTY, O_RDWR); // must be R/W - g_fd_sink = open(TWO_TTY, O_RDWR); // must be R/W - - if (g_fd_source <= 0) || (g_fd_sink <= 0) { - // doubt you'll want to use these exact error lines of code - printf("Error on open(). errno: %d\n",errno); - return errno; - } - - retval = ioctl(g_fd_sink, TIOCSETD, &sink_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - retval = ioctl(g_fd_source, TIOCSETD, &source_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - /////////// To disconnect n_tracerouter and n_tracesink //////// - - // First make sure data through the ldiscs has stopped. - - // Second, disconnect ldiscs. This provides a - // little cleaner shutdown on tty stack. - sink_ldisc_num = 0; - source_ldisc_num = 0; - ioctl(g_fd_uart, TIOCSETD, &sink_ldisc_num); - ioctl(g_fd_gadget, TIOCSETD, &source_ldisc_num); - - // Three, program closes connection, and cleanup: - close(g_fd_uart); - close(g_fd_gadget); - g_fd_uart = g_fd_gadget = NULL; diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 8085213913cc..f532c59bb59b 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -104,19 +104,6 @@ config PHANTOM If you choose to build module, its name will be phantom. If unsure, say N here. -config INTEL_MID_PTI - tristate "Parallel Trace Interface for MIPI P1149.7 cJTAG standard" - depends on PCI && TTY && (X86_INTEL_MID || COMPILE_TEST) - help - The PTI (Parallel Trace Interface) driver directs - trace data routed from various parts in the system out - through an Intel Penwell PTI port and out of the mobile - device for analysis with a debugging tool (Lauterbach or Fido). - - You should select this driver if the target kernel is meant for - an Intel Atom (non-netbook) mobile device containing a MIPI - P1149.7 standard implementation. - config TIFM_CORE tristate "TI Flash Media interface support" depends on PCI diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 456aa8a4c896..99b6f15a3c70 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -8,7 +8,6 @@ obj-$(CONFIG_IBMVMC) += ibmvmc.o obj-$(CONFIG_AD525X_DPOT) += ad525x_dpot.o obj-$(CONFIG_AD525X_DPOT_I2C) += ad525x_dpot-i2c.o obj-$(CONFIG_AD525X_DPOT_SPI) += ad525x_dpot-spi.o -obj-$(CONFIG_INTEL_MID_PTI) += pti.o obj-$(CONFIG_ATMEL_SSC) += atmel-ssc.o obj-$(CONFIG_DUMMY_IRQ) += dummy-irq.o obj-$(CONFIG_ICS932S401) += ics932s401.o diff --git a/drivers/misc/pti.c b/drivers/misc/pti.c deleted file mode 100644 index 7236ae527b19..000000000000 --- a/drivers/misc/pti.c +++ /dev/null @@ -1,978 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * pti.c - PTI driver for cJTAG data extration - * - * Copyright (C) Intel 2010 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The PTI (Parallel Trace Interface) driver directs trace data routed from - * various parts in the system out through the Intel Penwell PTI port and - * out of the mobile device for analysis with a debugging tool - * (Lauterbach, Fido). This is part of a solution for the MIPI P1149.7, - * compact JTAG, standard. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DRIVERNAME "pti" -#define PCINAME "pciPTI" -#define TTYNAME "ttyPTI" -#define CHARNAME "pti" -#define PTITTY_MINOR_START 0 -#define PTITTY_MINOR_NUM 2 -#define MAX_APP_IDS 16 /* 128 channel ids / u8 bit size */ -#define MAX_OS_IDS 16 /* 128 channel ids / u8 bit size */ -#define MAX_MODEM_IDS 16 /* 128 channel ids / u8 bit size */ -#define MODEM_BASE_ID 71 /* modem master ID address */ -#define CONTROL_ID 72 /* control master ID address */ -#define CONSOLE_ID 73 /* console master ID address */ -#define OS_BASE_ID 74 /* base OS master ID address */ -#define APP_BASE_ID 80 /* base App master ID address */ -#define CONTROL_FRAME_LEN 32 /* PTI control frame maximum size */ -#define USER_COPY_SIZE 8192 /* 8Kb buffer for user space copy */ -#define APERTURE_14 0x3800000 /* offset to first OS write addr */ -#define APERTURE_LEN 0x400000 /* address length */ - -struct pti_tty { - struct pti_masterchannel *mc; -}; - -struct pti_dev { - struct tty_port port[PTITTY_MINOR_NUM]; - unsigned long pti_addr; - unsigned long aperture_base; - void __iomem *pti_ioaddr; - u8 ia_app[MAX_APP_IDS]; - u8 ia_os[MAX_OS_IDS]; - u8 ia_modem[MAX_MODEM_IDS]; -}; - -/* - * This protects access to ia_app, ia_os, and ia_modem, - * which keeps track of channels allocated in - * an aperture write id. - */ -static DEFINE_MUTEX(alloclock); - -static const struct pci_device_id pci_ids[] = { - {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x82B)}, - {0} -}; - -static struct tty_driver *pti_tty_driver; -static struct pti_dev *drv_data; - -static unsigned int pti_console_channel; -static unsigned int pti_control_channel; - -/** - * pti_write_to_aperture()- The private write function to PTI HW. - * - * @mc: The 'aperture'. It's part of a write address that holds - * a master and channel ID. - * @buf: Data being written to the HW that will ultimately be seen - * in a debugging tool (Fido, Lauterbach). - * @len: Size of buffer. - * - * Since each aperture is specified by a unique - * master/channel ID, no two processes will be writing - * to the same aperture at the same time so no lock is required. The - * PTI-Output agent will send these out in the order that they arrived, and - * thus, it will intermix these messages. The debug tool can then later - * regroup the appropriate message segments together reconstituting each - * message. - */ -static void pti_write_to_aperture(struct pti_masterchannel *mc, - u8 *buf, - int len) -{ - int dwordcnt; - int final; - int i; - u32 ptiword; - u32 __iomem *aperture; - u8 *p = buf; - - /* - * calculate the aperture offset from the base using the master and - * channel id's. - */ - aperture = drv_data->pti_ioaddr + (mc->master << 15) - + (mc->channel << 8); - - dwordcnt = len >> 2; - final = len - (dwordcnt << 2); /* final = trailing bytes */ - if (final == 0 && dwordcnt != 0) { /* always need a final dword */ - final += 4; - dwordcnt--; - } - - for (i = 0; i < dwordcnt; i++) { - ptiword = be32_to_cpu(*(u32 *)p); - p += 4; - iowrite32(ptiword, aperture); - } - - aperture += PTI_LASTDWORD_DTS; /* adding DTS signals that is EOM */ - - ptiword = 0; - for (i = 0; i < final; i++) - ptiword |= *p++ << (24-(8*i)); - - iowrite32(ptiword, aperture); - return; -} - -/** - * pti_control_frame_built_and_sent()- control frame build and send function. - * - * @mc: The master / channel structure on which the function - * built a control frame. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * To be able to post process the PTI contents on host side, a control frame - * is added before sending any PTI content. So the host side knows on - * each PTI frame the name of the thread using a dedicated master / channel. - * The thread name is retrieved from 'current' global variable if 'thread_name' - * is 'NULL', else it is retrieved from 'thread_name' parameter. - * This function builds this frame and sends it to a master ID CONTROL_ID. - * The overhead is only 32 bytes since the driver only writes to HW - * in 32 byte chunks. - */ -static void pti_control_frame_built_and_sent(struct pti_masterchannel *mc, - const char *thread_name) -{ - /* - * Since we access the comm member in current's task_struct, we only - * need to be as large as what 'comm' in that structure is. - */ - char comm[TASK_COMM_LEN]; - struct pti_masterchannel mccontrol = {.master = CONTROL_ID, - .channel = 0}; - const char *thread_name_p; - const char *control_format = "%3d %3d %s"; - u8 control_frame[CONTROL_FRAME_LEN]; - - if (!thread_name) { - if (!in_interrupt()) - get_task_comm(comm, current); - else - strncpy(comm, "Interrupt", TASK_COMM_LEN); - - /* Absolutely ensure our buffer is zero terminated. */ - comm[TASK_COMM_LEN-1] = 0; - thread_name_p = comm; - } else { - thread_name_p = thread_name; - } - - mccontrol.channel = pti_control_channel; - pti_control_channel = (pti_control_channel + 1) & 0x7f; - - snprintf(control_frame, CONTROL_FRAME_LEN, control_format, mc->master, - mc->channel, thread_name_p); - pti_write_to_aperture(&mccontrol, control_frame, strlen(control_frame)); -} - -/** - * pti_write_full_frame_to_aperture()- high level function to - * write to PTI. - * - * @mc: The 'aperture'. It's part of a write address that holds - * a master and channel ID. - * @buf: Data being written to the HW that will ultimately be seen - * in a debugging tool (Fido, Lauterbach). - * @len: Size of buffer. - * - * All threads sending data (either console, user space application, ...) - * are calling the high level function to write to PTI meaning that it is - * possible to add a control frame before sending the content. - */ -static void pti_write_full_frame_to_aperture(struct pti_masterchannel *mc, - const unsigned char *buf, - int len) -{ - pti_control_frame_built_and_sent(mc, NULL); - pti_write_to_aperture(mc, (u8 *)buf, len); -} - -/** - * get_id()- Allocate a master and channel ID. - * - * @id_array: an array of bits representing what channel - * id's are allocated for writing. - * @max_ids: The max amount of available write IDs to use. - * @base_id: The starting SW channel ID, based on the Intel - * PTI arch. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * Returns: - * pti_masterchannel struct with master, channel ID address - * 0 for error - * - * Each bit in the arrays ia_app and ia_os correspond to a master and - * channel id. The bit is one if the id is taken and 0 if free. For - * every master there are 128 channel id's. - */ -static struct pti_masterchannel *get_id(u8 *id_array, - int max_ids, - int base_id, - const char *thread_name) -{ - struct pti_masterchannel *mc; - int i, j, mask; - - mc = kmalloc(sizeof(struct pti_masterchannel), GFP_KERNEL); - if (mc == NULL) - return NULL; - - /* look for a byte with a free bit */ - for (i = 0; i < max_ids; i++) - if (id_array[i] != 0xff) - break; - if (i == max_ids) { - kfree(mc); - return NULL; - } - /* find the bit in the 128 possible channel opportunities */ - mask = 0x80; - for (j = 0; j < 8; j++) { - if ((id_array[i] & mask) == 0) - break; - mask >>= 1; - } - - /* grab it */ - id_array[i] |= mask; - mc->master = base_id; - mc->channel = ((i & 0xf)<<3) + j; - /* write new master Id / channel Id allocation to channel control */ - pti_control_frame_built_and_sent(mc, thread_name); - return mc; -} - -/* - * The following three functions: - * pti_request_mastercahannel(), mipi_release_masterchannel() - * and pti_writedata() are an API for other kernel drivers to - * access PTI. - */ - -/** - * pti_request_masterchannel()- Kernel API function used to allocate - * a master, channel ID address - * to write to PTI HW. - * - * @type: 0- request Application master, channel aperture ID - * write address. - * 1- request OS master, channel aperture ID write - * address. - * 2- request Modem master, channel aperture ID - * write address. - * Other values, error. - * @thread_name: The thread name associated with the master / channel or - * 'NULL' if using the 'current' global variable. - * - * Returns: - * pti_masterchannel struct - * 0 for error - */ -struct pti_masterchannel *pti_request_masterchannel(u8 type, - const char *thread_name) -{ - struct pti_masterchannel *mc; - - mutex_lock(&alloclock); - - switch (type) { - - case 0: - mc = get_id(drv_data->ia_app, MAX_APP_IDS, - APP_BASE_ID, thread_name); - break; - - case 1: - mc = get_id(drv_data->ia_os, MAX_OS_IDS, - OS_BASE_ID, thread_name); - break; - - case 2: - mc = get_id(drv_data->ia_modem, MAX_MODEM_IDS, - MODEM_BASE_ID, thread_name); - break; - default: - mc = NULL; - } - - mutex_unlock(&alloclock); - return mc; -} -EXPORT_SYMBOL_GPL(pti_request_masterchannel); - -/** - * pti_release_masterchannel()- Kernel API function used to release - * a master, channel ID address - * used to write to PTI HW. - * - * @mc: master, channel apeture ID address to be released. This - * will de-allocate the structure via kfree(). - */ -void pti_release_masterchannel(struct pti_masterchannel *mc) -{ - u8 master, channel, i; - - mutex_lock(&alloclock); - - if (mc) { - master = mc->master; - channel = mc->channel; - - if (master == APP_BASE_ID) { - i = channel >> 3; - drv_data->ia_app[i] &= ~(0x80>>(channel & 0x7)); - } else if (master == OS_BASE_ID) { - i = channel >> 3; - drv_data->ia_os[i] &= ~(0x80>>(channel & 0x7)); - } else { - i = channel >> 3; - drv_data->ia_modem[i] &= ~(0x80>>(channel & 0x7)); - } - - kfree(mc); - } - - mutex_unlock(&alloclock); -} -EXPORT_SYMBOL_GPL(pti_release_masterchannel); - -/** - * pti_writedata()- Kernel API function used to write trace - * debugging data to PTI HW. - * - * @mc: Master, channel aperture ID address to write to. - * Null value will return with no write occurring. - * @buf: Trace debuging data to write to the PTI HW. - * Null value will return with no write occurring. - * @count: Size of buf. Value of 0 or a negative number will - * return with no write occuring. - */ -void pti_writedata(struct pti_masterchannel *mc, u8 *buf, int count) -{ - /* - * since this function is exported, this is treated like an - * API function, thus, all parameters should - * be checked for validity. - */ - if ((mc != NULL) && (buf != NULL) && (count > 0)) - pti_write_to_aperture(mc, buf, count); - return; -} -EXPORT_SYMBOL_GPL(pti_writedata); - -/* - * for the tty_driver_*() basic function descriptions, see tty_driver.h. - * Specific header comments made for PTI-related specifics. - */ - -/** - * pti_tty_driver_open()- Open an Application master, channel aperture - * ID to the PTI device via tty device. - * - * @tty: tty interface. - * @filp: filp interface pased to tty_port_open() call. - * - * Returns: - * int, 0 for success - * otherwise, fail value - * - * The main purpose of using the tty device interface is for - * each tty port to have a unique PTI write aperture. In an - * example use case, ttyPTI0 gets syslogd and an APP aperture - * ID and ttyPTI1 is where the n_tracesink ldisc hooks to route - * modem messages into PTI. Modem trace data does not have to - * go to ttyPTI1, but ttyPTI0 and ttyPTI1 do need to be distinct - * master IDs. These messages go through the PTI HW and out of - * the handheld platform and to the Fido/Lauterbach device. - */ -static int pti_tty_driver_open(struct tty_struct *tty, struct file *filp) -{ - /* - * we actually want to allocate a new channel per open, per - * system arch. HW gives more than plenty channels for a single - * system task to have its own channel to write trace data. This - * also removes a locking requirement for the actual write - * procedure. - */ - return tty_port_open(tty->port, tty, filp); -} - -/** - * pti_tty_driver_close()- close tty device and release Application - * master, channel aperture ID to the PTI device via tty device. - * - * @tty: tty interface. - * @filp: filp interface pased to tty_port_close() call. - * - * The main purpose of using the tty device interface is to route - * syslog daemon messages to the PTI HW and out of the handheld platform - * and to the Fido/Lauterbach device. - */ -static void pti_tty_driver_close(struct tty_struct *tty, struct file *filp) -{ - tty_port_close(tty->port, tty, filp); -} - -/** - * pti_tty_install()- Used to set up specific master-channels - * to tty ports for organizational purposes when - * tracing viewed from debuging tools. - * - * @driver: tty driver information. - * @tty: tty struct containing pti information. - * - * Returns: - * 0 for success - * otherwise, error - */ -static int pti_tty_install(struct tty_driver *driver, struct tty_struct *tty) -{ - int idx = tty->index; - struct pti_tty *pti_tty_data; - int ret = tty_standard_install(driver, tty); - - if (ret == 0) { - pti_tty_data = kmalloc(sizeof(struct pti_tty), GFP_KERNEL); - if (pti_tty_data == NULL) - return -ENOMEM; - - if (idx == PTITTY_MINOR_START) - pti_tty_data->mc = pti_request_masterchannel(0, NULL); - else - pti_tty_data->mc = pti_request_masterchannel(2, NULL); - - if (pti_tty_data->mc == NULL) { - kfree(pti_tty_data); - return -ENXIO; - } - tty->driver_data = pti_tty_data; - } - - return ret; -} - -/** - * pti_tty_cleanup()- Used to de-allocate master-channel resources - * tied to tty's of this driver. - * - * @tty: tty struct containing pti information. - */ -static void pti_tty_cleanup(struct tty_struct *tty) -{ - struct pti_tty *pti_tty_data = tty->driver_data; - if (pti_tty_data == NULL) - return; - pti_release_masterchannel(pti_tty_data->mc); - kfree(pti_tty_data); - tty->driver_data = NULL; -} - -/** - * pti_tty_driver_write()- Write trace debugging data through the char - * interface to the PTI HW. Part of the misc device implementation. - * - * @tty: tty struct containing pti information. - * @buf: trace data to be written. - * @len: # of byte to write. - * - * Returns: - * int, # of bytes written - * otherwise, error - */ -static int pti_tty_driver_write(struct tty_struct *tty, - const unsigned char *buf, int len) -{ - struct pti_tty *pti_tty_data = tty->driver_data; - if ((pti_tty_data != NULL) && (pti_tty_data->mc != NULL)) { - pti_write_to_aperture(pti_tty_data->mc, (u8 *)buf, len); - return len; - } - /* - * we can't write to the pti hardware if the private driver_data - * and the mc address is not there. - */ - else - return -EFAULT; -} - -/** - * pti_tty_write_room()- Always returns 2048. - * - * @tty: contains tty info of the pti driver. - */ -static int pti_tty_write_room(struct tty_struct *tty) -{ - return 2048; -} - -/** - * pti_char_open()- Open an Application master, channel aperture - * ID to the PTI device. Part of the misc device implementation. - * - * @inode: not used. - * @filp: Output- will have a masterchannel struct set containing - * the allocated application PTI aperture write address. - * - * Returns: - * int, 0 for success - * otherwise, a fail value - */ -static int pti_char_open(struct inode *inode, struct file *filp) -{ - struct pti_masterchannel *mc; - - /* - * We really do want to fail immediately if - * pti_request_masterchannel() fails, - * before assigning the value to filp->private_data. - * Slightly easier to debug if this driver needs debugging. - */ - mc = pti_request_masterchannel(0, NULL); - if (mc == NULL) - return -ENOMEM; - filp->private_data = mc; - return 0; -} - -/** - * pti_char_release()- Close a char channel to the PTI device. Part - * of the misc device implementation. - * - * @inode: Not used in this implementaiton. - * @filp: Contains private_data that contains the master, channel - * ID to be released by the PTI device. - * - * Returns: - * always 0 - */ -static int pti_char_release(struct inode *inode, struct file *filp) -{ - pti_release_masterchannel(filp->private_data); - filp->private_data = NULL; - return 0; -} - -/** - * pti_char_write()- Write trace debugging data through the char - * interface to the PTI HW. Part of the misc device implementation. - * - * @filp: Contains private data which is used to obtain - * master, channel write ID. - * @data: trace data to be written. - * @len: # of byte to write. - * @ppose: Not used in this function implementation. - * - * Returns: - * int, # of bytes written - * otherwise, error value - * - * Notes: From side discussions with Alan Cox and experimenting - * with PTI debug HW like Nokia's Fido box and Lauterbach - * devices, 8192 byte write buffer used by USER_COPY_SIZE was - * deemed an appropriate size for this type of usage with - * debugging HW. - */ -static ssize_t pti_char_write(struct file *filp, const char __user *data, - size_t len, loff_t *ppose) -{ - struct pti_masterchannel *mc; - void *kbuf; - const char __user *tmp; - size_t size = USER_COPY_SIZE; - size_t n = 0; - - tmp = data; - mc = filp->private_data; - - kbuf = kmalloc(size, GFP_KERNEL); - if (kbuf == NULL) { - pr_err("%s(%d): buf allocation failed\n", - __func__, __LINE__); - return -ENOMEM; - } - - do { - if (len - n > USER_COPY_SIZE) - size = USER_COPY_SIZE; - else - size = len - n; - - if (copy_from_user(kbuf, tmp, size)) { - kfree(kbuf); - return n ? n : -EFAULT; - } - - pti_write_to_aperture(mc, kbuf, size); - n += size; - tmp += size; - - } while (len > n); - - kfree(kbuf); - return len; -} - -static const struct tty_operations pti_tty_driver_ops = { - .open = pti_tty_driver_open, - .close = pti_tty_driver_close, - .write = pti_tty_driver_write, - .write_room = pti_tty_write_room, - .install = pti_tty_install, - .cleanup = pti_tty_cleanup -}; - -static const struct file_operations pti_char_driver_ops = { - .owner = THIS_MODULE, - .write = pti_char_write, - .open = pti_char_open, - .release = pti_char_release, -}; - -static struct miscdevice pti_char_driver = { - .minor = MISC_DYNAMIC_MINOR, - .name = CHARNAME, - .fops = &pti_char_driver_ops -}; - -/** - * pti_console_write()- Write to the console that has been acquired. - * - * @c: Not used in this implementaiton. - * @buf: Data to be written. - * @len: Length of buf. - */ -static void pti_console_write(struct console *c, const char *buf, unsigned len) -{ - static struct pti_masterchannel mc = {.master = CONSOLE_ID, - .channel = 0}; - - mc.channel = pti_console_channel; - pti_console_channel = (pti_console_channel + 1) & 0x7f; - - pti_write_full_frame_to_aperture(&mc, buf, len); -} - -/** - * pti_console_device()- Return the driver tty structure and set the - * associated index implementation. - * - * @c: Console device of the driver. - * @index: index associated with c. - * - * Returns: - * always value of pti_tty_driver structure when this function - * is called. - */ -static struct tty_driver *pti_console_device(struct console *c, int *index) -{ - *index = c->index; - return pti_tty_driver; -} - -/** - * pti_console_setup()- Initialize console variables used by the driver. - * - * @c: Not used. - * @opts: Not used. - * - * Returns: - * always 0. - */ -static int pti_console_setup(struct console *c, char *opts) -{ - pti_console_channel = 0; - pti_control_channel = 0; - return 0; -} - -/* - * pti_console struct, used to capture OS printk()'s and shift - * out to the PTI device for debugging. This cannot be - * enabled upon boot because of the possibility of eating - * any serial console printk's (race condition discovered). - * The console should be enabled upon when the tty port is - * used for the first time. Since the primary purpose for - * the tty port is to hook up syslog to it, the tty port - * will be open for a really long time. - */ -static struct console pti_console = { - .name = TTYNAME, - .write = pti_console_write, - .device = pti_console_device, - .setup = pti_console_setup, - .flags = CON_PRINTBUFFER, - .index = 0, -}; - -/** - * pti_port_activate()- Used to start/initialize any items upon - * first opening of tty_port(). - * - * @port: The tty port number of the PTI device. - * @tty: The tty struct associated with this device. - * - * Returns: - * always returns 0 - * - * Notes: The primary purpose of the PTI tty port 0 is to hook - * the syslog daemon to it; thus this port will be open for a - * very long time. - */ -static int pti_port_activate(struct tty_port *port, struct tty_struct *tty) -{ - if (port->tty->index == PTITTY_MINOR_START) - console_start(&pti_console); - return 0; -} - -/** - * pti_port_shutdown()- Used to stop/shutdown any items upon the - * last tty port close. - * - * @port: The tty port number of the PTI device. - * - * Notes: The primary purpose of the PTI tty port 0 is to hook - * the syslog daemon to it; thus this port will be open for a - * very long time. - */ -static void pti_port_shutdown(struct tty_port *port) -{ - if (port->tty->index == PTITTY_MINOR_START) - console_stop(&pti_console); -} - -static const struct tty_port_operations tty_port_ops = { - .activate = pti_port_activate, - .shutdown = pti_port_shutdown, -}; - -/* - * Note the _probe() call sets everything up and ties the char and tty - * to successfully detecting the PTI device on the pci bus. - */ - -/** - * pti_pci_probe()- Used to detect pti on the pci bus and set - * things up in the driver. - * - * @pdev: pci_dev struct values for pti. - * @ent: pci_device_id struct for pti driver. - * - * Returns: - * 0 for success - * otherwise, error - */ -static int pti_pci_probe(struct pci_dev *pdev, - const struct pci_device_id *ent) -{ - unsigned int a; - int retval; - int pci_bar = 1; - - dev_dbg(&pdev->dev, "%s %s(%d): PTI PCI ID %04x:%04x\n", __FILE__, - __func__, __LINE__, pdev->vendor, pdev->device); - - retval = misc_register(&pti_char_driver); - if (retval) { - pr_err("%s(%d): CHAR registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - goto err; - } - - retval = pci_enable_device(pdev); - if (retval != 0) { - dev_err(&pdev->dev, - "%s: pci_enable_device() returned error %d\n", - __func__, retval); - goto err_unreg_misc; - } - - drv_data = kzalloc(sizeof(*drv_data), GFP_KERNEL); - if (drv_data == NULL) { - retval = -ENOMEM; - dev_err(&pdev->dev, - "%s(%d): kmalloc() returned NULL memory.\n", - __func__, __LINE__); - goto err_disable_pci; - } - drv_data->pti_addr = pci_resource_start(pdev, pci_bar); - - retval = pci_request_region(pdev, pci_bar, dev_name(&pdev->dev)); - if (retval != 0) { - dev_err(&pdev->dev, - "%s(%d): pci_request_region() returned error %d\n", - __func__, __LINE__, retval); - goto err_free_dd; - } - drv_data->aperture_base = drv_data->pti_addr+APERTURE_14; - drv_data->pti_ioaddr = - ioremap((u32)drv_data->aperture_base, - APERTURE_LEN); - if (!drv_data->pti_ioaddr) { - retval = -ENOMEM; - goto err_rel_reg; - } - - pci_set_drvdata(pdev, drv_data); - - for (a = 0; a < PTITTY_MINOR_NUM; a++) { - struct tty_port *port = &drv_data->port[a]; - tty_port_init(port); - port->ops = &tty_port_ops; - - tty_port_register_device(port, pti_tty_driver, a, &pdev->dev); - } - - register_console(&pti_console); - - return 0; -err_rel_reg: - pci_release_region(pdev, pci_bar); -err_free_dd: - kfree(drv_data); -err_disable_pci: - pci_disable_device(pdev); -err_unreg_misc: - misc_deregister(&pti_char_driver); -err: - return retval; -} - -/** - * pti_pci_remove()- Driver exit method to remove PTI from - * PCI bus. - * @pdev: variable containing pci info of PTI. - */ -static void pti_pci_remove(struct pci_dev *pdev) -{ - struct pti_dev *drv_data = pci_get_drvdata(pdev); - unsigned int a; - - unregister_console(&pti_console); - - for (a = 0; a < PTITTY_MINOR_NUM; a++) { - tty_unregister_device(pti_tty_driver, a); - tty_port_destroy(&drv_data->port[a]); - } - - iounmap(drv_data->pti_ioaddr); - kfree(drv_data); - pci_release_region(pdev, 1); - pci_disable_device(pdev); - - misc_deregister(&pti_char_driver); -} - -static struct pci_driver pti_pci_driver = { - .name = PCINAME, - .id_table = pci_ids, - .probe = pti_pci_probe, - .remove = pti_pci_remove, -}; - -/** - * pti_init()- Overall entry/init call to the pti driver. - * It starts the registration process with the kernel. - * - * Returns: - * int __init, 0 for success - * otherwise value is an error - * - */ -static int __init pti_init(void) -{ - int retval; - - /* First register module as tty device */ - - pti_tty_driver = alloc_tty_driver(PTITTY_MINOR_NUM); - if (pti_tty_driver == NULL) { - pr_err("%s(%d): Memory allocation failed for ptiTTY driver\n", - __func__, __LINE__); - return -ENOMEM; - } - - pti_tty_driver->driver_name = DRIVERNAME; - pti_tty_driver->name = TTYNAME; - pti_tty_driver->major = 0; - pti_tty_driver->minor_start = PTITTY_MINOR_START; - pti_tty_driver->type = TTY_DRIVER_TYPE_SYSTEM; - pti_tty_driver->subtype = SYSTEM_TYPE_SYSCONS; - pti_tty_driver->flags = TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV; - pti_tty_driver->init_termios = tty_std_termios; - - tty_set_operations(pti_tty_driver, &pti_tty_driver_ops); - - retval = tty_register_driver(pti_tty_driver); - if (retval) { - pr_err("%s(%d): TTY registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - - goto put_tty; - } - - retval = pci_register_driver(&pti_pci_driver); - if (retval) { - pr_err("%s(%d): PCI registration failed of pti driver\n", - __func__, __LINE__); - pr_err("%s(%d): Error value returned: %d\n", - __func__, __LINE__, retval); - goto unreg_tty; - } - - return 0; -unreg_tty: - tty_unregister_driver(pti_tty_driver); -put_tty: - put_tty_driver(pti_tty_driver); - pti_tty_driver = NULL; - return retval; -} - -/** - * pti_exit()- Unregisters this module as a tty and pci driver. - */ -static void __exit pti_exit(void) -{ - tty_unregister_driver(pti_tty_driver); - pci_unregister_driver(&pti_pci_driver); - put_tty_driver(pti_tty_driver); -} - -module_init(pti_init); -module_exit(pti_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Ken Mills, Jay Freyensee"); -MODULE_DESCRIPTION("PTI Driver"); - diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile index b3ccae932660..730de6bf048b 100644 --- a/drivers/tty/Makefile +++ b/drivers/tty/Makefile @@ -9,8 +9,6 @@ obj-$(CONFIG_AUDIT) += tty_audit.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o obj-$(CONFIG_N_HDLC) += n_hdlc.o obj-$(CONFIG_N_GSM) += n_gsm.o -obj-$(CONFIG_TRACE_ROUTER) += n_tracerouter.o -obj-$(CONFIG_TRACE_SINK) += n_tracesink.o obj-$(CONFIG_R3964) += n_r3964.o obj-y += vt/ diff --git a/drivers/tty/n_tracerouter.c b/drivers/tty/n_tracerouter.c deleted file mode 100644 index 4479af4d2fa5..000000000000 --- a/drivers/tty/n_tracerouter.c +++ /dev/null @@ -1,233 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * n_tracerouter.c - Trace data router through tty space - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This trace router uses the Linux line discipline framework to route - * trace data coming from a HW Modem to a PTI (Parallel Trace Module) port. - * The solution is not specific to a HW modem and this line disciple can - * be used to route any stream of data in kernel space. - * This is part of a solution for the P1149.7, compact JTAG, standard. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "n_tracesink.h" - -/* - * Other ldisc drivers use 65536 which basically means, - * 'I can always accept 64k' and flow control is off. - * This number is deemed appropriate for this driver. - */ -#define RECEIVE_ROOM 65536 -#define DRIVERNAME "n_tracerouter" - -/* - * struct to hold private configuration data for this ldisc. - * opencalled is used to hold if this ldisc has been opened. - * kref_tty holds the tty reference the ldisc sits on top of. - */ -struct tracerouter_data { - u8 opencalled; - struct tty_struct *kref_tty; -}; -static struct tracerouter_data *tr_data; - -/* lock for when tty reference is being used */ -static DEFINE_MUTEX(routelock); - -/** - * n_tracerouter_open() - Called when a tty is opened by a SW entity. - * @tty: terminal device to the ldisc. - * - * Return: - * 0 for success. - * - * Caveats: This should only be opened one time per SW entity. - */ -static int n_tracerouter_open(struct tty_struct *tty) -{ - int retval = -EEXIST; - - mutex_lock(&routelock); - if (tr_data->opencalled == 0) { - - tr_data->kref_tty = tty_kref_get(tty); - if (tr_data->kref_tty == NULL) { - retval = -EFAULT; - } else { - tr_data->opencalled = 1; - tty->disc_data = tr_data; - tty->receive_room = RECEIVE_ROOM; - tty_driver_flush_buffer(tty); - retval = 0; - } - } - mutex_unlock(&routelock); - return retval; -} - -/** - * n_tracerouter_close() - close connection - * @tty: terminal device to the ldisc. - * - * Called when a software entity wants to close a connection. - */ -static void n_tracerouter_close(struct tty_struct *tty) -{ - struct tracerouter_data *tptr = tty->disc_data; - - mutex_lock(&routelock); - WARN_ON(tptr->kref_tty != tr_data->kref_tty); - tty_driver_flush_buffer(tty); - tty_kref_put(tr_data->kref_tty); - tr_data->kref_tty = NULL; - tr_data->opencalled = 0; - tty->disc_data = NULL; - mutex_unlock(&routelock); -} - -/** - * n_tracerouter_read() - read request from user space - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * function that allows read() functionality in userspace. By default if this - * is not implemented it returns -EIO. This module is functioning like a - * router via n_tracerouter_receivebuf(), and there is no real requirement - * to implement this function. However, an error return value other than - * -EIO should be used just to show that there was an intent not to have - * this function implemented. Return value based on read() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracerouter_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracerouter_write() - Function that allows write() in userspace. - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * By default if this is not implemented, it returns -EIO. - * This should not be implemented, ever, because - * 1. this driver is functioning like a router via - * n_tracerouter_receivebuf() - * 2. No writes to HW will ever go through this line discpline driver. - * However, an error return value other than -EIO should be used - * just to show that there was an intent not to have this function - * implemented. Return value based on write() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracerouter_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracerouter_receivebuf() - Routing function for driver. - * @tty: terminal device passed into the ldisc. It's assumed - * tty will never be NULL. - * @cp: buffer, block of characters to be eventually read by - * someone, somewhere (user read() call or some kernel function). - * @fp: flag buffer. - * @count: number of characters (aka, bytes) in cp. - * - * This function takes the input buffer, cp, and passes it to - * an external API function for processing. - */ -static void n_tracerouter_receivebuf(struct tty_struct *tty, - const unsigned char *cp, - char *fp, int count) -{ - mutex_lock(&routelock); - n_tracesink_datadrain((u8 *) cp, count); - mutex_unlock(&routelock); -} - -/* - * Flush buffer is not impelemented as the ldisc has no internal buffering - * so the tty_driver_flush_buffer() is sufficient for this driver's needs. - */ - -static struct tty_ldisc_ops tty_ptirouter_ldisc = { - .owner = THIS_MODULE, - .magic = TTY_LDISC_MAGIC, - .name = DRIVERNAME, - .open = n_tracerouter_open, - .close = n_tracerouter_close, - .read = n_tracerouter_read, - .write = n_tracerouter_write, - .receive_buf = n_tracerouter_receivebuf -}; - -/** - * n_tracerouter_init - module initialisation - * - * Registers this module as a line discipline driver. - * - * Return: - * 0 for success, any other value error. - */ -static int __init n_tracerouter_init(void) -{ - int retval; - - tr_data = kzalloc(sizeof(struct tracerouter_data), GFP_KERNEL); - if (tr_data == NULL) - return -ENOMEM; - - - /* Note N_TRACEROUTER is defined in linux/tty.h */ - retval = tty_register_ldisc(N_TRACEROUTER, &tty_ptirouter_ldisc); - if (retval < 0) { - pr_err("%s: Registration failed: %d\n", __func__, retval); - kfree(tr_data); - } - return retval; -} - -/** - * n_tracerouter_exit - module unload - * - * Removes this module as a line discipline driver. - */ -static void __exit n_tracerouter_exit(void) -{ - int retval = tty_unregister_ldisc(N_TRACEROUTER); - - if (retval < 0) - pr_err("%s: Unregistration failed: %d\n", __func__, retval); - else - kfree(tr_data); -} - -module_init(n_tracerouter_init); -module_exit(n_tracerouter_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jay Freyensee"); -MODULE_ALIAS_LDISC(N_TRACEROUTER); -MODULE_DESCRIPTION("Trace router ldisc driver"); diff --git a/drivers/tty/n_tracesink.c b/drivers/tty/n_tracesink.c deleted file mode 100644 index d96ba82cc356..000000000000 --- a/drivers/tty/n_tracesink.c +++ /dev/null @@ -1,228 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * n_tracesink.c - Trace data router and sink path through tty space. - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The trace sink uses the Linux line discipline framework to receive - * trace data coming from the PTI source line discipline driver - * to a user-desired tty port, like USB. - * This is to provide a way to extract modem trace data on - * devices that do not have a PTI HW module, or just need modem - * trace data to come out of a different HW output port. - * This is part of a solution for the P1149.7, compact JTAG, standard. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "n_tracesink.h" - -/* - * Other ldisc drivers use 65536 which basically means, - * 'I can always accept 64k' and flow control is off. - * This number is deemed appropriate for this driver. - */ -#define RECEIVE_ROOM 65536 -#define DRIVERNAME "n_tracesink" - -/* - * there is a quirk with this ldisc is he can write data - * to a tty from anyone calling his kernel API, which - * meets customer requirements in the drivers/misc/pti.c - * project. So he needs to know when he can and cannot write when - * the API is called. In theory, the API can be called - * after an init() but before a successful open() which - * would crash the system if tty is not checked. - */ -static struct tty_struct *this_tty; -static DEFINE_MUTEX(writelock); - -/** - * n_tracesink_open() - Called when a tty is opened by a SW entity. - * @tty: terminal device to the ldisc. - * - * Return: - * 0 for success, - * -EFAULT = couldn't get a tty kref n_tracesink will sit - * on top of - * -EEXIST = open() called successfully once and it cannot - * be called again. - * - * Caveats: open() should only be successful the first time a - * SW entity calls it. - */ -static int n_tracesink_open(struct tty_struct *tty) -{ - int retval = -EEXIST; - - mutex_lock(&writelock); - if (this_tty == NULL) { - this_tty = tty_kref_get(tty); - if (this_tty == NULL) { - retval = -EFAULT; - } else { - tty->disc_data = this_tty; - tty_driver_flush_buffer(tty); - retval = 0; - } - } - mutex_unlock(&writelock); - - return retval; -} - -/** - * n_tracesink_close() - close connection - * @tty: terminal device to the ldisc. - * - * Called when a software entity wants to close a connection. - */ -static void n_tracesink_close(struct tty_struct *tty) -{ - mutex_lock(&writelock); - tty_driver_flush_buffer(tty); - tty_kref_put(this_tty); - this_tty = NULL; - tty->disc_data = NULL; - mutex_unlock(&writelock); -} - -/** - * n_tracesink_read() - read request from user space - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * function that allows read() functionality in userspace. By default if this - * is not implemented it returns -EIO. This module is functioning like a - * router via n_tracesink_receivebuf(), and there is no real requirement - * to implement this function. However, an error return value other than - * -EIO should be used just to show that there was an intent not to have - * this function implemented. Return value based on read() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracesink_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracesink_write() - Function that allows write() in userspace. - * @tty: terminal device passed into the ldisc. - * @file: pointer to open file object. - * @buf: pointer to the data buffer that gets eventually returned. - * @nr: number of bytes of the data buffer that is returned. - * - * By default if this is not implemented, it returns -EIO. - * This should not be implemented, ever, because - * 1. this driver is functioning like a router via - * n_tracesink_receivebuf() - * 2. No writes to HW will ever go through this line discpline driver. - * However, an error return value other than -EIO should be used - * just to show that there was an intent not to have this function - * implemented. Return value based on write() man pages. - * - * Return: - * -EINVAL - */ -static ssize_t n_tracesink_write(struct tty_struct *tty, struct file *file, - const unsigned char *buf, size_t nr) { - return -EINVAL; -} - -/** - * n_tracesink_datadrain() - Kernel API function used to route - * trace debugging data to user-defined - * port like USB. - * - * @buf: Trace debuging data buffer to write to tty target - * port. Null value will return with no write occurring. - * @count: Size of buf. Value of 0 or a negative number will - * return with no write occuring. - * - * Caveat: If this line discipline does not set the tty it sits - * on top of via an open() call, this API function will not - * call the tty's write() call because it will have no pointer - * to call the write(). - */ -void n_tracesink_datadrain(u8 *buf, int count) -{ - mutex_lock(&writelock); - - if ((buf != NULL) && (count > 0) && (this_tty != NULL)) - this_tty->ops->write(this_tty, buf, count); - - mutex_unlock(&writelock); -} -EXPORT_SYMBOL_GPL(n_tracesink_datadrain); - -/* - * Flush buffer is not impelemented as the ldisc has no internal buffering - * so the tty_driver_flush_buffer() is sufficient for this driver's needs. - */ - -/* - * tty_ldisc function operations for this driver. - */ -static struct tty_ldisc_ops tty_n_tracesink = { - .owner = THIS_MODULE, - .magic = TTY_LDISC_MAGIC, - .name = DRIVERNAME, - .open = n_tracesink_open, - .close = n_tracesink_close, - .read = n_tracesink_read, - .write = n_tracesink_write -}; - -/** - * n_tracesink_init- module initialisation - * - * Registers this module as a line discipline driver. - * - * Return: - * 0 for success, any other value error. - */ -static int __init n_tracesink_init(void) -{ - /* Note N_TRACESINK is defined in linux/tty.h */ - int retval = tty_register_ldisc(N_TRACESINK, &tty_n_tracesink); - - if (retval < 0) - pr_err("%s: Registration failed: %d\n", __func__, retval); - - return retval; -} - -/** - * n_tracesink_exit - module unload - * - * Removes this module as a line discipline driver. - */ -static void __exit n_tracesink_exit(void) -{ - int retval = tty_unregister_ldisc(N_TRACESINK); - - if (retval < 0) - pr_err("%s: Unregistration failed: %d\n", __func__, retval); -} - -module_init(n_tracesink_init); -module_exit(n_tracesink_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jay Freyensee"); -MODULE_ALIAS_LDISC(N_TRACESINK); -MODULE_DESCRIPTION("Trace sink ldisc driver"); diff --git a/drivers/tty/n_tracesink.h b/drivers/tty/n_tracesink.h deleted file mode 100644 index 7031d515a700..000000000000 --- a/drivers/tty/n_tracesink.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * n_tracesink.h - Kernel driver API to route trace data in kernel space. - * - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The PTI (Parallel Trace Interface) driver directs trace data routed from - * various parts in the system out through the Intel Penwell PTI port and - * out of the mobile device for analysis with a debugging tool - * (Lauterbach, Fido). This is part of a solution for the MIPI P1149.7, - * compact JTAG, standard. - * - * This header file is used by n_tracerouter to be able to send the - * data of it's tty port to the tty port this module sits. This - * mechanism can also be used independent of the PTI module. - * - */ - -#ifndef N_TRACESINK_H_ -#define N_TRACESINK_H_ - -void n_tracesink_datadrain(u8 *buf, int count); - -#endif diff --git a/include/linux/intel-pti.h b/include/linux/intel-pti.h deleted file mode 100644 index fcd841a90f2f..000000000000 --- a/include/linux/intel-pti.h +++ /dev/null @@ -1,35 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) Intel 2011 - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * The PTI (Parallel Trace Interface) driver directs trace data routed from - * various parts in the system out through the Intel Penwell PTI port and - * out of the mobile device for analysis with a debugging tool - * (Lauterbach, Fido). This is part of a solution for the MIPI P1149.7, - * compact JTAG, standard. - * - * This header file will allow other parts of the OS to use the - * interface to write out it's contents for debugging a mobile system. - */ - -#ifndef LINUX_INTEL_PTI_H_ -#define LINUX_INTEL_PTI_H_ - -/* offset for last dword of any PTI message. Part of MIPI P1149.7 */ -#define PTI_LASTDWORD_DTS 0x30 - -/* basic structure used as a write address to the PTI HW */ -struct pti_masterchannel { - u8 master; - u8 channel; -}; - -/* the following functions are defined in misc/pti.c */ -void pti_writedata(struct pti_masterchannel *mc, u8 *buf, int count); -struct pti_masterchannel *pti_request_masterchannel(u8 type, - const char *thread_name); -void pti_release_masterchannel(struct pti_masterchannel *mc); - -#endif /* LINUX_INTEL_PTI_H_ */ From 487709fa1be2f27aa8e7de6c60587b4302a21467 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Sun, 10 Jan 2021 19:53:57 +0800 Subject: [PATCH 135/633] misc: pvpanic: introduce device capability According to pvpanic spec: https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/specs/pvpanic.txt The guest should determine pvpanic capability by RDPT, so initialize capability during device probing. There is no need to register panic notifier callback function if no events supported. Before sending event to host side, check capability firstly. Suggested by Greg KH, use sysfs to expose capability to user space, also add new sysfs attribute in document. Signed-off-by: zhenwei pi Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20210110115358.79100-2-pizhenwei@bytedance.com Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-bus-pci-devices-pvpanic | 12 +++++++ drivers/misc/pvpanic.c | 31 ++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic new file mode 100644 index 000000000000..79b7dc31cd55 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic @@ -0,0 +1,12 @@ +What: /sys/devices/pci0000:00/*/QEMU0001:00/capability +Date: Jan 2021 +Contact: zhenwei pi +Description: + Read-only attribute. Capabilities of pvpanic device which + are supported by QEMU. + + Format: %x. + + Detailed bit definition refers to section + from pvpanic device specification: + https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/specs/pvpanic.txt diff --git a/drivers/misc/pvpanic.c b/drivers/misc/pvpanic.c index 41cab297d66e..80cdfa8f951a 100644 --- a/drivers/misc/pvpanic.c +++ b/drivers/misc/pvpanic.c @@ -19,6 +19,20 @@ #include static void __iomem *base; +static unsigned int capability = PVPANIC_PANICKED | PVPANIC_CRASH_LOADED; + +static ssize_t capability_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%x", capability); +} +static DEVICE_ATTR_RO(capability); + +static struct attribute *pvpanic_dev_attrs[] = { + &dev_attr_capability.attr, + NULL +}; +ATTRIBUTE_GROUPS(pvpanic_dev); MODULE_AUTHOR("Hu Tao "); MODULE_DESCRIPTION("pvpanic device driver"); @@ -27,7 +41,8 @@ MODULE_LICENSE("GPL"); static void pvpanic_send_event(unsigned int event) { - iowrite8(event, base); + if (event & capability) + iowrite8(event, base); } static int @@ -73,8 +88,12 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) return -EINVAL; } - atomic_notifier_chain_register(&panic_notifier_list, - &pvpanic_panic_nb); + /* initlize capability by RDPT */ + capability &= ioread8(base); + + if (capability) + atomic_notifier_chain_register(&panic_notifier_list, + &pvpanic_panic_nb); return 0; } @@ -82,8 +101,9 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) static int pvpanic_mmio_remove(struct platform_device *pdev) { - atomic_notifier_chain_unregister(&panic_notifier_list, - &pvpanic_panic_nb); + if (capability) + atomic_notifier_chain_unregister(&panic_notifier_list, + &pvpanic_panic_nb); return 0; } @@ -104,6 +124,7 @@ static struct platform_driver pvpanic_mmio_driver = { .name = "pvpanic-mmio", .of_match_table = pvpanic_mmio_match, .acpi_match_table = pvpanic_device_ids, + .dev_groups = pvpanic_dev_groups, }, .probe = pvpanic_mmio_probe, .remove = pvpanic_mmio_remove, From 8d6da6575ffec171161d36a06c015142b0049637 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Sun, 10 Jan 2021 19:53:58 +0800 Subject: [PATCH 136/633] misc: pvpanic: introduce events device attribue Suggested by Paolo & Greg, add 'events' device attribute that can be used to limit which capabilities the driver uses. Finally, the pvpanic guest driver works by the limitation of both device capability and user setting. Signed-off-by: zhenwei pi Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20210110115358.79100-3-pizhenwei@bytedance.com Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-bus-pci-devices-pvpanic | 12 ++++++++ drivers/misc/pvpanic.c | 30 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic index 79b7dc31cd55..1936f7324155 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic +++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic @@ -10,3 +10,15 @@ Description: Detailed bit definition refers to section from pvpanic device specification: https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/specs/pvpanic.txt + +What: /sys/devices/pci0000:00/*/QEMU0001:00/events +Date: Jan 2021 +Contact: zhenwei pi +Description: + RW attribute. Set/get which features in-use. This attribute + is used to enable/disable feature(s) of pvpanic device. + Notice that this value should be a subset of capability. + + Format: %x. + + Also refer to pvpanic device specification. diff --git a/drivers/misc/pvpanic.c b/drivers/misc/pvpanic.c index 80cdfa8f951a..b1e4922a7fda 100644 --- a/drivers/misc/pvpanic.c +++ b/drivers/misc/pvpanic.c @@ -20,6 +20,7 @@ static void __iomem *base; static unsigned int capability = PVPANIC_PANICKED | PVPANIC_CRASH_LOADED; +static unsigned int events; static ssize_t capability_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -28,8 +29,34 @@ static ssize_t capability_show(struct device *dev, } static DEVICE_ATTR_RO(capability); +static ssize_t events_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%x", events); +} + +static ssize_t events_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned int tmp; + int err; + + err = kstrtouint(buf, 16, &tmp); + if (err) + return err; + + if ((tmp & capability) != tmp) + return -EINVAL; + + events = tmp; + + return count; + +} +static DEVICE_ATTR_RW(events); + static struct attribute *pvpanic_dev_attrs[] = { &dev_attr_capability.attr, + &dev_attr_events.attr, NULL }; ATTRIBUTE_GROUPS(pvpanic_dev); @@ -41,7 +68,7 @@ MODULE_LICENSE("GPL"); static void pvpanic_send_event(unsigned int event) { - if (event & capability) + if (event & capability & events) iowrite8(event, base); } @@ -90,6 +117,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev) /* initlize capability by RDPT */ capability &= ioread8(base); + events = capability; if (capability) atomic_notifier_chain_register(&panic_notifier_list, From c35901b39ddc20077f4ae7b9f7bf344487f62212 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 21 Jan 2021 16:07:54 +0100 Subject: [PATCH 137/633] virt: vbox: Do not use wait_event_interruptible when called from kernel context Do not use wait_event_interruptible when vbg_hgcm_call() gets called from kernel-context, such as it being called by the vboxsf filesystem code. This fixes some filesystem related system calls on shared folders unexpectedly failing with -EINTR. Fixes: 0532a1b0d045 ("virt: vbox: Implement passing requestor info to the host for VirtualBox 6.0.x") Reported-by: Ludovic Pouzenc Signed-off-by: Hans de Goede Cc: stable Link: https://lore.kernel.org/r/20210121150754.147598-1-hdegoede@redhat.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/vboxguest/vboxguest_utils.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/virt/vboxguest/vboxguest_utils.c b/drivers/virt/vboxguest/vboxguest_utils.c index ea05af41ec69..8d195e3f8301 100644 --- a/drivers/virt/vboxguest/vboxguest_utils.c +++ b/drivers/virt/vboxguest/vboxguest_utils.c @@ -468,7 +468,7 @@ static int hgcm_cancel_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call) * Cancellation fun. */ static int vbg_hgcm_do_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call, - u32 timeout_ms, bool *leak_it) + u32 timeout_ms, bool interruptible, bool *leak_it) { int rc, cancel_rc, ret; long timeout; @@ -495,10 +495,15 @@ static int vbg_hgcm_do_call(struct vbg_dev *gdev, struct vmmdev_hgcm_call *call, else timeout = msecs_to_jiffies(timeout_ms); - timeout = wait_event_interruptible_timeout( - gdev->hgcm_wq, - hgcm_req_done(gdev, &call->header), - timeout); + if (interruptible) { + timeout = wait_event_interruptible_timeout(gdev->hgcm_wq, + hgcm_req_done(gdev, &call->header), + timeout); + } else { + timeout = wait_event_timeout(gdev->hgcm_wq, + hgcm_req_done(gdev, &call->header), + timeout); + } /* timeout > 0 means hgcm_req_done has returned true, so success */ if (timeout > 0) @@ -631,7 +636,8 @@ int vbg_hgcm_call(struct vbg_dev *gdev, u32 requestor, u32 client_id, hgcm_call_init_call(call, client_id, function, parms, parm_count, bounce_bufs); - ret = vbg_hgcm_do_call(gdev, call, timeout_ms, &leak_it); + ret = vbg_hgcm_do_call(gdev, call, timeout_ms, + requestor & VMMDEV_REQUESTOR_USERMODE, &leak_it); if (ret == 0) { *vbox_status = call->header.result; ret = hgcm_call_copy_back_result(call, parms, parm_count, From d0259c42abff51b586496a0594933e394efefbc5 Mon Sep 17 00:00:00 2001 From: Bert Vermeulen Date: Thu, 21 Jan 2021 09:54:12 +0100 Subject: [PATCH 138/633] spdxcheck.py: Use Python 3 Python 2.x has been officially EOL'ed for some time, and in any case the git module for it is hard to come by. Signed-off-by: Bert Vermeulen Link: https://lore.kernel.org/r/20210121085412.265400-1-bert@biot.com Signed-off-by: Greg Kroah-Hartman --- scripts/spdxcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spdxcheck.py b/scripts/spdxcheck.py index bc87200f9c7c..cbdb5c83c08f 100755 --- a/scripts/spdxcheck.py +++ b/scripts/spdxcheck.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 # Copyright Thomas Gleixner From afe9017901761d04a106916e02619caf0d2afbf5 Mon Sep 17 00:00:00 2001 From: Alexander Kapshuk Date: Fri, 8 Jan 2021 13:26:26 +0200 Subject: [PATCH 139/633] ver_linux: Eliminate duplicate code in ldconfig processing logic The code that acquires the version strings for libc and libcpp is identical, as is the printversion call. The only difference being the name of the library being printed. Refactor the code by unifying the bits that are common to both libraries. Signed-off-by: Alexander Kapshuk Link: https://lore.kernel.org/r/20210108112626.8623-1-alexander.kapshuk@gmail.com Signed-off-by: Greg Kroah-Hartman --- scripts/ver_linux | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/ver_linux b/scripts/ver_linux index 0968a3070eff..a92acc703f9b 100755 --- a/scripts/ver_linux +++ b/scripts/ver_linux @@ -15,7 +15,7 @@ BEGIN { vernum = "[0-9]+([.]?[0-9]+)+" libc = "libc[.]so[.][0-9]+$" - libcpp = "(libg|stdc)[+]+[.]so[.][0-9]+$" + libcpp = "(libg|stdc)[+]+[.]so([.][0-9]+)+$" printversion("GNU C", version("gcc -dumpversion")) printversion("GNU Make", version("make --version")) @@ -37,12 +37,10 @@ BEGIN { printversion("Bison", version("bison --version")) printversion("Flex", version("flex --version")) - while ("ldconfig -p 2>/dev/null" | getline > 0) { - if ($NF ~ libc && !seen[ver = version("readlink " $NF)]++) - printversion("Linux C Library", ver) - else if ($NF ~ libcpp && !seen[ver = version("readlink " $NF)]++) - printversion("Linux C++ Library", ver) - } + while ("ldconfig -p 2>/dev/null" | getline > 0) + if ($NF ~ libc || $NF ~ libcpp) + if (!seen[ver = version("readlink " $NF)]++) + printversion("Linux C" ($NF ~ libcpp? "++" : "") " Library", ver) printversion("Dynamic linker (ldd)", version("ldd --version")) printversion("Procps", version("ps --version")) From 2c104a469a4ce276117acda94ccf922c3b82d735 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 14 Jan 2021 16:20:14 -0800 Subject: [PATCH 140/633] platform/goldfish: Convert pipe tasklet to threaded irq Tasklets have long been deprecated as being too heavy on the system by running in irq context - and this is not a performance critical path. If a higher priority process wants to run, it must wait for the tasklet to finish before doing so. A more suitable equivalent is to converted to threaded irq instead and deal with the signaled pipes in task context. Signed-off-by: Davidlohr Bueso Link: https://lore.kernel.org/r/20210115002014.117528-1-dave@stgolabs.net Signed-off-by: Greg Kroah-Hartman --- drivers/platform/goldfish/goldfish_pipe.c | 28 +++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/drivers/platform/goldfish/goldfish_pipe.c b/drivers/platform/goldfish/goldfish_pipe.c index 1ab207ec9c94..b67539f9848c 100644 --- a/drivers/platform/goldfish/goldfish_pipe.c +++ b/drivers/platform/goldfish/goldfish_pipe.c @@ -212,9 +212,6 @@ struct goldfish_pipe_dev { int version; unsigned char __iomem *base; - /* an irq tasklet to run goldfish_interrupt_task */ - struct tasklet_struct irq_tasklet; - struct miscdevice miscdev; }; @@ -577,10 +574,10 @@ static struct goldfish_pipe *signalled_pipes_pop_front( return pipe; } -static void goldfish_interrupt_task(unsigned long dev_addr) +static irqreturn_t goldfish_interrupt_task(int irq, void *dev_addr) { /* Iterate over the signalled pipes and wake them one by one */ - struct goldfish_pipe_dev *dev = (struct goldfish_pipe_dev *)dev_addr; + struct goldfish_pipe_dev *dev = dev_addr; struct goldfish_pipe *pipe; int wakes; @@ -599,13 +596,14 @@ static void goldfish_interrupt_task(unsigned long dev_addr) */ wake_up_interruptible(&pipe->wake_queue); } + return IRQ_HANDLED; } static void goldfish_pipe_device_deinit(struct platform_device *pdev, struct goldfish_pipe_dev *dev); /* - * The general idea of the interrupt handling: + * The general idea of the (threaded) interrupt handling: * * 1. device raises an interrupt if there's at least one signalled pipe * 2. IRQ handler reads the signalled pipes and their count from the device @@ -614,8 +612,8 @@ static void goldfish_pipe_device_deinit(struct platform_device *pdev, * otherwise it leaves it raised, so IRQ handler will be called * again for the next chunk * 4. IRQ handler adds all returned pipes to the device's signalled pipes list - * 5. IRQ handler launches a tasklet to process the signalled pipes from the - * list in a separate context + * 5. IRQ handler defers processing the signalled pipes from the list in a + * separate context */ static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) { @@ -645,8 +643,7 @@ static irqreturn_t goldfish_pipe_interrupt(int irq, void *dev_id) spin_unlock_irqrestore(&dev->lock, flags); - tasklet_schedule(&dev->irq_tasklet); - return IRQ_HANDLED; + return IRQ_WAKE_THREAD; } static int get_free_pipe_id_locked(struct goldfish_pipe_dev *dev) @@ -811,12 +808,10 @@ static int goldfish_pipe_device_init(struct platform_device *pdev, { int err; - tasklet_init(&dev->irq_tasklet, &goldfish_interrupt_task, - (unsigned long)dev); - - err = devm_request_irq(&pdev->dev, dev->irq, - goldfish_pipe_interrupt, - IRQF_SHARED, "goldfish_pipe", dev); + err = devm_request_threaded_irq(&pdev->dev, dev->irq, + goldfish_pipe_interrupt, + goldfish_interrupt_task, + IRQF_SHARED, "goldfish_pipe", dev); if (err) { dev_err(&pdev->dev, "unable to allocate IRQ for v2\n"); return err; @@ -874,7 +869,6 @@ static void goldfish_pipe_device_deinit(struct platform_device *pdev, struct goldfish_pipe_dev *dev) { misc_deregister(&dev->miscdev); - tasklet_kill(&dev->irq_tasklet); kfree(dev->pipes); free_page((unsigned long)dev->buffers); } From 2f6055c26f1913763eabc66c7c27d0693561e966 Mon Sep 17 00:00:00 2001 From: Ivan Zaentsev Date: Thu, 21 Jan 2021 12:30:21 +0300 Subject: [PATCH 141/633] w1: w1_therm: Fix conversion result for negative temperatures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DS18B20 device driver returns an incorrect value for negative temperatures due to a missing sign-extension in w1_DS18B20_convert_temp(). Fix by using s16 temperature value when converting to int. Fixes: 9ace0b4dab1c (w1: w1_therm: Add support for GXCAS GX20MH01 device.) Cc: stable Reported-by: Paweł Marciniak Signed-off-by: Ivan Zaentsev Link: https://lore.kernel.org/r/20210121093021.224764-1-ivan.zaentsev@wirenboard.ru Signed-off-by: Greg Kroah-Hartman --- drivers/w1/slaves/w1_therm.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c index 3712b1e6dc71..976eea28f268 100644 --- a/drivers/w1/slaves/w1_therm.c +++ b/drivers/w1/slaves/w1_therm.c @@ -667,28 +667,24 @@ static inline int w1_DS18B20_get_resolution(struct w1_slave *sl) */ static inline int w1_DS18B20_convert_temp(u8 rom[9]) { - int t; - u32 bv; + u16 bv; + s16 t; + + /* Signed 16-bit value to unsigned, cpu order */ + bv = le16_to_cpup((__le16 *)rom); /* Config register bit R2 = 1 - GX20MH01 in 13 or 14 bit resolution mode */ if (rom[4] & 0x80) { - /* Signed 16-bit value to unsigned, cpu order */ - bv = le16_to_cpup((__le16 *)rom); - /* Insert two temperature bits from config register */ /* Avoid arithmetic shift of signed value */ bv = (bv << 2) | (rom[4] & 3); - - t = (int) sign_extend32(bv, 17); /* Degrees, lowest bit is 2^-6 */ - return (t*1000)/64; /* Millidegrees */ + t = (s16) bv; /* Degrees, lowest bit is 2^-6 */ + return (int)t * 1000 / 64; /* Sign-extend to int; millidegrees */ } - - t = (int)le16_to_cpup((__le16 *)rom); - return t*1000/16; + t = (s16)bv; /* Degrees, lowest bit is 2^-4 */ + return (int)t * 1000 / 16; /* Sign-extend to int; millidegrees */ } - - /** * w1_DS18S20_convert_temp() - temperature computation for DS18S20 * @rom: data read from device RAM (8 data bytes + 1 CRC byte) From 8544717cdacc2f33f0f53a3b34c5125b37e13ce9 Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 14 Jan 2021 19:07:48 +0200 Subject: [PATCH 142/633] bus: fsl-mc: move fsl_mc_command struct in a uapi header Define "struct fsl_mc_command" as a structure that can cross the user/kernel boundary. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210114170752.2927915-2-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 1 + include/linux/fsl/mc.h | 8 +------- include/uapi/linux/fsl_mc.h | 25 +++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 include/uapi/linux/fsl_mc.h diff --git a/MAINTAINERS b/MAINTAINERS index 23273a59da6f..128455811980 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14652,6 +14652,7 @@ S: Maintained F: Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt F: Documentation/networking/device_drivers/ethernet/freescale/dpaa2/overview.rst F: drivers/bus/fsl-mc/ +F: include/uapi/linux/fsl_mc.h QT1010 MEDIA DRIVER M: Antti Palosaari diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index db244874e834..63b56aba925a 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -13,6 +13,7 @@ #include #include #include +#include #define FSL_MC_VENDOR_FREESCALE 0x1957 @@ -209,8 +210,6 @@ struct fsl_mc_device { #define to_fsl_mc_device(_dev) \ container_of(_dev, struct fsl_mc_device, dev) -#define MC_CMD_NUM_OF_PARAMS 7 - struct mc_cmd_header { u8 src_id; u8 flags_hw; @@ -220,11 +219,6 @@ struct mc_cmd_header { __le16 cmd_id; }; -struct fsl_mc_command { - __le64 header; - __le64 params[MC_CMD_NUM_OF_PARAMS]; -}; - enum mc_cmd_status { MC_CMD_STATUS_OK = 0x0, /* Completed successfully */ MC_CMD_STATUS_READY = 0x1, /* Ready to be processed */ diff --git a/include/uapi/linux/fsl_mc.h b/include/uapi/linux/fsl_mc.h new file mode 100644 index 000000000000..cf56d46f052e --- /dev/null +++ b/include/uapi/linux/fsl_mc.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Management Complex (MC) userspace public interface + * + * Copyright 2021 NXP + * + */ +#ifndef _UAPI_FSL_MC_H_ +#define _UAPI_FSL_MC_H_ + +#include + +#define MC_CMD_NUM_OF_PARAMS 7 + +/** + * struct fsl_mc_command - Management Complex (MC) command structure + * @header: MC command header + * @params: MC command parameters + */ +struct fsl_mc_command { + __le64 header; + __le64 params[MC_CMD_NUM_OF_PARAMS]; +}; + +#endif /* _UAPI_FSL_MC_H_ */ From 46707989269a251e5e4ca295975525605bc8afef Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 14 Jan 2021 19:07:49 +0200 Subject: [PATCH 143/633] bus: fsl-mc: export mc_cmd_hdr_read_cmdid() to the fsl-mc bus Export the mc_cmd_hdr_read_cmdid() function to the entire fsl-mc bus since it will be needed in the following patch. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210114170752.2927915-3-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/bus/fsl-mc/fsl-mc-private.h | 2 ++ drivers/bus/fsl-mc/mc-sys.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index c932387641fa..be686c363121 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -612,4 +612,6 @@ void fsl_mc_get_root_dprc(struct device *dev, struct fsl_mc_device *fsl_mc_device_lookup(struct fsl_mc_obj_desc *obj_desc, struct fsl_mc_device *mc_bus_dev); +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd); + #endif /* _FSL_MC_PRIVATE_H_ */ diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index 85a0225db522..b291b35e3884 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -35,7 +35,7 @@ static enum mc_cmd_status mc_cmd_hdr_read_status(struct fsl_mc_command *cmd) return (enum mc_cmd_status)hdr->status; } -static u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) { struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; u16 cmd_id = le16_to_cpu(hdr->cmd_id); From 2cf1e703f066cfa82eb5a358ae84c29fe15a3b3a Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 14 Jan 2021 19:07:50 +0200 Subject: [PATCH 144/633] bus: fsl-mc: add fsl-mc userspace support Adding userspace support for the MC (Management Complex) means exporting an ioctl capable device file representing the root resource container. This new functionality in the fsl-mc bus driver intends to provide userspace applications an interface to interact with the MC firmware. Commands that are composed in userspace are sent to the MC firmware through the FSL_MC_SEND_MC_COMMAND ioctl. By default the implicit MC I/O portal is used for this operation, but if the implicit one is busy, a dynamic portal is allocated and then freed upon execution. The command received through the ioctl interface is checked against a known whitelist of accepted MC commands. Commands that attempt a change in hardware configuration will need CAP_NET_ADMIN, while commands used in debugging do not need it. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210114170752.2927915-4-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- .../userspace-api/ioctl/ioctl-number.rst | 1 + drivers/bus/fsl-mc/Kconfig | 7 + drivers/bus/fsl-mc/Makefile | 3 + drivers/bus/fsl-mc/dprc-driver.c | 12 + drivers/bus/fsl-mc/fsl-mc-private.h | 39 ++ drivers/bus/fsl-mc/fsl-mc-uapi.c | 547 ++++++++++++++++++ include/uapi/linux/fsl_mc.h | 9 + 7 files changed, 618 insertions(+) create mode 100644 drivers/bus/fsl-mc/fsl-mc-uapi.c diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index a4c75a28c839..19d730f9d136 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -180,6 +180,7 @@ Code Seq# Include File Comments 'R' 00-1F linux/random.h conflict! 'R' 01 linux/rfkill.h conflict! 'R' C0-DF net/bluetooth/rfcomm.h +'R' E0 uapi/linux/fsl_mc.h 'S' all linux/cdrom.h conflict! 'S' 80-81 scsi/scsi_ioctl.h conflict! 'S' 82-FF scsi/scsi.h conflict! diff --git a/drivers/bus/fsl-mc/Kconfig b/drivers/bus/fsl-mc/Kconfig index c23c77c9b705..b1fd55901c50 100644 --- a/drivers/bus/fsl-mc/Kconfig +++ b/drivers/bus/fsl-mc/Kconfig @@ -14,3 +14,10 @@ config FSL_MC_BUS architecture. The fsl-mc bus driver handles discovery of DPAA2 objects (which are represented as Linux devices) and binding objects to drivers. + +config FSL_MC_UAPI_SUPPORT + bool "Management Complex (MC) userspace support" + depends on FSL_MC_BUS + help + Provides userspace support for interrogating, creating, destroying or + configuring DPAA2 objects exported by the Management Complex. diff --git a/drivers/bus/fsl-mc/Makefile b/drivers/bus/fsl-mc/Makefile index 3c518c7e8374..4ae292a30e53 100644 --- a/drivers/bus/fsl-mc/Makefile +++ b/drivers/bus/fsl-mc/Makefile @@ -16,3 +16,6 @@ mc-bus-driver-objs := fsl-mc-bus.o \ fsl-mc-allocator.o \ fsl-mc-msi.o \ dpmcp.o + +# MC userspace support +obj-$(CONFIG_FSL_MC_UAPI_SUPPORT) += fsl-mc-uapi.o diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 68488a7ad0d6..ccec375095f2 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -603,6 +603,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev) struct irq_domain *mc_msi_domain; bool mc_io_created = false; bool msi_domain_set = false; + bool uapi_created = false; u16 major_ver, minor_ver; size_t region_size; int error; @@ -635,6 +636,11 @@ int dprc_setup(struct fsl_mc_device *mc_dev) return error; mc_io_created = true; + } else { + error = fsl_mc_uapi_create_device_file(mc_bus); + if (error < 0) + return -EPROBE_DEFER; + uapi_created = true; } mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev); @@ -692,6 +698,9 @@ int dprc_setup(struct fsl_mc_device *mc_dev) mc_dev->mc_io = NULL; } + if (uapi_created) + fsl_mc_uapi_remove_device_file(mc_bus); + return error; } EXPORT_SYMBOL_GPL(dprc_setup); @@ -763,6 +772,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) int dprc_cleanup(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* this function should be called only for DPRCs, it @@ -793,6 +803,8 @@ int dprc_cleanup(struct fsl_mc_device *mc_dev) if (!fsl_mc_is_root_dprc(&mc_dev->dev)) { fsl_destroy_mc_io(mc_dev->mc_io); mc_dev->mc_io = NULL; + } else { + fsl_mc_uapi_remove_device_file(mc_bus); } return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index be686c363121..6293a24de456 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -10,6 +10,8 @@ #include #include +#include +#include /* * Data Path Management Complex (DPMNG) General API @@ -542,6 +544,22 @@ struct fsl_mc_resource_pool { struct fsl_mc_bus *mc_bus; }; +/** + * struct fsl_mc_uapi - information associated with a device file + * @misc: struct miscdevice linked to the root dprc + * @device: newly created device in /dev + * @mutex: mutex lock to serialize the open/release operations + * @local_instance_in_use: local MC I/O instance in use or not + * @static_mc_io: pointer to the static MC I/O object + */ +struct fsl_mc_uapi { + struct miscdevice misc; + struct device *device; + struct mutex mutex; /* serialize open/release operations */ + u32 local_instance_in_use; + struct fsl_mc_io *static_mc_io; +}; + /** * struct fsl_mc_bus - logical bus that corresponds to a physical DPRC * @mc_dev: fsl-mc device for the bus device itself. @@ -551,6 +569,7 @@ struct fsl_mc_resource_pool { * @irq_resources: Pointer to array of IRQ objects for the IRQ pool * @scan_mutex: Serializes bus scanning * @dprc_attr: DPRC attributes + * @uapi_misc: struct that abstracts the interaction with userspace */ struct fsl_mc_bus { struct fsl_mc_device mc_dev; @@ -558,6 +577,7 @@ struct fsl_mc_bus { struct fsl_mc_device_irq *irq_resources; struct mutex scan_mutex; /* serializes bus scanning */ struct dprc_attributes dprc_attr; + struct fsl_mc_uapi uapi_misc; }; #define to_fsl_mc_bus(_mc_dev) \ @@ -614,4 +634,23 @@ struct fsl_mc_device *fsl_mc_device_lookup(struct fsl_mc_obj_desc *obj_desc, u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd); +#ifdef CONFIG_FSL_MC_UAPI_SUPPORT + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus); + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus); + +#else + +static inline int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + return 0; +} + +static inline void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ +} + +#endif + #endif /* _FSL_MC_PRIVATE_H_ */ diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c new file mode 100644 index 000000000000..eeb988c9f4bb --- /dev/null +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Management Complex (MC) userspace support + * + * Copyright 2021 NXP + * + */ + +#include +#include +#include +#include + +#include "fsl-mc-private.h" + +struct uapi_priv_data { + struct fsl_mc_uapi *uapi; + struct fsl_mc_io *mc_io; +}; + +struct fsl_mc_cmd_desc { + u16 cmdid_value; + u16 cmdid_mask; + int size; + bool token; + int flags; +}; + +#define FSL_MC_CHECK_MODULE_ID BIT(0) +#define FSL_MC_CAP_NET_ADMIN_NEEDED BIT(1) + +enum fsl_mc_cmd_index { + DPDBG_DUMP = 0, + DPDBG_SET, + DPRC_GET_CONTAINER_ID, + DPRC_CREATE_CONT, + DPRC_DESTROY_CONT, + DPRC_ASSIGN, + DPRC_UNASSIGN, + DPRC_GET_OBJ_COUNT, + DPRC_GET_OBJ, + DPRC_GET_RES_COUNT, + DPRC_GET_RES_IDS, + DPRC_SET_OBJ_LABEL, + DPRC_SET_LOCKED, + DPRC_CONNECT, + DPRC_DISCONNECT, + DPRC_GET_POOL, + DPRC_GET_POOL_COUNT, + DPRC_GET_CONNECTION, + DPCI_GET_LINK_STATE, + DPCI_GET_PEER_ATTR, + DPAIOP_GET_SL_VERSION, + DPAIOP_GET_STATE, + DPMNG_GET_VERSION, + DPSECI_GET_TX_QUEUE, + DPMAC_GET_COUNTER, + DPMAC_GET_MAC_ADDR, + DPNI_SET_PRIM_MAC, + DPNI_GET_PRIM_MAC, + DPNI_GET_STATISTICS, + DPNI_GET_LINK_STATE, + GET_ATTR, + GET_IRQ_MASK, + GET_IRQ_STATUS, + CLOSE, + OPEN, + GET_API_VERSION, + DESTROY, + CREATE, +}; + +static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { + [DPDBG_DUMP] = { + .cmdid_value = 0x1300, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPDBG_SET] = { + .cmdid_value = 0x1400, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPRC_GET_CONTAINER_ID] = { + .cmdid_value = 0x8300, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPRC_CREATE_CONT] = { + .cmdid_value = 0x1510, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DESTROY_CONT] = { + .cmdid_value = 0x1520, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_ASSIGN] = { + .cmdid_value = 0x1570, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_UNASSIGN] = { + .cmdid_value = 0x1580, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_OBJ_COUNT] = { + .cmdid_value = 0x1590, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + }, + [DPRC_GET_OBJ] = { + .cmdid_value = 0x15A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_RES_COUNT] = { + .cmdid_value = 0x15B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + [DPRC_GET_RES_IDS] = { + .cmdid_value = 0x15C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + }, + [DPRC_SET_OBJ_LABEL] = { + .cmdid_value = 0x1610, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 48, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_SET_LOCKED] = { + .cmdid_value = 0x16B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_CONNECT] = { + .cmdid_value = 0x1670, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 56, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DISCONNECT] = { + .cmdid_value = 0x1680, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_POOL] = { + .cmdid_value = 0x1690, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_POOL_COUNT] = { + .cmdid_value = 0x16A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPRC_GET_CONNECTION] = { + .cmdid_value = 0x16C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + + [DPCI_GET_LINK_STATE] = { + .cmdid_value = 0x0E10, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPCI_GET_PEER_ATTR] = { + .cmdid_value = 0x0E20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_SL_VERSION] = { + .cmdid_value = 0x2820, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_STATE] = { + .cmdid_value = 0x2830, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPMNG_GET_VERSION] = { + .cmdid_value = 0x8310, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPSECI_GET_TX_QUEUE] = { + .cmdid_value = 0x1970, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPMAC_GET_COUNTER] = { + .cmdid_value = 0x0c40, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 9, + }, + [DPMAC_GET_MAC_ADDR] = { + .cmdid_value = 0x0c50, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_SET_PRIM_MAC] = { + .cmdid_value = 0x2240, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPNI_GET_PRIM_MAC] = { + .cmdid_value = 0x2250, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_STATISTICS] = { + .cmdid_value = 0x25D0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPNI_GET_LINK_STATE] = { + .cmdid_value = 0x2150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [GET_ATTR] = { + .cmdid_value = 0x0040, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [GET_IRQ_MASK] = { + .cmdid_value = 0x0150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [GET_IRQ_STATUS] = { + .cmdid_value = 0x0160, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [CLOSE] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + + /* Common commands amongst all types of objects. Must be checked last. */ + [OPEN] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [GET_API_VERSION] = { + .cmdid_value = 0xA000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 8, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [DESTROY] = { + .cmdid_value = 0x9800, + .cmdid_mask = 0xFC00, + .token = true, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [CREATE] = { + .cmdid_value = 0x9000, + .cmdid_mask = 0xFC00, + .token = true, + .size = 64, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, +}; + +#define FSL_MC_NUM_ACCEPTED_CMDS ARRAY_SIZE(fsl_mc_accepted_cmds) + +#define FSL_MC_MAX_MODULE_ID 0x10 + +static int fsl_mc_command_check(struct fsl_mc_device *mc_dev, + struct fsl_mc_command *mc_cmd) +{ + struct fsl_mc_cmd_desc *desc = NULL; + int mc_cmd_max_size, i; + bool token_provided; + u16 cmdid, module_id; + char *mc_cmd_end; + char sum = 0; + + /* Check if this is an accepted MC command */ + cmdid = mc_cmd_hdr_read_cmdid(mc_cmd); + for (i = 0; i < FSL_MC_NUM_ACCEPTED_CMDS; i++) { + desc = &fsl_mc_accepted_cmds[i]; + if ((cmdid & desc->cmdid_mask) == desc->cmdid_value) + break; + } + if (!desc) { + dev_err(&mc_dev->dev, "MC command 0x%04x: cmdid not accepted\n", cmdid); + return -EACCES; + } + + /* Check if the size of the command is honored. Anything beyond the + * last valid byte of the command should be zeroed. + */ + mc_cmd_max_size = sizeof(*mc_cmd); + mc_cmd_end = ((char *)mc_cmd) + desc->size; + for (i = desc->size; i < mc_cmd_max_size; i++) + sum |= *mc_cmd_end++; + if (sum) { + dev_err(&mc_dev->dev, "MC command 0x%04x: garbage beyond max size of %d bytes!\n", + cmdid, desc->size); + return -EACCES; + } + + /* Some MC commands request a token to be passed so that object + * identification is possible. Check if the token passed in the command + * is as expected. + */ + token_provided = mc_cmd_hdr_read_token(mc_cmd) ? true : false; + if (token_provided != desc->token) { + dev_err(&mc_dev->dev, "MC command 0x%04x: token 0x%04x is invalid!\n", + cmdid, mc_cmd_hdr_read_token(mc_cmd)); + return -EACCES; + } + + /* If needed, check if the module ID passed is valid */ + if (desc->flags & FSL_MC_CHECK_MODULE_ID) { + /* The module ID is represented by bits [4:9] from the cmdid */ + module_id = (cmdid & GENMASK(9, 4)) >> 4; + if (module_id == 0 || module_id > FSL_MC_MAX_MODULE_ID) { + dev_err(&mc_dev->dev, "MC command 0x%04x: unknown module ID 0x%x\n", + cmdid, module_id); + return -EACCES; + } + } + + /* Some commands alter how hardware resources are managed. For these + * commands, check for CAP_NET_ADMIN. + */ + if (desc->flags & FSL_MC_CAP_NET_ADMIN_NEEDED) { + if (!capable(CAP_NET_ADMIN)) { + dev_err(&mc_dev->dev, "MC command 0x%04x: needs CAP_NET_ADMIN!\n", + cmdid); + return -EPERM; + } + } + + return 0; +} + +static int fsl_mc_uapi_send_command(struct fsl_mc_device *mc_dev, unsigned long arg, + struct fsl_mc_io *mc_io) +{ + struct fsl_mc_command mc_cmd; + int error; + + error = copy_from_user(&mc_cmd, (void __user *)arg, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + error = fsl_mc_command_check(mc_dev, &mc_cmd); + if (error) + return error; + + error = mc_send_command(mc_io, &mc_cmd); + if (error) + return error; + + error = copy_to_user((void __user *)arg, &mc_cmd, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + return 0; +} + +static int fsl_mc_uapi_dev_open(struct inode *inode, struct file *filep) +{ + struct fsl_mc_device *root_mc_device; + struct uapi_priv_data *priv_data; + struct fsl_mc_io *dynamic_mc_io; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_bus *mc_bus; + int error; + + priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); + if (!priv_data) + return -ENOMEM; + + mc_uapi = container_of(filep->private_data, struct fsl_mc_uapi, misc); + mc_bus = container_of(mc_uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + mutex_lock(&mc_uapi->mutex); + + if (!mc_uapi->local_instance_in_use) { + priv_data->mc_io = mc_uapi->static_mc_io; + mc_uapi->local_instance_in_use = 1; + } else { + error = fsl_mc_portal_allocate(root_mc_device, 0, + &dynamic_mc_io); + if (error) { + dev_dbg(&root_mc_device->dev, + "Could not allocate MC portal\n"); + goto error_portal_allocate; + } + + priv_data->mc_io = dynamic_mc_io; + } + priv_data->uapi = mc_uapi; + filep->private_data = priv_data; + + mutex_unlock(&mc_uapi->mutex); + + return 0; + +error_portal_allocate: + mutex_unlock(&mc_uapi->mutex); + kfree(priv_data); + + return error; +} + +static int fsl_mc_uapi_dev_release(struct inode *inode, struct file *filep) +{ + struct uapi_priv_data *priv_data; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_io *mc_io; + + priv_data = filep->private_data; + mc_uapi = priv_data->uapi; + mc_io = priv_data->mc_io; + + mutex_lock(&mc_uapi->mutex); + + if (mc_io == mc_uapi->static_mc_io) + mc_uapi->local_instance_in_use = 0; + else + fsl_mc_portal_free(mc_io); + + kfree(filep->private_data); + filep->private_data = NULL; + + mutex_unlock(&mc_uapi->mutex); + + return 0; +} + +static long fsl_mc_uapi_dev_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct uapi_priv_data *priv_data = file->private_data; + struct fsl_mc_device *root_mc_device; + struct fsl_mc_bus *mc_bus; + int error; + + mc_bus = container_of(priv_data->uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + switch (cmd) { + case FSL_MC_SEND_MC_COMMAND: + error = fsl_mc_uapi_send_command(root_mc_device, arg, priv_data->mc_io); + break; + default: + dev_dbg(&root_mc_device->dev, "unexpected ioctl call number\n"); + error = -EINVAL; + } + + return error; +} + +static const struct file_operations fsl_mc_uapi_dev_fops = { + .owner = THIS_MODULE, + .open = fsl_mc_uapi_dev_open, + .release = fsl_mc_uapi_dev_release, + .unlocked_ioctl = fsl_mc_uapi_dev_ioctl, +}; + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + struct fsl_mc_device *mc_dev = &mc_bus->mc_dev; + struct fsl_mc_uapi *mc_uapi = &mc_bus->uapi_misc; + int error; + + mc_uapi->misc.minor = MISC_DYNAMIC_MINOR; + mc_uapi->misc.name = dev_name(&mc_dev->dev); + mc_uapi->misc.fops = &fsl_mc_uapi_dev_fops; + + error = misc_register(&mc_uapi->misc); + if (error) + return error; + + mc_uapi->static_mc_io = mc_bus->mc_dev.mc_io; + + mutex_init(&mc_uapi->mutex); + + return 0; +} + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ + misc_deregister(&mc_bus->uapi_misc.misc); +} diff --git a/include/uapi/linux/fsl_mc.h b/include/uapi/linux/fsl_mc.h index cf56d46f052e..e57451570033 100644 --- a/include/uapi/linux/fsl_mc.h +++ b/include/uapi/linux/fsl_mc.h @@ -16,10 +16,19 @@ * struct fsl_mc_command - Management Complex (MC) command structure * @header: MC command header * @params: MC command parameters + * + * Used by FSL_MC_SEND_MC_COMMAND */ struct fsl_mc_command { __le64 header; __le64 params[MC_CMD_NUM_OF_PARAMS]; }; +#define FSL_MC_SEND_CMD_IOCTL_TYPE 'R' +#define FSL_MC_SEND_CMD_IOCTL_SEQ 0xE0 + +#define FSL_MC_SEND_MC_COMMAND \ + _IOWR(FSL_MC_SEND_CMD_IOCTL_TYPE, FSL_MC_SEND_CMD_IOCTL_SEQ, \ + struct fsl_mc_command) + #endif /* _UAPI_FSL_MC_H_ */ From 3f6099438181d269d56f9d4040d93ffae65f9e4c Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 14 Jan 2021 19:07:51 +0200 Subject: [PATCH 145/633] bus: fsl-mc: add bus rescan attribute Introduce the rescan attribute as a bus attribute to synchronize the fsl-mc bus objects and the MC firmware. To rescan the fsl-mc bus, e.g., echo 1 > /sys/bus/fsl-mc/rescan Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210114170752.2927915-5-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/stable/sysfs-bus-fsl-mc | 9 +++++ MAINTAINERS | 1 + drivers/bus/fsl-mc/dprc-driver.c | 4 +-- drivers/bus/fsl-mc/fsl-mc-bus.c | 41 +++++++++++++++++++++++ drivers/bus/fsl-mc/fsl-mc-private.h | 3 ++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 Documentation/ABI/stable/sysfs-bus-fsl-mc diff --git a/Documentation/ABI/stable/sysfs-bus-fsl-mc b/Documentation/ABI/stable/sysfs-bus-fsl-mc new file mode 100644 index 000000000000..a4d384df9ba8 --- /dev/null +++ b/Documentation/ABI/stable/sysfs-bus-fsl-mc @@ -0,0 +1,9 @@ +What: /sys/bus/fsl-mc/rescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a non-zero value to this attribute will + force a rescan of fsl-mc bus in the system and + synchronize the objects under fsl-mc bus and the + Management Complex firmware. +Users: Userspace drivers and management tools diff --git a/MAINTAINERS b/MAINTAINERS index 128455811980..1d9d112af51a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14649,6 +14649,7 @@ M: Stuart Yoder M: Laurentiu Tudor L: linux-kernel@vger.kernel.org S: Maintained +F: Documentation/ABI/stable/sysfs-bus-fsl-mc F: Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt F: Documentation/networking/device_drivers/ethernet/freescale/dpaa2/overview.rst F: drivers/bus/fsl-mc/ diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index ccec375095f2..4adb6f318884 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -237,8 +237,8 @@ static void dprc_add_new_devices(struct fsl_mc_device *mc_bus_dev, * populated before they can get allocation requests from probe callbacks * of the device drivers for the non-allocatable devices. */ -static int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, - bool alloc_interrupts) +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts) { int num_child_objects; int dprc_get_obj_failures; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 5a8fc68d6109..ee3e6252ffc3 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -208,12 +208,53 @@ static struct attribute *fsl_mc_dev_attrs[] = { ATTRIBUTE_GROUPS(fsl_mc_dev); +static int scan_fsl_mc_bus(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + struct fsl_mc_bus *root_mc_bus; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + root_mc_bus = to_fsl_mc_bus(root_mc_dev); + mutex_lock(&root_mc_bus->scan_mutex); + dprc_scan_objects(root_mc_dev, NULL); + mutex_unlock(&root_mc_bus->scan_mutex); + +exit: + return 0; +} + +static ssize_t rescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + bus_for_each_dev(bus, NULL, NULL, scan_fsl_mc_bus); + + return count; +} +static BUS_ATTR_WO(rescan); + +static struct attribute *fsl_mc_bus_attrs[] = { + &bus_attr_rescan.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(fsl_mc_bus); + struct bus_type fsl_mc_bus_type = { .name = "fsl-mc", .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .dma_configure = fsl_mc_dma_configure, .dev_groups = fsl_mc_dev_groups, + .bus_groups = fsl_mc_bus_groups, }; EXPORT_SYMBOL_GPL(fsl_mc_bus_type); diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index 6293a24de456..42bdb8679a36 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -594,6 +594,9 @@ int __init dprc_driver_init(void); void dprc_driver_exit(void); +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts); + int __init fsl_mc_allocator_driver_init(void); void fsl_mc_allocator_driver_exit(void); From 296c6264d4b19554dc8367e3f409bd248f504c2d Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Thu, 14 Jan 2021 19:07:52 +0200 Subject: [PATCH 146/633] bus: fsl-mc: add autorescan sysfs Add the autorescan sysfs in order to enable/disable the DPRC IRQs on which automatic rescan of the bus is performed. This is important when dynamic creation of objects is needed to happen in a timely manner because object creation can be bundled together. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210114170752.2927915-6-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/stable/sysfs-bus-fsl-mc | 10 +++++ drivers/bus/fsl-mc/dprc-driver.c | 17 ++++++- drivers/bus/fsl-mc/fsl-mc-bus.c | 55 +++++++++++++++++++++++ drivers/bus/fsl-mc/fsl-mc-private.h | 5 +++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-bus-fsl-mc b/Documentation/ABI/stable/sysfs-bus-fsl-mc index a4d384df9ba8..58f06c7eeed7 100644 --- a/Documentation/ABI/stable/sysfs-bus-fsl-mc +++ b/Documentation/ABI/stable/sysfs-bus-fsl-mc @@ -7,3 +7,13 @@ Description: Writing a non-zero value to this attribute will synchronize the objects under fsl-mc bus and the Management Complex firmware. Users: Userspace drivers and management tools + +What: /sys/bus/fsl-mc/autorescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a zero value to this attribute will + disable the DPRC IRQs on which automatic rescan + of the fsl-mc bus is performed. A non-zero value + will enable the DPRC IRQs. +Users: Userspace drivers and management tools diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 4adb6f318884..e3e2ae41c22b 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -458,8 +458,9 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) /* * Disable and clear interrupt for a given DPRC object */ -static int disable_dprc_irq(struct fsl_mc_device *mc_dev) +int disable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; struct fsl_mc_io *mc_io = mc_dev->mc_io; @@ -496,9 +497,18 @@ static int disable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 0; + return 0; } +int get_dprc_irq_state(struct fsl_mc_device *mc_dev) +{ + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); + + return mc_bus->irq_enabled; +} + static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) { int error; @@ -525,8 +535,9 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) return 0; } -static int enable_dprc_irq(struct fsl_mc_device *mc_dev) +int enable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* @@ -554,6 +565,8 @@ static int enable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 1; + return 0; } diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index ee3e6252ffc3..70f4b69556f5 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -241,8 +241,63 @@ static ssize_t rescan_store(struct bus_type *bus, } static BUS_ATTR_WO(rescan); +static int fsl_mc_bus_set_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + unsigned long val; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + enable_dprc_irq(root_mc_dev); + else + disable_dprc_irq(root_mc_dev); + +exit: + return 0; +} + +static int fsl_mc_bus_get_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + sprintf(buf, "%d\n", get_dprc_irq_state(root_mc_dev)); +exit: + return 0; +} + +static ssize_t autorescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan); + + return count; +} + +static ssize_t autorescan_show(struct bus_type *bus, char *buf) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan); + return strlen(buf); +} + +static BUS_ATTR_RW(autorescan); + static struct attribute *fsl_mc_bus_attrs[] = { &bus_attr_rescan.attr, + &bus_attr_autorescan.attr, NULL, }; diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index 42bdb8679a36..1958fa065360 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -578,6 +578,7 @@ struct fsl_mc_bus { struct mutex scan_mutex; /* serializes bus scanning */ struct dprc_attributes dprc_attr; struct fsl_mc_uapi uapi_misc; + int irq_enabled; }; #define to_fsl_mc_bus(_mc_dev) \ @@ -656,4 +657,8 @@ static inline void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) #endif +int disable_dprc_irq(struct fsl_mc_device *mc_dev); +int enable_dprc_irq(struct fsl_mc_device *mc_dev); +int get_dprc_irq_state(struct fsl_mc_device *mc_dev); + #endif /* _FSL_MC_PRIVATE_H_ */ From 1423de718e6a0ce00372ab119fa40060ff50883e Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 2 Jun 2020 15:56:55 -0500 Subject: [PATCH 147/633] PCI/ACPI: Make acpi_pci_osc_control_set() static acpi_pci_osc_control_set() is only called from pci_root.c, so stop exporting it and make it static. Signed-off-by: Bjorn Helgaas Acked-by: Rafael J. Wysocki --- drivers/acpi/pci_root.c | 3 +-- include/linux/acpi.h | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 0bf072cef6cf..7d5a4bd4207c 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -353,7 +353,7 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev); * _OSC bits the BIOS has granted control of, but its contents are meaningless * on failure. **/ -acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) +static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) { struct acpi_pci_root *root; acpi_status status = AE_OK; @@ -406,7 +406,6 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) mutex_unlock(&osc_lock); return status; } -EXPORT_SYMBOL(acpi_pci_osc_control_set); static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, bool is_pcie) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 053bf05fb1f7..4703daafcce9 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -581,9 +581,6 @@ extern bool osc_pc_lpi_support_confirmed; #define ACPI_GSB_ACCESS_ATTRIB_RAW_BYTES 0x0000000E #define ACPI_GSB_ACCESS_ATTRIB_RAW_PROCESS 0x0000000F -extern acpi_status acpi_pci_osc_control_set(acpi_handle handle, - u32 *mask, u32 req); - /* Enable _OST when all relevant hotplug operations are enabled */ #if defined(CONFIG_ACPI_HOTPLUG_CPU) && \ defined(CONFIG_ACPI_HOTPLUG_MEMORY) && \ From 866e61fc40c96e7adba62c9e149e96912b10663c Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 2 Jun 2020 17:27:33 -0500 Subject: [PATCH 148/633] PCI/ACPI: Remove unnecessary osc_lock 9778c14b4ca2 ("ACPI/PCI: Fix possible race condition on _OSC evaluation") added locking around _OSC calls to protect the acpi_osc_data_list that stored the results. 63f10f0f6df4 ("PCI/ACPI: move _OSC code to pci_root.c") moved the results from acpi_osc_data_list to the struct acpi_pci_root, where it no longer needs locking, but did not remove the lock. Remove the unnecessary locking around _OSC calls. Signed-off-by: Bjorn Helgaas Acked-by: Rafael J. Wysocki --- drivers/acpi/pci_root.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 7d5a4bd4207c..ea08b3298e73 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -56,8 +56,6 @@ static struct acpi_scan_handler pci_root_handler = { }, }; -static DEFINE_MUTEX(osc_lock); - /** * acpi_is_root_bridge - determine whether an ACPI CA node is a PCI root bridge * @handle: the ACPI CA node in question. @@ -223,12 +221,7 @@ static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root, static acpi_status acpi_pci_osc_support(struct acpi_pci_root *root, u32 flags) { - acpi_status status; - - mutex_lock(&osc_lock); - status = acpi_pci_query_osc(root, flags, NULL); - mutex_unlock(&osc_lock); - return status; + return acpi_pci_query_osc(root, flags, NULL); } struct acpi_pci_root *acpi_pci_find_root(acpi_handle handle) @@ -356,7 +349,7 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev); static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) { struct acpi_pci_root *root; - acpi_status status = AE_OK; + acpi_status status; u32 ctrl, capbuf[3]; if (!mask) @@ -370,18 +363,16 @@ static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 r if (!root) return AE_NOT_EXIST; - mutex_lock(&osc_lock); - *mask = ctrl | root->osc_control_set; /* No need to evaluate _OSC if the control was already granted. */ if ((root->osc_control_set & ctrl) == ctrl) - goto out; + return AE_OK; /* Need to check the available controls bits before requesting them. */ while (*mask) { status = acpi_pci_query_osc(root, root->osc_support_set, mask); if (ACPI_FAILURE(status)) - goto out; + return status; if (ctrl == *mask) break; decode_osc_control(root, "platform does not support", @@ -392,19 +383,18 @@ static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 r if ((ctrl & req) != req) { decode_osc_control(root, "not requesting control; platform does not support", req & ~(ctrl)); - status = AE_SUPPORT; - goto out; + return AE_SUPPORT; } capbuf[OSC_QUERY_DWORD] = 0; capbuf[OSC_SUPPORT_DWORD] = root->osc_support_set; capbuf[OSC_CONTROL_DWORD] = ctrl; status = acpi_pci_run_osc(handle, capbuf, mask); - if (ACPI_SUCCESS(status)) - root->osc_control_set = *mask; -out: - mutex_unlock(&osc_lock); - return status; + if (ACPI_FAILURE(status)) + return status; + + root->osc_control_set = *mask; + return AE_OK; } static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, From 508d392ae0bb3729d4c18628e021968a1b11b32c Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 2 Jun 2020 16:42:33 -0500 Subject: [PATCH 149/633] PCI/ACPI: Clarify message about _OSC failure This message: acpi PNP0A08:02: _OSC failed (AE_NOT_FOUND); disabling ASPM is alarming and slightly misleading. Per the PCI Firmware Spec, r3.2, sec 4.5.1, _OSC is required for PCIe hierarchies. If _OSC is absent or fails, the OS must not attempt to use any of the features defined for _OSC. That includes native hotplug, native PME, AER, and other things as well as ASPM. Rephrase the message to: acpi PNP0A08:02: _OSC: platform retains control of PCIe features (AE_NOT_FOUND) Previous discussion at https://lore.kernel.org/r/20200602223618.GA845676@bjorn-Precision-5520/ Signed-off-by: Bjorn Helgaas Reviewed-by: Sinan Kaya Acked-by: Rafael J. Wysocki Cc: Yicong Yang --- drivers/acpi/pci_root.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index ea08b3298e73..dcd593766a64 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -441,9 +441,8 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, if ((status == AE_NOT_FOUND) && !is_pcie) return; - dev_info(&device->dev, "_OSC failed (%s)%s\n", - acpi_format_exception(status), - pcie_aspm_support_enabled() ? "; disabling ASPM" : ""); + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", + acpi_format_exception(status)); return; } @@ -499,7 +498,7 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, } else { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); - dev_info(&device->dev, "_OSC failed (%s); disabling ASPM\n", + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", acpi_format_exception(status)); /* From ef9e4005cbaf022c6251263aa27836acccaef65d Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sun, 24 Jan 2021 16:39:32 +0100 Subject: [PATCH 150/633] PCI: Align checking of syscall user config accessors After 34e3207205ef ("PCI: handle positive error codes"), pci_user_read_config_*() and pci_user_write_config_*() return 0 or negative errno values, not PCIBIOS_* values like PCIBIOS_SUCCESSFUL or PCIBIOS_BAD_REGISTER_NUMBER. Remove comparisons with PCIBIOS_SUCCESSFUL and check only for non-zero. It happens that PCIBIOS_SUCCESSFUL is zero, so this is not a functional change, but it aligns this code with the user accessors. [bhelgaas: commit log] Fixes: 34e3207205ef ("PCI: handle positive error codes") Link: https://lore.kernel.org/r/f1220314-e518-1e18-bf94-8e6f8c703758@gmail.com Signed-off-by: Heiner Kallweit Signed-off-by: Bjorn Helgaas --- drivers/pci/syscall.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/pci/syscall.c b/drivers/pci/syscall.c index 31e39558d49d..8b003c890b87 100644 --- a/drivers/pci/syscall.c +++ b/drivers/pci/syscall.c @@ -20,7 +20,7 @@ SYSCALL_DEFINE5(pciconfig_read, unsigned long, bus, unsigned long, dfn, u16 word; u32 dword; long err; - long cfg_ret; + int cfg_ret; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -46,7 +46,7 @@ SYSCALL_DEFINE5(pciconfig_read, unsigned long, bus, unsigned long, dfn, } err = -EIO; - if (cfg_ret != PCIBIOS_SUCCESSFUL) + if (cfg_ret) goto error; switch (len) { @@ -105,7 +105,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_byte(dev, off, byte); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; @@ -114,7 +114,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_word(dev, off, word); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; @@ -123,7 +123,7 @@ SYSCALL_DEFINE5(pciconfig_write, unsigned long, bus, unsigned long, dfn, if (err) break; err = pci_user_write_config_dword(dev, off, dword); - if (err != PCIBIOS_SUCCESSFUL) + if (err) err = -EIO; break; From 4c998836d413d8e4ca8749f876b75a74a6fb4c5d Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Fri, 4 Dec 2020 21:58:10 +0200 Subject: [PATCH 151/633] habanalabs: update firmware boot interface Update to latest firmware hl_boot_if.h file. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/include/common/hl_boot_if.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index b637dfd69f6e..1115456cca85 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -162,6 +162,10 @@ * statuses. * Initialized in: preboot * + * CPU_BOOT_DEV_STS0_SP_SRAM_EN SP SRAM is initialized and available + * for use. + * Initialized in: preboot + * */ #define CPU_BOOT_DEV_STS0_SECURITY_EN (1 << 0) #define CPU_BOOT_DEV_STS0_DEBUG_EN (1 << 1) @@ -175,6 +179,7 @@ #define CPU_BOOT_DEV_STS0_DRAM_SCR_EN (1 << 9) #define CPU_BOOT_DEV_STS0_FW_HARD_RST_EN (1 << 10) #define CPU_BOOT_DEV_STS0_PLL_INFO_EN (1 << 11) +#define CPU_BOOT_DEV_STS0_SP_SRAM_EN (1 << 12) #define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) From cb6ef0ee6d760281cc0b50711564415f39328bfb Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Thu, 26 Nov 2020 09:39:26 +0200 Subject: [PATCH 152/633] habanalabs: refactor MMU locks code remove mmu_cache_lock as it protects a section which is already protected by mmu_lock. in addition, wrap mmu cache invalidate calls in hl_vm_ctx_fini with mmu_lock. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/device.c | 2 -- drivers/misc/habanalabs/common/habanalabs.h | 2 -- drivers/misc/habanalabs/common/memory.c | 4 ++++ drivers/misc/habanalabs/gaudi/gaudi.c | 8 -------- drivers/misc/habanalabs/goya/goya.c | 8 -------- 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 69d04eca767f..ff2bab69e8fc 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -373,7 +373,6 @@ static int device_early_init(struct hl_device *hdev) mutex_init(&hdev->send_cpu_message_lock); mutex_init(&hdev->debug_lock); - mutex_init(&hdev->mmu_cache_lock); INIT_LIST_HEAD(&hdev->cs_mirror_list); spin_lock_init(&hdev->cs_mirror_lock); INIT_LIST_HEAD(&hdev->fpriv_list); @@ -414,7 +413,6 @@ static void device_early_fini(struct hl_device *hdev) { int i; - mutex_destroy(&hdev->mmu_cache_lock); mutex_destroy(&hdev->debug_lock); mutex_destroy(&hdev->send_cpu_message_lock); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 60e16dc4bcac..2dbee9440a5f 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1764,7 +1764,6 @@ struct hl_mmu_funcs { * @asic_funcs: ASIC specific functions. * @asic_specific: ASIC specific information to use only from ASIC files. * @vm: virtual memory manager for MMU. - * @mmu_cache_lock: protects MMU cache invalidation as it can serve one context. * @hwmon_dev: H/W monitor device. * @pm_mng_profile: current power management profile. * @hl_chip_info: ASIC's sensors information. @@ -1879,7 +1878,6 @@ struct hl_device { const struct hl_asic_funcs *asic_funcs; void *asic_specific; struct hl_vm vm; - struct mutex mmu_cache_lock; struct device *hwmon_dev; enum hl_pm_mng_profile pm_mng_profile; struct hwmon_chip_info *hl_chip_info; diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 5d4fbdcaefe3..eb09d63bb5a1 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1895,10 +1895,14 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) unmap_device_va(ctx, hnode->vaddr, true); } + mutex_lock(&ctx->mmu_lock); + /* invalidate the cache once after the unmapping loop */ hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_PHYS_PACK); + mutex_unlock(&ctx->mmu_lock); + spin_lock(&vm->idr_lock); idr_for_each_entry(&vm->phys_pg_pack_handles, phys_pg_list, i) if (phys_pg_list->asid == ctx->asid) { diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index b328ddaa64ee..ce81935fb28e 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7330,8 +7330,6 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* L0 & L1 invalidation */ WREG32(mmSTLB_INV_PS, 3); WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++); @@ -7347,8 +7345,6 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, WREG32(mmSTLB_INV_SET, 0); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -7371,8 +7367,6 @@ static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, hdev->hard_reset_pending) return 0; - mutex_lock(&hdev->mmu_cache_lock); - if (hdev->pldm) timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC; else @@ -7400,8 +7394,6 @@ static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 63679a747d2c..19b5bcc8b7ac 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5073,8 +5073,6 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* L0 & L1 invalidation */ WREG32(mmSTLB_INV_ALL_START, 1); @@ -5086,8 +5084,6 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); @@ -5117,8 +5113,6 @@ static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, else timeout_usec = MMU_CONFIG_TIMEOUT_USEC; - mutex_lock(&hdev->mmu_cache_lock); - /* * TODO: currently invalidate entire L0 & L1 as in regular hard * invalidation. Need to apply invalidation of specific cache lines with @@ -5141,8 +5135,6 @@ static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, 1000, timeout_usec); - mutex_unlock(&hdev->mmu_cache_lock); - if (rc) { dev_err_ratelimited(hdev->dev, "MMU cache invalidation timeout\n"); From 8e39e75a134f4cebc6503f21ba6e51c0cf434497 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 12 Nov 2020 11:03:32 +0200 Subject: [PATCH 153/633] habanalabs: Init the VM module for kernel context In order for reserving VA ranges for kernel memory, we need to allow the VM module to be initiated with kernel context. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/context.c | 16 +++++++++++++--- drivers/misc/habanalabs/common/device.c | 15 ++++++++++----- drivers/misc/habanalabs/common/memory.c | 3 ++- drivers/misc/habanalabs/gaudi/gaudi.c | 10 ++++------ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index f65e6559149b..3d86b83f4ca6 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -56,6 +56,8 @@ static void hl_ctx_fini(struct hl_ctx *ctx) idle_mask); } else { dev_dbg(hdev->dev, "closing kernel context\n"); + hdev->asic_funcs->ctx_fini(ctx); + hl_vm_ctx_fini(ctx); hl_mmu_ctx_fini(ctx); } } @@ -151,11 +153,18 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) if (is_kernel_ctx) { ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */ - rc = hl_mmu_ctx_init(ctx); + rc = hl_vm_ctx_init(ctx); if (rc) { - dev_err(hdev->dev, "Failed to init mmu ctx module\n"); + dev_err(hdev->dev, "Failed to init mem ctx module\n"); + rc = -ENOMEM; goto err_free_cs_pending; } + + rc = hdev->asic_funcs->ctx_init(ctx); + if (rc) { + dev_err(hdev->dev, "ctx_init failed\n"); + goto err_vm_ctx_fini; + } } else { ctx->asid = hl_asid_alloc(hdev); if (!ctx->asid) { @@ -194,7 +203,8 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) err_vm_ctx_fini: hl_vm_ctx_fini(ctx); err_asid_free: - hl_asid_free(hdev, ctx->asid); + if (ctx->asid != HL_KERNEL_ASID_ID) + hl_asid_free(hdev, ctx->asid); err_free_cs_pending: kfree(ctx->cs_pending); diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index ff2bab69e8fc..29358a949686 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -1312,11 +1312,16 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) hdev->compute_ctx = NULL; + hl_debugfs_add_device(hdev); + + /* debugfs nodes are created in hl_ctx_init so it must be called after + * hl_debugfs_add_device. + */ rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); if (rc) { dev_err(hdev->dev, "failed to initialize kernel context\n"); kfree(hdev->kernel_ctx); - goto mmu_fini; + goto remove_device_from_debugfs; } rc = hl_cb_pool_init(hdev); @@ -1325,8 +1330,6 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) goto release_ctx; } - hl_debugfs_add_device(hdev); - /* * From this point, in case of an error, add char devices and create * sysfs nodes as part of the error flow, to allow debugging. @@ -1415,6 +1418,8 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) if (hl_ctx_put(hdev->kernel_ctx) != 1) dev_err(hdev->dev, "kernel ctx is still alive on initialization failure\n"); +remove_device_from_debugfs: + hl_debugfs_remove_device(hdev); mmu_fini: hl_mmu_fini(hdev); eq_fini: @@ -1513,8 +1518,6 @@ void hl_device_fini(struct hl_device *hdev) device_late_fini(hdev); - hl_debugfs_remove_device(hdev); - /* * Halt the engines and disable interrupts so we won't get any more * completions from H/W and we won't have any accesses from the @@ -1546,6 +1549,8 @@ void hl_device_fini(struct hl_device *hdev) if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) dev_err(hdev->dev, "kernel ctx is still alive\n"); + hl_debugfs_remove_device(hdev); + hl_vm_fini(hdev); hl_mmu_fini(hdev); diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index eb09d63bb5a1..fb2c268a262b 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1929,7 +1929,8 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) * because the user notifies us on allocations. If the user is no more, * all DRAM is available */ - if (!ctx->hdev->asic_prop.dram_supports_virtual_memory) + if (ctx->asid != HL_KERNEL_ASID_ID && + !ctx->hdev->asic_prop.dram_supports_virtual_memory) atomic64_set(&ctx->hdev->dram_used_mem, 0); } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index ce81935fb28e..58ad1630192e 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7868,18 +7868,16 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, static int gaudi_ctx_init(struct hl_ctx *ctx) { + if (ctx->asid == HL_KERNEL_ASID_ID) + return 0; + gaudi_mmu_prepare(ctx->hdev, ctx->asid); return gaudi_internal_cb_pool_init(ctx->hdev, ctx); } static void gaudi_ctx_fini(struct hl_ctx *ctx) { - struct hl_device *hdev = ctx->hdev; - - /* Gaudi will NEVER support more then a single compute context. - * Therefore, don't clear anything unless it is the compute context - */ - if (hdev->compute_ctx != ctx) + if (ctx->asid == HL_KERNEL_ASID_ID) return; gaudi_internal_cb_pool_fini(ctx->hdev, ctx); From ac6fdbfe2ea833b610a51bc6fc078fbce8457d97 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 3 Dec 2020 16:59:28 +0200 Subject: [PATCH 154/633] habanalabs/gaudi: support CS with no completion As part of the staged submission feature, we need Gaudi to support command submissions that will never get a completion. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/command_submission.c | 1 + drivers/misc/habanalabs/common/habanalabs.h | 2 ++ drivers/misc/habanalabs/gaudi/gaudi.c | 10 +++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index b2b3d2b0f808..ccb4972a555f 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -225,6 +225,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) parser.queue_type = job->queue_type; parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb; job->patched_cb = NULL; + parser.completion = true; rc = hdev->asic_funcs->cs_parser(hdev, &parser); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 2dbee9440a5f..dee73a04411b 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1223,6 +1223,7 @@ struct hl_cs_job { * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't * have streams so the engine can't be busy by another * stream. + * @completion: true if we need completion for this CS. */ struct hl_cs_parser { struct hl_cb *user_cb; @@ -1237,6 +1238,7 @@ struct hl_cs_parser { u8 job_id; u8 is_kernel_allocated_cb; u8 contains_dma_pkt; + u8 completion; }; /* diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 58ad1630192e..0324114ab4ee 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -5060,7 +5060,8 @@ static int gaudi_validate_cb(struct hl_device *hdev, * 1. A packet that will act as a completion packet * 2. A packet that will generate MSI-X interrupt */ - parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; + if (parser->completion) + parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; return rc; } @@ -5287,8 +5288,11 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev, * 1. A packet that will act as a completion packet * 2. A packet that will generate MSI interrupt */ - parser->patched_cb_size = parser->user_cb_size + - sizeof(struct packet_msg_prot) * 2; + if (parser->completion) + parser->patched_cb_size = parser->user_cb_size + + sizeof(struct packet_msg_prot) * 2; + else + parser->patched_cb_size = parser->user_cb_size; rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, parser->patched_cb_size, false, false, From c209e742141bc0064ecdb73ef0dd825310de28cb Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 3 Dec 2020 17:12:09 +0200 Subject: [PATCH 155/633] habanalabs: allow user to pass a staged submission seq In order to support the staged submission feature, user must be allowed to use the same CS sequence for all submissions in the same staged submission. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../habanalabs/common/command_submission.c | 16 +++++++---- include/uapi/misc/habanalabs.h | 28 +++++++++++++------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index ccb4972a555f..348b0ecb4489 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -807,7 +807,7 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, } static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, - u32 num_chunks, u64 *cs_seq, bool timestamp) + u32 num_chunks, u64 *cs_seq, u32 flags) { bool int_queues_only = true; struct hl_device *hdev = hpriv->hdev; @@ -836,7 +836,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, goto free_cs_chunk_array; } - cs->timestamp = !!timestamp; + cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); *cs_seq = cs->sequence; hl_debugfs_add_cs(cs); @@ -1004,7 +1004,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, rc = 0; } else { rc = cs_ioctl_default(hpriv, chunks, num_chunks, - cs_seq, false); + cs_seq, 0); } mutex_unlock(&hpriv->restore_phase_mutex); @@ -1347,7 +1347,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) enum hl_cs_type cs_type; u64 cs_seq = ULONG_MAX; void __user *chunks; - u32 num_chunks; + u32 num_chunks, flags; int rc; rc = hl_cs_sanity_checks(hpriv, args); @@ -1362,6 +1362,12 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) ~HL_CS_FLAGS_FORCE_RESTORE); chunks = (void __user *) (uintptr_t) args->in.chunks_execute; num_chunks = args->in.num_chunks_execute; + flags = args->in.cs_flags; + + /* In case this is a staged CS, user should supply the CS sequence */ + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + cs_seq = args->in.seq; switch (cs_type) { case CS_TYPE_SIGNAL: @@ -1372,7 +1378,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) break; default: rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq, - args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP); + args->in.cs_flags); break; } diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index dba3827c43ca..b9afe1cdac24 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -604,11 +604,14 @@ struct hl_cs_chunk { }; /* SIGNAL and WAIT/COLLECTIVE_WAIT flags are mutually exclusive */ -#define HL_CS_FLAGS_FORCE_RESTORE 0x1 -#define HL_CS_FLAGS_SIGNAL 0x2 -#define HL_CS_FLAGS_WAIT 0x4 -#define HL_CS_FLAGS_COLLECTIVE_WAIT 0x8 -#define HL_CS_FLAGS_TIMESTAMP 0x20 +#define HL_CS_FLAGS_FORCE_RESTORE 0x1 +#define HL_CS_FLAGS_SIGNAL 0x2 +#define HL_CS_FLAGS_WAIT 0x4 +#define HL_CS_FLAGS_COLLECTIVE_WAIT 0x8 +#define HL_CS_FLAGS_TIMESTAMP 0x20 +#define HL_CS_FLAGS_STAGED_SUBMISSION 0x40 +#define HL_CS_FLAGS_STAGED_SUBMISSION_FIRST 0x80 +#define HL_CS_FLAGS_STAGED_SUBMISSION_LAST 0x100 #define HL_CS_STATUS_SUCCESS 0 @@ -622,10 +625,17 @@ struct hl_cs_in { /* holds address of array of hl_cs_chunk for execution phase */ __u64 chunks_execute; - /* this holds address of array of hl_cs_chunk for store phase - - * Currently not in use - */ - __u64 chunks_store; + union { + /* this holds address of array of hl_cs_chunk for store phase - + * Currently not in use + */ + __u64 chunks_store; + + /* Sequence number of a staged submission CS + * valid only if HL_CS_FLAGS_STAGED_SUBMISSION is set + */ + __u64 seq; + }; /* Number of chunks in restore phase array. Maximum number is * HL_MAX_JOBS_PER_CS From f8b0f2ecc57023c1988c592cfc02fbca987a068b Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 6 Dec 2020 10:22:32 +0200 Subject: [PATCH 156/633] habanalabs/gaudi: remove duplicated gaudi packets masks As all packets use the same CTL register masks, we remove duplicated masks and use common masks instead. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 32 +++++++++---------- .../habanalabs/include/gaudi/gaudi_packets.h | 24 -------------- 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 0324114ab4ee..5281cad4b552 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7922,10 +7922,10 @@ static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 3); /* W_S SOB base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, eb); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, eb); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -7942,10 +7942,10 @@ static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, addr); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 0); /* last pkt MB */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 0); /* last pkt MB */ pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -7991,10 +7991,10 @@ static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev, ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, msg_addr_offset); ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */ ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */ - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->value = cpu_to_le32(value); pkt->ctl = cpu_to_le32(ctl); @@ -8012,10 +8012,10 @@ static u32 gaudi_add_fence_pkt(struct packet_fence *pkt) cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_ID_MASK, 2); - ctl = FIELD_PREP(GAUDI_PKT_FENCE_CTL_OPCODE_MASK, PACKET_FENCE); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_EB_MASK, 0); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_RB_MASK, 1); - ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_MB_MASK, 1); + ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_FENCE); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); pkt->cfg = cpu_to_le32(cfg); pkt->ctl = cpu_to_le32(ctl); diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h index f30f2c0458d7..784b5bc8d0ba 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h @@ -111,18 +111,6 @@ struct packet_msg_long { #define GAUDI_PKT_SHORT_CTL_BASE_SHIFT 22 #define GAUDI_PKT_SHORT_CTL_BASE_MASK 0x00C00000 -#define GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT 24 -#define GAUDI_PKT_SHORT_CTL_OPCODE_MASK 0x1F000000 - -#define GAUDI_PKT_SHORT_CTL_EB_SHIFT 29 -#define GAUDI_PKT_SHORT_CTL_EB_MASK 0x20000000 - -#define GAUDI_PKT_SHORT_CTL_RB_SHIFT 30 -#define GAUDI_PKT_SHORT_CTL_RB_MASK 0x40000000 - -#define GAUDI_PKT_SHORT_CTL_MB_SHIFT 31 -#define GAUDI_PKT_SHORT_CTL_MB_MASK 0x80000000 - struct packet_msg_short { __le32 value; __le32 ctl; @@ -146,18 +134,6 @@ struct packet_msg_prot { #define GAUDI_PKT_FENCE_CTL_PRED_SHIFT 0 #define GAUDI_PKT_FENCE_CTL_PRED_MASK 0x0000001F -#define GAUDI_PKT_FENCE_CTL_OPCODE_SHIFT 24 -#define GAUDI_PKT_FENCE_CTL_OPCODE_MASK 0x1F000000 - -#define GAUDI_PKT_FENCE_CTL_EB_SHIFT 29 -#define GAUDI_PKT_FENCE_CTL_EB_MASK 0x20000000 - -#define GAUDI_PKT_FENCE_CTL_RB_SHIFT 30 -#define GAUDI_PKT_FENCE_CTL_RB_MASK 0x40000000 - -#define GAUDI_PKT_FENCE_CTL_MB_SHIFT 31 -#define GAUDI_PKT_FENCE_CTL_MB_MASK 0x80000000 - struct packet_fence { __le32 cfg; __le32 ctl; From e1b85dbaf06d2e2a61797318738e52abeafa4f6b Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Tue, 1 Dec 2020 14:06:27 +0200 Subject: [PATCH 157/633] habanalabs/goya: move mmu_prepare to context init Currently mmu_prepare is located at context switch. Since we support a single context, no reason to reconfigure the MMU registers every context switch. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/goya/goya.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 19b5bcc8b7ac..720484bffcab 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -4877,8 +4877,6 @@ int goya_context_switch(struct hl_device *hdev, u32 asid) WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020); - goya_mmu_prepare(hdev, asid); - goya_clear_sm_regs(hdev); return 0; @@ -5313,6 +5311,9 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data, static int goya_ctx_init(struct hl_ctx *ctx) { + if (ctx->asid != HL_KERNEL_ASID_ID) + goya_mmu_prepare(ctx->hdev, ctx->asid); + return 0; } From 0eda23d77e1beede9c61b0cba6d9267d3a92bd4e Mon Sep 17 00:00:00 2001 From: Moti Haimovski Date: Mon, 7 Dec 2020 09:10:34 +0200 Subject: [PATCH 158/633] habanalabs: report dram_page_size in hw_ip_info ioctl Instead of having it hard-coded as a define, pass it to the user in runtime. Signed-off-by: Moti Haimovski Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 1 + include/uapi/misc/habanalabs.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index d25892d61ec9..394a2e1767ce 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -63,6 +63,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.dram_size = prop->dram_size - dram_kmd_size; if (hw_ip.dram_size > PAGE_SIZE) hw_ip.dram_enabled = 1; + hw_ip.dram_page_size = prop->dram_page_size; hw_ip.num_of_events = prop->num_of_events; memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version, diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index b9afe1cdac24..b431a70e1b8b 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -320,6 +320,8 @@ struct hl_info_hw_ip_info { __u8 pad[2]; __u8 cpucp_version[HL_INFO_VERSION_MAX_LEN]; __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN]; + __u64 reserved2; + __u64 dram_page_size; }; struct hl_info_dram_usage { From 75d9a2a0aa28c4519c9648d501825e2c6860cb81 Mon Sep 17 00:00:00 2001 From: Alon Mizrahi Date: Thu, 3 Dec 2020 17:32:19 +0200 Subject: [PATCH 159/633] habanalabs: replace WARN/WARN_ON with dev_crit in driver Often WARN is defined in data-centers as BUG and we would like to avoid hanging the entire server on some internal error of the driver (important as it might be). Therefore, use dev_crit instead. Signed-off-by: Alon Mizrahi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/asid.c | 6 ++++-- drivers/misc/habanalabs/common/command_buffer.c | 8 +++++--- .../misc/habanalabs/common/command_submission.c | 4 ++-- drivers/misc/habanalabs/common/device.c | 3 ++- drivers/misc/habanalabs/common/mmu.c | 7 ++++--- drivers/misc/habanalabs/gaudi/gaudi.c | 14 +++++++------- drivers/misc/habanalabs/goya/goya.c | 14 +++++++------- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/drivers/misc/habanalabs/common/asid.c b/drivers/misc/habanalabs/common/asid.c index a2fdf31cf27c..ede04c032b6e 100644 --- a/drivers/misc/habanalabs/common/asid.c +++ b/drivers/misc/habanalabs/common/asid.c @@ -50,8 +50,10 @@ unsigned long hl_asid_alloc(struct hl_device *hdev) void hl_asid_free(struct hl_device *hdev, unsigned long asid) { - if (WARN((asid == 0 || asid >= hdev->asic_prop.max_asid), - "Invalid ASID %lu", asid)) + if (asid == HL_KERNEL_ASID_ID || asid >= hdev->asic_prop.max_asid) { + dev_crit(hdev->dev, "Invalid ASID %lu", asid); return; + } + clear_bit(asid, hdev->asid_bitmap); } diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c index 6f6a904ab6ca..d9adb9a5e4d8 100644 --- a/drivers/misc/habanalabs/common/command_buffer.c +++ b/drivers/misc/habanalabs/common/command_buffer.c @@ -635,10 +635,12 @@ struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, cb_handle >>= PAGE_SHIFT; cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle); - if (!cb) + /* hl_cb_get should never fail here */ + if (!cb) { + dev_crit(hdev->dev, "Kernel CB handle invalid 0x%x\n", + (u32) cb_handle); goto destroy_cb; + } return cb; diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 348b0ecb4489..f7fac82ac41d 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -48,8 +48,8 @@ void hl_sob_reset_error(struct kref *ref) struct hl_device *hdev = hw_sob->hdev; dev_crit(hdev->dev, - "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", - hw_sob->q_idx, hw_sob->sob_id); + "SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n", + hw_sob->q_idx, hw_sob->sob_id); } /** diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 29358a949686..13405d47d843 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -1485,7 +1485,8 @@ void hl_device_fini(struct hl_device *hdev) usleep_range(50, 200); rc = atomic_cmpxchg(&hdev->in_reset, 0, 1); if (ktime_compare(ktime_get(), timeout) > 0) { - WARN(1, "Failed to remove device because reset function did not finish\n"); + dev_crit(hdev->dev, + "Failed to remove device because reset function did not finish\n"); return; } } diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu.c index 28a4638741d8..62cfa4190fe4 100644 --- a/drivers/misc/habanalabs/common/mmu.c +++ b/drivers/misc/habanalabs/common/mmu.c @@ -261,9 +261,10 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, return -EFAULT; } - WARN_ONCE((phys_addr & (real_page_size - 1)), - "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size", - phys_addr, real_page_size); + if (phys_addr & (real_page_size - 1)) + dev_crit(hdev->dev, + "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size", + phys_addr, real_page_size); npages = page_size / real_page_size; real_virt_addr = virt_addr; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 5281cad4b552..19c9e3895983 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -5308,10 +5308,10 @@ static int gaudi_parse_cb_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -5380,10 +5380,10 @@ static int gaudi_parse_cb_no_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -5928,7 +5928,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid) return; if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) { - WARN(1, "asid %u is too big\n", asid); + dev_crit(hdev->dev, "asid %u is too big\n", asid); return; } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 720484bffcab..12d9e5205f52 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -3876,10 +3876,10 @@ static int goya_parse_cb_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -3948,10 +3948,10 @@ static int goya_parse_cb_no_mmu(struct hl_device *hdev, patched_cb_handle >>= PAGE_SHIFT; parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) patched_cb_handle); - /* hl_cb_get should never fail here so use kernel WARN */ - WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n", - (u32) patched_cb_handle); + /* hl_cb_get should never fail here */ if (!parser->patched_cb) { + dev_crit(hdev->dev, "DMA CB handle invalid 0x%x\n", + (u32) patched_cb_handle); rc = -EFAULT; goto out; } @@ -5042,7 +5042,7 @@ static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) return; if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) { - WARN(1, "asid %u is too big\n", asid); + dev_crit(hdev->dev, "asid %u is too big\n", asid); return; } From 3b762f55aa27df4cd3e2da0d42ad570d9018d86d Mon Sep 17 00:00:00 2001 From: Omer Shpigelman Date: Wed, 9 Dec 2020 13:28:46 +0200 Subject: [PATCH 160/633] habanalabs: kernel doc format in memory functions Change all memory functions documentation according to kernel doc format. Signed-off-by: Omer Shpigelman Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/memory.c | 377 +++++++++++------------- 1 file changed, 180 insertions(+), 197 deletions(-) diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index fb2c268a262b..53707e9c5c3e 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -38,15 +38,14 @@ */ /* - * alloc_device_memory - allocate device memory - * - * @ctx : current context - * @args : host parameters containing the requested size - * @ret_handle : result handle + * alloc_device_memory() - allocate device memory. + * @ctx: pointer to the context structure. + * @args: host parameters containing the requested size. + * @ret_handle: result handle. * * This function does the following: - * - Allocate the requested size rounded up to 'dram_page_size' pages - * - Return unique handle + * - Allocate the requested size rounded up to 'dram_page_size' pages. + * - Return unique handle for later map/unmap/free. */ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, u32 *ret_handle) @@ -182,17 +181,17 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, return rc; } -/* - * dma_map_host_va - DMA mapping of the given host virtual address. - * @hdev: habanalabs device structure - * @addr: the host virtual address of the memory area - * @size: the size of the memory area - * @p_userptr: pointer to result userptr structure +/** + * dma_map_host_va() - DMA mapping of the given host virtual address. + * @hdev: habanalabs device structure. + * @addr: the host virtual address of the memory area. + * @size: the size of the memory area. + * @p_userptr: pointer to result userptr structure. * * This function does the following: - * - Allocate userptr structure - * - Pin the given host memory using the userptr structure - * - Perform DMA mapping to have the DMA addresses of the pages + * - Allocate userptr structure. + * - Pin the given host memory using the userptr structure. + * - Perform DMA mapping to have the DMA addresses of the pages. */ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr **p_userptr) @@ -236,14 +235,14 @@ static int dma_map_host_va(struct hl_device *hdev, u64 addr, u64 size, return rc; } -/* - * dma_unmap_host_va - DMA unmapping of the given host virtual address. - * @hdev: habanalabs device structure - * @userptr: userptr to free +/** + * dma_unmap_host_va() - DMA unmapping of the given host virtual address. + * @hdev: habanalabs device structure. + * @userptr: userptr to free. * * This function does the following: - * - Unpins the physical pages - * - Frees the userptr structure + * - Unpins the physical pages. + * - Frees the userptr structure. */ static void dma_unmap_host_va(struct hl_device *hdev, struct hl_userptr *userptr) @@ -252,14 +251,13 @@ static void dma_unmap_host_va(struct hl_device *hdev, kfree(userptr); } -/* - * dram_pg_pool_do_release - free DRAM pages pool - * - * @ref : pointer to reference object +/** + * dram_pg_pool_do_release() - free DRAM pages pool + * @ref: pointer to reference object. * * This function does the following: - * - Frees the idr structure of physical pages handles - * - Frees the generic pool of DRAM physical pages + * - Frees the idr structure of physical pages handles. + * - Frees the generic pool of DRAM physical pages. */ static void dram_pg_pool_do_release(struct kref *ref) { @@ -274,15 +272,15 @@ static void dram_pg_pool_do_release(struct kref *ref) gen_pool_destroy(vm->dram_pg_pool); } -/* - * free_phys_pg_pack - free physical page pack - * @hdev: habanalabs device structure - * @phys_pg_pack: physical page pack to free +/** + * free_phys_pg_pack() - free physical page pack. + * @hdev: habanalabs device structure. + * @phys_pg_pack: physical page pack to free. * * This function does the following: * - For DRAM memory only, iterate over the pack and free each physical block - * structure by returning it to the general pool - * - Free the hl_vm_phys_pg_pack structure + * structure by returning it to the general pool. + * - Free the hl_vm_phys_pg_pack structure. */ static void free_phys_pg_pack(struct hl_device *hdev, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -313,14 +311,13 @@ static void free_phys_pg_pack(struct hl_device *hdev, kfree(phys_pg_pack); } -/* - * free_device_memory - free device memory - * - * @ctx : current context - * @handle : handle of the memory chunk to free +/** + * free_device_memory() - free device memory. + * @ctx: pointer to the context structure. + * @handle: handle of the memory chunk to free. * * This function does the following: - * - Free the device memory related to the given handle + * - Free the device memory related to the given handle. */ static int free_device_memory(struct hl_ctx *ctx, u32 handle) { @@ -361,16 +358,15 @@ static int free_device_memory(struct hl_ctx *ctx, u32 handle) return 0; } -/* - * clear_va_list_locked - free virtual addresses list - * - * @hdev : habanalabs device structure - * @va_list : list of virtual addresses to free +/** + * clear_va_list_locked() - free virtual addresses list. + * @hdev: habanalabs device structure. + * @va_list: list of virtual addresses to free. * * This function does the following: - * - Iterate over the list and free each virtual addresses block + * - Iterate over the list and free each virtual addresses block. * - * This function should be called only when va_list lock is taken + * This function should be called only when va_list lock is taken. */ static void clear_va_list_locked(struct hl_device *hdev, struct list_head *va_list) @@ -383,16 +379,15 @@ static void clear_va_list_locked(struct hl_device *hdev, } } -/* - * print_va_list_locked - print virtual addresses list - * - * @hdev : habanalabs device structure - * @va_list : list of virtual addresses to print +/** + * print_va_list_locked() - print virtual addresses list. + * @hdev: habanalabs device structure. + * @va_list: list of virtual addresses to print. * * This function does the following: - * - Iterate over the list and print each virtual addresses block + * - Iterate over the list and print each virtual addresses block. * - * This function should be called only when va_list lock is taken + * This function should be called only when va_list lock is taken. */ static void print_va_list_locked(struct hl_device *hdev, struct list_head *va_list) @@ -409,18 +404,17 @@ static void print_va_list_locked(struct hl_device *hdev, #endif } -/* - * merge_va_blocks_locked - merge a virtual block if possible - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @va_block : virtual block to merge with adjacent blocks +/** + * merge_va_blocks_locked() - merge a virtual block if possible. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @va_block: virtual block to merge with adjacent blocks. * * This function does the following: * - Merge the given blocks with the adjacent blocks if their virtual ranges - * create a contiguous virtual range + * create a contiguous virtual range. * - * This Function should be called only when va_list lock is taken + * This Function should be called only when va_list lock is taken. */ static void merge_va_blocks_locked(struct hl_device *hdev, struct list_head *va_list, struct hl_vm_va_block *va_block) @@ -445,19 +439,18 @@ static void merge_va_blocks_locked(struct hl_device *hdev, } } -/* - * add_va_block_locked - add a virtual block to the virtual addresses list - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @start : start virtual address - * @end : end virtual address +/** + * add_va_block_locked() - add a virtual block to the virtual addresses list. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Add the given block to the virtual blocks list and merge with other - * blocks if a contiguous virtual block can be created + * - Add the given block to the virtual blocks list and merge with other blocks + * if a contiguous virtual block can be created. * - * This Function should be called only when va_list lock is taken + * This Function should be called only when va_list lock is taken. */ static int add_va_block_locked(struct hl_device *hdev, struct list_head *va_list, u64 start, u64 end) @@ -501,16 +494,15 @@ static int add_va_block_locked(struct hl_device *hdev, return 0; } -/* - * add_va_block - wrapper for add_va_block_locked - * - * @hdev : pointer to the habanalabs device structure - * @va_list : pointer to the virtual addresses block list - * @start : start virtual address - * @end : end virtual address +/** + * add_va_block() - wrapper for add_va_block_locked. + * @hdev: pointer to the habanalabs device structure. + * @va_list: pointer to the virtual addresses block list. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Takes the list lock and calls add_va_block_locked + * - Takes the list lock and calls add_va_block_locked. */ static inline int add_va_block(struct hl_device *hdev, struct hl_va_range *va_range, u64 start, u64 end) @@ -524,7 +516,7 @@ static inline int add_va_block(struct hl_device *hdev, return rc; } -/* +/** * get_va_block() - get a virtual block for the given size and alignment. * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range. @@ -620,7 +612,7 @@ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, return res_valid_start; } -/* +/** * hl_reserve_va_block() - reserve a virtual block of a given size. * @hdev: pointer to the habanalabs device structure. * @ctx: current context @@ -644,9 +636,9 @@ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, /** * hl_get_va_range_type() - get va_range type for the given address and size. - * @address: The start address of the area we want to validate. - * @size: The size in bytes of the area we want to validate. - * @type: returned va_range type + * @address: the start address of the area we want to validate. + * @size: the size in bytes of the area we want to validate. + * @type: returned va_range type. * * Return: true if the area is inside a valid range, false otherwise. */ @@ -667,16 +659,15 @@ static int hl_get_va_range_type(struct hl_ctx *ctx, u64 address, u64 size, return -EINVAL; } -/* - * hl_unreserve_va_block - wrapper for add_va_block for unreserving a va block - * +/** + * hl_unreserve_va_block() - wrapper for add_va_block to unreserve a va block. * @hdev: pointer to the habanalabs device structure - * @ctx: current context - * @start: start virtual address - * @end: end virtual address + * @ctx: pointer to the context structure. + * @start: start virtual address. + * @end: end virtual address. * * This function does the following: - * - Takes the list lock and calls add_va_block_locked + * - Takes the list lock and calls add_va_block_locked. */ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, u64 start_addr, u64 size) @@ -701,11 +692,10 @@ int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, return rc; } -/* - * get_sg_info - get number of pages and the DMA address from SG list - * - * @sg : the SG list - * @dma_addr : pointer to DMA address to return +/** + * get_sg_info() - get number of pages and the DMA address from SG list. + * @sg: the SG list. + * @dma_addr: pointer to DMA address to return. * * Calculate the number of consecutive pages described by the SG list. Take the * offset of the address in the first page, add to it the length and round it up @@ -719,17 +709,17 @@ static u32 get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr) (PAGE_SIZE - 1)) >> PAGE_SHIFT; } -/* - * init_phys_pg_pack_from_userptr - initialize physical page pack from host - * memory - * @ctx: current context - * @userptr: userptr to initialize from - * @pphys_pg_pack: result pointer +/** + * init_phys_pg_pack_from_userptr() - initialize physical page pack from host + * memory + * @ctx: pointer to the context structure. + * @userptr: userptr to initialize from. + * @pphys_pg_pack: result pointer. * * This function does the following: - * - Pin the physical pages related to the given virtual block + * - Pin the physical pages related to the given virtual block. * - Create a physical page pack from the physical pages related to the given - * virtual block + * virtual block. */ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, struct hl_userptr *userptr, @@ -821,16 +811,16 @@ static int init_phys_pg_pack_from_userptr(struct hl_ctx *ctx, return rc; } -/* - * map_phys_pg_pack - maps the physical page pack. - * @ctx: current context - * @vaddr: start address of the virtual area to map from - * @phys_pg_pack: the pack of physical pages to map to +/** + * map_phys_pg_pack() - maps the physical page pack.. + * @ctx: pointer to the context structure. + * @vaddr: start address of the virtual area to map from. + * @phys_pg_pack: the pack of physical pages to map to. * * This function does the following: - * - Maps each chunk of virtual memory to matching physical chunk - * - Stores number of successful mappings in the given argument - * - Returns 0 on success, error code otherwise + * - Maps each chunk of virtual memory to matching physical chunk. + * - Stores number of successful mappings in the given argument. + * - Returns 0 on success, error code otherwise. */ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -875,11 +865,11 @@ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, return rc; } -/* - * unmap_phys_pg_pack - unmaps the physical page pack - * @ctx: current context - * @vaddr: start address of the virtual area to unmap - * @phys_pg_pack: the pack of physical pages to unmap +/** + * unmap_phys_pg_pack() - unmaps the physical page pack. + * @ctx: pointer to the context structure. + * @vaddr: start address of the virtual area to unmap. + * @phys_pg_pack: the pack of physical pages to unmap. */ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, struct hl_vm_phys_pg_pack *phys_pg_pack) @@ -913,7 +903,7 @@ static void unmap_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr, } static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args, - u64 *paddr) + u64 *paddr) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; @@ -936,19 +926,18 @@ static int get_paddr_from_handle(struct hl_ctx *ctx, struct hl_mem_in *args, return 0; } -/* - * map_device_va - map the given memory - * - * @ctx : current context - * @args : host parameters with handle/host virtual address - * @device_addr : pointer to result device virtual address +/** + * map_device_va() - map the given memory. + * @ctx: pointer to the context structure. + * @args: host parameters with handle/host virtual address. + * @device_addr: pointer to result device virtual address. * * This function does the following: * - If given a physical device memory handle, map to a device virtual block - * and return the start address of this block + * and return the start address of this block. * - If given a host virtual address and size, find the related physical pages, * map a device virtual block to this pages and return the start address of - * this block + * this block. */ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device_addr) @@ -1125,16 +1114,15 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, return rc; } -/* - * unmap_device_va - unmap the given device virtual address - * - * @ctx : current context - * @vaddr : device virtual address to unmap - * @ctx_free : true if in context free flow, false otherwise. +/** + * unmap_device_va() - unmap the given device virtual address. + * @ctx: pointer to the context structure. + * @vaddr: device virtual address to unmap. + * @ctx_free: true if in context free flow, false otherwise. * * This function does the following: - * - Unmap the physical pages related to the given virtual address - * - return the device virtual block to the virtual block list + * - unmap the physical pages related to the given virtual address. + * - return the device virtual block to the virtual block list. */ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free) { @@ -1301,7 +1289,7 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) rc = 0; } else { rc = get_paddr_from_handle(ctx, &args->in, - &device_addr); + &device_addr); } memset(args, 0, sizeof(*args)); @@ -1478,16 +1466,16 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, return rc; } -/* - * hl_pin_host_memory - pins a chunk of host memory. - * @hdev: pointer to the habanalabs device structure - * @addr: the host virtual address of the memory area - * @size: the size of the memory area - * @userptr: pointer to hl_userptr structure +/** + * hl_pin_host_memory() - pins a chunk of host memory. + * @hdev: pointer to the habanalabs device structure. + * @addr: the host virtual address of the memory area. + * @size: the size of the memory area. + * @userptr: pointer to hl_userptr structure. * * This function does the following: - * - Pins the physical pages - * - Create an SG list from those pages + * - Pins the physical pages. + * - Create an SG list from those pages. */ int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, struct hl_userptr *userptr) @@ -1585,11 +1573,10 @@ void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr) kfree(userptr->sgt); } -/* - * hl_userptr_delete_list - clear userptr list - * - * @hdev : pointer to the habanalabs device structure - * @userptr_list : pointer to the list to clear +/** + * hl_userptr_delete_list() - clear userptr list. + * @hdev: pointer to the habanalabs device structure. + * @userptr_list: pointer to the list to clear. * * This function does the following: * - Iterates over the list and unpins the host memory and frees the userptr @@ -1608,12 +1595,11 @@ void hl_userptr_delete_list(struct hl_device *hdev, INIT_LIST_HEAD(userptr_list); } -/* - * hl_userptr_is_pinned - returns whether the given userptr is pinned - * - * @hdev : pointer to the habanalabs device structure - * @userptr_list : pointer to the list to clear - * @userptr : pointer to userptr to check +/** + * hl_userptr_is_pinned() - returns whether the given userptr is pinned. + * @hdev: pointer to the habanalabs device structure. + * @userptr_list: pointer to the list to clear. + * @userptr: pointer to userptr to check. * * This function does the following: * - Iterates over the list and checks if the given userptr is in it, means is @@ -1631,12 +1617,12 @@ bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, return false; } -/* - * va_range_init - initialize virtual addresses range - * @hdev: pointer to the habanalabs device structure - * @va_range: pointer to the range to initialize - * @start: range start address - * @end: range end address +/** + * va_range_init() - initialize virtual addresses range. + * @hdev: pointer to the habanalabs device structure. + * @va_range: pointer to the range to initialize. + * @start: range start address. + * @end: range end address. * * This function does the following: * - Initializes the virtual addresses list of the given range with the given @@ -1678,13 +1664,13 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range, return 0; } -/* - * va_range_fini() - clear a virtual addresses range - * @hdev: pointer to the habanalabs structure - * va_range: pointer to virtual addresses range +/** + * va_range_fini() - clear a virtual addresses range. + * @hdev: pointer to the habanalabs structure. + * va_range: pointer to virtual addresses rang.e * * This function does the following: - * - Frees the virtual addresses block list and its lock + * - Frees the virtual addresses block list and its lock. */ static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range) { @@ -1696,22 +1682,22 @@ static void va_range_fini(struct hl_device *hdev, struct hl_va_range *va_range) kfree(va_range); } -/* - * vm_ctx_init_with_ranges() - initialize virtual memory for context - * @ctx: pointer to the habanalabs context structure +/** + * vm_ctx_init_with_ranges() - initialize virtual memory for context. + * @ctx: pointer to the habanalabs context structure. * @host_range_start: host virtual addresses range start. * @host_range_end: host virtual addresses range end. * @host_huge_range_start: host virtual addresses range start for memory - * allocated with huge pages. + * allocated with huge pages. * @host_huge_range_end: host virtual addresses range end for memory allocated * with huge pages. * @dram_range_start: dram virtual addresses range start. * @dram_range_end: dram virtual addresses range end. * * This function initializes the following: - * - MMU for context - * - Virtual address to area descriptor hashtable - * - Virtual block list of available virtual memory + * - MMU for context. + * - Virtual address to area descriptor hashtable. + * - Virtual block list of available virtual memory. */ static int vm_ctx_init_with_ranges(struct hl_ctx *ctx, u64 host_range_start, @@ -1846,15 +1832,14 @@ int hl_vm_ctx_init(struct hl_ctx *ctx) dram_range_start, dram_range_end, dram_page_size); } -/* - * hl_vm_ctx_fini - virtual memory teardown of context - * - * @ctx : pointer to the habanalabs context structure +/** + * hl_vm_ctx_fini() - virtual memory teardown of context. + * @ctx: pointer to the habanalabs context structure. * * This function perform teardown the following: - * - Virtual block list of available virtual memory - * - Virtual address to area descriptor hashtable - * - MMU for context + * - Virtual block list of available virtual memory. + * - Virtual address to area descriptor hashtable. + * - MMU for context. * * In addition this function does the following: * - Unmaps the existing hashtable nodes if the hashtable is not empty. The @@ -1875,7 +1860,7 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) struct hlist_node *tmp_node; int i; - if (!ctx->hdev->mmu_enable) + if (!hdev->mmu_enable) return; hl_debugfs_remove_ctx_mem_hash(hdev, ctx); @@ -1930,19 +1915,18 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) * all DRAM is available */ if (ctx->asid != HL_KERNEL_ASID_ID && - !ctx->hdev->asic_prop.dram_supports_virtual_memory) - atomic64_set(&ctx->hdev->dram_used_mem, 0); + !hdev->asic_prop.dram_supports_virtual_memory) + atomic64_set(&hdev->dram_used_mem, 0); } -/* - * hl_vm_init - initialize virtual memory module - * - * @hdev : pointer to the habanalabs device structure +/** + * hl_vm_init() - initialize virtual memory module. + * @hdev: pointer to the habanalabs device structure. * * This function initializes the following: - * - MMU module - * - DRAM physical pages pool of 2MB - * - Idr for device memory allocation handles + * - MMU module. + * - DRAM physical pages pool of 2MB. + * - Idr for device memory allocation handles. */ int hl_vm_init(struct hl_device *hdev) { @@ -1983,15 +1967,14 @@ int hl_vm_init(struct hl_device *hdev) return rc; } -/* - * hl_vm_fini - virtual memory module teardown - * - * @hdev : pointer to the habanalabs device structure +/** + * hl_vm_fini() - virtual memory module teardown. + * @hdev: pointer to the habanalabs device structure. * * This function perform teardown to the following: - * - Idr for device memory allocation handles - * - DRAM physical pages pool of 2MB - * - MMU module + * - Idr for device memory allocation handles. + * - DRAM physical pages pool of 2MB. + * - MMU module. */ void hl_vm_fini(struct hl_device *hdev) { From f19040ce418d6fb4837dba45a920c0ae0d5c698f Mon Sep 17 00:00:00 2001 From: Omer Shpigelman Date: Wed, 9 Dec 2020 13:34:11 +0200 Subject: [PATCH 161/633] habanalabs: modify memory functions signatures For consistency, modify all memory ioctl functions to get the ioctl arguments structure rather than the arguments themselves. Signed-off-by: Omer Shpigelman Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/memory.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 53707e9c5c3e..207fda36cac9 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -314,16 +314,17 @@ static void free_phys_pg_pack(struct hl_device *hdev, /** * free_device_memory() - free device memory. * @ctx: pointer to the context structure. - * @handle: handle of the memory chunk to free. + * @args: host parameters containing the requested size. * * This function does the following: * - Free the device memory related to the given handle. */ -static int free_device_memory(struct hl_ctx *ctx, u32 handle) +static int free_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args) { struct hl_device *hdev = ctx->hdev; struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; + u32 handle = args->free.handle; spin_lock(&vm->idr_lock); phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); @@ -1117,20 +1118,22 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, /** * unmap_device_va() - unmap the given device virtual address. * @ctx: pointer to the context structure. - * @vaddr: device virtual address to unmap. + * @args: host parameters with device virtual address to unmap. * @ctx_free: true if in context free flow, false otherwise. * * This function does the following: * - unmap the physical pages related to the given virtual address. * - return the device virtual block to the virtual block list. */ -static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free) +static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, + bool ctx_free) { struct hl_device *hdev = ctx->hdev; struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; struct hl_vm_hash_node *hnode = NULL; struct hl_userptr *userptr = NULL; struct hl_va_range *va_range; + u64 vaddr = args->unmap.device_virt_addr; enum vm_type_t *vm_type; bool is_userptr; int rc = 0; @@ -1280,7 +1283,7 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) break; case HL_MEM_OP_FREE: - rc = free_device_memory(ctx, args->in.free.handle); + rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: @@ -1388,7 +1391,7 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) goto out; } - rc = free_device_memory(ctx, args->in.free.handle); + rc = free_device_memory(ctx, &args->in); break; case HL_MEM_OP_MAP: @@ -1399,8 +1402,7 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) break; case HL_MEM_OP_UNMAP: - rc = unmap_device_va(ctx, args->in.unmap.device_virt_addr, - false); + rc = unmap_device_va(ctx, &args->in, false); break; default: @@ -1858,6 +1860,7 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) struct hl_vm_phys_pg_pack *phys_pg_list; struct hl_vm_hash_node *hnode; struct hlist_node *tmp_node; + struct hl_mem_in args; int i; if (!hdev->mmu_enable) @@ -1877,7 +1880,8 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) dev_dbg(hdev->dev, "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n", hnode->vaddr, ctx->asid); - unmap_device_va(ctx, hnode->vaddr, true); + args.unmap.device_virt_addr = hnode->vaddr; + unmap_device_va(ctx, &args, true); } mutex_lock(&ctx->mmu_lock); From 2e368560080cf020cf149578e70d0594ca4d19d0 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Wed, 16 Dec 2020 09:12:08 +0200 Subject: [PATCH 162/633] habanalabs/gaudi: add debug prints for security status In order to have more information while debugging boot issues, we should print the firmware security status at every boot stage. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index c9a12980218a..3cb3d32aa814 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -659,6 +659,9 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg, prop->fw_security_disabled = true; } + dev_dbg(hdev->dev, "Firmware preboot security status %#x\n", + security_status); + dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n", prop->hard_reset_done_by_fw ? "enabled" : "disabled"); @@ -753,6 +756,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (prop->fw_boot_cpu_security_map & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; + + dev_dbg(hdev->dev, + "Firmware boot CPU security status %#x\n", + prop->fw_boot_cpu_security_map); } dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n", @@ -837,6 +844,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, if (prop->fw_app_security_map & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; + + dev_dbg(hdev->dev, + "Firmware application CPU security status %#x\n", + prop->fw_app_security_map); } dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n", From 1530d468178d90e1a2870f0693e5f1a240e74dc5 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Fri, 18 Dec 2020 19:12:56 +0200 Subject: [PATCH 163/633] habanalabs: add ASIC property of functional HBMs The number of functional HBMs in the same ASIC can be different due to malfunctioning HBM banks. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index dee73a04411b..f083487ca563 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -416,6 +416,7 @@ struct hl_mmu_properties { * from BOOT_DEV_STS0 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow + * @num_functional_hbms: number of functional HBMs in each DCORE. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -474,6 +475,7 @@ struct asic_fixed_properties { u8 fw_security_status_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; + u8 num_functional_hbms; }; /** From 3abe1040ba61a0a7148364ee4b4d969d7afd375b Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Fri, 18 Dec 2020 21:27:54 +0200 Subject: [PATCH 164/633] habanalabs: update to latest hl_boot_if.h Update the latest version of this file that the F/W exports Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/include/common/hl_boot_if.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index 1115456cca85..93552d6b59b3 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -150,6 +150,10 @@ * CPU_BOOT_DEV_STS0_PLL_INFO_EN FW retrieval of PLL info is enabled. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_SP_SRAM_EN SP SRAM is initialized and available + * for use. + * Initialized in: preboot + * * CPU_BOOT_DEV_STS0_CLK_GATE_EN Clock Gating enabled. * FW initialized Clock Gating. * Initialized in: preboot @@ -162,10 +166,6 @@ * statuses. * Initialized in: preboot * - * CPU_BOOT_DEV_STS0_SP_SRAM_EN SP SRAM is initialized and available - * for use. - * Initialized in: preboot - * */ #define CPU_BOOT_DEV_STS0_SECURITY_EN (1 << 0) #define CPU_BOOT_DEV_STS0_DEBUG_EN (1 << 1) From 9402a3362462d080b998e00ed33fc193c12adbbf Mon Sep 17 00:00:00 2001 From: Alon Mizrahi Date: Wed, 23 Dec 2020 17:53:17 +0200 Subject: [PATCH 165/633] habanalabs: return dram virtual address in info ioctl When working with DRAM MMU, we should supply the userspace with the virtual start address of the DRAM instead of the physical one. This is because the physical one has no meaning for the user as he only knows the virtual address range. Signed-off-by: Alon Mizrahi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 394a2e1767ce..018320e0331b 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -57,7 +57,9 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev); hw_ip.sram_base_address = prop->sram_user_base_address; - hw_ip.dram_base_address = prop->dram_user_base_address; + hw_ip.dram_base_address = + hdev->mmu_enable && prop->dram_supports_virtual_memory ? + prop->dmmu.start_addr : prop->dram_user_base_address; hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask; hw_ip.sram_size = prop->sram_size - sram_kmd_size; hw_ip.dram_size = prop->dram_size - dram_kmd_size; From 266cdfa2b7bd2d2ae2fdc895f10254b71276c751 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 22 Dec 2020 15:56:12 +0200 Subject: [PATCH 166/633] habanalabs/gaudi: set uninitialized symbol Initialize local variable that is returned by the function, in case it is never assigned. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 19c9e3895983..e105f1c6ab74 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -1248,7 +1248,7 @@ static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, u32 queue_id, collective_queue, num_jobs; u32 stream, nic_queue, nic_idx = 0; bool skip; - int i, rc; + int i, rc = 0; /* Verify wait queue id is configured as master */ hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id]; From a1f8533269aa6e96154fddc33b2cb3696cde356e Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Mon, 28 Dec 2020 18:28:08 +0200 Subject: [PATCH 167/633] habanalabs: remove access to kernel memory using debugfs Accessing kernel allocated memory through debugfs should not be allowed as it introduces a security vulnerability. We remove the option to read/write kernel memory for all asics. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 8 -------- drivers/misc/habanalabs/goya/goya.c | 12 ------------ 2 files changed, 20 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index e105f1c6ab74..3d8414d3afa8 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -5734,8 +5734,6 @@ static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); } else { rc = -EFAULT; } @@ -5781,8 +5779,6 @@ static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; } else { rc = -EFAULT; } @@ -5832,8 +5828,6 @@ static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); } else { rc = -EFAULT; } @@ -5882,8 +5876,6 @@ static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) } if (hbm_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; } else { rc = -EFAULT; } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 12d9e5205f52..eeade9a45bc0 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -4122,9 +4122,6 @@ static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE); - } else { rc = -EFAULT; } @@ -4178,9 +4175,6 @@ static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; - } else { rc = -EFAULT; } @@ -4223,9 +4217,6 @@ static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE); - } else { rc = -EFAULT; } @@ -4266,9 +4257,6 @@ static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) if (ddr_bar_addr == U64_MAX) rc = -EIO; - } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) { - *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val; - } else { rc = -EFAULT; } From b19dc67aa8c008661b10b1c2372a29f403bf5dac Mon Sep 17 00:00:00 2001 From: Moti Haimovski Date: Wed, 18 Nov 2020 20:15:29 +0200 Subject: [PATCH 168/633] habanalabs: support non power-of-2 DRAM phys page sizes DRAM physical page sizes depend of the amount of HBMs available in the device. this number is device-dependent and may also be subject to binning when one or more of the DRAM controllers are found to to be faulty. Such a configuration may lead to partitioning the DRAM to non-power-of-2 pages. To support this feature we also need to add infrastructure of address scarmbling. Signed-off-by: Moti Haimovski Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/debugfs.c | 6 +- drivers/misc/habanalabs/common/habanalabs.h | 8 + drivers/misc/habanalabs/common/memory.c | 176 ++++++++++++++++---- drivers/misc/habanalabs/common/mmu.c | 61 ++++++- drivers/misc/habanalabs/gaudi/gaudi.c | 3 +- drivers/misc/habanalabs/goya/goya.c | 3 +- 6 files changed, 213 insertions(+), 44 deletions(-) diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index cef716643979..50ca8eea6648 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -333,8 +333,10 @@ static int mmu_show(struct seq_file *s, void *data) return 0; } - seq_printf(s, "asid: %u, virt_addr: 0x%llx\n", - dev_entry->mmu_asid, dev_entry->mmu_addr); + seq_printf(s, + "asid: %u, virt_addr: 0x%llx, scrambled virt_addr: 0x%llx\n", + dev_entry->mmu_asid, dev_entry->mmu_addr, + hops_info.scrambled_vaddr); for (i = 0 ; i < hops_info.used_hops ; i++) { seq_printf(s, "hop%d_addr: 0x%llx\n", diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index f083487ca563..5da26e81a251 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -848,6 +848,8 @@ enum div_select_defs { * @collective_wait_init_cs: Generate collective master/slave packets * and place them in the relevant cs jobs * @collective_wait_create_jobs: allocate collective wait cs jobs + * @scramble_vaddr: Routine to scramble the virtual address prior of mapping it + * in the MMU. */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -957,6 +959,7 @@ struct hl_asic_funcs { int (*collective_wait_create_jobs)(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id); + u64 (*scramble_vaddr)(struct hl_device *hdev, u64 virt_addr); }; @@ -1690,10 +1693,14 @@ struct hl_mmu_per_hop_info { * struct hl_mmu_hop_info - A structure describing the TLB hops and their * hop-entries that were created in order to translate a virtual address to a * physical one. + * @scrambled_vaddr: The value of the virtual address after scrambling. This + * address replaces the original virtual-address when mapped + * in the MMU tables. * @hop_info: Array holding the per-hop information used for the translation. * @used_hops: The number of hops used for the translation. */ struct hl_mmu_hop_info { + u64 scrambled_vaddr; struct hl_mmu_per_hop_info hop_info[MMU_ARCH_5_HOPS]; u32 used_hops; }; @@ -2184,6 +2191,7 @@ void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); +u64 hl_mmu_scramble_vaddr(struct hl_device *hdev, u64 virt_addr); bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 207fda36cac9..ada977eb136c 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -14,6 +14,9 @@ #define HL_MMU_DEBUG 0 +/* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */ +#define DRAM_POOL_PAGE_SIZE SZ_8M + /* * The va ranges in context object contain a list with the available chunks of * device virtual memory. @@ -54,15 +57,14 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, struct hl_vm *vm = &hdev->vm; struct hl_vm_phys_pg_pack *phys_pg_pack; u64 paddr = 0, total_size, num_pgs, i; - u32 num_curr_pgs, page_size, page_shift; + u32 num_curr_pgs, page_size; int handle, rc; bool contiguous; num_curr_pgs = 0; page_size = hdev->asic_prop.dram_page_size; - page_shift = __ffs(page_size); - num_pgs = (args->alloc.mem_size + (page_size - 1)) >> page_shift; - total_size = num_pgs << page_shift; + num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size); + total_size = num_pgs * page_size; if (!total_size) { dev_err(hdev->dev, "Cannot allocate 0 bytes\n"); @@ -518,7 +520,8 @@ static inline int add_va_block(struct hl_device *hdev, } /** - * get_va_block() - get a virtual block for the given size and alignment. + * get_va_block_pow2() - get a virtual block for the given size and alignment + * where alignment is a power of 2. * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range. * @size: requested block size. @@ -531,12 +534,13 @@ static inline int add_va_block(struct hl_device *hdev, * - Reserve the requested block and update the list. * - Return the start address of the virtual block. */ -static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, - u64 size, u64 hint_addr, u32 va_block_align) +static u64 get_va_block_pow2(struct hl_device *hdev, + struct hl_va_range *va_range, + u64 size, u64 hint_addr, u32 va_block_align) { struct hl_vm_va_block *va_block, *new_va_block = NULL; u64 valid_start, valid_size, prev_start, prev_end, align_mask, - res_valid_start = 0, res_valid_size = 0; + reserved_valid_start = 0, reserved_valid_size = 0; bool add_prev = false; align_mask = ~((u64)va_block_align - 1); @@ -562,34 +566,34 @@ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, valid_size = va_block->end - valid_start; - if (valid_size >= size && - (!new_va_block || valid_size < res_valid_size)) { + if (valid_size >= size && (!new_va_block || + valid_size < reserved_valid_size)) { new_va_block = va_block; - res_valid_start = valid_start; - res_valid_size = valid_size; + reserved_valid_start = valid_start; + reserved_valid_size = valid_size; } if (hint_addr && hint_addr >= valid_start && - ((hint_addr + size) <= va_block->end)) { + (hint_addr + size) <= va_block->end) { new_va_block = va_block; - res_valid_start = hint_addr; - res_valid_size = valid_size; + reserved_valid_start = hint_addr; + reserved_valid_size = valid_size; break; } } if (!new_va_block) { dev_err(hdev->dev, "no available va block for size %llu\n", - size); + size); goto out; } - if (res_valid_start > new_va_block->start) { + if (reserved_valid_start > new_va_block->start) { prev_start = new_va_block->start; - prev_end = res_valid_start - 1; + prev_end = reserved_valid_start - 1; - new_va_block->start = res_valid_start; - new_va_block->size = res_valid_size; + new_va_block->start = reserved_valid_start; + new_va_block->size = reserved_valid_size; add_prev = true; } @@ -610,10 +614,98 @@ static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, out: mutex_unlock(&va_range->lock); - return res_valid_start; + return reserved_valid_start; } /** + * get_va_block_non_pow2() - get a virtual block for the given size and + * alignment where alignment is not a power of 2. + * @hdev: pointer to the habanalabs device structure. + * @va_range: pointer to the virtual addresses range. + * @size: requested block size. + * @hint_addr: hint for requested address by the user. + * @va_block_align: required alignment of the virtual block start address. + * + * This function does the following: + * - Iterate on the virtual block list to find a suitable virtual block for the + * given size and alignment. + * - Reserve the requested block and update the list. + * - Return the start address of the virtual block. + */ +static u64 get_va_block_non_pow2(struct hl_device *hdev, + struct hl_va_range *va_range, + u64 size, u64 hint_addr, u32 va_block_align) +{ + struct hl_vm_va_block *va_block, *new_va_block = NULL; + u64 reserved_valid_start = 0; + + /* + * with non-power-of-2 range we work only with page granularity and the + * start address is page aligned, so no need for alignment checking. + */ + size = DIV_ROUND_UP_ULL(size, va_range->page_size) * + va_range->page_size; + + mutex_lock(&va_range->lock); + + print_va_list_locked(hdev, &va_range->list); + + list_for_each_entry(va_block, &va_range->list, node) { + if ((va_block->start + size) > va_block->end) + continue; + + new_va_block = va_block; + reserved_valid_start = va_block->start; + break; + } + + if (!new_va_block) { + dev_err(hdev->dev, "no available va block for size %llu\n", + size); + goto out; + } + + if (new_va_block->size > size) { + new_va_block->start += size; + new_va_block->size = new_va_block->end - new_va_block->start; + } else { + list_del(&new_va_block->node); + kfree(new_va_block); + } + + print_va_list_locked(hdev, &va_range->list); +out: + mutex_unlock(&va_range->lock); + + return reserved_valid_start; +} + +/* + * get_va_block() - get a virtual block for the given size and alignment. + * @hdev: pointer to the habanalabs device structure. + * @va_range: pointer to the virtual addresses range. + * @size: requested block size. + * @hint_addr: hint for requested address by the user. + * @va_block_align: required alignment of the virtual block start address. + * + * This function does the following: + * - Iterate on the virtual block list to find a suitable virtual block for the + * given size and alignment. + * - Reserve the requested block and update the list. + * - Return the start address of the virtual block. + */ +static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, + u64 size, u64 hint_addr, u32 va_block_align) +{ + if (is_power_of_2(va_range->page_size)) + return get_va_block_pow2(hdev, va_range, + size, hint_addr, va_block_align); + else + return get_va_block_non_pow2(hdev, va_range, + size, hint_addr, va_block_align); +} + +/* * hl_reserve_va_block() - reserve a virtual block of a given size. * @hdev: pointer to the habanalabs device structure. * @ctx: current context @@ -1024,7 +1116,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, hint_addr = args->map_device.hint_addr; - /* DRAM VA alignment is the same as the DRAM page size */ + /* DRAM VA alignment is the same as the MMU page size */ va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM]; va_block_align = hdev->asic_prop.dmmu.page_size; } @@ -1129,6 +1221,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, bool ctx_free) { struct hl_device *hdev = ctx->hdev; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; struct hl_vm_hash_node *hnode = NULL; struct hl_userptr *userptr = NULL; @@ -1192,7 +1285,13 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, goto mapping_cnt_err; } - vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); + if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size)) + vaddr = prop->dram_base_address + + DIV_ROUND_DOWN_ULL(vaddr - prop->dram_base_address, + phys_pg_pack->page_size) * + phys_pg_pack->page_size; + else + vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); mutex_lock(&ctx->mmu_lock); @@ -1637,16 +1736,22 @@ static int va_range_init(struct hl_device *hdev, struct hl_va_range *va_range, INIT_LIST_HEAD(&va_range->list); - /* PAGE_SIZE alignment */ + /* + * PAGE_SIZE alignment + * it is the callers responsibility to align the addresses if the + * page size is not a power of 2 + */ - if (start & (PAGE_SIZE - 1)) { - start &= PAGE_MASK; - start += PAGE_SIZE; + if (is_power_of_2(page_size)) { + if (start & (PAGE_SIZE - 1)) { + start &= PAGE_MASK; + start += PAGE_SIZE; + } + + if (end & (PAGE_SIZE - 1)) + end &= PAGE_MASK; } - if (end & (PAGE_SIZE - 1)) - end &= PAGE_MASK; - if (start >= end) { dev_err(hdev->dev, "too small vm range for va list\n"); return -EFAULT; @@ -1820,7 +1925,8 @@ int hl_vm_ctx_init(struct hl_ctx *ctx) dram_range_start = prop->dmmu.start_addr; dram_range_end = prop->dmmu.end_addr; - dram_page_size = prop->dmmu.page_size; + dram_page_size = prop->dram_page_size ? + prop->dram_page_size : prop->dmmu.page_size; host_range_start = prop->pmmu.start_addr; host_range_end = prop->pmmu.end_addr; host_page_size = prop->pmmu.page_size; @@ -1938,7 +2044,13 @@ int hl_vm_init(struct hl_device *hdev) struct hl_vm *vm = &hdev->vm; int rc; - vm->dram_pg_pool = gen_pool_create(__ffs(prop->dram_page_size), -1); + if (is_power_of_2(prop->dram_page_size)) + vm->dram_pg_pool = + gen_pool_create(__ffs(prop->dram_page_size), -1); + else + vm->dram_pg_pool = + gen_pool_create(__ffs(DRAM_POOL_PAGE_SIZE), -1); + if (!vm->dram_pg_pool) { dev_err(hdev->dev, "Failed to create dram page pool\n"); return -ENOMEM; diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu.c index 62cfa4190fe4..38234c243b21 100644 --- a/drivers/misc/habanalabs/common/mmu.c +++ b/drivers/misc/habanalabs/common/mmu.c @@ -166,7 +166,6 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, mmu_prop = &prop->pmmu; pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; - /* * The H/W handles mapping of specific page sizes. Hence if the page * size is bigger, we break it to sub-pages and unmap them separately. @@ -174,11 +173,21 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, if ((page_size % mmu_prop->page_size) == 0) { real_page_size = mmu_prop->page_size; } else { - dev_err(hdev->dev, - "page size of %u is not %uKB aligned, can't unmap\n", - page_size, mmu_prop->page_size >> 10); + /* + * MMU page size may differ from DRAM page size. + * In such case work with the DRAM page size and let the MMU + * scrambling routine to handle this mismatch when + * calculating the address to remove from the MMU page table + */ + if (is_dram_addr && ((page_size % prop->dram_page_size) == 0)) { + real_page_size = prop->dram_page_size; + } else { + dev_err(hdev->dev, + "page size of %u is not %uKB aligned, can't unmap\n", + page_size, mmu_prop->page_size >> 10); - return -EFAULT; + return -EFAULT; + } } npages = page_size / real_page_size; @@ -253,6 +262,17 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, */ if ((page_size % mmu_prop->page_size) == 0) { real_page_size = mmu_prop->page_size; + } else if (is_dram_addr && ((page_size % prop->dram_page_size) == 0) && + (prop->dram_page_size < mmu_prop->page_size)) { + /* + * MMU page size may differ from DRAM page size. + * In such case work with the DRAM page size and let the MMU + * scrambling routine handle this mismatch when calculating + * the address to place in the MMU page table. (in that case + * also make sure that the dram_page_size smaller than the + * mmu page size) + */ + real_page_size = prop->dram_page_size; } else { dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", @@ -261,10 +281,21 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, return -EFAULT; } - if (phys_addr & (real_page_size - 1)) + /* + * Verify that the phys and virt addresses are aligned with the + * MMU page size (in dram this means checking the address and MMU + * after scrambling) + */ + if ((is_dram_addr && + ((hdev->asic_funcs->scramble_vaddr(hdev, phys_addr) & + (mmu_prop->page_size - 1)) || + (hdev->asic_funcs->scramble_vaddr(hdev, virt_addr) & + (mmu_prop->page_size - 1)))) || + (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || + (virt_addr & (real_page_size - 1))))) dev_crit(hdev->dev, - "Mapping 0x%llx with page size of 0x%x is erroneous! Address must be divisible by page size", - phys_addr, real_page_size); + "Mapping address 0x%llx with virtual address 0x%llx and page size of 0x%x is erroneous! Addresses must be divisible by page size", + phys_addr, virt_addr, real_page_size); npages = page_size / real_page_size; real_virt_addr = virt_addr; @@ -474,6 +505,8 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, if (!hdev->mmu_enable) return -EOPNOTSUPP; + hops->scrambled_vaddr = virt_addr; /* assume no scrambling */ + is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, prop->dmmu.start_addr, prop->dmmu.end_addr); @@ -513,3 +546,15 @@ int hl_mmu_if_set_funcs(struct hl_device *hdev) return 0; } + +/** + * hl_mmu_scramble_vaddr() - The generic mmu virtual address scrambling routine. + * @hdev: pointer to device data. + * @virt_addr: The virtual address to scramble. + * + * Return: The scrambled virtual address. + */ +u64 hl_mmu_scramble_vaddr(struct hl_device *hdev, u64 virt_addr) +{ + return virt_addr; +} diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 3d8414d3afa8..66194b7d3806 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -8308,7 +8308,8 @@ static const struct hl_asic_funcs gaudi_funcs = { .set_dma_mask_from_fw = gaudi_set_dma_mask_from_fw, .get_device_time = gaudi_get_device_time, .collective_wait_init_cs = gaudi_collective_wait_init_cs, - .collective_wait_create_jobs = gaudi_collective_wait_create_jobs + .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, + .scramble_vaddr = hl_mmu_scramble_vaddr }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index eeade9a45bc0..80198c39985a 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5456,7 +5456,8 @@ static const struct hl_asic_funcs goya_funcs = { .set_dma_mask_from_fw = goya_set_dma_mask_from_fw, .get_device_time = goya_get_device_time, .collective_wait_init_cs = goya_collective_wait_init_cs, - .collective_wait_create_jobs = goya_collective_wait_create_jobs + .collective_wait_create_jobs = goya_collective_wait_create_jobs, + .scramble_vaddr = hl_mmu_scramble_vaddr }; /* From 6769cea8de590920c95603c4f3ce063e38033423 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 31 Dec 2020 12:00:55 +0200 Subject: [PATCH 169/633] habanalabs: report correct dram size in info ioctl In case MMU is enabled, we must take MMU page size into consideration when reporting dram size to the user. This is because the MMU page size can be a value which is NOT a power-of-2 value. As a result, the total DRAM size (which is always a power-of-2 value) needed to be rounded-down. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 018320e0331b..dfac5c8cadb3 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -62,7 +62,15 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) prop->dmmu.start_addr : prop->dram_user_base_address; hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask; hw_ip.sram_size = prop->sram_size - sram_kmd_size; - hw_ip.dram_size = prop->dram_size - dram_kmd_size; + + if (hdev->mmu_enable) + hw_ip.dram_size = + DIV_ROUND_DOWN_ULL(prop->dram_size - dram_kmd_size, + prop->dram_page_size) * + prop->dram_page_size; + else + hw_ip.dram_size = prop->dram_size - dram_kmd_size; + if (hw_ip.dram_size > PAGE_SIZE) hw_ip.dram_enabled = 1; hw_ip.dram_page_size = prop->dram_page_size; From edb07cb69caacb9be06a299cdf62f266292cc890 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 27 Dec 2020 17:09:09 +0200 Subject: [PATCH 170/633] habanalabs: read device boot errors after cpucp is up Boot cpu can report errors in various boot stages. Current implementaion does not take into consideration errors reported in late stages, hence we will check for errors at the most late stage when fetching cpucp information. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 126 ++++++++++-------- drivers/misc/habanalabs/common/habanalabs.h | 3 +- drivers/misc/habanalabs/gaudi/gaudi.c | 2 +- drivers/misc/habanalabs/goya/goya.c | 2 +- .../habanalabs/include/common/hl_boot_if.h | 3 + 5 files changed, 80 insertions(+), 56 deletions(-) diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 3cb3d32aa814..9f4ae21a126f 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -279,8 +279,68 @@ int hl_fw_send_heartbeat(struct hl_device *hdev) return rc; } +static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, + u32 cpu_security_boot_status_reg) +{ + u32 err_val, security_val; + + /* Some of the firmware status codes are deprecated in newer f/w + * versions. In those versions, the errors are reported + * in different registers. Therefore, we need to check those + * registers and print the exact errors. Moreover, there + * may be multiple errors, so we need to report on each error + * separately. Some of the error codes might indicate a state + * that is not an error per-se, but it is an error in production + * environment + */ + err_val = RREG32(boot_err0_reg); + if (!(err_val & CPU_BOOT_ERR0_ENABLED)) + return 0; + + if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) + dev_err(hdev->dev, + "Device boot error - DRAM initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) + dev_err(hdev->dev, "Device boot error - FIT image corrupted\n"); + if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) + dev_err(hdev->dev, + "Device boot error - Thermal Sensor initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) + dev_warn(hdev->dev, + "Device boot warning - Skipped DRAM initialization\n"); + if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) + dev_warn(hdev->dev, + "Device boot error - Skipped waiting for BMC\n"); + if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) + dev_err(hdev->dev, + "Device boot error - Serdes data from BMC not available\n"); + if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) + dev_err(hdev->dev, + "Device boot error - NIC F/W initialization failed\n"); + if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) + dev_warn(hdev->dev, + "Device boot warning - security not ready\n"); + if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) + dev_err(hdev->dev, "Device boot error - security failure\n"); + if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) + dev_err(hdev->dev, "Device boot error - eFuse failure\n"); + if (err_val & CPU_BOOT_ERR0_PLL_FAIL) + dev_err(hdev->dev, "Device boot error - PLL failure\n"); + + security_val = RREG32(cpu_security_boot_status_reg); + if (security_val & CPU_BOOT_DEV_STS0_ENABLED) + dev_dbg(hdev->dev, "Device security status %#x\n", + security_val); + + if (err_val & ~CPU_BOOT_ERR0_ENABLED) + return -EIO; + + return 0; +} + int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg) + u32 cpu_security_boot_status_reg, + u32 boot_err0_reg) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet pkt = {}; @@ -314,6 +374,12 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev, goto out; } + rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + if (rc) { + dev_err(hdev->dev, "Errors in device boot\n"); + goto out; + } + memcpy(&prop->cpucp_info, cpucp_info_cpu_addr, sizeof(prop->cpucp_info)); @@ -483,58 +549,6 @@ int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index, return rc; } -static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, - u32 cpu_security_boot_status_reg) -{ - u32 err_val, security_val; - - /* Some of the firmware status codes are deprecated in newer f/w - * versions. In those versions, the errors are reported - * in different registers. Therefore, we need to check those - * registers and print the exact errors. Moreover, there - * may be multiple errors, so we need to report on each error - * separately. Some of the error codes might indicate a state - * that is not an error per-se, but it is an error in production - * environment - */ - err_val = RREG32(boot_err0_reg); - if (!(err_val & CPU_BOOT_ERR0_ENABLED)) - return; - - if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) - dev_err(hdev->dev, - "Device boot error - DRAM initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) - dev_err(hdev->dev, "Device boot error - FIT image corrupted\n"); - if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) - dev_err(hdev->dev, - "Device boot error - Thermal Sensor initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) - dev_warn(hdev->dev, - "Device boot warning - Skipped DRAM initialization\n"); - if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) - dev_warn(hdev->dev, - "Device boot error - Skipped waiting for BMC\n"); - if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) - dev_err(hdev->dev, - "Device boot error - Serdes data from BMC not available\n"); - if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) - dev_err(hdev->dev, - "Device boot error - NIC F/W initialization failed\n"); - if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) - dev_warn(hdev->dev, - "Device boot warning - security not ready\n"); - if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) - dev_err(hdev->dev, "Device boot error - security failure\n"); - if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) - dev_err(hdev->dev, "Device boot error - eFuse failure\n"); - - security_val = RREG32(cpu_security_boot_status_reg); - if (security_val & CPU_BOOT_DEV_STS0_ENABLED) - dev_dbg(hdev->dev, "Device security status %#x\n", - security_val); -} - static void detect_cpu_boot_status(struct hl_device *hdev, u32 status) { /* Some of the status codes below are deprecated in newer f/w @@ -833,6 +847,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, goto out; } + rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); + if (rc) + return rc; + /* Clear reset status since we need to read again from app */ prop->hard_reset_done_by_fw = false; @@ -855,6 +873,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, dev_info(hdev->dev, "Successfully loaded firmware to device\n"); + return 0; + out: fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 5da26e81a251..eb43fb3065ce 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -2209,7 +2209,8 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); int hl_fw_send_heartbeat(struct hl_device *hdev); int hl_fw_cpucp_info_get(struct hl_device *hdev, - u32 cpu_security_boot_status_reg); + u32 cpu_security_boot_status_reg, + u32 boot_err0_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 66194b7d3806..f6cfd6e075e2 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7451,7 +7451,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); if (rc) return rc; diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 80198c39985a..cf0496b04044 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5150,7 +5150,7 @@ int goya_cpucp_info_get(struct hl_device *hdev) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0); if (rc) return rc; diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index 93552d6b59b3..e29c77bdea07 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -69,6 +69,8 @@ * image has failed to match expected * checksum. Trying to program image again * might solve this. + * CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one + * of the PLLs remained in REF_CLK * * CPU_BOOT_ERR0_ENABLED Error registers enabled. * This is a main indication that the @@ -88,6 +90,7 @@ #define CPU_BOOT_ERR0_EFUSE_FAIL (1 << 9) #define CPU_BOOT_ERR0_PRI_IMG_VER_FAIL (1 << 10) #define CPU_BOOT_ERR0_SEC_IMG_VER_FAIL (1 << 11) +#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12) #define CPU_BOOT_ERR0_ENABLED (1 << 31) /* From 8563e19159b02ec26a6ec7dc7bac9766a8ac494c Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Mon, 28 Dec 2020 14:36:47 +0200 Subject: [PATCH 171/633] habanalabs: separate common code to dedicated folders We separate some of the common code source files to different folders for a better maintainability and testability. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/Makefile | 10 ++++++++-- drivers/misc/habanalabs/common/mmu/Makefile | 2 ++ drivers/misc/habanalabs/common/{ => mmu}/mmu.c | 2 +- drivers/misc/habanalabs/common/{ => mmu}/mmu_v1.c | 4 ++-- drivers/misc/habanalabs/common/pci/Makefile | 2 ++ drivers/misc/habanalabs/common/{ => pci}/pci.c | 4 ++-- 6 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 drivers/misc/habanalabs/common/mmu/Makefile rename drivers/misc/habanalabs/common/{ => mmu}/mmu.c (99%) rename drivers/misc/habanalabs/common/{ => mmu}/mmu_v1.c (99%) create mode 100644 drivers/misc/habanalabs/common/pci/Makefile rename drivers/misc/habanalabs/common/{ => pci}/pci.c (99%) diff --git a/drivers/misc/habanalabs/common/Makefile b/drivers/misc/habanalabs/common/Makefile index eccd8c7dc62d..5d8b48288cf4 100644 --- a/drivers/misc/habanalabs/common/Makefile +++ b/drivers/misc/habanalabs/common/Makefile @@ -1,7 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only + +include $(src)/common/mmu/Makefile +habanalabs-y += $(HL_COMMON_MMU_FILES) + +include $(src)/common/pci/Makefile +habanalabs-y += $(HL_COMMON_PCI_FILES) + HL_COMMON_FILES := common/habanalabs_drv.o common/device.o common/context.o \ common/asid.o common/habanalabs_ioctl.o \ common/command_buffer.o common/hw_queue.o common/irq.o \ common/sysfs.o common/hwmon.o common/memory.o \ - common/command_submission.o common/mmu.o common/mmu_v1.o \ - common/firmware_if.o common/pci.o + common/command_submission.o common/firmware_if.o diff --git a/drivers/misc/habanalabs/common/mmu/Makefile b/drivers/misc/habanalabs/common/mmu/Makefile new file mode 100644 index 000000000000..d852c3874658 --- /dev/null +++ b/drivers/misc/habanalabs/common/mmu/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +HL_COMMON_MMU_FILES := common/mmu/mmu.o common/mmu/mmu_v1.o diff --git a/drivers/misc/habanalabs/common/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c similarity index 99% rename from drivers/misc/habanalabs/common/mmu.c rename to drivers/misc/habanalabs/common/mmu/mmu.c index 38234c243b21..97c51686fcfe 100644 --- a/drivers/misc/habanalabs/common/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -7,7 +7,7 @@ #include -#include "habanalabs.h" +#include "../habanalabs.h" bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr) { diff --git a/drivers/misc/habanalabs/common/mmu_v1.c b/drivers/misc/habanalabs/common/mmu/mmu_v1.c similarity index 99% rename from drivers/misc/habanalabs/common/mmu_v1.c rename to drivers/misc/habanalabs/common/mmu/mmu_v1.c index 06d8a44dd5d4..c5e93ff32586 100644 --- a/drivers/misc/habanalabs/common/mmu_v1.c +++ b/drivers/misc/habanalabs/common/mmu/mmu_v1.c @@ -5,8 +5,8 @@ * All Rights Reserved. */ -#include "habanalabs.h" -#include "../include/hw_ip/mmu/mmu_general.h" +#include "../habanalabs.h" +#include "../../include/hw_ip/mmu/mmu_general.h" #include diff --git a/drivers/misc/habanalabs/common/pci/Makefile b/drivers/misc/habanalabs/common/pci/Makefile new file mode 100644 index 000000000000..dc922a686683 --- /dev/null +++ b/drivers/misc/habanalabs/common/pci/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +HL_COMMON_PCI_FILES := common/pci/pci.o diff --git a/drivers/misc/habanalabs/common/pci.c b/drivers/misc/habanalabs/common/pci/pci.c similarity index 99% rename from drivers/misc/habanalabs/common/pci.c rename to drivers/misc/habanalabs/common/pci/pci.c index b4725e6101f6..c56ec1574127 100644 --- a/drivers/misc/habanalabs/common/pci.c +++ b/drivers/misc/habanalabs/common/pci/pci.c @@ -5,8 +5,8 @@ * All Rights Reserved. */ -#include "habanalabs.h" -#include "../include/hw_ip/pci/pci_general.h" +#include "../habanalabs.h" +#include "../../include/hw_ip/pci/pci_general.h" #include From 1e3f2536a8fc12d5b82164349715861ccc7968a7 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 3 Jan 2021 14:13:06 +0200 Subject: [PATCH 172/633] habanalabs: increment ctx ref from within a cs allocation A CS must increment the relevant context reference count. We want to increment the reference inside the CS allocation function as opposed for today where we increment it outside. This is logical since we want to avoid explicitly incrementing the context every time we call the CS allocate function. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index f7fac82ac41d..3affb350070c 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -479,6 +479,9 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, return -ENOMEM; } + /* increment refcnt for context */ + hl_ctx_get(hdev, ctx); + cs->ctx = ctx; cs->submitted = false; cs->completed = false; @@ -550,6 +553,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, kfree(cs_cmpl); free_cs: kfree(cs); + hl_ctx_put(ctx); return rc; } @@ -827,14 +831,9 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, if (rc) goto out; - /* increment refcnt for context */ - hl_ctx_get(hdev, hpriv->ctx); - rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs); - if (rc) { - hl_ctx_put(hpriv->ctx); + if (rc) goto free_cs_chunk_array; - } cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP); *cs_seq = cs->sequence; @@ -1276,15 +1275,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, } } - /* increment refcnt for context */ - hl_ctx_get(hdev, ctx); - rc = allocate_cs(hdev, ctx, cs_type, &cs); if (rc) { if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) hl_fence_put(sig_fence); - hl_ctx_put(ctx); goto free_cs_chunk_array; } From d3f139c46280035509600900a6ead3d7e680218f Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Wed, 18 Nov 2020 15:46:57 +0200 Subject: [PATCH 173/633] habanalabs: add driver support for internal cb scheduling In order to support scnenarios in which driver needs access to HW components but it cannot access them directly, we add support for scheduling command buffers internally. These command buffers will be transmitted upon next user command submission context. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../habanalabs/common/command_submission.c | 139 ++++++++++++++++++ drivers/misc/habanalabs/common/context.c | 8 + drivers/misc/habanalabs/common/habanalabs.h | 26 ++++ 3 files changed, 173 insertions(+) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 3affb350070c..a5e9bb0a4855 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -585,6 +585,18 @@ void hl_cs_rollback_all(struct hl_device *hdev) } } +void hl_pending_cb_list_flush(struct hl_ctx *ctx) +{ + struct hl_pending_cb *pending_cb, *tmp; + + list_for_each_entry_safe(pending_cb, tmp, + &ctx->pending_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + hl_cb_put(pending_cb->cb); + kfree(pending_cb); + } +} + static void job_wq_completion(struct work_struct *work) { struct hl_cs_job *job = container_of(work, struct hl_cs_job, @@ -954,6 +966,129 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, return rc; } +static int pending_cb_create_job(struct hl_device *hdev, struct hl_ctx *ctx, + struct hl_cs *cs, struct hl_cb *cb, u32 size, u32 hw_queue_id) +{ + struct hw_queue_properties *hw_queue_prop; + struct hl_cs_counters_atomic *cntr; + struct hl_cs_job *job; + + hw_queue_prop = &hdev->asic_prop.hw_queues_props[hw_queue_id]; + cntr = &hdev->aggregated_cs_counters; + + job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); + if (!job) { + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); + atomic64_inc(&cntr->out_of_mem_drop_cnt); + dev_err(hdev->dev, "Failed to allocate a new job\n"); + return -ENOMEM; + } + + job->id = 0; + job->cs = cs; + job->user_cb = cb; + atomic_inc(&job->user_cb->cs_cnt); + job->user_cb_size = size; + job->hw_queue_id = hw_queue_id; + job->patched_cb = job->user_cb; + job->job_cb_size = job->user_cb_size; + + /* increment refcount as for external queues we get completion */ + cs_get(cs); + + cs->jobs_in_queue_cnt[job->hw_queue_id]++; + + list_add_tail(&job->cs_node, &cs->job_list); + + hl_debugfs_add_job(hdev, job); + + return 0; +} + +static int hl_submit_pending_cb(struct hl_fpriv *hpriv) +{ + struct hl_device *hdev = hpriv->hdev; + struct hl_ctx *ctx = hpriv->ctx; + struct hl_pending_cb *pending_cb, *tmp; + struct list_head local_cb_list; + struct hl_cs *cs; + struct hl_cb *cb; + u32 hw_queue_id; + u32 cb_size; + int process_list, rc = 0; + + if (list_empty(&ctx->pending_cb_list)) + return 0; + + process_list = atomic_cmpxchg(&ctx->thread_pending_cb_token, 1, 0); + + /* Only a single thread is allowed to process the list */ + if (!process_list) + return 0; + + if (list_empty(&ctx->pending_cb_list)) + goto free_pending_cb_token; + + /* move all list elements to a local list */ + INIT_LIST_HEAD(&local_cb_list); + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe(pending_cb, tmp, &ctx->pending_cb_list, + cb_node) + list_move_tail(&pending_cb->cb_node, &local_cb_list); + spin_unlock(&ctx->pending_cb_lock); + + rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, &cs); + if (rc) + goto add_list_elements; + + hl_debugfs_add_cs(cs); + + /* Iterate through pending cb list, create jobs and add to CS */ + list_for_each_entry(pending_cb, &local_cb_list, cb_node) { + cb = pending_cb->cb; + cb_size = pending_cb->cb_size; + hw_queue_id = pending_cb->hw_queue_id; + + rc = pending_cb_create_job(hdev, ctx, cs, cb, cb_size, + hw_queue_id); + if (rc) + goto free_cs_object; + } + + rc = hl_hw_queue_schedule_cs(cs); + if (rc) { + if (rc != -EAGAIN) + dev_err(hdev->dev, + "Failed to submit CS %d.%llu (%d)\n", + ctx->asid, cs->sequence, rc); + goto free_cs_object; + } + + /* pending cb was scheduled successfully */ + list_for_each_entry_safe(pending_cb, tmp, &local_cb_list, cb_node) { + list_del(&pending_cb->cb_node); + kfree(pending_cb); + } + + cs_put(cs); + + goto free_pending_cb_token; + +free_cs_object: + cs_rollback(hdev, cs); + cs_put(cs); +add_list_elements: + spin_lock(&ctx->pending_cb_lock); + list_for_each_entry_safe_reverse(pending_cb, tmp, &local_cb_list, + cb_node) + list_move(&pending_cb->cb_node, &ctx->pending_cb_list); + spin_unlock(&ctx->pending_cb_lock); +free_pending_cb_token: + atomic_set(&ctx->thread_pending_cb_token, 1); + + return rc; +} + static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, u64 *cs_seq) { @@ -1353,6 +1488,10 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) if (rc) goto out; + rc = hl_submit_pending_cb(hpriv); + if (rc) + goto out; + cs_type = hl_cs_get_cs_type(args->in.cs_flags & ~HL_CS_FLAGS_FORCE_RESTORE); chunks = (void __user *) (uintptr_t) args->in.chunks_execute; diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index 3d86b83f4ca6..829fe98eed61 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -15,6 +15,11 @@ static void hl_ctx_fini(struct hl_ctx *ctx) u64 idle_mask = 0; int i; + /* Release all allocated pending cb's, those cb's were never + * scheduled so it is safe to release them here + */ + hl_pending_cb_list_flush(ctx); + /* * If we arrived here, there are no jobs waiting for this context * on its queues so we can safely remove it. @@ -142,8 +147,11 @@ int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx) kref_init(&ctx->refcount); ctx->cs_sequence = 1; + INIT_LIST_HEAD(&ctx->pending_cb_list); + spin_lock_init(&ctx->pending_cb_lock); spin_lock_init(&ctx->cs_lock); atomic_set(&ctx->thread_ctx_switch_token, 1); + atomic_set(&ctx->thread_pending_cb_token, 1); ctx->thread_ctx_switch_wait_token = 0; ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs, sizeof(struct hl_fence *), diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index eb43fb3065ce..8e0553bf3e0e 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1016,6 +1016,20 @@ struct hl_cs_counters_atomic { atomic64_t validation_drop_cnt; }; +/** + * struct hl_pending_cb - pending command buffer structure + * @cb_node: cb node in pending cb list + * @cb: command buffer to send in next submission + * @cb_size: command buffer size + * @hw_queue_id: destination queue id + */ +struct hl_pending_cb { + struct list_head cb_node; + struct hl_cb *cb; + u32 cb_size; + u32 hw_queue_id; +}; + /** * struct hl_ctx - user/kernel context. * @mem_hash: holds mapping from virtual address to virtual memory area @@ -1031,6 +1045,8 @@ struct hl_cs_counters_atomic { * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the * MMU hash or walking the PGT requires talking this lock. * @debugfs_list: node in debugfs list of contexts. + * pending_cb_list: list of pending command buffers waiting to be sent upon + * next user command submission context. * @cs_counters: context command submission counters. * @cb_va_pool: device VA pool for command buffers which are mapped to the * device's MMU. @@ -1039,11 +1055,17 @@ struct hl_cs_counters_atomic { * index to cs_pending array. * @dram_default_hops: array that holds all hops addresses needed for default * DRAM mapping. + * @pending_cb_lock: spinlock to protect pending cb list * @cs_lock: spinlock to protect cs_sequence. * @dram_phys_mem: amount of used physical DRAM memory by this context. * @thread_ctx_switch_token: token to prevent multiple threads of the same * context from running the context switch phase. * Only a single thread should run it. + * @thread_pending_cb_token: token to prevent multiple threads from processing + * the pending CB list. Only a single thread should + * process the list since it is protected by a + * spinlock and we don't want to halt the entire + * command submission sequence. * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run * the context switch phase from moving to their * execution phase before the context switch phase @@ -1062,13 +1084,16 @@ struct hl_ctx { struct mutex mem_hash_lock; struct mutex mmu_lock; struct list_head debugfs_list; + struct list_head pending_cb_list; struct hl_cs_counters_atomic cs_counters; struct gen_pool *cb_va_pool; u64 cs_sequence; u64 *dram_default_hops; + spinlock_t pending_cb_lock; spinlock_t cs_lock; atomic64_t dram_phys_mem; atomic_t thread_ctx_switch_token; + atomic_t thread_pending_cb_token; u32 thread_ctx_switch_wait_token; u32 asid; u32 handle; @@ -2143,6 +2168,7 @@ int hl_cb_va_pool_init(struct hl_ctx *ctx); void hl_cb_va_pool_fini(struct hl_ctx *ctx); void hl_cs_rollback_all(struct hl_device *hdev); +void hl_pending_cb_list_flush(struct hl_ctx *ctx); struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, enum hl_queue_type queue_type, bool is_kernel_allocated_cb); void hl_sob_reset_error(struct kref *ref); From 423815bf02e257091d5337be5c63b57fc29e4254 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 5 Jan 2021 09:04:07 +0200 Subject: [PATCH 174/633] habanalabs/gaudi: remove PCI access to SM block Due to HW limitation we must remove all direct access to SM registers, in order to do that we will access SM registers using the HW QMANS. When possible and no user context is present, we can directly access the HW QMANS. Whenever there is an active user, driver will prepare a pending command buffer list which will be sent upon user submissions. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 248 ++++++++++++++++-- drivers/misc/habanalabs/gaudi/gaudiP.h | 2 + .../habanalabs/include/gaudi/gaudi_packets.h | 3 + 3 files changed, 227 insertions(+), 26 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index f6cfd6e075e2..0d2edb25c92a 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -354,6 +354,10 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job); static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, u32 size, u64 val); +static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, + u32 num_regs, u32 val); +static int gaudi_schedule_register_memset(struct hl_device *hdev, + u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val); static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel, u32 tpc_id); static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev); @@ -913,11 +917,17 @@ static void gaudi_sob_group_hw_reset(struct kref *ref) struct gaudi_hw_sob_group *hw_sob_group = container_of(ref, struct gaudi_hw_sob_group, kref); struct hl_device *hdev = hw_sob_group->hdev; - int i; + u64 base_addr; + int rc; - for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + - (hw_sob_group->base_sob_id + i) * 4, 0); + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + hw_sob_group->base_sob_id * 4; + rc = gaudi_schedule_register_memset(hdev, hw_sob_group->queue_id, + base_addr, NUMBER_OF_SOBS_IN_GRP, 0); + if (rc) + dev_err(hdev->dev, + "failed resetting sob group - sob base %u, count %u", + hw_sob_group->base_sob_id, NUMBER_OF_SOBS_IN_GRP); kref_init(&hw_sob_group->kref); } @@ -1008,6 +1018,8 @@ static void gaudi_collective_master_init_job(struct hl_device *hdev, cprop->hw_sob_group[sob_group_offset].base_sob_id; master_monitor = prop->collective_mstr_mon_id[0]; + cprop->hw_sob_group[sob_group_offset].queue_id = queue_id; + dev_dbg(hdev->dev, "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", master_sob_base, cprop->mstr_sob_mask[0], @@ -5583,31 +5595,206 @@ static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, return rc; } -static void gaudi_restore_sm_registers(struct hl_device *hdev) +static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base, + u32 num_regs, u32 val) { + struct packet_msg_long *pkt; + struct hl_cs_job *job; + u32 cb_size, ctl; + struct hl_cb *cb; + int i, rc; + + cb_size = (sizeof(*pkt) * num_regs) + sizeof(struct packet_msg_prot); + + if (cb_size > SZ_2M) { + dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); + return -ENOMEM; + } + + cb = hl_cb_kernel_create(hdev, cb_size, false); + if (!cb) + return -EFAULT; + + pkt = cb->kernel_address; + + ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); + + for (i = 0; i < num_regs ; i++, pkt++) { + pkt->ctl = cpu_to_le32(ctl); + pkt->value = cpu_to_le32(val); + pkt->addr = cpu_to_le64(reg_base + (i * 4)); + } + + job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); + if (!job) { + dev_err(hdev->dev, "Failed to allocate a new job\n"); + rc = -ENOMEM; + goto release_cb; + } + + job->id = 0; + job->user_cb = cb; + atomic_inc(&job->user_cb->cs_cnt); + job->user_cb_size = cb_size; + job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; + job->patched_cb = job->user_cb; + job->job_cb_size = cb_size; + + hl_debugfs_add_job(hdev, job); + + rc = gaudi_send_job_on_qman0(hdev, job); + hl_debugfs_remove_job(hdev, job); + kfree(job); + atomic_dec(&cb->cs_cnt); + +release_cb: + hl_cb_put(cb); + hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); + + return rc; +} + +static int gaudi_schedule_register_memset(struct hl_device *hdev, + u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val) +{ + struct hl_ctx *ctx = hdev->compute_ctx; + struct hl_pending_cb *pending_cb; + struct packet_msg_long *pkt; + u32 cb_size, ctl; + struct hl_cb *cb; int i; - for (i = 0 ; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) { - WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); - WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); - WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); + /* If no compute context available or context is going down + * memset registers directly + */ + if (!ctx || kref_read(&ctx->refcount) == 0) + return gaudi_memset_registers(hdev, reg_base, num_regs, val); + + cb_size = (sizeof(*pkt) * num_regs) + + sizeof(struct packet_msg_prot) * 2; + + if (cb_size > SZ_2M) { + dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); + return -ENOMEM; } - for (i = 0 ; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) { - WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); - WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); - WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); + pending_cb = kzalloc(sizeof(*pending_cb), GFP_KERNEL); + if (!pending_cb) + return -ENOMEM; + + cb = hl_cb_kernel_create(hdev, cb_size, false); + if (!cb) { + kfree(pending_cb); + return -EFAULT; } - i = GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4; + pkt = cb->kernel_address; - for (; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0); + ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */ + ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); + ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); - i = GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4; + for (i = 0; i < num_regs ; i++, pkt++) { + pkt->ctl = cpu_to_le32(ctl); + pkt->value = cpu_to_le32(val); + pkt->addr = cpu_to_le64(reg_base + (i * 4)); + } - for (; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0); + hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); + + pending_cb->cb = cb; + pending_cb->cb_size = cb_size; + /* The queue ID MUST be an external queue ID. Otherwise, we will + * have undefined behavior + */ + pending_cb->hw_queue_id = hw_queue_id; + + spin_lock(&ctx->pending_cb_lock); + list_add_tail(&pending_cb->cb_node, &ctx->pending_cb_list); + spin_unlock(&ctx->pending_cb_lock); + + return 0; +} + +static int gaudi_restore_sm_registers(struct hl_device *hdev) +{ + u64 base_addr; + u32 num_regs; + int rc; + + base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0; + num_regs = NUM_OF_SOB_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0; + num_regs = NUM_OF_MONITORS_IN_BLOCK; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + (GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4); + num_regs = NUM_OF_SOB_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + + (GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4); + num_regs = NUM_OF_MONITORS_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_MONITOR; + rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0); + if (rc) { + dev_err(hdev->dev, "failed resetting SM registers"); + return -ENOMEM; + } + + return 0; } static void gaudi_restore_dma_registers(struct hl_device *hdev) @@ -5664,18 +5851,23 @@ static void gaudi_restore_qm_registers(struct hl_device *hdev) } } -static void gaudi_restore_user_registers(struct hl_device *hdev) +static int gaudi_restore_user_registers(struct hl_device *hdev) { - gaudi_restore_sm_registers(hdev); + int rc; + + rc = gaudi_restore_sm_registers(hdev); + if (rc) + return rc; + gaudi_restore_dma_registers(hdev); gaudi_restore_qm_registers(hdev); + + return 0; } static int gaudi_context_switch(struct hl_device *hdev, u32 asid) { - gaudi_restore_user_registers(hdev); - - return 0; + return gaudi_restore_user_registers(hdev); } static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev) @@ -8203,12 +8395,16 @@ static u32 gaudi_gen_wait_cb(struct hl_device *hdev, static void gaudi_reset_sob(struct hl_device *hdev, void *data) { struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data; + int rc; dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); - WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, - 0); + rc = gaudi_schedule_register_memset(hdev, hw_sob->q_idx, + CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + hw_sob->sob_id * 4, 1, 0); + if (rc) + dev_err(hdev->dev, "failed resetting sob %u", hw_sob->sob_id); kref_init(&hw_sob->kref); } diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index a7ab2d7e57d4..78830443341d 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -251,11 +251,13 @@ enum gaudi_nic_mask { * @hdev: habanalabs device structure. * @kref: refcount of this SOB group. group will reset once refcount is zero. * @base_sob_id: base sob id of this SOB group. + * @queue_id: id of the queue that waits on this sob group */ struct gaudi_hw_sob_group { struct hl_device *hdev; struct kref kref; u32 base_sob_id; + u32 queue_id; }; #define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS) diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h index 784b5bc8d0ba..6e097ace2e96 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h @@ -78,6 +78,9 @@ struct packet_wreg_bulk { __le64 values[0]; /* data starts here */ }; +#define GAUDI_PKT_LONG_CTL_OP_SHIFT 20 +#define GAUDI_PKT_LONG_CTL_OP_MASK 0x00300000 + struct packet_msg_long { __le32 value; __le32 ctl; From 825b30c4f37bd4a6846916a18379dcd211ea3fbf Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 4 Jan 2021 17:08:46 +0200 Subject: [PATCH 175/633] habanalabs: Use 'dma_set_mask_and_coherent()' Axe 'hl_pci_set_dma_mask()' and replace it with an equivalent 'dma_set_mask_and_coherent()' call. This makes the code a bit less verbose. It also removes an erroneous comment, because 'hl_pci_set_dma_mask()' does not try to use a fall-back value. Signed-off-by: Christophe JAILLET Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/pci/pci.c | 43 ++++-------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/drivers/misc/habanalabs/common/pci/pci.c b/drivers/misc/habanalabs/common/pci/pci.c index c56ec1574127..b799f9258fb0 100644 --- a/drivers/misc/habanalabs/common/pci/pci.c +++ b/drivers/misc/habanalabs/common/pci/pci.c @@ -307,40 +307,6 @@ int hl_pci_set_outbound_region(struct hl_device *hdev, return rc; } -/** - * hl_pci_set_dma_mask() - Set DMA masks for the device. - * @hdev: Pointer to hl_device structure. - * - * This function sets the DMA masks (regular and consistent) for a specified - * value. If it doesn't succeed, it tries to set it to a fall-back value - * - * Return: 0 on success, non-zero for failure. - */ -static int hl_pci_set_dma_mask(struct hl_device *hdev) -{ - struct pci_dev *pdev = hdev->pdev; - int rc; - - /* set DMA mask */ - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask)); - if (rc) { - dev_err(hdev->dev, - "Failed to set pci dma mask to %d bits, error %d\n", - hdev->dma_mask, rc); - return rc; - } - - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask)); - if (rc) { - dev_err(hdev->dev, - "Failed to set pci consistent dma mask to %d bits, error %d\n", - hdev->dma_mask, rc); - return rc; - } - - return 0; -} - /** * hl_pci_init() - PCI initialization code. * @hdev: Pointer to hl_device structure. @@ -377,9 +343,14 @@ int hl_pci_init(struct hl_device *hdev) goto unmap_pci_bars; } - rc = hl_pci_set_dma_mask(hdev); - if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, + DMA_BIT_MASK(hdev->dma_mask)); + if (rc) { + dev_err(hdev->dev, + "Failed to set dma mask to %d bits, error %d\n", + hdev->dma_mask, rc); goto unmap_pci_bars; + } return 0; From f8bc7f091cc0ef115bd284e68df247b56c422d93 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 3 Jan 2021 20:52:40 +0200 Subject: [PATCH 176/633] habanalabs/gaudi: print sync manager SEI interrupt info Driver must print sync manager SEI information upon receiving interrupt from FW. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 41 +++++++++++++++++++ .../misc/habanalabs/include/common/cpucp_if.h | 7 ++++ .../include/gaudi/gaudi_async_events.h | 4 ++ 3 files changed, 52 insertions(+) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 0d2edb25c92a..d9f1b646aac9 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -225,6 +225,12 @@ gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = { "MSG AXI LBW returned with error" }; +enum gaudi_sm_sei_cause { + GAUDI_SM_SEI_SO_OVERFLOW, + GAUDI_SM_SEI_LBW_4B_UNALIGNED, + GAUDI_SM_SEI_AXI_RESPONSE_ERR +}; + static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */ QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */ @@ -6846,6 +6852,34 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, } } +static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, + struct hl_eq_sm_sei_data *sei_data) +{ + u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0; + + switch (sei_data->sei_cause) { + case GAUDI_SM_SEI_SO_OVERFLOW: + dev_err(hdev->dev, + "SM %u SEI Error: SO %u overflow/underflow", + index, le16_to_cpu(sei_data->sei_log)); + break; + case GAUDI_SM_SEI_LBW_4B_UNALIGNED: + dev_err(hdev->dev, + "SM %u SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x", + index, le16_to_cpu(sei_data->sei_log)); + break; + case GAUDI_SM_SEI_AXI_RESPONSE_ERR: + dev_err(hdev->dev, + "SM %u SEI Error: AXI ID %u response error", + index, le16_to_cpu(sei_data->sei_log)); + break; + default: + dev_err(hdev->dev, "Unknown SM SEI cause %u", + le16_to_cpu(sei_data->sei_log)); + break; + } +} + static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type, struct hl_eq_ecc_data *ecc_data) { @@ -7469,6 +7503,13 @@ static void gaudi_handle_eqe(struct hl_device *hdev, hl_fw_unmask_irq(hdev, event_type); break; + case GAUDI_EVENT_DMA_IF_SEI_0 ... GAUDI_EVENT_DMA_IF_SEI_3: + gaudi_print_irq_info(hdev, event_type, false); + gaudi_print_sm_sei_info(hdev, event_type, + &eq_entry->sm_sei_data); + hl_fw_unmask_irq(hdev, event_type); + break; + case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E: gaudi_print_clk_change_info(hdev, event_type); hl_fw_unmask_irq(hdev, event_type); diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index 00bd9b392f93..d75d1077461b 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -58,11 +58,18 @@ struct hl_eq_ecc_data { __u8 pad[7]; }; +struct hl_eq_sm_sei_data { + __le16 sei_log; + __u8 sei_cause; + __u8 pad[5]; +}; + struct hl_eq_entry { struct hl_eq_header hdr; union { struct hl_eq_ecc_data ecc_data; struct hl_eq_hbm_ecc_data hbm_ecc_data; + struct hl_eq_sm_sei_data sm_sei_data; __le64 data[7]; }; }; diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h index 9ccba8437ec9..49335e8334b4 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h @@ -212,6 +212,10 @@ enum gaudi_async_event_id { GAUDI_EVENT_NIC_SEI_2 = 266, GAUDI_EVENT_NIC_SEI_3 = 267, GAUDI_EVENT_NIC_SEI_4 = 268, + GAUDI_EVENT_DMA_IF_SEI_0 = 277, + GAUDI_EVENT_DMA_IF_SEI_1 = 278, + GAUDI_EVENT_DMA_IF_SEI_2 = 279, + GAUDI_EVENT_DMA_IF_SEI_3 = 280, GAUDI_EVENT_PCIE_FLR = 290, GAUDI_EVENT_TPC0_BMON_SPMU = 300, GAUDI_EVENT_TPC0_KRN_ERR = 301, From eea4c2557cb986584c4960eb15dd17cb3f4f99a2 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 10 Jan 2021 10:34:42 +0200 Subject: [PATCH 177/633] habanalabs: ignore F/W BMC errors in case no BMC present In order to support operation mode in which BMC is not active, driver must not take BMC errors into consideration. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 9f4ae21a126f..ba6920f2b4ab 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -308,9 +308,15 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg, if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) dev_warn(hdev->dev, "Device boot warning - Skipped DRAM initialization\n"); - if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) - dev_warn(hdev->dev, - "Device boot error - Skipped waiting for BMC\n"); + + if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) { + if (hdev->bmc_enable) + dev_warn(hdev->dev, + "Device boot error - Skipped waiting for BMC\n"); + else + err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED; + } + if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) dev_err(hdev->dev, "Device boot error - Serdes data from BMC not available\n"); From d2b980f3299e9bdef55399c9cde6a72e0b6d446c Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 7 Jan 2021 12:14:17 +0200 Subject: [PATCH 178/633] habanalabs: add security violations dump to debugfs In order to improve driver security debuggability, we add security violations dump to debugfs. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../ABI/testing/debugfs-driver-habanalabs | 8 +++++++ drivers/misc/habanalabs/common/debugfs.c | 22 +++++++++++++++++++ drivers/misc/habanalabs/common/habanalabs.h | 2 ++ drivers/misc/habanalabs/gaudi/gaudi.c | 3 ++- drivers/misc/habanalabs/gaudi/gaudiP.h | 1 + .../misc/habanalabs/gaudi/gaudi_security.c | 5 +++++ drivers/misc/habanalabs/goya/goya.c | 3 ++- drivers/misc/habanalabs/goya/goyaP.h | 1 + drivers/misc/habanalabs/goya/goya_security.c | 5 +++++ 9 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs index c5d678d39144..3979bfdaa080 100644 --- a/Documentation/ABI/testing/debugfs-driver-habanalabs +++ b/Documentation/ABI/testing/debugfs-driver-habanalabs @@ -182,3 +182,11 @@ KernelVersion: 5.6 Contact: oded.gabbay@gmail.com Description: Sets the stop-on_error option for the device engines. Value of "0" is for disable, otherwise enable. + +What: /sys/kernel/debug/habanalabs/hl/dump_security_violations +Date: Jan 2021 +KernelVersion: 5.12 +Contact: oded.gabbay@gmail.com +Description: Dumps all security violations to dmesg. This will also ack + all security violations meanings those violations will not be + dumped next time user calls this API diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 50ca8eea6648..323d0381a60a 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -867,6 +867,17 @@ static ssize_t hl_stop_on_err_write(struct file *f, const char __user *buf, return count; } +static ssize_t hl_security_violations_read(struct file *f, char __user *buf, + size_t count, loff_t *ppos) +{ + struct hl_dbg_device_entry *entry = file_inode(f)->i_private; + struct hl_device *hdev = entry->hdev; + + hdev->asic_funcs->ack_protection_bits_errors(hdev); + + return 0; +} + static const struct file_operations hl_data32b_fops = { .owner = THIS_MODULE, .read = hl_data_read32, @@ -924,6 +935,11 @@ static const struct file_operations hl_stop_on_err_fops = { .write = hl_stop_on_err_write }; +static const struct file_operations hl_security_violations_fops = { + .owner = THIS_MODULE, + .read = hl_security_violations_read +}; + static const struct hl_info_list hl_debugfs_list[] = { {"command_buffers", command_buffers_show, NULL}, {"command_submission", command_submission_show, NULL}, @@ -1073,6 +1089,12 @@ void hl_debugfs_add_device(struct hl_device *hdev) dev_entry, &hl_stop_on_err_fops); + debugfs_create_file("dump_security_violations", + 0644, + dev_entry->root, + dev_entry, + &hl_security_violations_fops); + for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) { ent = debugfs_create_file(hl_debugfs_list[i].name, diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 8e0553bf3e0e..3923b03e99aa 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -850,6 +850,7 @@ enum div_select_defs { * @collective_wait_create_jobs: allocate collective wait cs jobs * @scramble_vaddr: Routine to scramble the virtual address prior of mapping it * in the MMU. + * @ack_protection_bits_errors: ack and dump all security violations */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -960,6 +961,7 @@ struct hl_asic_funcs { struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id); u64 (*scramble_vaddr)(struct hl_device *hdev, u64 virt_addr); + void (*ack_protection_bits_errors)(struct hl_device *hdev); }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index d9f1b646aac9..2b01c081404a 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -8546,7 +8546,8 @@ static const struct hl_asic_funcs gaudi_funcs = { .get_device_time = gaudi_get_device_time, .collective_wait_init_cs = gaudi_collective_wait_init_cs, .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, - .scramble_vaddr = hl_mmu_scramble_vaddr + .scramble_vaddr = hl_mmu_scramble_vaddr, + .ack_protection_bits_errors = gaudi_ack_protection_bits_errors }; /** diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 78830443341d..50bb4ad570fd 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -335,6 +335,7 @@ struct gaudi_device { }; void gaudi_init_security(struct hl_device *hdev); +void gaudi_ack_protection_bits_errors(struct hl_device *hdev); void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_attr_grp); void gaudi_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq); diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c index e10181692d0b..7085f45814ae 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_security.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c @@ -13052,3 +13052,8 @@ void gaudi_init_security(struct hl_device *hdev) gaudi_init_protection_bits(hdev); } + +void gaudi_ack_protection_bits_errors(struct hl_device *hdev) +{ + +} diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index cf0496b04044..50dcefc02cdd 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5457,7 +5457,8 @@ static const struct hl_asic_funcs goya_funcs = { .get_device_time = goya_get_device_time, .collective_wait_init_cs = goya_collective_wait_init_cs, .collective_wait_create_jobs = goya_collective_wait_create_jobs, - .scramble_vaddr = hl_mmu_scramble_vaddr + .scramble_vaddr = hl_mmu_scramble_vaddr, + .ack_protection_bits_errors = goya_ack_protection_bits_errors }; /* diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h index 8b3408211af6..23fe099ed218 100644 --- a/drivers/misc/habanalabs/goya/goyaP.h +++ b/drivers/misc/habanalabs/goya/goyaP.h @@ -173,6 +173,7 @@ void goya_init_mme_qmans(struct hl_device *hdev); void goya_init_tpc_qmans(struct hl_device *hdev); int goya_init_cpu_queues(struct hl_device *hdev); void goya_init_security(struct hl_device *hdev); +void goya_ack_protection_bits_errors(struct hl_device *hdev); int goya_late_init(struct hl_device *hdev); void goya_late_fini(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/goya/goya_security.c b/drivers/misc/habanalabs/goya/goya_security.c index 14701836f92b..14c3bae3ccdc 100644 --- a/drivers/misc/habanalabs/goya/goya_security.c +++ b/drivers/misc/habanalabs/goya/goya_security.c @@ -3120,3 +3120,8 @@ void goya_init_security(struct hl_device *hdev) goya_init_protection_bits(hdev); } + +void goya_ack_protection_bits_errors(struct hl_device *hdev) +{ + +} From 2f8db5a1754416ba869ce200e1de37a0ccfb3087 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 12 Jan 2021 15:13:06 +0200 Subject: [PATCH 179/633] habanalabs: update email address in sysfs/debugfs docs Use my kernel.org address for contact point instead of my private email address. Signed-off-by: Oded Gabbay --- .../ABI/testing/debugfs-driver-habanalabs | 44 +++++++------- .../ABI/testing/sysfs-driver-habanalabs | 58 +++++++++---------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs index 3979bfdaa080..d447a611c41b 100644 --- a/Documentation/ABI/testing/debugfs-driver-habanalabs +++ b/Documentation/ABI/testing/debugfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/kernel/debug/habanalabs/hl/addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the device address to be used for read or write through PCI bar, or the device VA of a host mapped memory to be read or written directly from the host. The latter option is allowed @@ -11,7 +11,7 @@ Description: Sets the device address to be used for read or write through What: /sys/kernel/debug/habanalabs/hl/clk_gate Date: May 2020 KernelVersion: 5.8 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allow the root user to disable/enable in runtime the clock gating mechanism in Gaudi. Due to how Gaudi is built, the clock gating needs to be disabled in order to access the @@ -34,28 +34,28 @@ Description: Allow the root user to disable/enable in runtime the clock What: /sys/kernel/debug/habanalabs/hl/command_buffers Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently allocated command buffers What: /sys/kernel/debug/habanalabs/hl/command_submission Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently active command submissions What: /sys/kernel/debug/habanalabs/hl/command_submission_jobs Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with detailed information about each JOB (CB) of each active command submission What: /sys/kernel/debug/habanalabs/hl/data32 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -70,7 +70,7 @@ Description: Allows the root user to read or write directly through the What: /sys/kernel/debug/habanalabs/hl/data64 Date: Jan 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write 64 bit data directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -85,7 +85,7 @@ Description: Allows the root user to read or write 64 bit data directly What: /sys/kernel/debug/habanalabs/hl/device Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Enables the root user to set the device to specific state. Valid values are "disable", "enable", "suspend", "resume". User can read this property to see the valid values @@ -93,28 +93,28 @@ Description: Enables the root user to set the device to specific state. What: /sys/kernel/debug/habanalabs/hl/engines Date: Jul 2019 KernelVersion: 5.3 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the status registers values of the device engines and their derived idle status What: /sys/kernel/debug/habanalabs/hl/i2c_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C device address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_bus Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C bus address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_data Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Triggers an I2C transaction that is generated by the device's CPU. Writing to this file generates a write transaction while reading from the file generates a read transcation @@ -122,32 +122,32 @@ Description: Triggers an I2C transaction that is generated by the device's What: /sys/kernel/debug/habanalabs/hl/i2c_reg Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C register id for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/led0 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the first S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led1 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the second S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led2 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the third S/W led on the device What: /sys/kernel/debug/habanalabs/hl/mmu Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the hop values and physical address for a given ASID and virtual address. The user should write the ASID and VA into the file and then read the file to get the result. @@ -157,14 +157,14 @@ Description: Displays the hop values and physical address for a given ASID What: /sys/kernel/debug/habanalabs/hl/set_power_state Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the PCI power state. Valid values are "1" for D0 and "2" for D3Hot What: /sys/kernel/debug/habanalabs/hl/userptr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently user pointers (user virtual addresses) that are pinned and mapped to DMA addresses @@ -172,21 +172,21 @@ Description: Displays a list with information about the currently user What: /sys/kernel/debug/habanalabs/hl/vm Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about all the active virtual address mappings per ASID What: /sys/kernel/debug/habanalabs/hl/stop_on_err Date: Mar 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the stop-on_error option for the device engines. Value of "0" is for disable, otherwise enable. What: /sys/kernel/debug/habanalabs/hl/dump_security_violations Date: Jan 2021 KernelVersion: 5.12 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Dumps all security violations to dmesg. This will also ack all security violations meanings those violations will not be dumped next time user calls this API diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs index 169ae4b2a180..1f127f71d2b4 100644 --- a/Documentation/ABI/testing/sysfs-driver-habanalabs +++ b/Documentation/ABI/testing/sysfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/class/habanalabs/hl/armcp_kernel_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU. Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_kernel_ver @@ -9,7 +9,7 @@ Description: Version of the Linux kernel running on the device's CPU. What: /sys/class/habanalabs/hl/armcp_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_ver @@ -17,7 +17,7 @@ Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/clk_max_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in MHz. The device clock might be set to lower value than the maximum. The user should read the clk_cur_freq_mhz to see the actual @@ -27,52 +27,52 @@ Description: Allows the user to set the maximum clock frequency, in MHz. What: /sys/class/habanalabs/hl/clk_cur_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current frequency, in MHz, of the device clock. This property is valid only for the Gaudi ASIC family What: /sys/class/habanalabs/hl/cpld_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's CPLD F/W What: /sys/class/habanalabs/hl/cpucp_kernel_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU What: /sys/class/habanalabs/hl/cpucp_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/device_type Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the code name of the device according to its type. The supported values are: "GOYA" What: /sys/class/habanalabs/hl/eeprom Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: A binary file attribute that contains the contents of the on-board EEPROM What: /sys/class/habanalabs/hl/fuse_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the device's version from the eFuse What: /sys/class/habanalabs/hl/hard_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a hard-reset operation for the device. Hard-reset will reset ALL internal components of the device except for the PCI interface and the internal PLLs @@ -80,14 +80,14 @@ Description: Interface to trigger a hard-reset operation for the device. What: /sys/class/habanalabs/hl/hard_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a hard-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/high_pll Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency for MME, TPC and IC when the power management profile is set to "automatic". This property is valid only for the Goya ASIC family @@ -95,7 +95,7 @@ Description: Allows the user to set the maximum clock frequency for MME, TPC What: /sys/class/habanalabs/hl/ic_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the Interconnect fabric. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -107,27 +107,27 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/ic_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the Interconnect fabric. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/infineon_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's power supply F/W code What: /sys/class/habanalabs/hl/max_power Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum power consumption of the device in milliwatts. What: /sys/class/habanalabs/hl/mme_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the MME compute engine. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -139,21 +139,21 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/mme_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the MME compute engine. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/pci_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the PCI address of the device. This is needed so the user would be able to open a device based on its PCI address What: /sys/class/habanalabs/hl/pm_mng_profile Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Power management profile. Values are "auto", "manual". In "auto" mode, the driver will set the maximum clock frequency to a high value when a user-space process opens the device's file (unless @@ -167,13 +167,13 @@ Description: Power management profile. Values are "auto", "manual". In "auto" What: /sys/class/habanalabs/hl/preboot_btl_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the device's preboot F/W code What: /sys/class/habanalabs/hl/soft_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a soft-reset operation for the device. Soft-reset will reset only the compute and DMA engines of the device @@ -181,26 +181,26 @@ Description: Interface to trigger a soft-reset operation for the device. What: /sys/class/habanalabs/hl/soft_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a soft-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/status Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Status of the card: "Operational", "Malfunction", "In reset". What: /sys/class/habanalabs/hl/thermal_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's thermal daemon What: /sys/class/habanalabs/hl/tpc_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the TPC compute engines. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -212,12 +212,12 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/tpc_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the TPC compute engines. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/uboot_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the u-boot running on the device's CPU \ No newline at end of file From a24ab89001aa4d7a0e6db7078b3870cae8de6681 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Wed, 13 Jan 2021 09:15:09 +0200 Subject: [PATCH 180/633] CREDITS: update email address and home address Update my email address to kernel.org account and my home address to my new house. Signed-off-by: Oded Gabbay --- CREDITS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CREDITS b/CREDITS index 9add7e6a4fa0..be097156bd71 100644 --- a/CREDITS +++ b/CREDITS @@ -1244,10 +1244,10 @@ S: 80050-430 - Curitiba - Paraná S: Brazil N: Oded Gabbay -E: oded.gabbay@gmail.com -D: HabanaLabs and AMD KFD maintainer -S: 12 Shraga Raphaeli -S: Petah-Tikva, 4906418 +E: ogabbay@kernel.org +D: HabanaLabs maintainer +S: 29 Duchifat St. +S: Ra'anana 4372029 S: Israel N: Kumar Gala From 8d79ce162e2b9bc57d8dca814f784654efe179da Mon Sep 17 00:00:00 2001 From: farah kassabri Date: Mon, 11 Jan 2021 10:10:00 +0200 Subject: [PATCH 181/633] habanalabs: always try to use the hint address Currently hint address is ignored in case va block page size is not power of 2. We need to support th user hint address also in this case, but only if the hint address is aligned to page size. Signed-off-by: farah kassabri Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/memory.c | 139 +++++++----------------- 1 file changed, 37 insertions(+), 102 deletions(-) diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index ada977eb136c..36de0d05d3a2 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -520,8 +520,8 @@ static inline int add_va_block(struct hl_device *hdev, } /** - * get_va_block_pow2() - get a virtual block for the given size and alignment - * where alignment is a power of 2. + * get_va_block() - get a virtual block for the given size and alignment. + * * @hdev: pointer to the habanalabs device structure. * @va_range: pointer to the virtual addresses range. * @size: requested block size. @@ -530,34 +530,51 @@ static inline int add_va_block(struct hl_device *hdev, * * This function does the following: * - Iterate on the virtual block list to find a suitable virtual block for the - * given size and alignment. + * given size, hint address and alignment. * - Reserve the requested block and update the list. * - Return the start address of the virtual block. */ -static u64 get_va_block_pow2(struct hl_device *hdev, +static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, u64 size, u64 hint_addr, u32 va_block_align) { struct hl_vm_va_block *va_block, *new_va_block = NULL; - u64 valid_start, valid_size, prev_start, prev_end, align_mask, - reserved_valid_start = 0, reserved_valid_size = 0; + u64 tmp_hint_addr, valid_start, valid_size, prev_start, prev_end, + align_mask, reserved_valid_start = 0, reserved_valid_size = 0; bool add_prev = false; + bool is_align_pow_2 = is_power_of_2(va_range->page_size); - align_mask = ~((u64)va_block_align - 1); + if (is_align_pow_2) + align_mask = ~((u64)va_block_align - 1); + else + /* + * with non-power-of-2 range we work only with page granularity + * and the start address is page aligned, + * so no need for alignment checking. + */ + size = DIV_ROUND_UP_ULL(size, va_range->page_size) * + va_range->page_size; - /* check if hint_addr is aligned */ - if (hint_addr & (va_block_align - 1)) + tmp_hint_addr = hint_addr; + + /* Check if we need to ignore hint address */ + if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) || + (!is_align_pow_2 && + do_div(tmp_hint_addr, va_range->page_size))) { + dev_info(hdev->dev, "Hint address 0x%llx will be ignored\n", + hint_addr); hint_addr = 0; + } mutex_lock(&va_range->lock); print_va_list_locked(hdev, &va_range->list); list_for_each_entry(va_block, &va_range->list, node) { - /* calc the first possible aligned addr */ + /* Calc the first possible aligned addr */ valid_start = va_block->start; - if (valid_start & (va_block_align - 1)) { + if (is_align_pow_2 && (valid_start & (va_block_align - 1))) { valid_start &= align_mask; valid_start += va_block_align; if (valid_start > va_block->end) @@ -565,9 +582,11 @@ static u64 get_va_block_pow2(struct hl_device *hdev, } valid_size = va_block->end - valid_start; + if (valid_size < size) + continue; - if (valid_size >= size && (!new_va_block || - valid_size < reserved_valid_size)) { + /* Pick the minimal length block which has the required size */ + if (!new_va_block || (valid_size < reserved_valid_size)) { new_va_block = va_block; reserved_valid_start = valid_start; reserved_valid_size = valid_size; @@ -584,10 +603,14 @@ static u64 get_va_block_pow2(struct hl_device *hdev, if (!new_va_block) { dev_err(hdev->dev, "no available va block for size %llu\n", - size); + size); goto out; } + /* + * Check if there is some leftover range due to reserving the new + * va block, then return it to the main virtual addresses list. + */ if (reserved_valid_start > new_va_block->start) { prev_start = new_va_block->start; prev_end = reserved_valid_start - 1; @@ -617,94 +640,6 @@ static u64 get_va_block_pow2(struct hl_device *hdev, return reserved_valid_start; } -/** - * get_va_block_non_pow2() - get a virtual block for the given size and - * alignment where alignment is not a power of 2. - * @hdev: pointer to the habanalabs device structure. - * @va_range: pointer to the virtual addresses range. - * @size: requested block size. - * @hint_addr: hint for requested address by the user. - * @va_block_align: required alignment of the virtual block start address. - * - * This function does the following: - * - Iterate on the virtual block list to find a suitable virtual block for the - * given size and alignment. - * - Reserve the requested block and update the list. - * - Return the start address of the virtual block. - */ -static u64 get_va_block_non_pow2(struct hl_device *hdev, - struct hl_va_range *va_range, - u64 size, u64 hint_addr, u32 va_block_align) -{ - struct hl_vm_va_block *va_block, *new_va_block = NULL; - u64 reserved_valid_start = 0; - - /* - * with non-power-of-2 range we work only with page granularity and the - * start address is page aligned, so no need for alignment checking. - */ - size = DIV_ROUND_UP_ULL(size, va_range->page_size) * - va_range->page_size; - - mutex_lock(&va_range->lock); - - print_va_list_locked(hdev, &va_range->list); - - list_for_each_entry(va_block, &va_range->list, node) { - if ((va_block->start + size) > va_block->end) - continue; - - new_va_block = va_block; - reserved_valid_start = va_block->start; - break; - } - - if (!new_va_block) { - dev_err(hdev->dev, "no available va block for size %llu\n", - size); - goto out; - } - - if (new_va_block->size > size) { - new_va_block->start += size; - new_va_block->size = new_va_block->end - new_va_block->start; - } else { - list_del(&new_va_block->node); - kfree(new_va_block); - } - - print_va_list_locked(hdev, &va_range->list); -out: - mutex_unlock(&va_range->lock); - - return reserved_valid_start; -} - -/* - * get_va_block() - get a virtual block for the given size and alignment. - * @hdev: pointer to the habanalabs device structure. - * @va_range: pointer to the virtual addresses range. - * @size: requested block size. - * @hint_addr: hint for requested address by the user. - * @va_block_align: required alignment of the virtual block start address. - * - * This function does the following: - * - Iterate on the virtual block list to find a suitable virtual block for the - * given size and alignment. - * - Reserve the requested block and update the list. - * - Return the start address of the virtual block. - */ -static u64 get_va_block(struct hl_device *hdev, struct hl_va_range *va_range, - u64 size, u64 hint_addr, u32 va_block_align) -{ - if (is_power_of_2(va_range->page_size)) - return get_va_block_pow2(hdev, va_range, - size, hint_addr, va_block_align); - else - return get_va_block_non_pow2(hdev, va_range, - size, hint_addr, va_block_align); -} - /* * hl_reserve_va_block() - reserve a virtual block of a given size. * @hdev: pointer to the habanalabs device structure. From e1fa724dd17a6a9b9934636226e683912d12c876 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Wed, 6 Jan 2021 15:40:37 +0200 Subject: [PATCH 182/633] habanalabs: add user available interrupt to hw_ip In order to support completions that arrive directly to the user, the driver needs to supply the user with the first available msix interrupt available. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs.h | 3 +++ drivers/misc/habanalabs/common/habanalabs_ioctl.c | 2 ++ drivers/misc/habanalabs/gaudi/gaudi.c | 2 ++ drivers/misc/habanalabs/goya/goya.c | 2 ++ include/uapi/misc/habanalabs.h | 6 ++++-- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 3923b03e99aa..ce1a1e70a6d5 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -408,6 +408,8 @@ struct hl_mmu_properties { * @sync_stream_first_mon: first monitor available for sync stream use * @first_available_user_sob: first sob available for the user * @first_available_user_mon: first monitor available for the user + * @first_available_user_msix_interrupt: first available msix interrupt + * reserved for the user * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. * @fw_security_disabled: true if security measures are disabled in firmware, @@ -469,6 +471,7 @@ struct asic_fixed_properties { u16 sync_stream_first_mon; u16 first_available_user_sob[HL_MAX_DCORES]; u16 first_available_user_mon[HL_MAX_DCORES]; + u16 first_available_user_msix_interrupt; u8 tpc_enabled_mask; u8 completion_queues_count; u8 fw_security_disabled; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index dfac5c8cadb3..628bdc56dca3 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -90,6 +90,8 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od; hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor; + hw_ip.first_available_interrupt_id = + prop->first_available_user_msix_interrupt; return copy_to_user(out, &hw_ip, min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 2b01c081404a..69b3867bc151 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -527,6 +527,8 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->sync_stream_first_mon + (num_sync_stream_queues * HL_RSVD_MONS); + prop->first_available_user_msix_interrupt = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 50dcefc02cdd..82f69274def7 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -455,6 +455,8 @@ int goya_get_fixed_properties(struct hl_device *hdev) prop->max_pending_cs = GOYA_MAX_PENDING_CS; + prop->first_available_user_msix_interrupt = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index b431a70e1b8b..866355a53188 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -309,7 +309,9 @@ struct hl_info_hw_ip_info { __u32 num_of_events; __u32 device_id; /* PCI Device ID */ __u32 module_id; /* For mezzanine cards in servers (From OCP spec.) */ - __u32 reserved[2]; + __u32 reserved; + __u16 first_available_interrupt_id; + __u16 reserved2; __u32 cpld_version; __u32 psoc_pci_pll_nr; __u32 psoc_pci_pll_nf; @@ -320,7 +322,7 @@ struct hl_info_hw_ip_info { __u8 pad[2]; __u8 cpucp_version[HL_INFO_VERSION_MAX_LEN]; __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN]; - __u64 reserved2; + __u64 reserved3; __u64 dram_page_size; }; From 89473a1fc3607d9ee5a4f859a2684d0abd0c4ded Mon Sep 17 00:00:00 2001 From: farah kassabri Date: Tue, 12 Jan 2021 17:24:00 +0200 Subject: [PATCH 183/633] habanalabs: fix MMU debugfs related nodes In mmu debugfs node show un-scrambled physical addresses. before read/write through data nodes, need to unscramble the physical address before using it for pci transaction. Signed-off-by: farah kassabri Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/debugfs.c | 21 ++++-- drivers/misc/habanalabs/common/habanalabs.h | 14 +++- drivers/misc/habanalabs/common/mmu/mmu.c | 74 ++++++++++++++++++--- drivers/misc/habanalabs/gaudi/gaudi.c | 3 +- drivers/misc/habanalabs/goya/goya.c | 3 +- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 323d0381a60a..9e3c1efe56ba 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -310,8 +310,8 @@ static int mmu_show(struct seq_file *s, void *data) struct hl_dbg_device_entry *dev_entry = entry->dev_entry; struct hl_device *hdev = dev_entry->hdev; struct hl_ctx *ctx; - struct hl_mmu_hop_info hops_info; - u64 virt_addr = dev_entry->mmu_addr; + struct hl_mmu_hop_info hops_info = {0}; + u64 virt_addr = dev_entry->mmu_addr, phys_addr; int i; if (!hdev->mmu_enable) @@ -333,10 +333,19 @@ static int mmu_show(struct seq_file *s, void *data) return 0; } - seq_printf(s, - "asid: %u, virt_addr: 0x%llx, scrambled virt_addr: 0x%llx\n", - dev_entry->mmu_asid, dev_entry->mmu_addr, - hops_info.scrambled_vaddr); + phys_addr = hops_info.hop_info[hops_info.used_hops - 1].hop_pte_val; + + if (hops_info.scrambled_vaddr && + (dev_entry->mmu_addr != hops_info.scrambled_vaddr)) + seq_printf(s, + "asid: %u, virt_addr: 0x%llx, scrambled virt_addr: 0x%llx,\nphys_addr: 0x%llx, scrambled_phys_addr: 0x%llx\n", + dev_entry->mmu_asid, dev_entry->mmu_addr, + hops_info.scrambled_vaddr, + hops_info.unscrambled_paddr, phys_addr); + else + seq_printf(s, + "asid: %u, virt_addr: 0x%llx, phys_addr: 0x%llx\n", + dev_entry->mmu_asid, dev_entry->mmu_addr, phys_addr); for (i = 0 ; i < hops_info.used_hops ; i++) { seq_printf(s, "hop%d_addr: 0x%llx\n", diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index ce1a1e70a6d5..4129e4e6a7b5 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -851,8 +851,10 @@ enum div_select_defs { * @collective_wait_init_cs: Generate collective master/slave packets * and place them in the relevant cs jobs * @collective_wait_create_jobs: allocate collective wait cs jobs - * @scramble_vaddr: Routine to scramble the virtual address prior of mapping it + * @scramble_addr: Routine to scramble the address prior of mapping it * in the MMU. + * @descramble_addr: Routine to de-scramble the address prior of + * showing it to users. * @ack_protection_bits_errors: ack and dump all security violations */ struct hl_asic_funcs { @@ -963,7 +965,8 @@ struct hl_asic_funcs { int (*collective_wait_create_jobs)(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id); - u64 (*scramble_vaddr)(struct hl_device *hdev, u64 virt_addr); + u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); + u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); void (*ack_protection_bits_errors)(struct hl_device *hdev); }; @@ -1726,13 +1729,17 @@ struct hl_mmu_per_hop_info { * @scrambled_vaddr: The value of the virtual address after scrambling. This * address replaces the original virtual-address when mapped * in the MMU tables. + * @unscrambled_paddr: The un-scrambled physical address. * @hop_info: Array holding the per-hop information used for the translation. * @used_hops: The number of hops used for the translation. + * @range_type: virtual address range type. */ struct hl_mmu_hop_info { u64 scrambled_vaddr; + u64 unscrambled_paddr; struct hl_mmu_per_hop_info hop_info[MMU_ARCH_5_HOPS]; u32 used_hops; + enum hl_va_range_type range_type; }; /** @@ -2222,7 +2229,8 @@ void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); -u64 hl_mmu_scramble_vaddr(struct hl_device *hdev, u64 virt_addr); +u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr); +u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr); bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index 97c51686fcfe..27dc0d116db5 100644 --- a/drivers/misc/habanalabs/common/mmu/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -287,9 +287,9 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, * after scrambling) */ if ((is_dram_addr && - ((hdev->asic_funcs->scramble_vaddr(hdev, phys_addr) & + ((hdev->asic_funcs->scramble_addr(hdev, phys_addr) & (mmu_prop->page_size - 1)) || - (hdev->asic_funcs->scramble_vaddr(hdev, virt_addr) & + (hdev->asic_funcs->scramble_addr(hdev, virt_addr) & (mmu_prop->page_size - 1)))) || (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || (virt_addr & (real_page_size - 1))))) @@ -476,19 +476,53 @@ void hl_mmu_swap_in(struct hl_ctx *ctx) hdev->mmu_func[MMU_HR_PGT].swap_in(ctx); } +static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, + struct hl_mmu_hop_info *hops, + u64 *phys_addr) +{ + struct hl_device *hdev = ctx->hdev; + struct asic_fixed_properties *prop = &hdev->asic_prop; + u64 offset_mask, addr_mask, hop_shift, tmp_phys_addr; + u32 hop0_shift_off; + void *p; + + /* last hop holds the phys address and flags */ + if (hops->unscrambled_paddr) + tmp_phys_addr = hops->unscrambled_paddr; + else + tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val; + + if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE) + p = &prop->pmmu_huge; + else if (hops->range_type == HL_VA_RANGE_TYPE_HOST) + p = &prop->pmmu; + else /* HL_VA_RANGE_TYPE_DRAM */ + p = &prop->dmmu; + + /* + * find the correct hop shift field in hl_mmu_properties structure + * in order to determine the right maks for the page offset. + */ + hop0_shift_off = offsetof(struct hl_mmu_properties, hop0_shift); + p = (char *)p + hop0_shift_off; + p = (char *)p + ((hops->used_hops - 1) * sizeof(u64)); + hop_shift = *(u64 *)p; + offset_mask = (1 << hop_shift) - 1; + addr_mask = ~(offset_mask); + *phys_addr = (tmp_phys_addr & addr_mask) | + (virt_addr & offset_mask); +} + int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr) { struct hl_mmu_hop_info hops; - u64 tmp_addr; int rc; rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops); if (rc) return rc; - /* last hop holds the phys address and flags */ - tmp_addr = hops.hop_info[hops.used_hops - 1].hop_pte_val; - *phys_addr = (tmp_addr & HOP_PHYS_ADDR_MASK) | (virt_addr & FLAGS_MASK); + hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops, phys_addr); return 0; } @@ -525,6 +559,11 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, mutex_unlock(&ctx->mmu_lock); + /* add page offset to physical address */ + if (hops->unscrambled_paddr) + hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, + &hops->unscrambled_paddr); + return rc; } @@ -548,13 +587,26 @@ int hl_mmu_if_set_funcs(struct hl_device *hdev) } /** - * hl_mmu_scramble_vaddr() - The generic mmu virtual address scrambling routine. + * hl_mmu_scramble_addr() - The generic mmu address scrambling routine. * @hdev: pointer to device data. - * @virt_addr: The virtual address to scramble. + * @addr: The address to scramble. * - * Return: The scrambled virtual address. + * Return: The scrambled address. */ -u64 hl_mmu_scramble_vaddr(struct hl_device *hdev, u64 virt_addr) +u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr) { - return virt_addr; + return addr; +} + +/** + * hl_mmu_descramble_addr() - The generic mmu address descrambling + * routine. + * @hdev: pointer to device data. + * @addr: The address to descramble. + * + * Return: The un-scrambled address. + */ +u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr) +{ + return addr; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 69b3867bc151..b49e10394ed4 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -8548,7 +8548,8 @@ static const struct hl_asic_funcs gaudi_funcs = { .get_device_time = gaudi_get_device_time, .collective_wait_init_cs = gaudi_collective_wait_init_cs, .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, - .scramble_vaddr = hl_mmu_scramble_vaddr, + .scramble_addr = hl_mmu_scramble_addr, + .descramble_addr = hl_mmu_descramble_addr, .ack_protection_bits_errors = gaudi_ack_protection_bits_errors }; diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 82f69274def7..db951d622ad5 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5459,7 +5459,8 @@ static const struct hl_asic_funcs goya_funcs = { .get_device_time = goya_get_device_time, .collective_wait_init_cs = goya_collective_wait_init_cs, .collective_wait_create_jobs = goya_collective_wait_create_jobs, - .scramble_vaddr = hl_mmu_scramble_vaddr, + .scramble_addr = hl_mmu_scramble_addr, + .descramble_addr = hl_mmu_descramble_addr, .ack_protection_bits_errors = goya_ack_protection_bits_errors }; From d00697fbe13ccc1dfb7adb82204e1469a7783b07 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 5 Jan 2021 12:55:06 +0200 Subject: [PATCH 184/633] habanalabs: add new mem ioctl op for mapping hw blocks For future ASIC support the driver allows user to map certain regions in the device's configuration space for direct access from userspace. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/device.c | 3 + drivers/misc/habanalabs/common/habanalabs.h | 16 +++- drivers/misc/habanalabs/common/memory.c | 93 ++++++++++++++++++++- drivers/misc/habanalabs/gaudi/gaudi.c | 17 +++- drivers/misc/habanalabs/goya/goya.c | 16 +++- include/uapi/misc/habanalabs.h | 18 +++- 6 files changed, 153 insertions(+), 10 deletions(-) diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 13405d47d843..59219c862ca0 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -142,6 +142,9 @@ static int hl_mmap(struct file *filp, struct vm_area_struct *vma) switch (vm_pgoff & HL_MMAP_TYPE_MASK) { case HL_MMAP_TYPE_CB: return hl_cb_mmap(hpriv, vma); + + case HL_MMAP_TYPE_BLOCK: + return hl_hw_block_mmap(hpriv, vma); } return -EINVAL; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 4129e4e6a7b5..e105612ed577 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -28,17 +28,18 @@ #define HL_NAME "habanalabs" /* Use upper bits of mmap offset to store habana driver specific information. - * bits[63:62] - Encode mmap type + * bits[63:61] - Encode mmap type * bits[45:0] - mmap offset value * * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these * defines are w.r.t to PAGE_SIZE */ -#define HL_MMAP_TYPE_SHIFT (62 - PAGE_SHIFT) -#define HL_MMAP_TYPE_MASK (0x3ull << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_SHIFT (61 - PAGE_SHIFT) +#define HL_MMAP_TYPE_MASK (0x7ull << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT) #define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT) -#define HL_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFull >> PAGE_SHIFT) +#define HL_MMAP_OFFSET_VALUE_MASK (0x1FFFFFFFFFFFull >> PAGE_SHIFT) #define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK) #define HL_PENDING_RESET_PER_SEC 10 @@ -856,6 +857,8 @@ enum div_select_defs { * @descramble_addr: Routine to de-scramble the address prior of * showing it to users. * @ack_protection_bits_errors: ack and dump all security violations + * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. + * @hw_block_mmap: mmap a HW block with a given id. */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -968,6 +971,10 @@ struct hl_asic_funcs { u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); void (*ack_protection_bits_errors)(struct hl_device *hdev); + int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr, + u32 *block_id); + int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, + u32 block_id, u32 block_size); }; @@ -2167,6 +2174,7 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr, bool map_cb, u64 *handle); int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle); int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); +int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr, u32 handle); void hl_cb_put(struct hl_cb *cb); diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 36de0d05d3a2..7171e8820a2d 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1289,11 +1289,88 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, return rc; } +static int map_block(struct hl_device *hdev, u64 address, u64 *handle) +{ + u32 block_id = 0; + int rc; + + rc = hdev->asic_funcs->get_hw_block_id(hdev, address, &block_id); + + *handle = block_id | HL_MMAP_TYPE_BLOCK; + *handle <<= PAGE_SHIFT; + + return rc; +} + +static void hw_block_vm_close(struct vm_area_struct *vma) +{ + struct hl_ctx *ctx = (struct hl_ctx *) vma->vm_private_data; + + hl_ctx_put(ctx); + vma->vm_private_data = NULL; +} + +static const struct vm_operations_struct hw_block_vm_ops = { + .close = hw_block_vm_close +}; + +/** + * hl_hw_block_mmap() - mmap a hw block to user. + * @hpriv: pointer to the private data of the fd + * @vma: pointer to vm_area_struct of the process + * + * Driver increments context reference for every HW block mapped in order + * to prevent user from closing FD without unmapping first + */ +int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) +{ + struct hl_device *hdev = hpriv->hdev; + u32 block_id, block_size; + int rc; + + /* We use the page offset to hold the block id and thus we need to clear + * it before doing the mmap itself + */ + block_id = vma->vm_pgoff; + vma->vm_pgoff = 0; + + /* Driver only allows mapping of a complete HW block */ + block_size = vma->vm_end - vma->vm_start; + +#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK + if (!access_ok(VERIFY_WRITE, + (void __user *) (uintptr_t) vma->vm_start, block_size)) { +#else + if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) { +#endif + dev_err(hdev->dev, + "user pointer is invalid - 0x%lx\n", + vma->vm_start); + + return -EINVAL; + } + + vma->vm_ops = &hw_block_vm_ops; + vma->vm_private_data = hpriv->ctx; + + hl_ctx_get(hdev, hpriv->ctx); + + rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size); + if (rc) { + hl_ctx_put(hpriv->ctx); + return rc; + } + + vma->vm_pgoff = block_id; + + return 0; +} + static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) { struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; - u64 device_addr = 0; + u64 block_handle, device_addr = 0; u32 handle = 0; int rc; @@ -1337,6 +1414,12 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) rc = 0; break; + case HL_MEM_OP_MAP_BLOCK: + rc = map_block(hdev, args->in.map_block.block_addr, + &block_handle); + args->out.handle = block_handle; + break; + default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -ENOTTY; @@ -1353,7 +1436,7 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) union hl_mem_args *args = data; struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; - u64 device_addr = 0; + u64 block_handle, device_addr = 0; u32 handle = 0; int rc; @@ -1439,6 +1522,12 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) rc = unmap_device_va(ctx, &args->in, false); break; + case HL_MEM_OP_MAP_BLOCK: + rc = map_block(hdev, args->in.map_block.block_addr, + &block_handle); + args->out.handle = block_handle; + break; + default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -ENOTTY; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index b49e10394ed4..e1c6072e5fb3 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -8471,6 +8471,19 @@ static u64 gaudi_get_device_time(struct hl_device *hdev) return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); } +static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr, + u32 *block_id) +{ + return -EPERM; +} + +static int gaudi_block_mmap(struct hl_device *hdev, + struct vm_area_struct *vma, + u32 block_id, u32 block_size) +{ + return -EPERM; +} + static const struct hl_asic_funcs gaudi_funcs = { .early_init = gaudi_early_init, .early_fini = gaudi_early_fini, @@ -8550,7 +8563,9 @@ static const struct hl_asic_funcs gaudi_funcs = { .collective_wait_create_jobs = gaudi_collective_wait_create_jobs, .scramble_addr = hl_mmu_scramble_addr, .descramble_addr = hl_mmu_descramble_addr, - .ack_protection_bits_errors = gaudi_ack_protection_bits_errors + .ack_protection_bits_errors = gaudi_ack_protection_bits_errors, + .get_hw_block_id = gaudi_get_hw_block_id, + .hw_block_mmap = gaudi_block_mmap }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index db951d622ad5..6b4c41188495 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5382,6 +5382,18 @@ static void goya_ctx_fini(struct hl_ctx *ctx) } +static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, + u32 *block_id) +{ + return -EPERM; +} + +static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, + u32 block_id, u32 block_size) +{ + return -EPERM; +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5461,7 +5473,9 @@ static const struct hl_asic_funcs goya_funcs = { .collective_wait_create_jobs = goya_collective_wait_create_jobs, .scramble_addr = hl_mmu_scramble_addr, .descramble_addr = hl_mmu_descramble_addr, - .ack_protection_bits_errors = goya_ack_protection_bits_errors + .ack_protection_bits_errors = goya_ack_protection_bits_errors, + .get_hw_block_id = goya_get_hw_block_id, + .hw_block_mmap = goya_block_mmap }; /* diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 866355a53188..b1c09eba8ac2 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -718,6 +718,8 @@ union hl_wait_cs_args { #define HL_MEM_OP_MAP 2 /* Opcode to unmap previously mapped host and device memory */ #define HL_MEM_OP_UNMAP 3 +/* Opcode to map a hw block */ +#define HL_MEM_OP_MAP_BLOCK 4 /* Memory flags */ #define HL_MEM_CONTIGUOUS 0x1 @@ -772,6 +774,17 @@ struct hl_mem_in { __u64 mem_size; } map_host; + /* HL_MEM_OP_MAP_BLOCK - map a hw block */ + struct { + /* + * HW block address to map, a handle will be returned + * to the user and will be used to mmap the relevant + * block. Only addresses from configuration space are + * allowed. + */ + __u64 block_addr; + } map_block; + /* HL_MEM_OP_UNMAP - unmap host memory */ struct { /* Virtual address returned from HL_MEM_OP_MAP */ @@ -798,8 +811,9 @@ struct hl_mem_out { __u64 device_virt_addr; /* - * Used for HL_MEM_OP_ALLOC. This is the assigned - * handle for the allocated memory + * Used for HL_MEM_OP_ALLOC and HL_MEM_OP_MAP_BLOCK. + * This is the assigned handle for the allocated memory + * or mapped block */ __u64 handle; }; From 0811b391469510447a2263f706389b610e454177 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Sun, 6 Dec 2020 17:18:02 +0200 Subject: [PATCH 185/633] habanalabs: add CS completion and timeout properties In order to support staged submission feature, we need to distinguish on which command submission we want to receive timeout and for which we want to receive completion. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../habanalabs/common/command_submission.c | 104 ++++++++++++++---- drivers/misc/habanalabs/common/habanalabs.h | 15 ++- drivers/misc/habanalabs/common/hw_queue.c | 24 +++- 3 files changed, 117 insertions(+), 26 deletions(-) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index a5e9bb0a4855..57daff0e59ae 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -149,9 +149,10 @@ void hl_fence_get(struct hl_fence *fence) kref_get(&fence->refcount); } -static void hl_fence_init(struct hl_fence *fence) +static void hl_fence_init(struct hl_fence *fence, u64 sequence) { kref_init(&fence->refcount); + fence->cs_sequence = sequence; fence->error = 0; fence->timestamp = ktime_set(0, 0); init_completion(&fence->completion); @@ -184,6 +185,28 @@ static void cs_job_put(struct hl_cs_job *job) kref_put(&job->refcount, cs_job_do_release); } +bool cs_needs_completion(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the last CS in sequence should + * get a completion, any non staged CS will always get a completion + */ + if (cs->staged_cs && !cs->staged_last) + return false; + + return true; +} + +bool cs_needs_timeout(struct hl_cs *cs) +{ + /* In case this is a staged CS, only the first CS in sequence should + * get a timeout, any non staged CS will always get a timeout + */ + if (cs->staged_cs && !cs->staged_first) + return false; + + return true; +} + static bool is_cb_patched(struct hl_device *hdev, struct hl_cs_job *job) { /* @@ -225,7 +248,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) parser.queue_type = job->queue_type; parser.is_kernel_allocated_cb = job->is_kernel_allocated_cb; job->patched_cb = NULL; - parser.completion = true; + parser.completion = cs_needs_completion(job->cs); rc = hdev->asic_funcs->cs_parser(hdev, &parser); @@ -291,8 +314,21 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) hl_debugfs_remove_job(hdev, job); - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + /* We decrement reference only for a CS that gets completion + * because the reference was incremented only for this kind of CS + * right before it was scheduled. + * + * In staged submission, only the last CS marked as 'staged_last' + * gets completion, hence its release function will be called from here. + * As for all the rest CS's in the staged submission which do not get + * completion, their CS reference will be decremented by the + * 'staged_last' CS during the CS release flow. + * All relevant PQ CI counters will be incremented during the CS release + * flow by calling 'hl_hw_queue_update_ci'. + */ + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_put(cs); cs_job_put(job); @@ -347,8 +383,8 @@ static void cs_do_release(struct kref *ref) hdev->asic_funcs->hw_queues_unlock(hdev); - /* Need to update CI for internal queues */ - hl_int_hw_queue_update_ci(cs); + /* Need to update CI for all queue jobs that does not get completion */ + hl_hw_queue_update_ci(cs); /* remove CS from CS mirror list */ spin_lock(&hdev->cs_mirror_lock); @@ -359,6 +395,7 @@ static void cs_do_release(struct kref *ref) * running from the TDR context */ if (!cs->timedout && hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { + bool next_entry_found = false; struct hl_cs *next; if (cs->tdr_active) @@ -367,10 +404,13 @@ static void cs_do_release(struct kref *ref) spin_lock(&hdev->cs_mirror_lock); /* queue TDR for next CS */ - next = list_first_entry_or_null(&hdev->cs_mirror_list, - struct hl_cs, mirror_node); + list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node) + if (cs_needs_timeout(next)) { + next_entry_found = true; + break; + } - if (next && !next->tdr_active) { + if (next_entry_found && !next->tdr_active) { next->tdr_active = true; schedule_delayed_work(&next->work_tdr, hdev->timeout_jiffies); @@ -462,7 +502,8 @@ static void cs_timedout(struct work_struct *work) } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, - enum hl_cs_type cs_type, struct hl_cs **cs_new) + enum hl_cs_type cs_type, u64 user_sequence, + struct hl_cs **cs_new) { struct hl_cs_counters_atomic *cntr; struct hl_fence *other = NULL; @@ -511,6 +552,18 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, (hdev->asic_prop.max_pending_cs - 1)]; if (other && !completion_done(&other->completion)) { + /* If the following statement is true, it means we have reached + * a point in which only part of the staged submission was + * submitted and we don't have enough room in the 'cs_pending' + * array for the rest of the submission. + * This causes a deadlock because this CS will never be + * completed as it depends on future CS's for completion. + */ + if (other->cs_sequence == user_sequence) + dev_crit_ratelimited(hdev->dev, + "Staged CS %llu deadlock due to lack of resources", + user_sequence); + dev_dbg_ratelimited(hdev->dev, "Rejecting CS because of too many in-flights CS\n"); atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt); @@ -529,7 +582,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, } /* init hl_fence */ - hl_fence_init(&cs_cmpl->base_fence); + hl_fence_init(&cs_cmpl->base_fence, cs_cmpl->cs_seq); cs->sequence = cs_cmpl->cs_seq; @@ -825,7 +878,7 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, u32 num_chunks, u64 *cs_seq, u32 flags) { - bool int_queues_only = true; + bool staged_mid, int_queues_only = true; struct hl_device *hdev = hpriv->hdev; struct hl_cs_chunk *cs_chunk_array; struct hl_cs_counters_atomic *cntr; @@ -833,9 +886,11 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, struct hl_cs_job *job; struct hl_cs *cs; struct hl_cb *cb; + u64 user_sequence; int rc, i; cntr = &hdev->aggregated_cs_counters; + user_sequence = *cs_seq; *cs_seq = ULLONG_MAX; rc = hl_cs_copy_chunk_array(hdev, &cs_chunk_array, chunks, num_chunks, @@ -843,7 +898,14 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, if (rc) goto out; - rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs); + if ((flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST)) + staged_mid = true; + else + staged_mid = false; + + rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, + staged_mid ? user_sequence : ULLONG_MAX, &cs); if (rc) goto free_cs_chunk_array; @@ -911,8 +973,9 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, * Only increment for JOB on external or H/W queues, because * only for those JOBs we get completion */ - if (job->queue_type == QUEUE_TYPE_EXT || - job->queue_type == QUEUE_TYPE_HW) + if (cs_needs_completion(cs) && + (job->queue_type == QUEUE_TYPE_EXT || + job->queue_type == QUEUE_TYPE_HW)) cs_get(cs); hl_debugfs_add_job(hdev, job); @@ -928,11 +991,14 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, } } - if (int_queues_only) { + /* We allow a CS with any queue type combination as long as it does + * not get a completion + */ + if (int_queues_only && cs_needs_completion(cs)) { atomic64_inc(&ctx->cs_counters.validation_drop_cnt); atomic64_inc(&cntr->validation_drop_cnt); dev_err(hdev->dev, - "Reject CS %d.%llu because only internal queues jobs are present\n", + "Reject CS %d.%llu since it contains only internal queues jobs and needs completion\n", cs->ctx->asid, cs->sequence); rc = -EINVAL; goto free_cs_object; @@ -1037,7 +1103,7 @@ static int hl_submit_pending_cb(struct hl_fpriv *hpriv) list_move_tail(&pending_cb->cb_node, &local_cb_list); spin_unlock(&ctx->pending_cb_lock); - rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, &cs); + rc = allocate_cs(hdev, ctx, CS_TYPE_DEFAULT, ULLONG_MAX, &cs); if (rc) goto add_list_elements; @@ -1410,7 +1476,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, } } - rc = allocate_cs(hdev, ctx, cs_type, &cs); + rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs); if (rc) { if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index e105612ed577..fd2fffd20ba1 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -486,6 +486,7 @@ struct asic_fixed_properties { * struct hl_fence - software synchronization primitive * @completion: fence is implemented using completion * @refcount: refcount for this fence + * @cs_sequence: sequence of the corresponding command submission * @error: mark this fence with error * @timestamp: timestamp upon completion * @@ -493,6 +494,7 @@ struct asic_fixed_properties { struct hl_fence { struct completion completion; struct kref refcount; + u64 cs_sequence; int error; ktime_t timestamp; }; @@ -1176,7 +1178,11 @@ struct hl_userptr { * @tdr_active: true if TDR was activated for this CS (to prevent * double TDR activation). * @aborted: true if CS was aborted due to some device error. - * @timestamp: true if a timestmap must be captured upon completion + * @timestamp: true if a timestmap must be captured upon completion. + * @staged_last: true if this is the last staged CS and needs completion. + * @staged_first: true if this is the first staged CS and we need to receive + * timeout for this CS. + * @staged_cs: true if this CS is part of a staged submission. */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1198,6 +1204,9 @@ struct hl_cs { u8 tdr_active; u8 aborted; u8 timestamp; + u8 staged_last; + u8 staged_first; + u8 staged_cs; }; /** @@ -2118,7 +2127,7 @@ int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, int hl_hw_queue_schedule_cs(struct hl_cs *cs); u32 hl_hw_queue_add_ptr(u32 ptr, u16 val); void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id); -void hl_int_hw_queue_update_ci(struct hl_cs *cs); +void hl_hw_queue_update_ci(struct hl_cs *cs); void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset); #define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1) @@ -2196,6 +2205,8 @@ int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask); void hl_fence_put(struct hl_fence *fence); void hl_fence_get(struct hl_fence *fence); void cs_get(struct hl_cs *cs); +bool cs_needs_completion(struct hl_cs *cs); +bool cs_needs_timeout(struct hl_cs *cs); void goya_set_asic_funcs(struct hl_device *hdev); void gaudi_set_asic_funcs(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index 76217258780a..ad440ae785a3 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -38,7 +38,7 @@ static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len) return (abs(delta) - queue_len); } -void hl_int_hw_queue_update_ci(struct hl_cs *cs) +void hl_hw_queue_update_ci(struct hl_cs *cs) { struct hl_device *hdev = cs->ctx->hdev; struct hl_hw_queue *q; @@ -53,8 +53,13 @@ void hl_int_hw_queue_update_ci(struct hl_cs *cs) if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW) return; + /* We must increment CI for every queue that will never get a + * completion, there are 2 scenarios this can happen: + * 1. All queues of a non completion CS will never get a completion. + * 2. Internal queues never gets completion. + */ for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) { - if (q->queue_type == QUEUE_TYPE_INT) + if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT) atomic_add(cs->jobs_in_queue_cnt[i], &q->ci); } } @@ -292,6 +297,10 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) len = job->job_cb_size; ptr = cb->bus_address; + /* Skip completion flow in case this is a non completion CS */ + if (!cs_needs_completion(job->cs)) + goto submit_bd; + cq_pkt.data = cpu_to_le32( ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT) & CQ_ENTRY_SHADOW_INDEX_MASK) | @@ -318,6 +327,7 @@ static void ext_queue_schedule_job(struct hl_cs_job *job) cq->pi = hl_cq_inc_ptr(cq->pi); +submit_bd: ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr); } @@ -525,6 +535,7 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) struct hl_cs_job *job, *tmp; struct hl_hw_queue *q; int rc = 0, i, cq_cnt; + bool first_entry; u32 max_queues; cntr = &hdev->aggregated_cs_counters; @@ -548,7 +559,9 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) switch (q->queue_type) { case QUEUE_TYPE_EXT: rc = ext_queue_sanity_checks(hdev, q, - cs->jobs_in_queue_cnt[i], true); + cs->jobs_in_queue_cnt[i], + cs_needs_completion(cs) ? + true : false); break; case QUEUE_TYPE_INT: rc = int_queue_sanity_checks(hdev, q, @@ -586,9 +599,10 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list); /* Queue TDR if the CS is the first entry and if timeout is wanted */ + first_entry = list_first_entry(&hdev->cs_mirror_list, + struct hl_cs, mirror_node) == cs; if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) && - (list_first_entry(&hdev->cs_mirror_list, - struct hl_cs, mirror_node) == cs)) { + first_entry && cs_needs_timeout(cs)) { cs->tdr_active = true; schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies); From cf30339d3f44a64115e88d46a932fdc3d3644785 Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Sun, 17 Jan 2021 16:01:56 +0200 Subject: [PATCH 186/633] habanalabs: modify device_idle interface Currently this API uses single 64 bits mask for engines idle indication. Recently, it was observed that more bits are needed for some ASICs. This patch modifies the use of the idle mask and the idle_extensions mask. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/context.c | 9 +++-- drivers/misc/habanalabs/common/debugfs.c | 2 +- drivers/misc/habanalabs/common/habanalabs.h | 4 +- .../misc/habanalabs/common/habanalabs_ioctl.c | 5 ++- drivers/misc/habanalabs/gaudi/gaudi.c | 37 ++++++++----------- drivers/misc/habanalabs/goya/goya.c | 21 +++++------ include/uapi/misc/habanalabs.h | 4 +- 7 files changed, 40 insertions(+), 42 deletions(-) diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index 829fe98eed61..cda871afb8f4 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -12,7 +12,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx) { struct hl_device *hdev = ctx->hdev; - u64 idle_mask = 0; + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; int i; /* Release all allocated pending cb's, those cb's were never @@ -55,10 +55,11 @@ static void hl_ctx_fini(struct hl_ctx *ctx) if ((!hdev->pldm) && (hdev->pdev) && (!hdev->asic_funcs->is_device_idle(hdev, - &idle_mask, NULL))) + idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) dev_notice(hdev->dev, - "device not idle after user context is closed (0x%llx)\n", - idle_mask); + "device not idle after user context is closed (0x%llx, 0x%llx)\n", + idle_mask[0], idle_mask[1]); } else { dev_dbg(hdev->dev, "closing kernel context\n"); hdev->asic_funcs->ctx_fini(ctx); diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 9e3c1efe56ba..df847a6d19f4 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -414,7 +414,7 @@ static int engines_show(struct seq_file *s, void *data) return 0; } - hdev->asic_funcs->is_device_idle(hdev, NULL, s); + hdev->asic_funcs->is_device_idle(hdev, NULL, 0, s); return 0; } diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index fd2fffd20ba1..be7947d69dfa 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -933,8 +933,8 @@ struct hl_asic_funcs { void (*set_clock_gating)(struct hl_device *hdev); void (*disable_clock_gating)(struct hl_device *hdev); int (*debug_coresight)(struct hl_device *hdev, void *data); - bool (*is_device_idle)(struct hl_device *hdev, u64 *mask, - struct seq_file *s); + bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s); int (*soft_reset_late_init)(struct hl_device *hdev); void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 628bdc56dca3..e86f46d4b613 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -145,9 +145,10 @@ static int hw_idle(struct hl_device *hdev, struct hl_info_args *args) return -EINVAL; hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev, - &hw_idle.busy_engines_mask_ext, NULL); + hw_idle.busy_engines_mask_ext, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL); hw_idle.busy_engines_mask = - lower_32_bits(hw_idle.busy_engines_mask_ext); + lower_32_bits(hw_idle.busy_engines_mask_ext[0]); return copy_to_user(out, &hw_idle, min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index e1c6072e5fb3..9a3d2fb477a8 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -4538,7 +4538,6 @@ static int gaudi_scrub_device_mem(struct hl_device *hdev, u64 addr, u64 size) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct gaudi_device *gaudi = hdev->asic_specific; - u64 idle_mask = 0; int rc = 0; u64 val = 0; @@ -4551,8 +4550,8 @@ static int gaudi_scrub_device_mem(struct hl_device *hdev, u64 addr, u64 size) hdev, mmDMA0_CORE_STS0/* dummy */, val/* dummy */, - (hdev->asic_funcs->is_device_idle(hdev, - &idle_mask, NULL)), + (hdev->asic_funcs->is_device_idle(hdev, NULL, + 0, NULL)), 1000, HBM_SCRUBBING_TIMEOUT_US); if (rc) { @@ -6423,7 +6422,7 @@ static int gaudi_send_job_on_qman0(struct hl_device *hdev, else timeout = HL_DEVICE_TIMEOUT_USEC; - if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { + if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { dev_err_ratelimited(hdev->dev, "Can't send driver job on QMAN0 because the device is not idle\n"); return -EBUSY; @@ -7706,13 +7705,14 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) return 0; } -static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, - struct seq_file *s) +static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s) { struct gaudi_device *gaudi = hdev->asic_specific; const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n"; const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n"; const char *nic_fmt = "%-5d%-9s%#-14x%#x\n"; + unsigned long *mask = (unsigned long *)mask_arr; u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts; bool is_idle = true, is_eng_idle, is_slave; u64 offset; @@ -7738,9 +7738,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, IS_DMA_IDLE(dma_core_sts0); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_DMA_0 + dma_id); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_DMA_0 + dma_id, mask); if (s) seq_printf(s, fmt, dma_id, is_eng_idle ? "Y" : "N", qm_glbl_sts0, @@ -7761,9 +7760,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, IS_TPC_IDLE(tpc_cfg_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_TPC_0 + i); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_TPC_0 + i, mask); if (s) seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", @@ -7790,9 +7788,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_MME_0 + i); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_MME_0 + i, mask); if (s) { if (!is_slave) seq_printf(s, fmt, i, @@ -7818,9 +7815,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_NIC_0 + port); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); if (s) seq_printf(s, nic_fmt, port, is_eng_idle ? "Y" : "N", @@ -7834,9 +7830,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask, is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GAUDI_ENGINE_ID_NIC_0 + port); + if (mask && !is_eng_idle) + set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); if (s) seq_printf(s, nic_fmt, port, is_eng_idle ? "Y" : "N", diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 6b4c41188495..a954e7c02375 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -2916,7 +2916,7 @@ static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) else timeout = HL_DEVICE_TIMEOUT_USEC; - if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { + if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { dev_err_ratelimited(hdev->dev, "Can't send driver job on QMAN0 because the device is not idle\n"); return -EBUSY; @@ -5187,11 +5187,12 @@ static void goya_disable_clock_gating(struct hl_device *hdev) /* clock gating not supported in Goya */ } -static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, - struct seq_file *s) +static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, + u8 mask_len, struct seq_file *s) { const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; + unsigned long *mask = (unsigned long *)mask_arr; u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts, mme_arch_sts; bool is_idle = true, is_eng_idle; @@ -5211,9 +5212,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_DMA_IDLE(dma_core_sts0); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GOYA_ENGINE_ID_DMA_0 + i); + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_DMA_0 + i, mask); if (s) seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, dma_core_sts0); @@ -5235,9 +5235,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_TPC_IDLE(tpc_cfg_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << - (GOYA_ENGINE_ID_TPC_0 + i); + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_TPC_0 + i, mask); if (s) seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts); @@ -5256,8 +5255,8 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, IS_MME_IDLE(mme_arch_sts); is_idle &= is_eng_idle; - if (mask) - *mask |= ((u64) !is_eng_idle) << GOYA_ENGINE_ID_MME_0; + if (mask && !is_eng_idle) + set_bit(GOYA_ENGINE_ID_MME_0, mask); if (s) { seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, mme_arch_sts); diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index b1c09eba8ac2..ebde42b37b43 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -331,6 +331,8 @@ struct hl_info_dram_usage { __u64 ctx_dram_mem; }; +#define HL_BUSY_ENGINES_MASK_EXT_SIZE 2 + struct hl_info_hw_idle { __u32 is_idle; /* @@ -343,7 +345,7 @@ struct hl_info_hw_idle { * Extended Bitmask of busy engines. * Bits definition is according to `enum _enging_id'. */ - __u64 busy_engines_mask_ext; + __u64 busy_engines_mask_ext[HL_BUSY_ENGINES_MASK_EXT_SIZE]; }; struct hl_info_device_status { From 2795c8891577c4f1493dd4e3abc298c60009ec42 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 8 Dec 2020 13:47:05 +0200 Subject: [PATCH 187/633] habanalabs: staged submission support We introduce a new mechanism named Staged Submission. This mechanism allows the user to send a whole CS in pieces. Each CS will not require completion rather than the last CS. Timeout timer will be triggered upon reception of the first CS in group. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../habanalabs/common/command_submission.c | 214 ++++++++++++++++-- drivers/misc/habanalabs/common/habanalabs.h | 9 + drivers/misc/habanalabs/common/hw_queue.c | 27 +++ drivers/misc/habanalabs/gaudi/gaudi.c | 1 + 4 files changed, 227 insertions(+), 24 deletions(-) diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 57daff0e59ae..7bd4a03b3429 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -334,6 +334,133 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) cs_job_put(job); } +/* + * hl_staged_cs_find_first - locate the first CS in this staged submission + * + * @hdev: pointer to device structure + * @cs_seq: staged submission sequence number + * + * @note: This function must be called under 'hdev->cs_mirror_lock' + * + * Find and return a CS pointer with the given sequence + */ +struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq) +{ + struct hl_cs *cs; + + list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node) + if (cs->staged_cs && cs->staged_first && + cs->sequence == cs_seq) + return cs; + + return NULL; +} + +/* + * is_staged_cs_last_exists - returns true if the last CS in sequence exists + * + * @hdev: pointer to device structure + * @cs: staged submission member + * + */ +bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs) +{ + struct hl_cs *last_entry; + + last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs, + staged_cs_node); + + if (last_entry->staged_last) + return true; + + return false; +} + +/* + * staged_cs_get - get CS reference if this CS is a part of a staged CS + * + * @hdev: pointer to device structure + * @cs: current CS + * @cs_seq: staged submission sequence number + * + * Increment CS reference for every CS in this staged submission except for + * the CS which get completion. + */ +static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs) +{ + /* Only the last CS in this staged submission will get a completion. + * We must increment the reference for all other CS's in this + * staged submission. + * Once we get a completion we will release the whole staged submission. + */ + if (!cs->staged_last) + cs_get(cs); +} + +/* + * staged_cs_put - put a CS in case it is part of staged submission + * + * @hdev: pointer to device structure + * @cs: CS to put + * + * This function decrements a CS reference (for a non completion CS) + */ +static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs) +{ + /* We release all CS's in a staged submission except the last + * CS which we have never incremented its reference. + */ + if (!cs_needs_completion(cs)) + cs_put(cs); +} + +static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs) +{ + bool next_entry_found = false; + struct hl_cs *next; + + if (!cs_needs_timeout(cs)) + return; + + spin_lock(&hdev->cs_mirror_lock); + + /* We need to handle tdr only once for the complete staged submission. + * Hence, we choose the CS that reaches this function first which is + * the CS marked as 'staged_last'. + */ + if (cs->staged_cs && cs->staged_last) + cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); + + spin_unlock(&hdev->cs_mirror_lock); + + /* Don't cancel TDR in case this CS was timedout because we might be + * running from the TDR context + */ + if (cs && (cs->timedout || + hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT)) + return; + + if (cs && cs->tdr_active) + cancel_delayed_work_sync(&cs->work_tdr); + + spin_lock(&hdev->cs_mirror_lock); + + /* queue TDR for next CS */ + list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node) + if (cs_needs_timeout(next)) { + next_entry_found = true; + break; + } + + if (next_entry_found && !next->tdr_active) { + next->tdr_active = true; + schedule_delayed_work(&next->work_tdr, + hdev->timeout_jiffies); + } + + spin_unlock(&hdev->cs_mirror_lock); +} + static void cs_do_release(struct kref *ref) { struct hl_cs *cs = container_of(ref, struct hl_cs, refcount); @@ -391,32 +518,29 @@ static void cs_do_release(struct kref *ref) list_del_init(&cs->mirror_node); spin_unlock(&hdev->cs_mirror_lock); - /* Don't cancel TDR in case this CS was timedout because we might be - * running from the TDR context - */ - if (!cs->timedout && hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { - bool next_entry_found = false; - struct hl_cs *next; + cs_handle_tdr(hdev, cs); - if (cs->tdr_active) - cancel_delayed_work_sync(&cs->work_tdr); + if (cs->staged_cs) { + /* the completion CS decrements reference for the entire + * staged submission + */ + if (cs->staged_last) { + struct hl_cs *staged_cs, *tmp; - spin_lock(&hdev->cs_mirror_lock); - - /* queue TDR for next CS */ - list_for_each_entry(next, &hdev->cs_mirror_list, mirror_node) - if (cs_needs_timeout(next)) { - next_entry_found = true; - break; - } - - if (next_entry_found && !next->tdr_active) { - next->tdr_active = true; - schedule_delayed_work(&next->work_tdr, - hdev->timeout_jiffies); + list_for_each_entry_safe(staged_cs, tmp, + &cs->staged_cs_node, staged_cs_node) + staged_cs_put(hdev, staged_cs); } - spin_unlock(&hdev->cs_mirror_lock); + /* A staged CS will be a member in the list only after it + * was submitted. We used 'cs_mirror_lock' when inserting + * it to list so we will use it again when removing it + */ + if (cs->submitted) { + spin_lock(&hdev->cs_mirror_lock); + list_del(&cs->staged_cs_node); + spin_unlock(&hdev->cs_mirror_lock); + } } out: @@ -614,6 +738,8 @@ static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs) { struct hl_cs_job *job, *tmp; + staged_cs_put(hdev, cs); + list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node) complete_job(hdev, job); } @@ -623,7 +749,9 @@ void hl_cs_rollback_all(struct hl_device *hdev) int i; struct hl_cs *cs, *tmp; - /* flush all completions */ + /* flush all completions before iterating over the CS mirror list in + * order to avoid a race with the release functions + */ for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) flush_workqueue(hdev->cq_wq[i]); @@ -632,7 +760,7 @@ void hl_cs_rollback_all(struct hl_device *hdev) cs_get(cs); cs->aborted = true; dev_warn_ratelimited(hdev->dev, "Killing CS %d.%llu\n", - cs->ctx->asid, cs->sequence); + cs->ctx->asid, cs->sequence); cs_rollback(hdev, cs); cs_put(cs); } @@ -804,6 +932,12 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) return -EBUSY; } + if ((args->in.cs_flags & HL_CS_FLAGS_STAGED_SUBMISSION) && + !hdev->supports_staged_submission) { + dev_err(hdev->dev, "staged submission not supported"); + return -EPERM; + } + cs_type_flags = args->in.cs_flags & HL_CS_FLAGS_TYPE_MASK; if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) { @@ -875,6 +1009,34 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, return 0; } +static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs, + u64 sequence, u32 flags) +{ + if (!(flags & HL_CS_FLAGS_STAGED_SUBMISSION)) + return 0; + + cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST); + cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST); + + if (cs->staged_first) { + /* Staged CS sequence is the first CS sequence */ + INIT_LIST_HEAD(&cs->staged_cs_node); + cs->staged_sequence = cs->sequence; + } else { + /* User sequence will be validated in 'hl_hw_queue_schedule_cs' + * under the cs_mirror_lock + */ + cs->staged_sequence = sequence; + } + + /* Increment CS reference if needed */ + staged_cs_get(hdev, cs); + + cs->staged_cs = true; + + return 0; +} + static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, u32 num_chunks, u64 *cs_seq, u32 flags) { @@ -914,6 +1076,10 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, hl_debugfs_add_cs(cs); + rc = cs_staged_submission(hdev, cs, user_sequence, flags); + if (rc) + goto free_cs_object; + /* Validate ALL the CS chunks before submitting the CS */ for (i = 0 ; i < num_chunks ; i++) { struct hl_cs_chunk *chunk = &cs_chunk_array[i]; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index be7947d69dfa..30f32f2edb8a 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1169,8 +1169,11 @@ struct hl_userptr { * @finish_work: workqueue object to run when CS is completed by H/W. * @work_tdr: delayed work node for TDR. * @mirror_node : node in device mirror list of command submissions. + * @staged_cs_node: node in the staged cs list. * @debugfs_list: node in debugfs list of command submissions. * @sequence: the sequence number of this CS. + * @staged_sequence: the sequence of the staged submission this CS is part of, + * relevant only if staged_cs is set. * @type: CS_TYPE_*. * @submitted: true if CS was submitted to H/W. * @completed: true if CS was completed by device. @@ -1195,8 +1198,10 @@ struct hl_cs { struct work_struct finish_work; struct delayed_work work_tdr; struct list_head mirror_node; + struct list_head staged_cs_node; struct list_head debugfs_list; u64 sequence; + u64 staged_sequence; enum hl_cs_type type; u8 submitted; u8 completed; @@ -1905,6 +1910,7 @@ struct hl_mmu_funcs { * user processes * @device_fini_pending: true if device_fini was called and might be * waiting for the reset thread to finish + * @supports_staged_submission: true if staged submissions are supported */ struct hl_device { struct pci_dev *pdev; @@ -2010,6 +2016,7 @@ struct hl_device { u8 needs_reset; u8 process_kill_trial_cnt; u8 device_fini_pending; + u8 supports_staged_submission; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -2207,6 +2214,8 @@ void hl_fence_get(struct hl_fence *fence); void cs_get(struct hl_cs *cs); bool cs_needs_completion(struct hl_cs *cs); bool cs_needs_timeout(struct hl_cs *cs); +bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs); +struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq); void goya_set_asic_funcs(struct hl_device *hdev); void gaudi_set_asic_funcs(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index ad440ae785a3..0f335182267f 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -596,6 +596,31 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) hdev->asic_funcs->collective_wait_init_cs(cs); spin_lock(&hdev->cs_mirror_lock); + + /* Verify staged CS exists and add to the staged list */ + if (cs->staged_cs && !cs->staged_first) { + struct hl_cs *staged_cs; + + staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence); + if (!staged_cs) { + dev_err(hdev->dev, + "Cannot find staged submission sequence %llu", + cs->staged_sequence); + rc = -EINVAL; + goto unlock_cs_mirror; + } + + if (is_staged_cs_last_exists(hdev, staged_cs)) { + dev_err(hdev->dev, + "Staged submission sequence %llu already submitted", + cs->staged_sequence); + rc = -EINVAL; + goto unlock_cs_mirror; + } + + list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node); + } + list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list); /* Queue TDR if the CS is the first entry and if timeout is wanted */ @@ -637,6 +662,8 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) goto out; +unlock_cs_mirror: + spin_unlock(&hdev->cs_mirror_lock); unroll_cq_resv: q = &hdev->kernel_queues[0]; for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) { diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 9a3d2fb477a8..1348016309e3 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -1627,6 +1627,7 @@ static int gaudi_sw_init(struct hl_device *hdev) hdev->supports_sync_stream = true; hdev->supports_coresight = true; + hdev->supports_staged_submission = true; return 0; From 663a301d75b86fbb23e02628c31136e0b8698a1a Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Thu, 21 Jan 2021 22:25:52 +0200 Subject: [PATCH 188/633] habanalabs: fix ETR security issue ETR should always be non-secured as it is used by the users to record profiling/trace data. This patch fixes the configuration to match those requirements. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/gaudi/gaudi_coresight.c | 18 +++++++++++++++--- drivers/misc/habanalabs/goya/goya_coresight.c | 11 +++++++++-- .../habanalabs/include/gaudi/gaudi_masks.h | 5 ++++- .../include/goya/asic_reg/goya_masks.h | 5 ++++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c index 88a09d42e111..6e56fa1c6c69 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c @@ -634,9 +634,21 @@ static int gaudi_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - /* Workaround for H3 #HW-2075 bug: use small data chunks */ - WREG32(mmPSOC_ETR_AXICTL, (is_host ? 0 : 0x700) | - PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT); + if (hdev->asic_prop.fw_security_disabled) { + /* make ETR not privileged */ + val = FIELD_PREP( + PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); + /* make ETR non-secured (inverted logic) */ + val |= FIELD_PREP( + PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK, 1); + /* + * Workaround for H3 #HW-2075 bug: use small data + * chunks + */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_WRBURSTLEN_MASK, + is_host ? 0 : 7); + WREG32(mmPSOC_ETR_AXICTL, val); + } WREG32(mmPSOC_ETR_DBALO, lower_32_bits(input->buffer_address)); WREG32(mmPSOC_ETR_DBAHI, diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c index 6fa03933b438..6b7445cca580 100644 --- a/drivers/misc/habanalabs/goya/goya_coresight.c +++ b/drivers/misc/habanalabs/goya/goya_coresight.c @@ -434,8 +434,15 @@ static int goya_config_etr(struct hl_device *hdev, WREG32(mmPSOC_ETR_BUFWM, 0x3FFC); WREG32(mmPSOC_ETR_RSZ, input->buffer_size); WREG32(mmPSOC_ETR_MODE, input->sink_mode); - WREG32(mmPSOC_ETR_AXICTL, - 0x700 | PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT); + if (hdev->asic_prop.fw_security_disabled) { + /* make ETR not privileged */ + val = FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0); + /* make ETR non-secured (inverted logic) */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK, 1); + /* burst size 8 */ + val |= FIELD_PREP(PSOC_ETR_AXICTL_WRBURSTLEN_MASK, 7); + WREG32(mmPSOC_ETR_AXICTL, val); + } WREG32(mmPSOC_ETR_DBALO, lower_32_bits(input->buffer_address)); WREG32(mmPSOC_ETR_DBAHI, diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h index b9b90d079e23..b53aeda9a982 100644 --- a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h +++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h @@ -388,7 +388,10 @@ enum axi_id { #define RAZWI_INITIATOR_ID_X_Y_TPC6 RAZWI_INITIATOR_ID_X_Y(7, 6) #define RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5 RAZWI_INITIATOR_ID_X_Y(8, 6) -#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK 0x1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK 0x2 +#define PSOC_ETR_AXICTL_WRBURSTLEN_MASK 0xF00 /* STLB_CACHE_INV */ #define STLB_CACHE_INV_PRODUCER_INDEX_SHIFT 0 diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h index 067489bd048e..9ff3cb245580 100644 --- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h +++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h @@ -259,6 +259,9 @@ #define DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT #define DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT -#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT 1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK 0x1 +#define PSOC_ETR_AXICTL_PROTCTRLBIT1_MASK 0x2 +#define PSOC_ETR_AXICTL_WRBURSTLEN_MASK 0xF00 #endif /* ASIC_REG_GOYA_MASKS_H_ */ From 7838504171d910a0934c0b4043386a471309649e Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 26 Jan 2021 22:56:56 +0200 Subject: [PATCH 189/633] habanalabs: update SyncManager interrupt handling The firmware provides more information about SyncManager events. Adjust the code to the latest firmware interface file. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 14 +++++++------- drivers/misc/habanalabs/include/common/cpucp_if.h | 11 +++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 1348016309e3..a9bd5aef6c02 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -6860,24 +6860,24 @@ static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type, u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0; switch (sei_data->sei_cause) { - case GAUDI_SM_SEI_SO_OVERFLOW: + case SM_SEI_SO_OVERFLOW: dev_err(hdev->dev, "SM %u SEI Error: SO %u overflow/underflow", - index, le16_to_cpu(sei_data->sei_log)); + index, le32_to_cpu(sei_data->sei_log)); break; - case GAUDI_SM_SEI_LBW_4B_UNALIGNED: + case SM_SEI_LBW_4B_UNALIGNED: dev_err(hdev->dev, "SM %u SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x", - index, le16_to_cpu(sei_data->sei_log)); + index, le32_to_cpu(sei_data->sei_log)); break; - case GAUDI_SM_SEI_AXI_RESPONSE_ERR: + case SM_SEI_AXI_RESPONSE_ERR: dev_err(hdev->dev, "SM %u SEI Error: AXI ID %u response error", - index, le16_to_cpu(sei_data->sei_log)); + index, le32_to_cpu(sei_data->sei_log)); break; default: dev_err(hdev->dev, "Unknown SM SEI cause %u", - le16_to_cpu(sei_data->sei_log)); + le32_to_cpu(sei_data->sei_log)); break; } } diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index d75d1077461b..b77c1c16c32c 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -58,10 +58,17 @@ struct hl_eq_ecc_data { __u8 pad[7]; }; +enum hl_sm_sei_cause { + SM_SEI_SO_OVERFLOW, + SM_SEI_LBW_4B_UNALIGNED, + SM_SEI_AXI_RESPONSE_ERR +}; + struct hl_eq_sm_sei_data { - __le16 sei_log; + __le32 sei_log; + /* enum hl_sm_sei_cause */ __u8 sei_cause; - __u8 pad[5]; + __u8 pad[3]; }; struct hl_eq_entry { From 230cd89480d322ff76ed241973158059a170fc5c Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 26 Jan 2021 22:58:13 +0200 Subject: [PATCH 190/633] habanalabs/gaudi: unmask HBM interrupts after handling As the driver does with all interrupts, we need to tell F/W to unmask the HBM interrupts after the driver handled them. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index a9bd5aef6c02..52fcaf25531a 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7377,6 +7377,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, gaudi_hbm_read_interrupts(hdev, gaudi_hbm_event_to_dev(event_type), &eq_entry->hbm_ecc_data); + hl_fw_unmask_irq(hdev, event_type); break; case GAUDI_EVENT_TPC0_DEC: From f1aebf5e3d606a2a910eed54bc8d17655f12b606 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 26 Jan 2021 22:59:35 +0200 Subject: [PATCH 191/633] habanalabs: update to latest hl_boot_if.h spec from F/W It adds the definition for indication that the F/W handles HBM ECC events. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/include/common/hl_boot_if.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index e29c77bdea07..57785478a4ef 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -69,8 +69,9 @@ * image has failed to match expected * checksum. Trying to program image again * might solve this. + * * CPU_BOOT_ERR0_PLL_FAIL PLL settings failed, meaning that one - * of the PLLs remained in REF_CLK + * of the PLLs remains in REF_CLK * * CPU_BOOT_ERR0_ENABLED Error registers enabled. * This is a main indication that the @@ -161,6 +162,10 @@ * FW initialized Clock Gating. * Initialized in: preboot * + * CPU_BOOT_DEV_STS0_HBM_ECC_EN HBM ECC handling Enabled. + * FW handles HBM ECC indications. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -184,6 +189,7 @@ #define CPU_BOOT_DEV_STS0_PLL_INFO_EN (1 << 11) #define CPU_BOOT_DEV_STS0_SP_SRAM_EN (1 << 12) #define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13) +#define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << 14) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) enum cpu_boot_status { From cc4a08cd09e4766066eee86ce501fbed42d8ff75 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 26 Jan 2021 15:35:03 -0600 Subject: [PATCH 192/633] PCI: xgene: Fix CRS SV comment Configuration Request Retry Status ("CRS") must be supported by all PCIe devices. CRS Software Visibility is an optional feature that enables a Root Port to make CRS visible to software by returning a special data value to complete a config read. Clarify a comment to say that it is "CRS SV", not "CRS", that can be enabled. Link: https://lore.kernel.org/r/20210126213503.2922848-1-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/pci-xgene.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 85e7c98265e8..2afdc865253e 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -173,12 +173,13 @@ static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, /* * The v1 controller has a bug in its Configuration Request - * Retry Status (CRS) logic: when CRS is enabled and we read the - * Vendor and Device ID of a non-existent device, the controller - * fabricates return data of 0xFFFF0001 ("device exists but is not - * ready") instead of 0xFFFFFFFF ("device does not exist"). This - * causes the PCI core to retry the read until it times out. - * Avoid this by not claiming to support CRS. + * Retry Status (CRS) logic: when CRS Software Visibility is + * enabled and we read the Vendor and Device ID of a non-existent + * device, the controller fabricates return data of 0xFFFF0001 + * ("device exists but is not ready") instead of 0xFFFFFFFF + * ("device does not exist"). This causes the PCI core to retry + * the read until it times out. Avoid this by not claiming to + * support CRS SV. */ if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) && ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL)) From c77bfb5417430832aae54a251a340c8f05682b3f Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 26 Jan 2021 15:38:55 -0600 Subject: [PATCH 193/633] PCI: hv: Fix typo Fix misspelling of "silently". Link: https://lore.kernel.org/r/20210126213855.2923461-1-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Signed-off-by: Lorenzo Pieralisi Reviewed-by: Michael Kelley --- drivers/pci/controller/pci-hyperv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 6db8d96a78eb..da0c22eb4315 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1714,7 +1714,7 @@ static void prepopulate_bars(struct hv_pcibus_device *hbus) * resumed and suspended again: see hibernation_snapshot() and * hibernation_platform_enter(). * - * If the memory enable bit is already set, Hyper-V sliently ignores + * If the memory enable bit is already set, Hyper-V silently ignores * the below BAR updates, and the related PCI device driver can not * work, because reading from the device register(s) always returns * 0xFFFFFFFF. From e4612ecd6f36e60c7bf0ad8922f11f6c3d557aea Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 29 Jan 2021 13:27:29 +0200 Subject: [PATCH 194/633] misc: pti: Remove a leftover in documentation Driver is gone, so is the documentation. Remove a leftover in documentation. Fixes: 8ba59e9dee31 ("misc: pti: Remove driver for deprecated platform") Reported-by: Stephen Rothwell Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20210129112729.65363-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 2456d0a97ed8..ef4bed8f3758 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -93,7 +93,6 @@ available subsections can be seen below. pps ptp phy/index - pti_intel_mid pwm pldmfw/index rfkill From d71277dc9bd6789964405e0a55d70c1fb2098f84 Mon Sep 17 00:00:00 2001 From: Desmond Yan Date: Thu, 28 Jan 2021 22:04:03 -0800 Subject: [PATCH 195/633] misc: bcm-vk: fix set_q_num API precedence issue Change set_q_num API to use if-else to make it more explicit, and avoid a precedence rule issue. Reported-by: kernel test robot Signed-off-by: Desmond Yan Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210129060403.14801-1-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk_msg.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index eec90494777d..a363e2e4f7bc 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -52,8 +52,14 @@ static u32 get_q_num(const struct vk_msg_blk *msg) static void set_q_num(struct vk_msg_blk *msg, u32 q_num) { - msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | - (q_num >= VK_MSGQ_PER_CHAN_MAX) ? VK_MSGQ_NUM_DEFAULT : q_num; + u32 trans_q; + + if (q_num >= VK_MSGQ_PER_CHAN_MAX) + trans_q = VK_MSGQ_NUM_DEFAULT; + else + trans_q = q_num; + + msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q; } static u32 get_msg_id(const struct vk_msg_blk *msg) From 1309ecc90f16ee9cc3077761e7f4474369747e6e Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 29 Jan 2021 14:07:46 +0200 Subject: [PATCH 196/633] mei: fix transfer over dma with extended header The size in header field for packet transferred over DMA includes size of the extended header. Include extended header in size check. Add size and sanity checks on extended header. Cc: # v5.10+ Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-1-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/interrupt.c | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c index 326955b04fda..2161c1234ad7 100644 --- a/drivers/misc/mei/interrupt.c +++ b/drivers/misc/mei/interrupt.c @@ -295,12 +295,17 @@ static inline bool hdr_is_fixed(struct mei_msg_hdr *mei_hdr) static inline int hdr_is_valid(u32 msg_hdr) { struct mei_msg_hdr *mei_hdr; + u32 expected_len = 0; mei_hdr = (struct mei_msg_hdr *)&msg_hdr; if (!msg_hdr || mei_hdr->reserved) return -EBADMSG; - if (mei_hdr->dma_ring && mei_hdr->length != MEI_SLOT_SIZE) + if (mei_hdr->dma_ring) + expected_len += MEI_SLOT_SIZE; + if (mei_hdr->extended) + expected_len += MEI_SLOT_SIZE; + if (mei_hdr->length < expected_len) return -EBADMSG; return 0; @@ -324,6 +329,8 @@ int mei_irq_read_handler(struct mei_device *dev, struct mei_cl *cl; int ret; u32 ext_meta_hdr_u32; + u32 hdr_size_left; + u32 hdr_size_ext; int i; int ext_hdr_end; @@ -353,6 +360,7 @@ int mei_irq_read_handler(struct mei_device *dev, } ext_hdr_end = 1; + hdr_size_left = mei_hdr->length; if (mei_hdr->extended) { if (!dev->rd_msg_hdr[1]) { @@ -363,8 +371,21 @@ int mei_irq_read_handler(struct mei_device *dev, dev_dbg(dev->dev, "extended header is %08x\n", ext_meta_hdr_u32); } - meta_hdr = ((struct mei_ext_meta_hdr *) - dev->rd_msg_hdr + 1); + meta_hdr = ((struct mei_ext_meta_hdr *)dev->rd_msg_hdr + 1); + if (check_add_overflow((u32)sizeof(*meta_hdr), + mei_slots2data(meta_hdr->size), + &hdr_size_ext)) { + dev_err(dev->dev, "extended message size too big %d\n", + meta_hdr->size); + return -EBADMSG; + } + if (hdr_size_left < hdr_size_ext) { + dev_err(dev->dev, "corrupted message header len %d\n", + mei_hdr->length); + return -EBADMSG; + } + hdr_size_left -= hdr_size_ext; + ext_hdr_end = meta_hdr->size + 2; for (i = dev->rd_msg_hdr_count; i < ext_hdr_end; i++) { dev->rd_msg_hdr[i] = mei_read_hdr(dev); @@ -376,6 +397,12 @@ int mei_irq_read_handler(struct mei_device *dev, } if (mei_hdr->dma_ring) { + if (hdr_size_left != sizeof(dev->rd_msg_hdr[ext_hdr_end])) { + dev_err(dev->dev, "corrupted message header len %d\n", + mei_hdr->length); + return -EBADMSG; + } + dev->rd_msg_hdr[ext_hdr_end] = mei_read_hdr(dev); dev->rd_msg_hdr_count++; (*slots)--; From 7615da2be0069254099bbb596583b46e9623e385 Mon Sep 17 00:00:00 2001 From: Tomas Winkler Date: Fri, 29 Jan 2021 14:07:47 +0200 Subject: [PATCH 197/633] mei: document that mei_msg_hdr_init returns ERR_PTR Document that mei_msg_hdr_init returns ERR_PTR. Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-2-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index a56d41321f32..beb6cdff20fb 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -1737,7 +1737,7 @@ static inline u8 mei_ext_hdr_set_vtag(struct mei_ext_hdr *ext, u8 vtag) * * @cb: message callback structure * - * Return: a pointer to initialized header + * Return: a pointer to initialized header or ERR_PTR on failure */ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb) { From 3a77df62deb2e62de0dc26c1cb763cc152329287 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 29 Jan 2021 14:07:48 +0200 Subject: [PATCH 198/633] mei: hbm: call mei_set_devstate() on hbm stop response Use mei_set_devstate() wrapper upon hbm stop command response, to trigger sysfs event. Fixes: 43b8a7ed4739 ("mei: expose device state in sysfs") Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-3-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hbm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index 686e8b6a4c55..0cba3c6dfb14 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -1373,7 +1373,7 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) return -EPROTO; } - dev->dev_state = MEI_DEV_POWER_DOWN; + mei_set_devstate(dev, MEI_DEV_POWER_DOWN); dev_info(dev->dev, "hbm: stop response: resetting.\n"); /* force the reset */ return -EPROTO; From da3eb47c90d4f951b83573d7203c40c0888521e5 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 29 Jan 2021 14:07:49 +0200 Subject: [PATCH 199/633] mei: hbm: drop hbm responses on shutdown Ignore and drop HBM responses from init phase in shutdown phase. Fixes stall if driver starting to stop in the middle of link init. Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-4-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hbm.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index 0cba3c6dfb14..df0f62de3dca 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -1177,6 +1177,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_STARTING) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: start: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: start: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1215,7 +1219,12 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) dev->init_clients_timer = 0; - if (dev->hbm_state != MEI_HBM_CAP_SETUP) { + if (dev->dev_state != MEI_DEV_INIT_CLIENTS || + dev->hbm_state != MEI_HBM_CAP_SETUP) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: capabilities response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: capabilities response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1247,7 +1256,12 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) dev->init_clients_timer = 0; - if (dev->hbm_state != MEI_HBM_DR_SETUP) { + if (dev->dev_state != MEI_DEV_INIT_CLIENTS || + dev->hbm_state != MEI_HBM_DR_SETUP) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: dma setup response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: dma setup response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1311,6 +1325,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_CLIENT_PROPERTIES) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: properties response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: properties response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; @@ -1349,6 +1367,10 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) if (dev->dev_state != MEI_DEV_INIT_CLIENTS || dev->hbm_state != MEI_HBM_ENUM_CLIENTS) { + if (dev->dev_state == MEI_DEV_POWER_DOWN) { + dev_dbg(dev->dev, "hbm: enumeration response: on shutdown, ignoring\n"); + return 0; + } dev_err(dev->dev, "hbm: enumeration response: state mismatch, [%d, %d]\n", dev->dev_state, dev->hbm_state); return -EPROTO; From 372726cb3957dbd69ded9a4e3419d5c6c3bc648e Mon Sep 17 00:00:00 2001 From: Tomas Winkler Date: Fri, 29 Jan 2021 14:07:50 +0200 Subject: [PATCH 200/633] mei: me: emmitsburg workstation DID Add Emmitsburg workstation DID. Cc: Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-5-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hw-me-regs.h | 2 ++ drivers/misc/mei/pci-me.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 9cf8d8f60cfe..a368849278b2 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -101,6 +101,8 @@ #define MEI_DEV_ID_MCC 0x4B70 /* Mule Creek Canyon (EHL) */ #define MEI_DEV_ID_MCC_4 0x4B75 /* Mule Creek Canyon 4 (EHL) */ +#define MEI_DEV_ID_EBG 0x1BE0 /* Emmitsburg WS */ + /* * MEI HW Section */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 1de9ef7a272b..12dc99ee4b58 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -107,6 +107,8 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_CDF, MEI_ME_PCH8_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_EBG, MEI_ME_PCH15_SPS_CFG)}, + /* required last entry */ {0, } }; From f7545efaf7950b240de6b8a20b9c3ffd7278538e Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 29 Jan 2021 14:07:51 +0200 Subject: [PATCH 201/633] mei: me: add adler lake point S DID Add Adler Lake S device id. Cc: Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-6-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hw-me-regs.h | 2 ++ drivers/misc/mei/pci-me.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index a368849278b2..50a82a38e66d 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -103,6 +103,8 @@ #define MEI_DEV_ID_EBG 0x1BE0 /* Emmitsburg WS */ +#define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */ + /* * MEI HW Section */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 12dc99ee4b58..94c122007504 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -109,6 +109,8 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_EBG, MEI_ME_PCH15_SPS_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)}, + /* required last entry */ {0, } }; From 930c922a987a02936000f15ea62988b7a39c27f5 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Fri, 29 Jan 2021 14:07:52 +0200 Subject: [PATCH 202/633] mei: me: add adler lake point LP DID Add Adler Lake LP device id. Cc: Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210129120752.850325-7-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hw-me-regs.h | 1 + drivers/misc/mei/pci-me.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 50a82a38e66d..14be76d4c2e6 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -104,6 +104,7 @@ #define MEI_DEV_ID_EBG 0x1BE0 /* Emmitsburg WS */ #define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */ +#define MEI_DEV_ID_ADP_LP 0x7A60 /* Alder Lake Point LP */ /* * MEI HW Section diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 94c122007504..a7e179626b63 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -110,6 +110,7 @@ static const struct pci_device_id mei_me_pci_tbl[] = { {MEI_PCI_DEVICE(MEI_DEV_ID_EBG, MEI_ME_PCH15_SPS_CFG)}, {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)}, + {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_LP, MEI_ME_PCH15_CFG)}, /* required last entry */ {0, } From 87525610b40212c96b09e24d6c5f025b5ff11cb1 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 25 Aug 2020 16:44:48 -0500 Subject: [PATCH 203/633] Fix "ordering" comment typos Fix comment typos in "ordering". Signed-off-by: Bjorn Helgaas Acked-by: Kalle Valo Acked-by: Vasily Gorbik # s390 --- arch/s390/include/asm/facility.h | 2 +- drivers/gpu/drm/qxl/qxl_drv.c | 2 +- drivers/net/wireless/intel/iwlwifi/fw/file.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index 68c476b20b57..91b5d714d28f 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -44,7 +44,7 @@ static inline int __test_facility(unsigned long nr, void *facilities) } /* - * The test_facility function uses the bit odering where the MSB is bit 0. + * The test_facility function uses the bit ordering where the MSB is bit 0. * That makes it easier to query facility bits with the bit number as * documented in the Principles of Operation. */ diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c index 6e7f16f4cec7..dab190a547cc 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.c +++ b/drivers/gpu/drm/qxl/qxl_drv.c @@ -141,7 +141,7 @@ static void qxl_drm_release(struct drm_device *dev) /* * TODO: qxl_device_fini() call should be in qxl_pci_remove(), - * reodering qxl_modeset_fini() + qxl_device_fini() calls is + * reordering qxl_modeset_fini() + qxl_device_fini() calls is * non-trivial though. */ qxl_modeset_fini(qdev); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h index 597bc88479ba..04fbfe5cbeb0 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/file.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h @@ -866,7 +866,7 @@ struct iwl_fw_dbg_trigger_time_event { * tx_bar: tid bitmap to configure on what tid the trigger should occur * when a BAR is send (for an Rx BlocAck session). * frame_timeout: tid bitmap to configure on what tid the trigger should occur - * when a frame times out in the reodering buffer. + * when a frame times out in the reordering buffer. */ struct iwl_fw_dbg_trigger_ba { __le16 rx_ba_start; From 2a0c106a671b2582f7d74cce4e48fd1a9dc7e8c9 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Thu, 21 Jan 2021 17:33:50 -0600 Subject: [PATCH 204/633] MAINTAINERS: Fix 'ARM/TEXAS INSTRUMENT KEYSTONE CLOCKSOURCE' capitalization Fix capitalization typo in 'ARM/TEXAS INSTRUMENT KEYSTONE CLOCKSOURCE'. Signed-off-by: Bjorn Helgaas --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cc1e6a5ee6e6..52311efad03e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2603,7 +2603,7 @@ L: linux-kernel@vger.kernel.org S: Maintained F: drivers/clk/keystone/ -ARM/TEXAS INSTRUMENT KEYSTONE ClOCKSOURCE +ARM/TEXAS INSTRUMENT KEYSTONE CLOCKSOURCE M: Santosh Shilimkar L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-kernel@vger.kernel.org From f1b690261247c9895f7a4b05d14a4026739134eb Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 26 Jan 2021 16:54:02 +0800 Subject: [PATCH 205/633] soundwire: bus: add better dev_dbg to track complete() calls Add a dev_dbg() log for both enumeration and initialization completion to better track suspend-resume issues. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Reviewed-by: Chao Song Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210126085402.4264-1-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 6e1c988f3845..82df088c9333 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -784,7 +784,7 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, if (status == SDW_SLAVE_UNATTACHED) { dev_dbg(&slave->dev, - "%s: initializing completion for Slave %d\n", + "%s: initializing enumeration and init completion for Slave %d\n", __func__, slave->dev_num); init_completion(&slave->enumeration_complete); @@ -793,7 +793,7 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, } else if ((status == SDW_SLAVE_ATTACHED) && (slave->status == SDW_SLAVE_UNATTACHED)) { dev_dbg(&slave->dev, - "%s: signaling completion for Slave %d\n", + "%s: signaling enumeration completion for Slave %d\n", __func__, slave->dev_num); complete(&slave->enumeration_complete); @@ -1758,8 +1758,13 @@ int sdw_handle_slave_status(struct sdw_bus *bus, if (ret) dev_err(slave->bus->dev, "Update Slave status failed:%d\n", ret); - if (attached_initializing) + if (attached_initializing) { + dev_dbg(&slave->dev, + "%s: signaling initialization completion for Slave %d\n", + __func__, slave->dev_num); + complete(&slave->initialization_complete); + } } return ret; From 18de2f72b74aa38dd4e561f6a80571190064dd61 Mon Sep 17 00:00:00 2001 From: Chao Song Date: Tue, 26 Jan 2021 16:54:39 +0800 Subject: [PATCH 206/633] soundwire: return earlier if no slave is attached If there is no slave attached to soundwire bus, we can return earlier from sdw_bus_prep_clk_stop() and sdw_bus_exit_clk_stop(), this saves a redundant value check. Signed-off-by: Chao Song Reviewed-by: Rander Wang Reviewed-by: Guennadi Liakhovetski Reviewed-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210126085439.4349-1-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 82df088c9333..d9deafdcf495 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -951,17 +951,17 @@ int sdw_bus_prep_clk_stop(struct sdw_bus *bus) simple_clk_stop = false; } - if (is_slave && !simple_clk_stop) { + /* Skip remaining clock stop preparation if no Slave is attached */ + if (!is_slave) + return ret; + + if (!simple_clk_stop) { ret = sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); if (ret < 0) return ret; } - /* Don't need to inform slaves if there is no slave attached */ - if (!is_slave) - return ret; - /* Inform slaves that prep is done */ list_for_each_entry(slave, &bus->slaves, node) { if (!slave->dev_num) @@ -1075,16 +1075,13 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus) "clk stop deprep failed:%d", ret); } - if (is_slave && !simple_clk_stop) - sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); - - /* - * Don't need to call slave callback function if there is no slave - * attached - */ + /* Skip remaining clock stop de-preparation if no Slave is attached */ if (!is_slave) return 0; + if (!simple_clk_stop) + sdw_bus_wait_for_clk_prep_deprep(bus, SDW_BROADCAST_DEV_NUM); + list_for_each_entry(slave, &bus->slaves, node) { if (!slave->dev_num) continue; From c98fe7c2a2038af2bc24f039ccdb5a65c22ba11a Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:04 -0800 Subject: [PATCH 207/633] vfio: option to unmap all For the UNMAP_DMA ioctl, delete all mappings if VFIO_DMA_UNMAP_FLAG_ALL is set. Define the corresponding VFIO_UNMAP_ALL extension. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- include/uapi/linux/vfio.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index d1812777139f..78462599e5ed 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -46,6 +46,9 @@ */ #define VFIO_NOIOMMU_IOMMU 8 +/* Supports VFIO_DMA_UNMAP_FLAG_ALL */ +#define VFIO_UNMAP_ALL 9 + /* * The IOCTL interface is designed for extensibility by embedding the * structure length (argsz) and flags into structures passed between @@ -1102,6 +1105,7 @@ struct vfio_bitmap { * field. No guarantee is made to the user that arbitrary unmaps of iova * or size different from those used in the original mapping call will * succeed. + * * VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP should be set to get the dirty bitmap * before unmapping IO virtual addresses. When this flag is set, the user must * provide a struct vfio_bitmap in data[]. User must provide zero-allocated @@ -1111,11 +1115,15 @@ struct vfio_bitmap { * indicates that the page at that offset from iova is dirty. A Bitmap of the * pages in the range of unmapped size is returned in the user-provided * vfio_bitmap.data. + * + * If flags & VFIO_DMA_UNMAP_FLAG_ALL, unmap all addresses. iova and size + * must be 0. This cannot be combined with the get-dirty-bitmap flag. */ struct vfio_iommu_type1_dma_unmap { __u32 argsz; __u32 flags; #define VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP (1 << 0) +#define VFIO_DMA_UNMAP_FLAG_ALL (1 << 1) __u64 iova; /* IO virtual address */ __u64 size; /* Size of mapping (bytes) */ __u8 data[]; From 0f53afa12baec8c00f5d1d6afb49325ada105253 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:05 -0800 Subject: [PATCH 208/633] vfio/type1: unmap cleanup Minor changes in vfio_dma_do_unmap to improve readability, which also simplify the subsequent unmap-all patch. No functional change. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 38 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 0b4dedaa9128..e31c926b9040 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -1074,34 +1074,28 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, { struct vfio_dma *dma, *dma_last = NULL; size_t unmapped = 0, pgsize; - int ret = 0, retries = 0; + int ret = -EINVAL, retries = 0; unsigned long pgshift; + dma_addr_t iova = unmap->iova; + unsigned long size = unmap->size; mutex_lock(&iommu->lock); pgshift = __ffs(iommu->pgsize_bitmap); pgsize = (size_t)1 << pgshift; - if (unmap->iova & (pgsize - 1)) { - ret = -EINVAL; + if (iova & (pgsize - 1)) goto unlock; - } - if (!unmap->size || unmap->size & (pgsize - 1)) { - ret = -EINVAL; + if (!size || size & (pgsize - 1)) goto unlock; - } - if (unmap->iova + unmap->size - 1 < unmap->iova || - unmap->size > SIZE_MAX) { - ret = -EINVAL; + if (iova + size - 1 < iova || size > SIZE_MAX) goto unlock; - } /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { - ret = -EINVAL; goto unlock; } @@ -1139,20 +1133,18 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, * mappings within the range. */ if (iommu->v2) { - dma = vfio_find_dma(iommu, unmap->iova, 1); - if (dma && dma->iova != unmap->iova) { - ret = -EINVAL; + dma = vfio_find_dma(iommu, iova, 1); + if (dma && dma->iova != iova) goto unlock; - } - dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0); - if (dma && dma->iova + dma->size != unmap->iova + unmap->size) { - ret = -EINVAL; + + dma = vfio_find_dma(iommu, iova + size - 1, 0); + if (dma && dma->iova + dma->size != iova + size) goto unlock; - } } - while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) { - if (!iommu->v2 && unmap->iova > dma->iova) + ret = 0; + while ((dma = vfio_find_dma(iommu, iova, size))) { + if (!iommu->v2 && iova > dma->iova) break; /* * Task with same address space who mapped this iova range is @@ -1190,7 +1182,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - unmap->iova, pgsize); + iova, pgsize); if (ret) break; } From c196509953741a8a472127c80070a0795a072f87 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:06 -0800 Subject: [PATCH 209/633] vfio/type1: implement unmap all Implement VFIO_DMA_UNMAP_FLAG_ALL. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e31c926b9040..5b388e5d2972 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -1078,6 +1078,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, unsigned long pgshift; dma_addr_t iova = unmap->iova; unsigned long size = unmap->size; + bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; mutex_lock(&iommu->lock); @@ -1087,8 +1088,13 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (iova & (pgsize - 1)) goto unlock; - if (!size || size & (pgsize - 1)) + if (unmap_all) { + if (iova || size) + goto unlock; + size = SIZE_MAX; + } else if (!size || size & (pgsize - 1)) { goto unlock; + } if (iova + size - 1 < iova || size > SIZE_MAX) goto unlock; @@ -1132,7 +1138,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, * will only return success and a size of zero if there were no * mappings within the range. */ - if (iommu->v2) { + if (iommu->v2 && !unmap_all) { dma = vfio_find_dma(iommu, iova, 1); if (dma && dma->iova != iova) goto unlock; @@ -2509,6 +2515,7 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: case VFIO_TYPE1_NESTING_IOMMU: + case VFIO_UNMAP_ALL: return 1; case VFIO_DMA_CC_IOMMU: if (!iommu) @@ -2698,6 +2705,8 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, { struct vfio_iommu_type1_dma_unmap unmap; struct vfio_bitmap bitmap = { 0 }; + uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP | + VFIO_DMA_UNMAP_FLAG_ALL; unsigned long minsz; int ret; @@ -2706,8 +2715,11 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (copy_from_user(&unmap, (void __user *)arg, minsz)) return -EFAULT; - if (unmap.argsz < minsz || - unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) + if (unmap.argsz < minsz || unmap.flags & ~mask) + return -EINVAL; + + if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && + (unmap.flags & VFIO_DMA_UNMAP_FLAG_ALL)) return -EINVAL; if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { From 441e8106a238920f28e2e79cff462044551f60e2 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:07 -0800 Subject: [PATCH 210/633] vfio: interfaces to update vaddr Define interfaces that allow the underlying memory object of an iova range to be mapped to a new host virtual address in the host process: - VFIO_DMA_UNMAP_FLAG_VADDR for VFIO_IOMMU_UNMAP_DMA - VFIO_DMA_MAP_FLAG_VADDR flag for VFIO_IOMMU_MAP_DMA - VFIO_UPDATE_VADDR extension for VFIO_CHECK_EXTENSION Unmap vaddr invalidates the host virtual address in an iova range, and blocks vfio translation of host virtual addresses. DMA to already-mapped pages continues. Map vaddr updates the base VA and resumes translation. See comments in uapi/linux/vfio.h for more details. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- include/uapi/linux/vfio.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 78462599e5ed..8ce36c1d53ca 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -49,6 +49,9 @@ /* Supports VFIO_DMA_UNMAP_FLAG_ALL */ #define VFIO_UNMAP_ALL 9 +/* Supports the vaddr flag for DMA map and unmap */ +#define VFIO_UPDATE_VADDR 10 + /* * The IOCTL interface is designed for extensibility by embedding the * structure length (argsz) and flags into structures passed between @@ -1077,12 +1080,22 @@ struct vfio_iommu_type1_info_dma_avail { * * Map process virtual addresses to IO virtual addresses using the * provided struct vfio_dma_map. Caller sets argsz. READ &/ WRITE required. + * + * If flags & VFIO_DMA_MAP_FLAG_VADDR, update the base vaddr for iova, and + * unblock translation of host virtual addresses in the iova range. The vaddr + * must have previously been invalidated with VFIO_DMA_UNMAP_FLAG_VADDR. To + * maintain memory consistency within the user application, the updated vaddr + * must address the same memory object as originally mapped. Failure to do so + * will result in user memory corruption and/or device misbehavior. iova and + * size must match those in the original MAP_DMA call. Protection is not + * changed, and the READ & WRITE flags must be 0. */ struct vfio_iommu_type1_dma_map { __u32 argsz; __u32 flags; #define VFIO_DMA_MAP_FLAG_READ (1 << 0) /* readable from device */ #define VFIO_DMA_MAP_FLAG_WRITE (1 << 1) /* writable from device */ +#define VFIO_DMA_MAP_FLAG_VADDR (1 << 2) __u64 vaddr; /* Process virtual address */ __u64 iova; /* IO virtual address */ __u64 size; /* Size of mapping (bytes) */ @@ -1118,12 +1131,18 @@ struct vfio_bitmap { * * If flags & VFIO_DMA_UNMAP_FLAG_ALL, unmap all addresses. iova and size * must be 0. This cannot be combined with the get-dirty-bitmap flag. + * + * If flags & VFIO_DMA_UNMAP_FLAG_VADDR, do not unmap, but invalidate host + * virtual addresses in the iova range. Tasks that attempt to translate an + * iova's vaddr will block. DMA to already-mapped pages continues. This + * cannot be combined with the get-dirty-bitmap flag. */ struct vfio_iommu_type1_dma_unmap { __u32 argsz; __u32 flags; #define VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP (1 << 0) #define VFIO_DMA_UNMAP_FLAG_ALL (1 << 1) +#define VFIO_DMA_UNMAP_FLAG_VADDR (1 << 2) __u64 iova; /* IO virtual address */ __u64 size; /* Size of mapping (bytes) */ __u8 data[]; From 40ae9b807b89e60f86a40f33b931e6cde7eed2b7 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:08 -0800 Subject: [PATCH 211/633] vfio/type1: massage unmap iteration Modify the iteration in vfio_dma_do_unmap so it does not depend on deletion of each dma entry. Add a variant of vfio_find_dma that returns the entry with the lowest iova in the search range to initialize the iteration. No externally visible change, but this behavior is needed in the subsequent update-vaddr patch. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 5b388e5d2972..ce97eda756fc 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -173,6 +173,31 @@ static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, return NULL; } +static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu, + dma_addr_t start, size_t size) +{ + struct rb_node *res = NULL; + struct rb_node *node = iommu->dma_list.rb_node; + struct vfio_dma *dma_res = NULL; + + while (node) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + if (start < dma->iova + dma->size) { + res = node; + dma_res = dma; + if (start >= dma->iova) + break; + node = node->rb_left; + } else { + node = node->rb_right; + } + } + if (res && size && dma_res->iova >= start + size) + res = NULL; + return res; +} + static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new) { struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; @@ -1079,6 +1104,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, dma_addr_t iova = unmap->iova; unsigned long size = unmap->size; bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; + struct rb_node *n; mutex_lock(&iommu->lock); @@ -1149,7 +1175,13 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, } ret = 0; - while ((dma = vfio_find_dma(iommu, iova, size))) { + n = vfio_find_dma_first_node(iommu, iova, size); + + while (n) { + dma = rb_entry(n, struct vfio_dma, node); + if (dma->iova >= iova + size) + break; + if (!iommu->v2 && iova > dma->iova) break; /* @@ -1194,6 +1226,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, } unmapped += dma->size; + n = rb_next(n); vfio_remove_dma(iommu, dma); } From c3cbab24db3860d68924d8a3f752a97d3cca1623 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:09 -0800 Subject: [PATCH 212/633] vfio/type1: implement interfaces to update vaddr Implement VFIO_DMA_UNMAP_FLAG_VADDR, VFIO_DMA_MAP_FLAG_VADDR, and VFIO_UPDATE_VADDR. This is a partial implementation. Blocking is added in a subsequent patch. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 59 +++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index ce97eda756fc..55b49bfac993 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -69,6 +69,7 @@ struct vfio_iommu { struct rb_root dma_list; struct blocking_notifier_head notifier; unsigned int dma_avail; + unsigned int vaddr_invalid_count; uint64_t pgsize_bitmap; bool v2; bool nesting; @@ -92,6 +93,7 @@ struct vfio_dma { int prot; /* IOMMU_READ/WRITE */ bool iommu_mapped; bool lock_cap; /* capable(CAP_IPC_LOCK) */ + bool vaddr_invalid; struct task_struct *task; struct rb_root pfn_list; /* Ex-user pinned pfn list */ unsigned long *bitmap; @@ -974,6 +976,8 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); vfio_dma_bitmap_free(dma); + if (dma->vaddr_invalid) + iommu->vaddr_invalid_count--; kfree(dma); iommu->dma_avail++; } @@ -1104,7 +1108,8 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, dma_addr_t iova = unmap->iova; unsigned long size = unmap->size; bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL; - struct rb_node *n; + bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR; + struct rb_node *n, *first_n; mutex_lock(&iommu->lock); @@ -1175,7 +1180,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, } ret = 0; - n = vfio_find_dma_first_node(iommu, iova, size); + n = first_n = vfio_find_dma_first_node(iommu, iova, size); while (n) { dma = rb_entry(n, struct vfio_dma, node); @@ -1191,6 +1196,27 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (dma->task->mm != current->mm) break; + if (invalidate_vaddr) { + if (dma->vaddr_invalid) { + struct rb_node *last_n = n; + + for (n = first_n; n != last_n; n = rb_next(n)) { + dma = rb_entry(n, + struct vfio_dma, node); + dma->vaddr_invalid = false; + iommu->vaddr_invalid_count--; + } + ret = -EINVAL; + unmapped = 0; + break; + } + dma->vaddr_invalid = true; + iommu->vaddr_invalid_count++; + unmapped += dma->size; + n = rb_next(n); + continue; + } + if (!RB_EMPTY_ROOT(&dma->pfn_list)) { struct vfio_iommu_type1_dma_unmap nb_unmap; @@ -1330,6 +1356,7 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { + bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR; dma_addr_t iova = map->iova; unsigned long vaddr = map->vaddr; size_t size = map->size; @@ -1347,13 +1374,16 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, if (map->flags & VFIO_DMA_MAP_FLAG_READ) prot |= IOMMU_READ; + if ((prot && set_vaddr) || (!prot && !set_vaddr)) + return -EINVAL; + mutex_lock(&iommu->lock); pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); WARN_ON((pgsize - 1) & PAGE_MASK); - if (!prot || !size || (size | iova | vaddr) & (pgsize - 1)) { + if (!size || (size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; } @@ -1364,7 +1394,20 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, goto out_unlock; } - if (vfio_find_dma(iommu, iova, size)) { + dma = vfio_find_dma(iommu, iova, size); + if (set_vaddr) { + if (!dma) { + ret = -ENOENT; + } else if (!dma->vaddr_invalid || dma->iova != iova || + dma->size != size) { + ret = -EINVAL; + } else { + dma->vaddr = vaddr; + dma->vaddr_invalid = false; + iommu->vaddr_invalid_count--; + } + goto out_unlock; + } else if (dma) { ret = -EEXIST; goto out_unlock; } @@ -2549,6 +2592,7 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, case VFIO_TYPE1v2_IOMMU: case VFIO_TYPE1_NESTING_IOMMU: case VFIO_UNMAP_ALL: + case VFIO_UPDATE_VADDR: return 1; case VFIO_DMA_CC_IOMMU: if (!iommu) @@ -2720,7 +2764,8 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, { struct vfio_iommu_type1_dma_map map; unsigned long minsz; - uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE | + VFIO_DMA_MAP_FLAG_VADDR; minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); @@ -2739,6 +2784,7 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_unmap unmap; struct vfio_bitmap bitmap = { 0 }; uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP | + VFIO_DMA_UNMAP_FLAG_VADDR | VFIO_DMA_UNMAP_FLAG_ALL; unsigned long minsz; int ret; @@ -2752,7 +2798,8 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, return -EINVAL; if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (unmap.flags & VFIO_DMA_UNMAP_FLAG_ALL)) + (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL | + VFIO_DMA_UNMAP_FLAG_VADDR))) return -EINVAL; if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { From ec5e32940cc9d2a8a321cb7d756fb6ae45702d03 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:10 -0800 Subject: [PATCH 213/633] vfio: iommu driver notify callback Define a vfio_iommu_driver_ops notify callback, for sending events to the driver. Drivers are not required to provide the callback, and may ignore any events. The handling of events is driver specific. Define the CONTAINER_CLOSE event, called when the container's file descriptor is closed. This event signifies that no further state changes will occur via container ioctl's. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio.c | 5 +++++ include/linux/vfio.h | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 4ad8a35667a7..38779e6fd80c 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1220,6 +1220,11 @@ static int vfio_fops_open(struct inode *inode, struct file *filep) static int vfio_fops_release(struct inode *inode, struct file *filep) { struct vfio_container *container = filep->private_data; + struct vfio_iommu_driver *driver = container->iommu_driver; + + if (driver && driver->ops->notify) + driver->ops->notify(container->iommu_data, + VFIO_IOMMU_CONTAINER_CLOSE); filep->private_data = NULL; diff --git a/include/linux/vfio.h b/include/linux/vfio.h index f45940b38a02..b7e18bde5aa8 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -57,6 +57,11 @@ extern struct vfio_device *vfio_device_get_from_dev(struct device *dev); extern void vfio_device_put(struct vfio_device *device); extern void *vfio_device_data(struct vfio_device *device); +/* events for the backend driver notify callback */ +enum vfio_iommu_notify_type { + VFIO_IOMMU_CONTAINER_CLOSE = 0, +}; + /** * struct vfio_iommu_driver_ops - VFIO IOMMU driver callbacks */ @@ -92,6 +97,8 @@ struct vfio_iommu_driver_ops { void *data, size_t count, bool write); struct iommu_domain *(*group_iommu_domain)(void *iommu_data, struct iommu_group *group); + void (*notify)(void *iommu_data, + enum vfio_iommu_notify_type event); }; extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops); From 487ace1340536ea9b8daf8c783a397a467fe55fc Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:11 -0800 Subject: [PATCH 214/633] vfio/type1: implement notify callback Implement a notify callback that remembers if the container's file descriptor has been closed. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 55b49bfac993..2109803bfb15 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -75,6 +75,7 @@ struct vfio_iommu { bool nesting; bool dirty_page_tracking; bool pinned_page_dirty_scope; + bool container_open; }; struct vfio_domain { @@ -2520,6 +2521,7 @@ static void *vfio_iommu_type1_open(unsigned long arg) INIT_LIST_HEAD(&iommu->iova_list); iommu->dma_list = RB_ROOT; iommu->dma_avail = dma_entry_limit; + iommu->container_open = true; mutex_init(&iommu->lock); BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); @@ -3087,6 +3089,18 @@ vfio_iommu_type1_group_iommu_domain(void *iommu_data, return domain; } +static void vfio_iommu_type1_notify(void *iommu_data, + enum vfio_iommu_notify_type event) +{ + struct vfio_iommu *iommu = iommu_data; + + if (event != VFIO_IOMMU_CONTAINER_CLOSE) + return; + mutex_lock(&iommu->lock); + iommu->container_open = false; + mutex_unlock(&iommu->lock); +} + static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { .name = "vfio-iommu-type1", .owner = THIS_MODULE, @@ -3101,6 +3115,7 @@ static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { .unregister_notifier = vfio_iommu_type1_unregister_notifier, .dma_rw = vfio_iommu_type1_dma_rw, .group_iommu_domain = vfio_iommu_type1_group_iommu_domain, + .notify = vfio_iommu_type1_notify, }; static int __init vfio_iommu_type1_init(void) From 898b9eaeb3fe5eb1ac510f2c4775c8277206752c Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 29 Jan 2021 08:54:12 -0800 Subject: [PATCH 215/633] vfio/type1: block on invalid vaddr Block translation of host virtual address while an iova range has an invalid vaddr. Signed-off-by: Steve Sistare Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 95 +++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 2109803bfb15..6cf1dad501d0 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,7 @@ struct vfio_iommu { unsigned int dma_avail; unsigned int vaddr_invalid_count; uint64_t pgsize_bitmap; + wait_queue_head_t vaddr_wait; bool v2; bool nesting; bool dirty_page_tracking; @@ -146,6 +148,8 @@ struct vfio_regions { #define DIRTY_BITMAP_PAGES_MAX ((u64)INT_MAX) #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) +#define WAITED 1 + static int put_pfn(unsigned long pfn, int prot); static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -507,6 +511,61 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, return ret; } +static int vfio_wait(struct vfio_iommu *iommu) +{ + DEFINE_WAIT(wait); + + prepare_to_wait(&iommu->vaddr_wait, &wait, TASK_KILLABLE); + mutex_unlock(&iommu->lock); + schedule(); + mutex_lock(&iommu->lock); + finish_wait(&iommu->vaddr_wait, &wait); + if (kthread_should_stop() || !iommu->container_open || + fatal_signal_pending(current)) { + return -EFAULT; + } + return WAITED; +} + +/* + * Find dma struct and wait for its vaddr to be valid. iommu lock is dropped + * if the task waits, but is re-locked on return. Return result in *dma_p. + * Return 0 on success with no waiting, WAITED on success if waited, and -errno + * on error. + */ +static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start, + size_t size, struct vfio_dma **dma_p) +{ + int ret; + + do { + *dma_p = vfio_find_dma(iommu, start, size); + if (!*dma_p) + ret = -EINVAL; + else if (!(*dma_p)->vaddr_invalid) + ret = 0; + else + ret = vfio_wait(iommu); + } while (ret > 0); + + return ret; +} + +/* + * Wait for all vaddr in the dma_list to become valid. iommu lock is dropped + * if the task waits, but is re-locked on return. Return 0 on success with no + * waiting, WAITED on success if waited, and -errno on error. + */ +static int vfio_wait_all_valid(struct vfio_iommu *iommu) +{ + int ret = 0; + + while (iommu->vaddr_invalid_count && ret >= 0) + ret = vfio_wait(iommu); + + return ret; +} + /* * Attempt to pin pages. We really don't want to track all the pfns and * the iommu can only map chunks of consecutive pfns anyway, so get the @@ -668,6 +727,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, unsigned long remote_vaddr; struct vfio_dma *dma; bool do_accounting; + dma_addr_t iova; if (!iommu || !user_pfn || !phys_pfn) return -EINVAL; @@ -678,6 +738,22 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, mutex_lock(&iommu->lock); + /* + * Wait for all necessary vaddr's to be valid so they can be used in + * the main loop without dropping the lock, to avoid racing vs unmap. + */ +again: + if (iommu->vaddr_invalid_count) { + for (i = 0; i < npage; i++) { + iova = user_pfn[i] << PAGE_SHIFT; + ret = vfio_find_dma_valid(iommu, iova, PAGE_SIZE, &dma); + if (ret < 0) + goto pin_done; + if (ret == WAITED) + goto again; + } + } + /* Fail if notifier list is empty */ if (!iommu->notifier.head) { ret = -EINVAL; @@ -692,7 +768,6 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu); for (i = 0; i < npage; i++) { - dma_addr_t iova; struct vfio_pfn *vpfn; iova = user_pfn[i] << PAGE_SHIFT; @@ -977,8 +1052,10 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); vfio_dma_bitmap_free(dma); - if (dma->vaddr_invalid) + if (dma->vaddr_invalid) { iommu->vaddr_invalid_count--; + wake_up_all(&iommu->vaddr_wait); + } kfree(dma); iommu->dma_avail++; } @@ -1406,6 +1483,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->vaddr = vaddr; dma->vaddr_invalid = false; iommu->vaddr_invalid_count--; + wake_up_all(&iommu->vaddr_wait); } goto out_unlock; } else if (dma) { @@ -1505,6 +1583,10 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret; + ret = vfio_wait_all_valid(iommu); + if (ret < 0) + return ret; + /* Arbitrarily pick the first domain in the list for lookups */ if (!list_empty(&iommu->domain_list)) d = list_first_entry(&iommu->domain_list, @@ -2524,6 +2606,7 @@ static void *vfio_iommu_type1_open(unsigned long arg) iommu->container_open = true; mutex_init(&iommu->lock); BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier); + init_waitqueue_head(&iommu->vaddr_wait); return iommu; } @@ -2992,12 +3075,13 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, struct vfio_dma *dma; bool kthread = current->mm == NULL; size_t offset; + int ret; *copied = 0; - dma = vfio_find_dma(iommu, user_iova, 1); - if (!dma) - return -EINVAL; + ret = vfio_find_dma_valid(iommu, user_iova, 1, &dma); + if (ret < 0) + return ret; if ((write && !(dma->prot & IOMMU_WRITE)) || !(dma->prot & IOMMU_READ)) @@ -3099,6 +3183,7 @@ static void vfio_iommu_type1_notify(void *iommu_data, mutex_lock(&iommu->lock); iommu->container_open = false; mutex_unlock(&iommu->lock); + wake_up_all(&iommu->vaddr_wait); } static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = { From d0a78f91761fcd837da1e7a4b0f8368873adc646 Mon Sep 17 00:00:00 2001 From: Keqian Zhu Date: Fri, 22 Jan 2021 17:26:34 +0800 Subject: [PATCH 216/633] vfio/iommu_type1: Populate full dirty when detach non-pinned group If a group with non-pinned-page dirty scope is detached with dirty logging enabled, we should fully populate the dirty bitmaps at the time it's removed since we don't know the extent of its previous DMA, nor will the group be present to trigger the full bitmap when the user retrieves the dirty bitmap. Fixes: d6a4c185660c ("vfio iommu: Implementation of ioctl for dirty pages tracking") Suggested-by: Alex Williamson Signed-off-by: Keqian Zhu Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 0b4dedaa9128..161725395f2f 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -236,6 +236,18 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) } } +static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) +{ + struct rb_node *n; + unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + + for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { + struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); + + bitmap_set(dma->bitmap, 0, dma->size >> pgshift); + } +} + static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize) { struct rb_node *n; @@ -2415,8 +2427,11 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, * Removal of a group without dirty tracking may allow the iommu scope * to be promoted. */ - if (update_dirty_scope) + if (update_dirty_scope) { update_pinned_page_dirty_scope(iommu); + if (iommu->dirty_page_tracking) + vfio_iommu_populate_bitmap_full(iommu); + } mutex_unlock(&iommu->lock); } From 4a19f37a3dd3f29997735e61b25ddad24b8abe73 Mon Sep 17 00:00:00 2001 From: Keqian Zhu Date: Fri, 22 Jan 2021 17:26:35 +0800 Subject: [PATCH 217/633] vfio/iommu_type1: Fix some sanity checks in detach group vfio_sanity_check_pfn_list() is used to check whether pfn_list and notifier are empty when remove the external domain, so it makes a wrong assumption that only external domain will use the pinning interface. Now we apply the pfn_list check when a vfio_dma is removed and apply the notifier check when all domains are removed. Fixes: a54eb55045ae ("vfio iommu type1: Add support for mediated devices") Signed-off-by: Keqian Zhu Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 161725395f2f..78bd28873945 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -957,6 +957,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { + WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); @@ -2250,23 +2251,6 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) } } -static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu) -{ - struct rb_node *n; - - n = rb_first(&iommu->dma_list); - for (; n; n = rb_next(n)) { - struct vfio_dma *dma; - - dma = rb_entry(n, struct vfio_dma, node); - - if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list))) - break; - } - /* mdev vendor driver must unregister notifier */ - WARN_ON(iommu->notifier.head); -} - /* * Called when a domain is removed in detach. It is possible that * the removed domain decided the iova aperture window. Modify the @@ -2366,10 +2350,10 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, kfree(group); if (list_empty(&iommu->external_domain->group_list)) { - vfio_sanity_check_pfn_list(iommu); - - if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) + if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) { + WARN_ON(iommu->notifier.head); vfio_iommu_unmap_unpin_all(iommu); + } kfree(iommu->external_domain); iommu->external_domain = NULL; @@ -2403,10 +2387,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, */ if (list_empty(&domain->group_list)) { if (list_is_singular(&iommu->domain_list)) { - if (!iommu->external_domain) + if (!iommu->external_domain) { + WARN_ON(iommu->notifier.head); vfio_iommu_unmap_unpin_all(iommu); - else + } else { vfio_iommu_unmap_unpin_reaccount(iommu); + } } iommu_domain_free(domain->domain); list_del(&domain->next); @@ -2490,7 +2476,6 @@ static void vfio_iommu_type1_release(void *iommu_data) if (iommu->external_domain) { vfio_release_domain(iommu->external_domain, true); - vfio_sanity_check_pfn_list(iommu); kfree(iommu->external_domain); } From 010321565a7d2cd73eeed6f589693ce752e154bd Mon Sep 17 00:00:00 2001 From: Keqian Zhu Date: Mon, 25 Jan 2021 10:46:42 +0800 Subject: [PATCH 218/633] vfio/iommu_type1: Mantain a counter for non_pinned_groups With this counter, we never need to traverse all groups to update pinned_scope of vfio_iommu. Suggested-by: Alex Williamson Signed-off-by: Keqian Zhu Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 40 +++++---------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 78bd28873945..706b88b5d4ee 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -70,10 +70,10 @@ struct vfio_iommu { struct blocking_notifier_head notifier; unsigned int dma_avail; uint64_t pgsize_bitmap; + uint64_t num_non_pinned_groups; bool v2; bool nesting; bool dirty_page_tracking; - bool pinned_page_dirty_scope; }; struct vfio_domain { @@ -148,7 +148,6 @@ static int put_pfn(unsigned long pfn, int prot); static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, struct iommu_group *iommu_group); -static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu); /* * This code handles mapping and unmapping of user data buffers * into DMA'ble space using the IOMMU @@ -726,7 +725,7 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; - update_pinned_page_dirty_scope(iommu); + iommu->num_non_pinned_groups--; } goto pin_done; @@ -1004,7 +1003,7 @@ static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, * mark all pages dirty if any IOMMU capable device is not able * to report dirty pages and all pages are pinned and mapped. */ - if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped) + if (iommu->num_non_pinned_groups && dma->iommu_mapped) bitmap_set(dma->bitmap, 0, nbits); if (shift) { @@ -1635,33 +1634,6 @@ static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, return group; } -static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu) -{ - struct vfio_domain *domain; - struct vfio_group *group; - - list_for_each_entry(domain, &iommu->domain_list, next) { - list_for_each_entry(group, &domain->group_list, next) { - if (!group->pinned_page_dirty_scope) { - iommu->pinned_page_dirty_scope = false; - return; - } - } - } - - if (iommu->external_domain) { - domain = iommu->external_domain; - list_for_each_entry(group, &domain->group_list, next) { - if (!group->pinned_page_dirty_scope) { - iommu->pinned_page_dirty_scope = false; - return; - } - } - } - - iommu->pinned_page_dirty_scope = true; -} - static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions, phys_addr_t *base) { @@ -2070,8 +2042,6 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, * addition of a dirty tracking group. */ group->pinned_page_dirty_scope = true; - if (!iommu->pinned_page_dirty_scope) - update_pinned_page_dirty_scope(iommu); mutex_unlock(&iommu->lock); return 0; @@ -2201,7 +2171,7 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, * demotes the iommu scope until it declares itself dirty tracking * capable via the page pinning interface. */ - iommu->pinned_page_dirty_scope = false; + iommu->num_non_pinned_groups++; mutex_unlock(&iommu->lock); vfio_iommu_resv_free(&group_resv_regions); @@ -2414,7 +2384,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data, * to be promoted. */ if (update_dirty_scope) { - update_pinned_page_dirty_scope(iommu); + iommu->num_non_pinned_groups--; if (iommu->dirty_page_tracking) vfio_iommu_populate_bitmap_full(iommu); } From 37a682ffbe2ab31b51123b67bd30ab42d1131cc1 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sun, 24 Jan 2021 16:35:41 +0100 Subject: [PATCH 219/633] vfio/pci: Fix handling of pci use accessor return codes The pci user accessors return negative errno's on error. Signed-off-by: Heiner Kallweit Reviewed-by: Cornelia Huck [aw: drop Fixes tag, pcibios_err_to_errno() behaves correctly for -errno] Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_igd.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci_igd.c b/drivers/vfio/pci/vfio_pci_igd.c index 53d97f459252..e66dfb0178ed 100644 --- a/drivers/vfio/pci/vfio_pci_igd.c +++ b/drivers/vfio/pci/vfio_pci_igd.c @@ -127,7 +127,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_byte(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; if (copy_to_user(buf + count - size, &val, 1)) return -EFAULT; @@ -141,7 +141,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_word(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le16(val); if (copy_to_user(buf + count - size, &val, 2)) @@ -156,7 +156,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_dword(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le32(val); if (copy_to_user(buf + count - size, &val, 4)) @@ -171,7 +171,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_word(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; val = cpu_to_le16(val); if (copy_to_user(buf + count - size, &val, 2)) @@ -186,7 +186,7 @@ static size_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, ret = pci_user_read_config_byte(pdev, pos, &val); if (ret) - return pcibios_err_to_errno(ret); + return ret; if (copy_to_user(buf + count - size, &val, 1)) return -EFAULT; From 46c474666094448ee843266adfc38bde43ef8f68 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Mon, 1 Feb 2021 16:28:24 +0000 Subject: [PATCH 220/633] vfio-pci/zdev: remove unused vdev argument Zdev static functions do not use vdev argument. Remove it. Signed-off-by: Max Gurtovoy Reviewed-by: Matthew Rosato Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_zdev.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index 229685634031..53084521cc80 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -24,8 +24,7 @@ /* * Add the Base PCI Function information to the device info region. */ -static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_base cap = { .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_BASE, @@ -45,8 +44,7 @@ static int zpci_base_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the Base PCI Function Group information to the device info region. */ -static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_group cap = { .header.id = VFIO_DEVICE_INFO_CAP_ZPCI_GROUP, @@ -66,8 +64,7 @@ static int zpci_group_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the device utility string to the device info region. */ -static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_util *cap; int cap_size = sizeof(*cap) + CLP_UTIL_STR_LEN; @@ -90,8 +87,7 @@ static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, /* * Add the function path string to the device info region. */ -static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_pci_device *vdev, - struct vfio_info_cap *caps) +static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) { struct vfio_device_info_cap_zpci_pfip *cap; int cap_size = sizeof(*cap) + CLP_PFIP_NR_SEGMENTS; @@ -123,21 +119,21 @@ int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, if (!zdev) return -ENODEV; - ret = zpci_base_cap(zdev, vdev, caps); + ret = zpci_base_cap(zdev, caps); if (ret) return ret; - ret = zpci_group_cap(zdev, vdev, caps); + ret = zpci_group_cap(zdev, caps); if (ret) return ret; if (zdev->util_str_avail) { - ret = zpci_util_cap(zdev, vdev, caps); + ret = zpci_util_cap(zdev, caps); if (ret) return ret; } - ret = zpci_pfip_cap(zdev, vdev, caps); + ret = zpci_pfip_cap(zdev, caps); return ret; } From 7e31d6dc2c78b2a0ba0039ca97ca98a581e8db82 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Mon, 1 Feb 2021 16:28:25 +0000 Subject: [PATCH 221/633] vfio-pci/zdev: fix possible segmentation fault issue In case allocation fails, we must behave correctly and exit with error. Fixes: e6b817d4b821 ("vfio-pci/zdev: Add zPCI capabilities to VFIO_DEVICE_GET_INFO") Signed-off-by: Max Gurtovoy Reviewed-by: Cornelia Huck Reviewed-by: Matthew Rosato Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_zdev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index 53084521cc80..7b011b62c766 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -71,6 +71,8 @@ static int zpci_util_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) int ret; cap = kmalloc(cap_size, GFP_KERNEL); + if (!cap) + return -ENOMEM; cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_UTIL; cap->header.version = 1; @@ -94,6 +96,8 @@ static int zpci_pfip_cap(struct zpci_dev *zdev, struct vfio_info_cap *caps) int ret; cap = kmalloc(cap_size, GFP_KERNEL); + if (!cap) + return -ENOMEM; cap->header.id = VFIO_DEVICE_INFO_CAP_ZPCI_PFIP; cap->header.version = 1; From b4c7d2076b4e767dd2e075a2b3a9e57753fc67f5 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 2 Feb 2021 14:17:54 -0600 Subject: [PATCH 222/633] PCI/LINK: Remove bandwidth notification The PCIe Bandwidth Change Notification feature logs messages when the link bandwidth changes. Some users have reported that these messages occur often enough to significantly reduce NVMe performance. GPUs also seem to generate these messages. We don't know why the link bandwidth changes, but in the reported cases there's no indication that it's caused by hardware failures. Remove the bandwidth change notifications for now. Hopefully we can add this back when we have a better understanding of why this happens and how we can make the messages useful instead of overwhelming. Link: https://lore.kernel.org/r/20200115221008.GA191037@google.com/ Link: https://lore.kernel.org/r/155605909349.3575.13433421148215616375.stgit@gimli.home/ Link: https://bugzilla.kernel.org/show_bug.cgi?id=206197 Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/Kconfig | 8 -- drivers/pci/pcie/Makefile | 1 - drivers/pci/pcie/bw_notification.c | 138 ----------------------------- drivers/pci/pcie/portdrv.h | 6 -- drivers/pci/pcie/portdrv_pci.c | 1 - 5 files changed, 154 deletions(-) delete mode 100644 drivers/pci/pcie/bw_notification.c diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 3946555a6042..45a2ef702b45 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -133,14 +133,6 @@ config PCIE_PTM This is only useful if you have devices that support PTM, but it is safe to enable even if you don't. -config PCIE_BW - bool "PCI Express Bandwidth Change Notification" - depends on PCIEPORTBUS - help - This enables PCI Express Bandwidth Change Notification. If - you know link width or rate changes occur only to correct - unreliable links, you may answer Y. - config PCIE_EDR bool "PCI Express Error Disconnect Recover support" depends on PCIE_DPC && ACPI diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index d9697892fa3e..b2980db88cc0 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -12,5 +12,4 @@ obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o obj-$(CONFIG_PCIE_PME) += pme.o obj-$(CONFIG_PCIE_DPC) += dpc.o obj-$(CONFIG_PCIE_PTM) += ptm.o -obj-$(CONFIG_PCIE_BW) += bw_notification.o obj-$(CONFIG_PCIE_EDR) += edr.o diff --git a/drivers/pci/pcie/bw_notification.c b/drivers/pci/pcie/bw_notification.c deleted file mode 100644 index 565d23cccb8b..000000000000 --- a/drivers/pci/pcie/bw_notification.c +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * PCI Express Link Bandwidth Notification services driver - * Author: Alexandru Gagniuc - * - * Copyright (C) 2019, Dell Inc - * - * The PCIe Link Bandwidth Notification provides a way to notify the - * operating system when the link width or data rate changes. This - * capability is required for all root ports and downstream ports - * supporting links wider than x1 and/or multiple link speeds. - * - * This service port driver hooks into the bandwidth notification interrupt - * and warns when links become degraded in operation. - */ - -#define dev_fmt(fmt) "bw_notification: " fmt - -#include "../pci.h" -#include "portdrv.h" - -static bool pcie_link_bandwidth_notification_supported(struct pci_dev *dev) -{ - int ret; - u32 lnk_cap; - - ret = pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnk_cap); - return (ret == PCIBIOS_SUCCESSFUL) && (lnk_cap & PCI_EXP_LNKCAP_LBNC); -} - -static void pcie_enable_link_bandwidth_notification(struct pci_dev *dev) -{ - u16 lnk_ctl; - - pcie_capability_write_word(dev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); - - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl); - lnk_ctl |= PCI_EXP_LNKCTL_LBMIE; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl); -} - -static void pcie_disable_link_bandwidth_notification(struct pci_dev *dev) -{ - u16 lnk_ctl; - - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl); - lnk_ctl &= ~PCI_EXP_LNKCTL_LBMIE; - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl); -} - -static irqreturn_t pcie_bw_notification_irq(int irq, void *context) -{ - struct pcie_device *srv = context; - struct pci_dev *port = srv->port; - u16 link_status, events; - int ret; - - ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); - events = link_status & PCI_EXP_LNKSTA_LBMS; - - if (ret != PCIBIOS_SUCCESSFUL || !events) - return IRQ_NONE; - - pcie_capability_write_word(port, PCI_EXP_LNKSTA, events); - pcie_update_link_speed(port->subordinate, link_status); - return IRQ_WAKE_THREAD; -} - -static irqreturn_t pcie_bw_notification_handler(int irq, void *context) -{ - struct pcie_device *srv = context; - struct pci_dev *port = srv->port; - struct pci_dev *dev; - - /* - * Print status from downstream devices, not this root port or - * downstream switch port. - */ - down_read(&pci_bus_sem); - list_for_each_entry(dev, &port->subordinate->devices, bus_list) - pcie_report_downtraining(dev); - up_read(&pci_bus_sem); - - return IRQ_HANDLED; -} - -static int pcie_bandwidth_notification_probe(struct pcie_device *srv) -{ - int ret; - - /* Single-width or single-speed ports do not have to support this. */ - if (!pcie_link_bandwidth_notification_supported(srv->port)) - return -ENODEV; - - ret = request_threaded_irq(srv->irq, pcie_bw_notification_irq, - pcie_bw_notification_handler, - IRQF_SHARED, "PCIe BW notif", srv); - if (ret) - return ret; - - pcie_enable_link_bandwidth_notification(srv->port); - pci_info(srv->port, "enabled with IRQ %d\n", srv->irq); - - return 0; -} - -static void pcie_bandwidth_notification_remove(struct pcie_device *srv) -{ - pcie_disable_link_bandwidth_notification(srv->port); - free_irq(srv->irq, srv); -} - -static int pcie_bandwidth_notification_suspend(struct pcie_device *srv) -{ - pcie_disable_link_bandwidth_notification(srv->port); - return 0; -} - -static int pcie_bandwidth_notification_resume(struct pcie_device *srv) -{ - pcie_enable_link_bandwidth_notification(srv->port); - return 0; -} - -static struct pcie_port_service_driver pcie_bandwidth_notification_driver = { - .name = "pcie_bw_notification", - .port_type = PCIE_ANY_PORT, - .service = PCIE_PORT_SERVICE_BWNOTIF, - .probe = pcie_bandwidth_notification_probe, - .suspend = pcie_bandwidth_notification_suspend, - .resume = pcie_bandwidth_notification_resume, - .remove = pcie_bandwidth_notification_remove, -}; - -int __init pcie_bandwidth_notification_init(void) -{ - return pcie_port_service_register(&pcie_bandwidth_notification_driver); -} diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index af7cf237432a..2ff5724b8f13 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -53,12 +53,6 @@ int pcie_dpc_init(void); static inline int pcie_dpc_init(void) { return 0; } #endif -#ifdef CONFIG_PCIE_BW -int pcie_bandwidth_notification_init(void); -#else -static inline int pcie_bandwidth_notification_init(void) { return 0; } -#endif - /* Port Type */ #define PCIE_ANY_PORT (~0) diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 0b250bc5f405..8bd4992a4f32 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -255,7 +255,6 @@ static void __init pcie_init_services(void) pcie_pme_init(); pcie_dpc_init(); pcie_hp_init(); - pcie_bandwidth_notification_init(); } static int __init pcie_portdrv_init(void) From d7a4bfcac9a5f229aebfe307280e773b5f565443 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 1 Feb 2021 15:22:07 +0300 Subject: [PATCH 223/633] misc: bcm-vk: unlock on error in bcm_to_h_msg_dequeue() Unlock before returning on this error path. Fixes: 111d746bb476 ("misc: bcm-vk: add VK messaging support") Acked-by: Desmond Yan Acked-by: Scott Branden Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/YBfyb+jU5lDUe+5g@mwanda Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk_msg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index a363e2e4f7bc..f40cf08a6192 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -855,7 +855,8 @@ s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk) * that is fatal. */ dev_crit(dev, "Kernel mem allocation failure.\n"); - return -ENOMEM; + total = -ENOMEM; + goto idx_err; } /* flush rd pointer after a message is dequeued */ From 94e6a5b9e3bbb0cec6850b3662b7e3d44a008371 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 1 Feb 2021 15:30:40 +0800 Subject: [PATCH 224/633] misc: rtsx: Remove unneeded return variable This patch removes unneeded return variables, using only '0' instead. It fixes the following warning detected by coccinelle: ./drivers/misc/cardreader/rtsx_pcr.c:1808:5-8: Unneeded variable: "ret". Return "0" on line 1833. Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1612164640-84541-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cardreader/rtsx_pcr.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index 5e94e87c80b7..273311184669 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -1799,7 +1799,6 @@ static int rtsx_pci_runtime_resume(struct device *device) struct pci_dev *pcidev = to_pci_dev(device); struct pcr_handle *handle; struct rtsx_pcr *pcr; - int ret = 0; handle = pci_get_drvdata(pcidev); pcr = handle->pcr; @@ -1824,7 +1823,7 @@ static int rtsx_pci_runtime_resume(struct device *device) schedule_delayed_work(&pcr->idle_work, msecs_to_jiffies(200)); mutex_unlock(&pcr->pcr_mutex); - return ret; + return 0; } #else /* CONFIG_PM */ From 8078efff8d4dda4a84deddcc3172b5820593147a Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 3 Feb 2021 17:42:53 +0300 Subject: [PATCH 225/633] misc: bcm-vk: Fix a couple error codes in probe() These errors should return negative error codes instead of returning success. Fixes: 064ffc7c3939 ("misc: bcm-vk: add autoload support") Fixes: 522f692686a7 ("misc: bcm-vk: add Broadcom VK driver") Acked-by: Scott Branden Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/YBpyEbmz00rjvT9S@mwanda Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/bcm_vk_dev.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index c3d2bba68ef1..a82a8927d92b 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -1358,6 +1358,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) vk->bar[i] = pci_ioremap_bar(pdev, i * 2); if (!vk->bar[i]) { dev_err(dev, "failed to remap BAR%d\n", i); + err = -ENOMEM; goto err_iounmap; } } @@ -1463,7 +1464,8 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); if (auto_load) { if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { - if (bcm_vk_trigger_autoload(vk)) + err = bcm_vk_trigger_autoload(vk); + if (err) goto err_bcm_vk_tty_exit; } else { dev_err(dev, From 3a11b0b5d8d2b3f7d4b44945ef9226a3115bb15f Mon Sep 17 00:00:00 2001 From: Scott Branden Date: Wed, 3 Feb 2021 14:38:26 -0800 Subject: [PATCH 226/633] misc: bcm-vk: only support ttyVK if CONFIG_TTY is set Correct compile issue if CONFIG_TTY is not set by only adding ttyVK devices if CONFIG_BCM_VK_TTY is set. Reported-by: Randy Dunlap Acked-by: Randy Dunlap # build-tested Signed-off-by: Scott Branden Link: https://lore.kernel.org/r/20210203223826.21674-1-scott.branden@broadcom.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/bcm-vk/Kconfig | 12 +++++++++ drivers/misc/bcm-vk/Makefile | 4 +-- drivers/misc/bcm-vk/bcm_vk.h | 42 +++++++++++++++++++++++++++++--- drivers/misc/bcm-vk/bcm_vk_dev.c | 5 ++-- drivers/misc/bcm-vk/bcm_vk_tty.c | 6 +++++ 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/drivers/misc/bcm-vk/Kconfig b/drivers/misc/bcm-vk/Kconfig index 052f6f28b540..68a972772b99 100644 --- a/drivers/misc/bcm-vk/Kconfig +++ b/drivers/misc/bcm-vk/Kconfig @@ -15,3 +15,15 @@ config BCM_VK accelerators via /dev/bcm-vk.N devices. If unsure, say N. + +config BCM_VK_TTY + bool "Enable tty ports on a Broadcom VK Accelerator device" + depends on TTY + depends on BCM_VK + help + Select this option to enable tty support to allow console + access to Broadcom VK Accelerator cards from host. + + Device node will in the form /dev/bcm-vk.x_ttyVKy where: + x is the instance of the VK card + y is the tty device number on the VK card. diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile index e4a1486f7209..1df2ebe851ca 100644 --- a/drivers/misc/bcm-vk/Makefile +++ b/drivers/misc/bcm-vk/Makefile @@ -7,6 +7,6 @@ obj-$(CONFIG_BCM_VK) += bcm_vk.o bcm_vk-objs := \ bcm_vk_dev.o \ bcm_vk_msg.o \ - bcm_vk_sg.o \ - bcm_vk_tty.o + bcm_vk_sg.o +bcm_vk-$(CONFIG_BCM_VK_TTY) += bcm_vk_tty.o diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h index 3f37c640a814..a1338f375589 100644 --- a/drivers/misc/bcm-vk/bcm_vk.h +++ b/drivers/misc/bcm-vk/bcm_vk.h @@ -258,7 +258,11 @@ enum pci_barno { BAR_2 }; +#ifdef CONFIG_BCM_VK_TTY #define BCM_VK_NUM_TTY 2 +#else +#define BCM_VK_NUM_TTY 0 +#endif struct bcm_vk_tty { struct tty_port port; @@ -366,11 +370,13 @@ struct bcm_vk { struct miscdevice miscdev; int devid; /* dev id allocated */ +#ifdef CONFIG_BCM_VK_TTY struct tty_driver *tty_drv; struct timer_list serial_timer; struct bcm_vk_tty tty[BCM_VK_NUM_TTY]; struct workqueue_struct *tty_wq_thread; struct work_struct tty_wq_work; +#endif /* Reference-counting to handle file operations */ struct kref kref; @@ -501,13 +507,43 @@ int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, const pid_t pid, const u32 q_num); void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val); int bcm_vk_auto_load_all_images(struct bcm_vk *vk); -int bcm_vk_tty_init(struct bcm_vk *vk, char *name); -void bcm_vk_tty_exit(struct bcm_vk *vk); -void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk); void bcm_vk_hb_init(struct bcm_vk *vk); void bcm_vk_hb_deinit(struct bcm_vk *vk); void bcm_vk_handle_notf(struct bcm_vk *vk); bool bcm_vk_drv_access_ok(struct bcm_vk *vk); void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask); +#ifdef CONFIG_BCM_VK_TTY +int bcm_vk_tty_init(struct bcm_vk *vk, char *name); +void bcm_vk_tty_exit(struct bcm_vk *vk); +void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk); +void bcm_vk_tty_wq_exit(struct bcm_vk *vk); + +static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index) +{ + vk->tty[index].irq_enabled = true; +} +#else +static inline int bcm_vk_tty_init(struct bcm_vk *vk, char *name) +{ + return 0; +} + +static inline void bcm_vk_tty_exit(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_wq_exit(struct bcm_vk *vk) +{ +} + +static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index) +{ +} +#endif /* CONFIG_BCM_VK_TTY */ + #endif diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index a82a8927d92b..6bfea3210389 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -1397,7 +1397,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pdev->irq + vk->num_irqs, vk->num_irqs + 1); goto err_irq; } - vk->tty[i].irq_enabled = true; + bcm_vk_tty_set_irq_enabled(vk, i); } id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); @@ -1582,8 +1582,7 @@ static void bcm_vk_remove(struct pci_dev *pdev) cancel_work_sync(&vk->wq_work); destroy_workqueue(vk->wq_thread); - cancel_work_sync(&vk->tty_wq_work); - destroy_workqueue(vk->tty_wq_thread); + bcm_vk_tty_wq_exit(vk); for (i = 0; i < MAX_BAR; i++) { if (vk->bar[i]) diff --git a/drivers/misc/bcm-vk/bcm_vk_tty.c b/drivers/misc/bcm-vk/bcm_vk_tty.c index be3964949b63..4d02692ecfc7 100644 --- a/drivers/misc/bcm-vk/bcm_vk_tty.c +++ b/drivers/misc/bcm-vk/bcm_vk_tty.c @@ -331,3 +331,9 @@ void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk) kill_pid(find_vpid(vktty->pid), SIGKILL, 1); } } + +void bcm_vk_tty_wq_exit(struct bcm_vk *vk) +{ + cancel_work_sync(&vk->tty_wq_work); + destroy_workqueue(vk->tty_wq_thread); +} From a618c47a816fa522fa2c8e0a17e8a291a967e44b Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 20 Jan 2021 14:45:30 -0800 Subject: [PATCH 227/633] dt-bindings: phy: qcom,qmp: Add SC8180X UFS to the QMP binding Add compatible for the SC8180x UFS PHY to the QMP binding. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210120224531.1610709-1-bjorn.andersson@linaro.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index f578677886c4..723194d938ba 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -25,6 +25,7 @@ properties: - qcom,msm8998-qmp-pcie-phy - qcom,msm8998-qmp-ufs-phy - qcom,msm8998-qmp-usb3-phy + - qcom,sc8180x-qmp-ufs-phy - qcom,sdm845-qhp-pcie-phy - qcom,sdm845-qmp-pcie-phy - qcom,sdm845-qmp-ufs-phy From 4dd8c1c7f2bd88a4777f87e2a12735a8404235f0 Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 20 Jan 2021 17:43:38 -0800 Subject: [PATCH 228/633] dt-bindings: phy: qcom,qmp: Add SC8180X USB phy Add compatibles for the Qualcomm QMP PHY binding for the SuperSpeed USB phys found in the SC8180x platform. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210121014339.1612525-1-bjorn.andersson@linaro.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index 723194d938ba..c38061de242a 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -26,6 +26,7 @@ properties: - qcom,msm8998-qmp-ufs-phy - qcom,msm8998-qmp-usb3-phy - qcom,sc8180x-qmp-ufs-phy + - qcom,sc8180x-qmp-usb3-phy - qcom,sdm845-qhp-pcie-phy - qcom,sdm845-qmp-pcie-phy - qcom,sdm845-qmp-ufs-phy From a5a621ad0ab44aa672385e0dddc730c3e50a908f Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 20 Jan 2021 14:45:31 -0800 Subject: [PATCH 229/633] phy: qcom-qmp: Add SC8180X UFS phy The UFS phy found in the Qualcomm SC8180X is either the same or very similar to the phy present in SM8150, so add a compatible and reuse the SM8150 configuration. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210120224531.1610709-2-bjorn.andersson@linaro.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index a679b5fb7b48..0f72f0bdd028 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -4414,6 +4414,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { }, { .compatible = "qcom,sc7180-qmp-usb3-dp-phy", /* It's a combo phy */ + }, { + .compatible = "qcom,sc8180x-qmp-ufs-phy", + .data = &sm8150_ufsphy_cfg, }, { .compatible = "qcom,sdm845-qhp-pcie-phy", .data = &sdm845_qhp_pciephy_cfg, From 4d1a6404e91efffe3192e1405e175fc7fb3fa3de Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 20 Jan 2021 17:43:39 -0800 Subject: [PATCH 230/633] phy: qcom-qmp: Add SC8180X USB phy The Qualcomm SC8180X has two QMP phys used for SuperSpeed USB, which are either the same or very similar to the same found in SM8150. Add a compatible for this, reusing the existing SM8150 USB phy config. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210121014339.1612525-2-bjorn.andersson@linaro.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index 0f72f0bdd028..5eaf6dee1e45 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -4417,6 +4417,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { }, { .compatible = "qcom,sc8180x-qmp-ufs-phy", .data = &sm8150_ufsphy_cfg, + }, { + .compatible = "qcom,sc8180x-qmp-usb3-phy", + .data = &sm8150_usb3phy_cfg, }, { .compatible = "qcom,sdm845-qhp-pcie-phy", .data = &sdm845_qhp_pciephy_cfg, From 2cfbe6765b7a61feebbbcfdd4aecb6e6de38b361 Mon Sep 17 00:00:00 2001 From: Kathiravan T Date: Wed, 27 Jan 2021 16:20:27 +0200 Subject: [PATCH 231/633] phy: qcom-qusb2: add QUSB2 support for IPQ6018 Add the phy init sequence for the Super Speed ports found on IPQ6018. Signed-off-by: Kathiravan T [baruch: add ipq6018_regs_layout[], drop binding change] Signed-off-by: Baruch Siach Link: https://lore.kernel.org/r/b8c22dddf1f70d89e135fe1ae705ddc68e295ebb.1611756920.git.baruch@tkos.co.il Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qusb2.c | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index 719e0888ed87..8f1bf7e2186b 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -22,6 +22,7 @@ #include +#define QUSB2PHY_PLL 0x0 #define QUSB2PHY_PLL_TEST 0x04 #define CLK_REF_SEL BIT(7) @@ -135,6 +136,35 @@ enum qusb2phy_reg_layout { QUSB2PHY_INTR_CTRL, }; +static const struct qusb2_phy_init_tbl ipq6018_init_tbl[] = { + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL, 0x14), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE1, 0xF8), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE2, 0xB3), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE3, 0x83), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE4, 0xC0), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_TUNE, 0x30), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_USER_CTL1, 0x79), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_USER_CTL2, 0x21), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TUNE5, 0x00), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_PWR_CTRL, 0x00), + QUSB2_PHY_INIT_CFG_L(QUSB2PHY_PORT_TEST2, 0x14), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_TEST, 0x80), + QUSB2_PHY_INIT_CFG(QUSB2PHY_PLL_AUTOPGM_CTL1, 0x9F), +}; + +static const unsigned int ipq6018_regs_layout[] = { + [QUSB2PHY_PLL_STATUS] = 0x38, + [QUSB2PHY_PORT_TUNE1] = 0x80, + [QUSB2PHY_PORT_TUNE2] = 0x84, + [QUSB2PHY_PORT_TUNE3] = 0x88, + [QUSB2PHY_PORT_TUNE4] = 0x8C, + [QUSB2PHY_PORT_TUNE5] = 0x90, + [QUSB2PHY_PORT_TEST1] = 0x98, + [QUSB2PHY_PORT_TEST2] = 0x9C, + [QUSB2PHY_PORT_POWERDOWN] = 0xB4, + [QUSB2PHY_INTR_CTRL] = 0xBC, +}; + static const unsigned int msm8996_regs_layout[] = { [QUSB2PHY_PLL_STATUS] = 0x38, [QUSB2PHY_PORT_TUNE1] = 0x80, @@ -275,6 +305,17 @@ static const struct qusb2_phy_cfg msm8998_phy_cfg = { .update_tune1_with_efuse = true, }; +static const struct qusb2_phy_cfg ipq6018_phy_cfg = { + .tbl = ipq6018_init_tbl, + .tbl_num = ARRAY_SIZE(ipq6018_init_tbl), + .regs = ipq6018_regs_layout, + + .disable_ctrl = POWER_DOWN, + .mask_core_ready = PLL_LOCKED, + /* autoresume not used */ + .autoresume_en = BIT(0), +}; + static const struct qusb2_phy_cfg qusb2_v2_phy_cfg = { .tbl = qusb2_v2_init_tbl, .tbl_num = ARRAY_SIZE(qusb2_v2_init_tbl), @@ -833,6 +874,9 @@ static const struct phy_ops qusb2_phy_gen_ops = { static const struct of_device_id qusb2_phy_of_match_table[] = { { + .compatible = "qcom,ipq6018-qusb2-phy", + .data = &ipq6018_phy_cfg, + }, { .compatible = "qcom,ipq8074-qusb2-phy", .data = &msm8996_phy_cfg, }, { From 53dd01da9729a9ece9ef1e47d248b420467836ae Mon Sep 17 00:00:00 2001 From: Baruch Siach Date: Wed, 27 Jan 2021 16:20:28 +0200 Subject: [PATCH 232/633] dt-bindings: phy: qcom,qusb2: document ipq6018 compatible This compatible string is for the USB PHY on IPQ60xx systems. Signed-off-by: Baruch Siach Link: https://lore.kernel.org/r/7e1e7bda6ccdaab9adeeb956fac1acc39908a8dc.1611756920.git.baruch@tkos.co.il Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index 582abbbd8b32..9f9cf07b7d45 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -22,6 +22,7 @@ properties: - qcom,msm8996-qusb2-phy - qcom,msm8998-qusb2-phy - qcom,sdm660-qusb2-phy + - qcom,ipq6018-qusb2-phy - items: - enum: - qcom,sc7180-qusb2-phy From 6b46e60a6943d629d69924be3169d8f214624ab2 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 29 Jan 2021 14:17:53 +0100 Subject: [PATCH 233/633] phy: USB_LGM_PHY should depend on X86 The Intel Lightning Mountain (LGM) USB3 USB is only present on Intel Lightning Mountain SoCs. Hence add a dependency on X86, to prevent asking the user about this driver when configuring a kernel without Intel Lightning Mountain platform support. Fixes: 1cce8f73a561c944 ("phy: Add USB3 PHY support for Intel LGM SoC") Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20210129131753.2656306-1-geert+renesas@glider.be Signed-off-by: Vinod Koul --- drivers/phy/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 00dabe5fab8a..68d9c2f6a5ca 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -52,6 +52,7 @@ config PHY_XGENE config USB_LGM_PHY tristate "INTEL Lightning Mountain USB PHY Driver" depends on USB_SUPPORT + depends on X86 || COMPILE_TEST select USB_PHY select REGULATOR select REGULATOR_FIXED_VOLTAGE From 557a28811c7e0286d3816842032db5eb7bb5f156 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Sun, 31 Jan 2021 02:31:24 +0100 Subject: [PATCH 234/633] phy: qualcomm: usb28nm: Add MDM9607 init sequence This is required to bring up the PHY on MDM9607-based boards. Signed-off-by: Konrad Dybcio Link: https://lore.kernel.org/r/20210131013124.54484-1-konrad.dybcio@somainline.org Signed-off-by: Vinod Koul --- .../devicetree/bindings/phy/qcom,usb-hs-28nm.yaml | 1 + drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml index ca6a0836b53c..abcc4373f39e 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml @@ -16,6 +16,7 @@ properties: compatible: enum: - qcom,usb-hs-28nm-femtophy + - qcom,usb-hs-28nm-mdm9607 reg: maxItems: 1 diff --git a/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c b/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c index a52a9bf13b75..8807e59a1162 100644 --- a/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c +++ b/drivers/phy/qualcomm/phy-qcom-usb-hs-28nm.c @@ -401,13 +401,26 @@ static const struct hsphy_init_seq init_seq_femtophy[] = { HSPHY_INIT_CFG(0x90, 0x60, 0), }; +static const struct hsphy_init_seq init_seq_mdm9607[] = { + HSPHY_INIT_CFG(0x80, 0x44, 0), + HSPHY_INIT_CFG(0x81, 0x38, 0), + HSPHY_INIT_CFG(0x82, 0x24, 0), + HSPHY_INIT_CFG(0x83, 0x13, 0), +}; + static const struct hsphy_data hsphy_data_femtophy = { .init_seq = init_seq_femtophy, .init_seq_num = ARRAY_SIZE(init_seq_femtophy), }; +static const struct hsphy_data hsphy_data_mdm9607 = { + .init_seq = init_seq_mdm9607, + .init_seq_num = ARRAY_SIZE(init_seq_mdm9607), +}; + static const struct of_device_id qcom_snps_hsphy_match[] = { { .compatible = "qcom,usb-hs-28nm-femtophy", .data = &hsphy_data_femtophy, }, + { .compatible = "qcom,usb-hs-28nm-mdm9607", .data = &hsphy_data_mdm9607, }, { }, }; MODULE_DEVICE_TABLE(of, qcom_snps_hsphy_match); From 25e3ee590f62772f6016a5ec1a37367a1813e198 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 3 Feb 2021 10:58:07 +0800 Subject: [PATCH 235/633] phy: phy-brcm-sata: remove unneeded semicolon Eliminate the following coccicheck warning: ./drivers/phy/broadcom/phy-brcm-sata.c:654:2-3: Unneeded semicolon Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1612321087-14743-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Vinod Koul --- drivers/phy/broadcom/phy-brcm-sata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/broadcom/phy-brcm-sata.c b/drivers/phy/broadcom/phy-brcm-sata.c index 3ecf41359591..769c707d9b71 100644 --- a/drivers/phy/broadcom/phy-brcm-sata.c +++ b/drivers/phy/broadcom/phy-brcm-sata.c @@ -651,7 +651,7 @@ static int brcm_dsl_sata_init(struct brcm_sata_port *port) break; msleep(20); try--; - }; + } if (!try) { /* PLL did not lock; give up */ From 9a8b9434c60f40e4d2603c822a68af6a9ca710df Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Wed, 3 Feb 2021 12:06:30 +0100 Subject: [PATCH 236/633] phy: mediatek: Add missing MODULE_DEVICE_TABLE() This patch adds the missing MODULE_DEVICE_TABLE definitions on different Mediatek phy drivers which generates correct modalias for automatic loading when these drivers are compiled as an external module. Signed-off-by: Boris Brezillon Signed-off-by: Enric Balletbo i Serra Link: https://lore.kernel.org/r/20210203110631.686003-1-enric.balletbo@collabora.com Signed-off-by: Vinod Koul --- drivers/phy/mediatek/phy-mtk-hdmi.c | 1 + drivers/phy/mediatek/phy-mtk-mipi-dsi.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/phy/mediatek/phy-mtk-hdmi.c b/drivers/phy/mediatek/phy-mtk-hdmi.c index 45be8aa724f3..8313bd517e4c 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi.c @@ -201,6 +201,7 @@ static const struct of_device_id mtk_hdmi_phy_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, mtk_hdmi_phy_match); static struct platform_driver mtk_hdmi_phy_driver = { .probe = mtk_hdmi_phy_probe, diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c index 18c481251f04..9c7815bb9000 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c @@ -233,6 +233,7 @@ static const struct of_device_id mtk_mipi_tx_match[] = { .data = &mt8183_mipitx_data }, { }, }; +MODULE_DEVICE_TABLE(of, mtk_mipi_tx_match); struct platform_driver mtk_mipi_tx_driver = { .probe = mtk_mipi_tx_probe, From a74ab2ed0def52a993d0cbd9df9d1c4de33c6fbd Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Fri, 29 Jan 2021 11:08:01 -0800 Subject: [PATCH 237/633] misc: pvpanic: sysfs_emit uses should have a newline Add newline terminations to the sysfs_emit uses added by -next commit 8d6da6575ffe ("misc: pvpanic: introduce events device attribue") Signed-off-by: Joe Perches Signed-off-by: zhenwei pi Link: https://lore.kernel.org/r/13b1c892d52c27d4caeccc89506aadda74f61365.camel@perches.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/pvpanic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/misc/pvpanic.c b/drivers/misc/pvpanic.c index b1e4922a7fda..9f350e05ef68 100644 --- a/drivers/misc/pvpanic.c +++ b/drivers/misc/pvpanic.c @@ -25,13 +25,13 @@ static unsigned int events; static ssize_t capability_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%x", capability); + return sysfs_emit(buf, "%x\n", capability); } static DEVICE_ATTR_RO(capability); static ssize_t events_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sysfs_emit(buf, "%x", events); + return sysfs_emit(buf, "%x\n", events); } static ssize_t events_store(struct device *dev, struct device_attribute *attr, From 11a0b5e0ec8c13bef06f7414f9e914506140d5cb Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 12 Jan 2021 11:28:18 -0800 Subject: [PATCH 238/633] random: fix the RNDRESEEDCRNG ioctl The RNDRESEEDCRNG ioctl reseeds the primary_crng from itself, which doesn't make sense. Reseed it from the input_pool instead. Fixes: d848e5f8e1eb ("random: add new ioctl RNDRESEEDCRNG") Cc: stable@vger.kernel.org Cc: linux-crypto@vger.kernel.org Cc: Andy Lutomirski Cc: Jann Horn Cc: Theodore Ts'o Reviewed-by: Jann Horn Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20210112192818.69921-1-ebiggers@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 5f3b8ac9d97b..a894c0559a8c 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1972,7 +1972,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng, NULL); + crng_reseed(&primary_crng, &input_pool); crng_global_init_time = jiffies - 1; return 0; default: From 65b2728145771b0df18f85303b8ae152cd53f2de Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Mon, 1 Feb 2021 11:13:21 -0700 Subject: [PATCH 239/633] coresight: cti: Reduce scope for the variable 'cs_fwnode' in cti_plat_create_connection() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local variable was used only within an else branch. Thus move the definition for the variable “cs_fwnode” into the corresponding code block. This issue was detected by using the Coccinelle software. Link: https://lore.kernel.org/r/c1b09b27-9012-324f-28d0-ba820dc468a5@web.de Signed-off-by: Markus Elfring Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-2-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-cti-platform.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c index 98f830c6ed50..ccef04f27f12 100644 --- a/drivers/hwtracing/coresight/coresight-cti-platform.c +++ b/drivers/hwtracing/coresight/coresight-cti-platform.c @@ -343,7 +343,6 @@ static int cti_plat_create_connection(struct device *dev, { struct cti_trig_con *tc = NULL; int cpuid = -1, err = 0; - struct fwnode_handle *cs_fwnode = NULL; struct coresight_device *csdev = NULL; const char *assoc_name = "unknown"; char cpu_name_str[16]; @@ -397,8 +396,9 @@ static int cti_plat_create_connection(struct device *dev, assoc_name = cpu_name_str; } else { /* associated device ? */ - cs_fwnode = fwnode_find_reference(fwnode, - CTI_DT_CSDEV_ASSOC, 0); + struct fwnode_handle *cs_fwnode = fwnode_find_reference(fwnode, + CTI_DT_CSDEV_ASSOC, + 0); if (!IS_ERR(cs_fwnode)) { assoc_name = cti_plat_get_csdev_or_node_name(cs_fwnode, &csdev); From b8336ad947e1913b9bb5cdf4f54b687654160d42 Mon Sep 17 00:00:00 2001 From: Chunyan Zhang Date: Mon, 1 Feb 2021 11:13:22 -0700 Subject: [PATCH 240/633] coresight: etm4x: add AMBA id for Cortex-A55 and Cortex-A75 Add AMBA UCI id to support Cortex-A55(Ananke) and Cortex-A75(Promethus). Reviewed by: Mike Leach Link: https://lore.kernel.org/r/20210118065549.197489-1-zhang.lyra@gmail.com Signed-off-by: Bin Ji Signed-off-by: Chunyan Zhang Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-3-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index b20b6ff17cf6..8c4b0c46c8f3 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1713,6 +1713,8 @@ static const struct amba_id etm4_ids[] = { CS_AMBA_ID(0x000bb95a), /* Cortex-A72 */ CS_AMBA_ID(0x000bb959), /* Cortex-A73 */ CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */ + CS_AMBA_UCI_ID(0x000bbd05, uci_id_etm4),/* Cortex-A55 */ + CS_AMBA_UCI_ID(0x000bbd0a, uci_id_etm4),/* Cortex-A75 */ CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */ CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */ CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */ From f6a18f354c587b6a77e71df40c715152328b34ff Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:23 -0700 Subject: [PATCH 241/633] coresight: etm4x: Handle access to TRCSSPCICRn TRCSSPCICR is present only if all of the following are true: TRCIDR4.NUMSSCC > n. TRCIDR4.NUMPC > 0b0000 . TRCSSCSR.PC == 0b1 Add a helper function to check all the conditions. Link: https://lore.kernel.org/r/20210110224850.1880240-2-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-4-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 29 +++++++++++++++---- drivers/hwtracing/coresight/coresight-etm4x.h | 2 ++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 8c4b0c46c8f3..4b615e9f3d76 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -59,6 +59,22 @@ static u64 etm4_get_access_type(struct etmv4_config *config); static enum cpuhp_state hp_online; +/* + * Check if TRCSSPCICRn(i) is implemented for a given instance. + * + * TRCSSPCICRn is implemented only if : + * TRCSSPCICR is present only if all of the following are true: + * TRCIDR4.NUMSSCC > n. + * TRCIDR4.NUMPC > 0b0000 . + * TRCSSCSR.PC == 0b1 + */ +static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) +{ + return (n < drvdata->nr_ss_cmp) && + drvdata->nr_pe && + (drvdata->config.ss_status[n] & TRCSSCSRn_PC); +} + static void etm4_os_unlock(struct etmv4_drvdata *drvdata) { /* Writing 0 to TRCOSLAR unlocks the trace registers */ @@ -270,8 +286,9 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) drvdata->base + TRCSSCCRn(i)); writel_relaxed(config->ss_status[i], drvdata->base + TRCSSCSRn(i)); - writel_relaxed(config->ss_pe_cmp[i], - drvdata->base + TRCSSPCICRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + writel_relaxed(config->ss_pe_cmp[i], + drvdata->base + TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp; i++) { writeq_relaxed(config->addr_val[i], @@ -1324,7 +1341,8 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) for (i = 0; i < drvdata->nr_ss_cmp; i++) { state->trcssccr[i] = readl(drvdata->base + TRCSSCCRn(i)); state->trcsscsr[i] = readl(drvdata->base + TRCSSCSRn(i)); - state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { @@ -1440,8 +1458,9 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) drvdata->base + TRCSSCCRn(i)); writel_relaxed(state->trcsscsr[i], drvdata->base + TRCSSCSRn(i)); - writel_relaxed(state->trcsspcicr[i], - drvdata->base + TRCSSPCICRn(i)); + if (etm4x_sspcicrn_present(drvdata, i)) + writel_relaxed(state->trcsspcicr[i], + drvdata->base + TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 3dd3e0633328..80e480c7fe5c 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -179,6 +179,8 @@ #define TRCSTATR_PMSTABLE_BIT 1 #define ETM_DEFAULT_ADDR_COMP 0 +#define TRCSSCSRn_PC BIT(3) + /* PowerDown Control Register bits */ #define TRCPDCR_PU BIT(3) From df81b43802f43c0954a55e5d513e8750a1ab4d31 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:24 -0700 Subject: [PATCH 242/633] coresight: etm4x: Skip accessing TRCPDCR in save/restore When the ETM is affected by Qualcomm errata, modifying the TRCPDCR could cause the system hang. Even though this is taken care of during enable/disable ETM, the ETM state save/restore could still access the TRCPDCR. Make sure we skip the access during the save/restore. Found by code inspection. Link: https://lore.kernel.org/r/20210110224850.1880240-3-suzuki.poulose@arm.com Fixes: 02510a5aa78d ("coresight: etm4x: Add support to skip trace unit power up") Cc: Mathieu Poirier Cc: Mike Leach Cc: Sai Prakash Ranjan Cc: Tingwei Zhang Tested-by: Sai Prakash Ranjan Reviewed-by: Sai Prakash Ranjan Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-5-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 4b615e9f3d76..0924c376e35a 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1373,7 +1373,8 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcclaimset = readl(drvdata->base + TRCCLAIMCLR); - state->trcpdcr = readl(drvdata->base + TRCPDCR); + if (!drvdata->skip_power_up) + state->trcpdcr = readl(drvdata->base + TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { @@ -1391,9 +1392,9 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * potentially save power on systems that respect the TRCPDCR_PU * despite requesting software to save/restore state. */ - writel_relaxed((state->trcpdcr & ~TRCPDCR_PU), - drvdata->base + TRCPDCR); - + if (!drvdata->skip_power_up) + writel_relaxed((state->trcpdcr & ~TRCPDCR_PU), + drvdata->base + TRCPDCR); out: CS_LOCK(drvdata->base); return ret; @@ -1488,7 +1489,8 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); - writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR); + if (!drvdata->skip_power_up) + writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR); drvdata->state_needs_restore = false; From 6e736c60a9fe905b3e4f4b07171a31ad602d56d2 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:25 -0700 Subject: [PATCH 243/633] coresight: Introduce device access abstraction We are about to introduce support for sysreg access to ETMv4.4+ component. Since there are generic routines that access the registers (e.g, CS_LOCK/UNLOCK , claim/disclaim operations, timeout) and in order to preserve the logic of these operations at a single place we introduce an abstraction layer for the accesses to a given device. Link: https://lore.kernel.org/r/20210110224850.1880240-4-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-6-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-catu.c | 1 + drivers/hwtracing/coresight/coresight-core.c | 43 ++++ .../hwtracing/coresight/coresight-cti-core.c | 1 + drivers/hwtracing/coresight/coresight-etb10.c | 1 + .../coresight/coresight-etm3x-core.c | 1 + .../coresight/coresight-etm4x-core.c | 1 + .../hwtracing/coresight/coresight-funnel.c | 1 + .../coresight/coresight-replicator.c | 1 + drivers/hwtracing/coresight/coresight-stm.c | 1 + .../hwtracing/coresight/coresight-tmc-core.c | 1 + drivers/hwtracing/coresight/coresight-tpiu.c | 1 + include/linux/coresight.h | 191 +++++++++++++++++- 12 files changed, 241 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index a61313f320bd..867c932c7b26 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -551,6 +551,7 @@ static int catu_probe(struct amba_device *adev, const struct amba_id *id) dev->platform_data = pdata; drvdata->base = base; + catu_desc.access = CSDEV_ACCESS_IOMEM(base); catu_desc.pdata = pdata; catu_desc.dev = dev; catu_desc.groups = catu_groups; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 4ba801dffcb7..a38af8f0831b 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1458,6 +1458,48 @@ int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) } EXPORT_SYMBOL_GPL(coresight_timeout); +u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read32(&csdev->access, offset); +} + +u32 coresight_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read32(&csdev->access, offset); +} + +void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset) +{ + csdev_access_relaxed_write32(&csdev->access, val, offset); +} + +void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) +{ + csdev_access_write32(&csdev->access, val, offset); +} + +u64 coresight_relaxed_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read64(&csdev->access, offset); +} + +u64 coresight_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read64(&csdev->access, offset); +} + +void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset) +{ + csdev_access_relaxed_write64(&csdev->access, val, offset); +} + +void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) +{ + csdev_access_write64(&csdev->access, val, offset); +} + /* * coresight_release_platform_data: Release references to the devices connected * to the output port of this device. @@ -1522,6 +1564,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) csdev->type = desc->type; csdev->subtype = desc->subtype; csdev->ops = desc->ops; + csdev->access = desc->access; csdev->orphan = false; csdev->dev.type = &coresight_dev_type[desc->type]; diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 61dbc1afd8da..b38a8db5f252 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -870,6 +870,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + cti_desc.access = CSDEV_ACCESS_IOMEM(base); dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 0cf6f0b947b6..cc742561a986 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -757,6 +757,7 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 5bf5a5a4ce6d..3b7837cbe376 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -839,6 +839,7 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 0924c376e35a..b9e01357ffad 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1626,6 +1626,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index 071c723227db..8f7c40d7d8d6 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -242,6 +242,7 @@ static int funnel_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = coresight_funnel_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index 7e2a2b7f503f..205756fab729 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -254,6 +254,7 @@ static int replicator_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = replicator_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } if (fwnode_property_present(dev_fwnode(dev), diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 99791773f682..41d9a922c2d4 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -884,6 +884,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); ret = stm_get_stimulus_area(dev, &ch_res); if (ret) diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index 8169dff5a9f6..e61b75be66b6 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -456,6 +456,7 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id) } drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index d5dfee9ee556..6ca396799883 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -149,6 +149,7 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); /* Disable tpiu to support older devices */ tpiu_disable_hw(drvdata); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 7d3c87e5b97c..6107cf4021d3 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -7,6 +7,7 @@ #define _LINUX_CORESIGHT_H #include +#include #include #include @@ -114,6 +115,32 @@ struct coresight_platform_data { struct coresight_connection *conns; }; +/** + * struct csdev_access - Abstraction of a CoreSight device access. + * + * @io_mem : True if the device has memory mapped I/O + * @base : When io_mem == true, base address of the component + * @read : Read from the given "offset" of the given instance. + * @write : Write "val" to the given "offset". + */ +struct csdev_access { + bool io_mem; + union { + void __iomem *base; + struct { + u64 (*read)(u32 offset, bool relaxed, bool _64bit); + void (*write)(u64 val, u32 offset, bool relaxed, + bool _64bit); + }; + }; +}; + +#define CSDEV_ACCESS_IOMEM(_addr) \ + ((struct csdev_access) { \ + .io_mem = true, \ + .base = (_addr), \ + }) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -125,6 +152,7 @@ struct coresight_platform_data { * @groups: operations specific to this component. These will end up * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. + * @access: Describe access to the device */ struct coresight_desc { enum coresight_dev_type type; @@ -134,6 +162,7 @@ struct coresight_desc { struct device *dev; const struct attribute_group **groups; const char *name; + struct csdev_access access; }; /** @@ -173,7 +202,8 @@ struct coresight_sysfs_link { * @type: as defined by @coresight_dev_type. * @subtype: as defined by @coresight_dev_subtype. * @ops: generic operations for this component, as defined - by @coresight_ops. + * by @coresight_ops. + * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. * @refcnt: keep track of what is in use. * @orphan: true if the component has connections that haven't been linked. @@ -195,6 +225,7 @@ struct coresight_device { enum coresight_dev_type type; union coresight_dev_subtype subtype; const struct coresight_ops *ops; + struct csdev_access access; struct device dev; atomic_t *refcnt; bool orphan; @@ -326,6 +357,104 @@ struct coresight_ops { }; #if IS_ENABLED(CONFIG_CORESIGHT) + +static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa, + u32 offset) +{ + if (likely(csa->io_mem)) + return readl_relaxed(csa->base + offset); + + return csa->read(offset, true, false); +} + +static inline u32 csdev_access_read32(struct csdev_access *csa, u32 offset) +{ + if (likely(csa->io_mem)) + return readl(csa->base + offset); + + return csa->read(offset, false, false); +} + +static inline void csdev_access_relaxed_write32(struct csdev_access *csa, + u32 val, u32 offset) +{ + if (likely(csa->io_mem)) + writel_relaxed(val, csa->base + offset); + else + csa->write(val, offset, true, false); +} + +static inline void csdev_access_write32(struct csdev_access *csa, u32 val, u32 offset) +{ + if (likely(csa->io_mem)) + writel(val, csa->base + offset); + else + csa->write(val, offset, false, false); +} + +#ifdef CONFIG_64BIT + +static inline u64 csdev_access_relaxed_read64(struct csdev_access *csa, + u32 offset) +{ + if (likely(csa->io_mem)) + return readq_relaxed(csa->base + offset); + + return csa->read(offset, true, true); +} + +static inline u64 csdev_access_read64(struct csdev_access *csa, u32 offset) +{ + if (likely(csa->io_mem)) + return readq(csa->base + offset); + + return csa->read(offset, false, true); +} + +static inline void csdev_access_relaxed_write64(struct csdev_access *csa, + u64 val, u32 offset) +{ + if (likely(csa->io_mem)) + writeq_relaxed(val, csa->base + offset); + else + csa->write(val, offset, true, true); +} + +static inline void csdev_access_write64(struct csdev_access *csa, u64 val, u32 offset) +{ + if (likely(csa->io_mem)) + writeq(val, csa->base + offset); + else + csa->write(val, offset, false, true); +} + +#else /* !CONFIG_64BIT */ + +static inline u64 csdev_access_relaxed_read64(struct csdev_access *csa, + u32 offset) +{ + WARN_ON(1); + return 0; +} + +static inline u64 csdev_access_read64(struct csdev_access *csa, u32 offset) +{ + WARN_ON(1); + return 0; +} + +static inline void csdev_access_relaxed_write64(struct csdev_access *csa, + u64 val, u32 offset) +{ + WARN_ON(1); +} + +static inline void csdev_access_write64(struct csdev_access *csa, u64 val, u32 offset) +{ + WARN_ON(1); +} +#endif /* CONFIG_64BIT */ + extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); @@ -343,6 +472,18 @@ extern char *coresight_alloc_device_name(struct coresight_dev_list *devs, struct device *dev); extern bool coresight_loses_context_with_cpu(struct device *dev); + +u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset); +u32 coresight_read32(struct coresight_device *csdev, u32 offset); +void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset); +void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset); +u64 coresight_relaxed_read64(struct coresight_device *csdev, u32 offset); +u64 coresight_read64(struct coresight_device *csdev, u32 offset); +void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset); +void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset); + #else static inline struct coresight_device * coresight_register(struct coresight_desc *desc) { return NULL; } @@ -369,10 +510,54 @@ static inline bool coresight_loses_context_with_cpu(struct device *dev) { return false; } -#endif + +static inline u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline u32 coresight_read32(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) +{ +} + +static inline void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset) +{ +} + +static inline u64 coresight_relaxed_read64(struct coresight_device *csdev, + u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline u64 coresight_read64(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset) +{ +} + +static inline void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) +{ +} + +#endif /* IS_ENABLED(CONFIG_CORESIGHT) */ extern int coresight_get_cpu(struct device *dev); struct coresight_platform_data *coresight_get_platform_data(struct device *dev); -#endif +#endif /* _LINUX_COREISGHT_H */ From 4eb1d85cfda84d7e9cce4575323a5ecd72b2327d Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:26 -0700 Subject: [PATCH 244/633] coresight: tpiu: Prepare for using coresight device access abstraction Prepare the TPIU driver to make use of the CoreSight device access abstraction layer. The driver touches the device even before the coresight device is registered. Thus we could be accessing the devices without a csdev. As we are about to use the abstraction layer for accessing the device, pass in the access directly to avoid having to deal with the un-initialised csdev. Link: https://lore.kernel.org/r/20210110224850.1880240-5-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-7-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-tpiu.c | 30 +++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index 6ca396799883..a12b6ee0a576 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -60,49 +60,45 @@ struct tpiu_drvdata { struct coresight_device *csdev; }; -static void tpiu_enable_hw(struct tpiu_drvdata *drvdata) +static void tpiu_enable_hw(struct csdev_access *csa) { - CS_UNLOCK(drvdata->base); + CS_UNLOCK(csa->base); /* TODO: fill this up */ - CS_LOCK(drvdata->base); + CS_LOCK(csa->base); } static int tpiu_enable(struct coresight_device *csdev, u32 mode, void *__unused) { - struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - - tpiu_enable_hw(drvdata); + tpiu_enable_hw(&csdev->access); atomic_inc(csdev->refcnt); dev_dbg(&csdev->dev, "TPIU enabled\n"); return 0; } -static void tpiu_disable_hw(struct tpiu_drvdata *drvdata) +static void tpiu_disable_hw(struct csdev_access *csa) { - CS_UNLOCK(drvdata->base); + CS_UNLOCK(csa->base); /* Clear formatter and stop on flush */ - writel_relaxed(FFCR_STOP_FI, drvdata->base + TPIU_FFCR); + csdev_access_relaxed_write32(csa, FFCR_STOP_FI, TPIU_FFCR); /* Generate manual flush */ - writel_relaxed(FFCR_STOP_FI | FFCR_FON_MAN, drvdata->base + TPIU_FFCR); + csdev_access_relaxed_write32(csa, FFCR_STOP_FI | FFCR_FON_MAN, TPIU_FFCR); /* Wait for flush to complete */ - coresight_timeout(drvdata->base, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); + coresight_timeout(csa->base, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); /* Wait for formatter to stop */ - coresight_timeout(drvdata->base, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); + coresight_timeout(csa->base, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); - CS_LOCK(drvdata->base); + CS_LOCK(csa->base); } static int tpiu_disable(struct coresight_device *csdev) { - struct tpiu_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); - if (atomic_dec_return(csdev->refcnt)) return -EBUSY; - tpiu_disable_hw(drvdata); + tpiu_disable_hw(&csdev->access); dev_dbg(&csdev->dev, "TPIU disabled\n"); return 0; @@ -152,7 +148,7 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) desc.access = CSDEV_ACCESS_IOMEM(base); /* Disable tpiu to support older devices */ - tpiu_disable_hw(drvdata); + tpiu_disable_hw(&desc.access); pdata = coresight_get_platform_data(dev); if (IS_ERR(pdata)) From 020052825e49128d381d6444d1ce079e8ca82386 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:27 -0700 Subject: [PATCH 245/633] coresight: Convert coresight_timeout to use access abstraction Convert the generic routines to use the new access abstraction layer gradually, starting with coresigth_timeout. Link: https://lore.kernel.org/r/20210110224850.1880240-6-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-8-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-catu.c | 5 ++-- drivers/hwtracing/coresight/coresight-core.c | 13 ++++---- drivers/hwtracing/coresight/coresight-etb10.c | 5 ++-- .../coresight/coresight-etm4x-core.c | 30 ++++++++++++------- drivers/hwtracing/coresight/coresight-stm.c | 3 +- .../hwtracing/coresight/coresight-tmc-core.c | 15 ++++++---- drivers/hwtracing/coresight/coresight-tpiu.c | 4 +-- include/linux/coresight.h | 11 +++++-- 8 files changed, 54 insertions(+), 32 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index 867c932c7b26..d6097454d399 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -401,8 +401,9 @@ static const struct attribute_group *catu_groups[] = { static inline int catu_wait_for_ready(struct catu_drvdata *drvdata) { - return coresight_timeout(drvdata->base, - CATU_STATUS, CATU_STATUS_READY, 1); + struct csdev_access *csa = &drvdata->csdev->access; + + return coresight_timeout(csa, CATU_STATUS, CATU_STATUS_READY, 1); } static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index a38af8f0831b..74985068f325 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1418,23 +1418,24 @@ static void coresight_remove_conns(struct coresight_device *csdev) } /** - * coresight_timeout - loop until a bit has changed to a specific state. - * @addr: base address of the area of interest. - * @offset: address of a register, starting from @addr. + * coresight_timeout - loop until a bit has changed to a specific register + * state. + * @csa: coresight device access for the device + * @offset: Offset of the register from the base of the device. * @position: the position of the bit of interest. * @value: the value the bit should have. * * Return: 0 as soon as the bit has taken the desired state or -EAGAIN if * TIMEOUT_US has elapsed, which ever happens first. */ - -int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) +int coresight_timeout(struct csdev_access *csa, u32 offset, + int position, int value) { int i; u32 val; for (i = TIMEOUT_US; i > 0; i--) { - val = __raw_readl(addr + offset); + val = csdev_access_read32(csa, offset); /* waiting on the bit to go from 0 to 1 */ if (value) { if (val & BIT(position)) diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index cc742561a986..0f664aeeda93 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -252,6 +252,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) { u32 ffcr; struct device *dev = &drvdata->csdev->dev; + struct csdev_access *csa = &drvdata->csdev->access; CS_UNLOCK(drvdata->base); @@ -263,7 +264,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) ffcr |= ETB_FFCR_FON_MAN; writel_relaxed(ffcr, drvdata->base + ETB_FFCR); - if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) { + if (coresight_timeout(csa, ETB_FFCR, ETB_FFCR_BIT, 0)) { dev_err(dev, "timeout while waiting for completion of Manual Flush\n"); } @@ -271,7 +272,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) /* disable trace capture */ writel_relaxed(0x0, drvdata->base + ETB_CTL_REG); - if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) { + if (coresight_timeout(csa, ETB_FFSR, ETB_FFSR_BIT, 1)) { dev_err(dev, "timeout while waiting for Formatter to Stop\n"); } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index b9e01357ffad..180bb6ed9090 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -217,7 +217,9 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) { int i, rc; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; CS_UNLOCK(drvdata->base); etm4_enable_arch_specific(drvdata); @@ -232,7 +234,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) writel_relaxed(0, drvdata->base + TRCPRGCTLR); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); if (drvdata->nr_pe) @@ -323,7 +325,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) writel_relaxed(1, drvdata->base + TRCPRGCTLR); /* wait for TRCSTATR.IDLE to go back down to '0' */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); @@ -587,7 +589,9 @@ static void etm4_disable_hw(void *info) u32 control; struct etmv4_drvdata *drvdata = info; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; int i; CS_UNLOCK(drvdata->base); @@ -615,8 +619,7 @@ static void etm4_disable_hw(void *info) writel_relaxed(control, drvdata->base + TRCPRGCTLR); /* wait for TRCSTATR.PMSTABLE to go to '1' */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for PM stable Trace Status\n"); @@ -1272,7 +1275,15 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) { int i, ret = 0; struct etmv4_save_state *state; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa; + struct device *etm_dev; + + if (WARN_ON(!csdev)) + return -ENODEV; + + etm_dev = &csdev->dev; + csa = &csdev->access; /* * As recommended by 3.4.1 ("The procedure when powering down the PE") @@ -1287,8 +1298,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) etm4_os_lock(drvdata); /* wait for TRCSTATR.PMSTABLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for PM Stable Status\n"); etm4_os_unlock(drvdata); @@ -1377,7 +1387,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcpdcr = readl(drvdata->base + TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); etm4_os_unlock(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 41d9a922c2d4..5927316d7a03 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -258,6 +258,7 @@ static void stm_disable(struct coresight_device *csdev, struct perf_event *event) { struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct csdev_access *csa = &csdev->access; /* * For as long as the tracer isn't disabled another entity can't @@ -270,7 +271,7 @@ static void stm_disable(struct coresight_device *csdev, spin_unlock(&drvdata->spinlock); /* Wait until the engine has completely stopped */ - coresight_timeout(drvdata->base, STMTCSR, STMTCSR_BUSY_BIT, 0); + coresight_timeout(csa, STMTCSR, STMTCSR_BUSY_BIT, 0); pm_runtime_put(csdev->dev.parent); diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index e61b75be66b6..4dc1ea2c19b5 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -33,16 +33,20 @@ DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr"); void tmc_wait_for_tmcready(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; + /* Ensure formatter, unformatter and hardware fifo are empty */ - if (coresight_timeout(drvdata->base, - TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { + dev_err(&csdev->dev, "timeout while waiting for TMC to be Ready\n"); } } void tmc_flush_and_stop(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; u32 ffcr; ffcr = readl_relaxed(drvdata->base + TMC_FFCR); @@ -51,9 +55,8 @@ void tmc_flush_and_stop(struct tmc_drvdata *drvdata) ffcr |= BIT(TMC_FFCR_FLUSHMAN_BIT); writel_relaxed(ffcr, drvdata->base + TMC_FFCR); /* Ensure flush completes */ - if (coresight_timeout(drvdata->base, - TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { + dev_err(&csdev->dev, "timeout while waiting for completion of Manual Flush\n"); } diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index a12b6ee0a576..2ec057892799 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -86,9 +86,9 @@ static void tpiu_disable_hw(struct csdev_access *csa) /* Generate manual flush */ csdev_access_relaxed_write32(csa, FFCR_STOP_FI | FFCR_FON_MAN, TPIU_FFCR); /* Wait for flush to complete */ - coresight_timeout(csa->base, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); + coresight_timeout(csa, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); /* Wait for formatter to stop */ - coresight_timeout(csa->base, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); + coresight_timeout(csa, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); CS_LOCK(csa->base); } diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 6107cf4021d3..18bc7f9fb041 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -460,7 +460,7 @@ coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); extern int coresight_enable(struct coresight_device *csdev); extern void coresight_disable(struct coresight_device *csdev); -extern int coresight_timeout(void __iomem *addr, u32 offset, +extern int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value); extern int coresight_claim_device(void __iomem *base); @@ -491,8 +491,13 @@ static inline void coresight_unregister(struct coresight_device *csdev) {} static inline int coresight_enable(struct coresight_device *csdev) { return -ENOSYS; } static inline void coresight_disable(struct coresight_device *csdev) {} -static inline int coresight_timeout(void __iomem *addr, u32 offset, - int position, int value) { return 1; } + +static inline int coresight_timeout(struct csdev_access *csa, u32 offset, + int position, int value) +{ + return 1; +} + static inline int coresight_claim_device_unlocked(void __iomem *base) { return -EINVAL; From 8ce0029658ba16c52670e869b9ccde02ae7dec6b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:28 -0700 Subject: [PATCH 246/633] coresight: Convert claim/disclaim operations to use access wrappers Convert the generic CLAIM tag management APIs to use the device access layer abstraction. Link: https://lore.kernel.org/r/20210110224850.1880240-7-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-9-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-catu.c | 6 +- drivers/hwtracing/coresight/coresight-core.c | 66 +++++++++++-------- .../hwtracing/coresight/coresight-cti-core.c | 17 +++-- drivers/hwtracing/coresight/coresight-etb10.c | 4 +- .../coresight/coresight-etm3x-core.c | 8 ++- .../coresight/coresight-etm4x-core.c | 4 +- .../hwtracing/coresight/coresight-funnel.c | 6 +- .../coresight/coresight-replicator.c | 12 ++-- .../hwtracing/coresight/coresight-tmc-etf.c | 10 +-- .../hwtracing/coresight/coresight-tmc-etr.c | 4 +- include/linux/coresight.h | 16 ++--- 11 files changed, 91 insertions(+), 62 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index d6097454d399..9a0b9ce4a7da 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -412,6 +412,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) u32 control, mode; struct etr_buf *etr_buf = data; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; if (catu_wait_for_ready(drvdata)) dev_warn(dev, "Timeout while waiting for READY\n"); @@ -422,7 +423,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) return -EBUSY; } - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) return rc; @@ -466,9 +467,10 @@ static int catu_disable_hw(struct catu_drvdata *drvdata) { int rc = 0; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; catu_write_control(drvdata, 0); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); if (catu_wait_for_ready(drvdata)) { dev_info(dev, "Timeout while waiting for READY\n"); rc = -EAGAIN; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 74985068f325..0062c8935653 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -145,30 +145,32 @@ static int coresight_find_link_outport(struct coresight_device *csdev, return -ENODEV; } -static inline u32 coresight_read_claim_tags(void __iomem *base) +static inline u32 coresight_read_claim_tags(struct coresight_device *csdev) { - return readl_relaxed(base + CORESIGHT_CLAIMCLR); + return csdev_access_relaxed_read32(&csdev->access, CORESIGHT_CLAIMCLR); } -static inline bool coresight_is_claimed_self_hosted(void __iomem *base) +static inline bool coresight_is_claimed_self_hosted(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) == CORESIGHT_CLAIM_SELF_HOSTED; + return coresight_read_claim_tags(csdev) == CORESIGHT_CLAIM_SELF_HOSTED; } -static inline bool coresight_is_claimed_any(void __iomem *base) +static inline bool coresight_is_claimed_any(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) != 0; + return coresight_read_claim_tags(csdev) != 0; } -static inline void coresight_set_claim_tags(void __iomem *base) +static inline void coresight_set_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMSET); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMSET); isb(); } -static inline void coresight_clear_claim_tags(void __iomem *base) +static inline void coresight_clear_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMCLR); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMCLR); isb(); } @@ -182,27 +184,33 @@ static inline void coresight_clear_claim_tags(void __iomem *base) * Called with CS_UNLOCKed for the component. * Returns : 0 on success */ -int coresight_claim_device_unlocked(void __iomem *base) +int coresight_claim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_any(base)) + if (WARN_ON(!csdev)) + return -EINVAL; + + if (coresight_is_claimed_any(csdev)) return -EBUSY; - coresight_set_claim_tags(base); - if (coresight_is_claimed_self_hosted(base)) + coresight_set_claim_tags(csdev); + if (coresight_is_claimed_self_hosted(csdev)) return 0; /* There was a race setting the tags, clean up and fail */ - coresight_clear_claim_tags(base); + coresight_clear_claim_tags(csdev); return -EBUSY; } EXPORT_SYMBOL_GPL(coresight_claim_device_unlocked); -int coresight_claim_device(void __iomem *base) +int coresight_claim_device(struct coresight_device *csdev) { int rc; - CS_UNLOCK(base); - rc = coresight_claim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return -EINVAL; + + CS_UNLOCK(csdev->access.base); + rc = coresight_claim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); return rc; } @@ -212,11 +220,14 @@ EXPORT_SYMBOL_GPL(coresight_claim_device); * coresight_disclaim_device_unlocked : Clear the claim tags for the device. * Called with CS_UNLOCKed for the component. */ -void coresight_disclaim_device_unlocked(void __iomem *base) +void coresight_disclaim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_self_hosted(base)) - coresight_clear_claim_tags(base); + if (WARN_ON(!csdev)) + return; + + if (coresight_is_claimed_self_hosted(csdev)) + coresight_clear_claim_tags(csdev); else /* * The external agent may have not honoured our claim @@ -227,11 +238,14 @@ void coresight_disclaim_device_unlocked(void __iomem *base) } EXPORT_SYMBOL_GPL(coresight_disclaim_device_unlocked); -void coresight_disclaim_device(void __iomem *base) +void coresight_disclaim_device(struct coresight_device *csdev) { - CS_UNLOCK(base); - coresight_disclaim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return; + + CS_UNLOCK(csdev->access.base); + coresight_disclaim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); } EXPORT_SYMBOL_GPL(coresight_disclaim_device); diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index b38a8db5f252..0c81eb9603ae 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -102,7 +102,7 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) goto cti_state_unchanged; /* claim the device */ - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (rc) goto cti_err_not_enabled; @@ -136,7 +136,7 @@ static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata) goto cti_hp_not_enabled; /* try to claim the device */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(drvdata->csdev)) goto cti_hp_not_enabled; cti_write_all_hw_regs(drvdata); @@ -154,6 +154,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) { struct cti_config *config = &drvdata->config; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; spin_lock(&drvdata->spinlock); @@ -171,7 +172,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) writel_relaxed(0, drvdata->base + CTICONTROL); config->hw_enabled = false; - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); pm_runtime_put(dev); @@ -655,6 +656,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { struct cti_drvdata *drvdata; + struct coresight_device *csdev; unsigned int cpu = smp_processor_id(); int notify_res = NOTIFY_OK; @@ -662,6 +664,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, return NOTIFY_OK; drvdata = cti_cpu_drvdata[cpu]; + csdev = drvdata->csdev; if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu)) return NOTIFY_BAD; @@ -673,13 +676,13 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* CTI regs all static - we have a copy & nothing to save */ drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(csdev); break; case CPU_PM_ENTER_FAILED: drvdata->config.hw_powered = true; if (drvdata->config.hw_enabled) { - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) drvdata->config.hw_enabled = false; } break; @@ -692,7 +695,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* check enable reference count to enable HW */ if (atomic_read(&drvdata->config.enable_req_count)) { /* check we can claim the device as we re-power */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) goto cti_notify_exit; drvdata->config.hw_enabled = true; @@ -736,7 +739,7 @@ static int cti_dying_cpu(unsigned int cpu) spin_lock(&drvdata->spinlock); drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); spin_unlock(&drvdata->spinlock); return 0; } diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 0f664aeeda93..74922c94f4b1 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -132,7 +132,7 @@ static void __etb_enable_hw(struct etb_drvdata *drvdata) static int etb_enable_hw(struct etb_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -345,7 +345,7 @@ static void etb_disable_hw(struct etb_drvdata *drvdata) { __etb_disable_hw(drvdata); etb_dump_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static int etb_disable(struct coresight_device *csdev) diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 3b7837cbe376..29d4dba4bee9 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -358,10 +358,11 @@ static int etm_enable_hw(struct etm_drvdata *drvdata) int i, rc; u32 etmcr; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; @@ -566,6 +567,7 @@ static void etm_disable_hw(void *info) int i; struct etm_drvdata *drvdata = info; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); etm_set_prog(drvdata); @@ -577,7 +579,7 @@ static void etm_disable_hw(void *info) config->cntr_val[i] = etm_readl(drvdata, ETMCNTVRn(i)); etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); @@ -602,7 +604,7 @@ static void etm_disable_perf(struct coresight_device *csdev) * power down the tracer. */ etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 180bb6ed9090..a041ad52737f 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -226,7 +226,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4_os_unlock(drvdata); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; @@ -635,7 +635,7 @@ static void etm4_disable_hw(void *info) readl_relaxed(drvdata->base + TRCCNTVRn(i)); } - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index 8f7c40d7d8d6..38bebdddcbff 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -52,13 +52,14 @@ static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) { u32 functl; int rc = 0; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); /* Claim the device only when we enable the first slave */ if (!(functl & FUNNEL_ENSx_MASK)) { - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; } @@ -101,6 +102,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) { u32 functl; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -110,7 +112,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, /* Disclaim the device if none of the slaves are now active */ if (!(functl & FUNNEL_ENSx_MASK)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index 205756fab729..a73fea9185b6 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -45,12 +45,14 @@ struct replicator_drvdata { static void dynamic_replicator_reset(struct replicator_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); - if (!coresight_claim_device_unlocked(drvdata->base)) { + if (!coresight_claim_device_unlocked(csdev)) { writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); } CS_LOCK(drvdata->base); @@ -70,6 +72,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, { int rc = 0; u32 id0val, id1val; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -84,7 +87,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, id0val = id1val = 0xff; if (id0val == 0xff && id1val == 0xff) - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (!rc) { switch (outport) { @@ -140,6 +143,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, int inport, int outport) { u32 reg; + struct coresight_device *csdev = drvdata->csdev; switch (outport) { case 0: @@ -160,7 +164,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, if ((readl_relaxed(drvdata->base + REPLICATOR_IDFILTER0) == 0xff) && (readl_relaxed(drvdata->base + REPLICATOR_IDFILTER1) == 0xff)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 989d965f3d90..45b85edfc690 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -37,7 +37,7 @@ static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -88,7 +88,7 @@ static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { __tmc_etb_disable_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) @@ -109,7 +109,7 @@ static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -120,11 +120,13 @@ static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); tmc_disable_hw(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index bf5230e39c5b..acdb59e0e661 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1040,7 +1040,7 @@ static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, rc = tmc_etr_enable_catu(drvdata, etr_buf); if (rc) return rc; - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (!rc) { drvdata->etr_buf = etr_buf; __tmc_etr_enable_hw(drvdata); @@ -1134,7 +1134,7 @@ void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) __tmc_etr_disable_hw(drvdata); /* Disable CATU device if this ETR is connected to one */ tmc_etr_disable_catu(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); /* Reset the ETR buf used by hardware */ drvdata->etr_buf = NULL; } diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 18bc7f9fb041..976ec2697610 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -463,11 +463,11 @@ extern void coresight_disable(struct coresight_device *csdev); extern int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value); -extern int coresight_claim_device(void __iomem *base); -extern int coresight_claim_device_unlocked(void __iomem *base); +extern int coresight_claim_device(struct coresight_device *csdev); +extern int coresight_claim_device_unlocked(struct coresight_device *csdev); -extern void coresight_disclaim_device(void __iomem *base); -extern void coresight_disclaim_device_unlocked(void __iomem *base); +extern void coresight_disclaim_device(struct coresight_device *csdev); +extern void coresight_disclaim_device_unlocked(struct coresight_device *csdev); extern char *coresight_alloc_device_name(struct coresight_dev_list *devs, struct device *dev); @@ -498,18 +498,18 @@ static inline int coresight_timeout(struct csdev_access *csa, u32 offset, return 1; } -static inline int coresight_claim_device_unlocked(void __iomem *base) +static inline int coresight_claim_device_unlocked(struct coresight_device *csdev) { return -EINVAL; } -static inline int coresight_claim_device(void __iomem *base) +static inline int coresight_claim_device(struct coresight_device *csdev) { return -EINVAL; } -static inline void coresight_disclaim_device(void __iomem *base) {} -static inline void coresight_disclaim_device_unlocked(void __iomem *base) {} +static inline void coresight_disclaim_device(struct coresight_device *csdev) {} +static inline void coresight_disclaim_device_unlocked(struct coresight_device *csdev) {} static inline bool coresight_loses_context_with_cpu(struct device *dev) { From 5e2acf9d5d2dffd668dab8899d9fc904f1051e07 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:29 -0700 Subject: [PATCH 247/633] coresight: etm4x: Always read the registers on the host CPU As we are about to add support for sysreg access to ETM4.4+ components, make sure that we read the registers only on the host CPU. Link: https://lore.kernel.org/r/20210110224850.1880240-8-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-10-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-sysfs.c | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index 989ce7b8ade7..c4781d4e5886 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2344,23 +2344,20 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset) return reg.data; } -#define coresight_etm4x_reg(name, offset) \ - coresight_simple_reg32(struct etmv4_drvdata, name, offset) - #define coresight_etm4x_cross_read(name, offset) \ coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read, \ name, offset) -coresight_etm4x_reg(trcpdcr, TRCPDCR); -coresight_etm4x_reg(trcpdsr, TRCPDSR); -coresight_etm4x_reg(trclsr, TRCLSR); -coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS); -coresight_etm4x_reg(trcdevid, TRCDEVID); -coresight_etm4x_reg(trcdevtype, TRCDEVTYPE); -coresight_etm4x_reg(trcpidr0, TRCPIDR0); -coresight_etm4x_reg(trcpidr1, TRCPIDR1); -coresight_etm4x_reg(trcpidr2, TRCPIDR2); -coresight_etm4x_reg(trcpidr3, TRCPIDR3); +coresight_etm4x_cross_read(trcpdcr, TRCPDCR); +coresight_etm4x_cross_read(trcpdsr, TRCPDSR); +coresight_etm4x_cross_read(trclsr, TRCLSR); +coresight_etm4x_cross_read(trcauthstatus, TRCAUTHSTATUS); +coresight_etm4x_cross_read(trcdevid, TRCDEVID); +coresight_etm4x_cross_read(trcdevtype, TRCDEVTYPE); +coresight_etm4x_cross_read(trcpidr0, TRCPIDR0); +coresight_etm4x_cross_read(trcpidr1, TRCPIDR1); +coresight_etm4x_cross_read(trcpidr2, TRCPIDR2); +coresight_etm4x_cross_read(trcpidr3, TRCPIDR3); coresight_etm4x_cross_read(trcoslsr, TRCOSLSR); coresight_etm4x_cross_read(trcconfig, TRCCONFIGR); coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR); From f5bd523690d2ff7bc4bd3f535888eaf9349be176 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:30 -0700 Subject: [PATCH 248/633] coresight: etm4x: Convert all register accesses Convert all register accesses from etm4x driver to use a wrapper to allow switching the access at runtime with little overhead. co-developed by sed tool ;-), mostly equivalent to : s/readl\(_relaxed\)\?(drvdata->base + \(.*\))/etm4x_\1_read32(csdev, \2) s/writel\(_relaxed\)\?(\(.*\), drvdata->base + \(.*\))/etm4x_\1_write32(csdev, \2, \3) We don't want to replace them with the csdev_access_* to avoid a function call for every register access for system register access. This is a prepartory step to add system register access later where the support is available. Link: https://lore.kernel.org/r/20210110224850.1880240-9-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-11-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 323 +++++++++--------- .../coresight/coresight-etm4x-sysfs.c | 9 +- drivers/hwtracing/coresight/coresight-etm4x.h | 24 ++ 3 files changed, 186 insertions(+), 170 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index a041ad52737f..9331281b931f 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -75,18 +75,28 @@ static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) (drvdata->config.ss_status[n] & TRCSSCSRn_PC); } -static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +static void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { /* Writing 0 to TRCOSLAR unlocks the trace registers */ - writel_relaxed(0x0, drvdata->base + TRCOSLAR); + etm4x_relaxed_write32(csa, 0x0, TRCOSLAR); drvdata->os_unlock = true; isb(); } +static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +{ + if (!WARN_ON(!drvdata->csdev)) + etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); + +} + static void etm4_os_lock(struct etmv4_drvdata *drvdata) { + if (WARN_ON(!drvdata->csdev)) + return; + /* Writing 0x1 to TRCOSLAR locks the trace registers */ - writel_relaxed(0x1, drvdata->base + TRCOSLAR); + etm4x_relaxed_write32(&drvdata->csdev->access, 0x1, TRCOSLAR); drvdata->os_unlock = false; isb(); } @@ -231,45 +241,39 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) goto done; /* Disable the trace unit before programming trace registers */ - writel_relaxed(0, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); /* wait for TRCSTATR.IDLE to go up */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); if (drvdata->nr_pe) - writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR); - writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR); + etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); + etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); /* nothing specific implemented */ - writel_relaxed(0x0, drvdata->base + TRCAUXCTLR); - writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R); - writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR); - writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR); - writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR); - writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR); - writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR); - writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR); - writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(config->vissctlr, - drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); + etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); + etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); + etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); + etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); + etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); + etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); + etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(config->vipcssctlr, - drvdata->base + TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(config->seq_ctrl[i], - drvdata->base + TRCSEQEVRn(i)); - writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR); - writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); + etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(config->cntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(config->cntr_ctrl[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(config->cntr_val[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); } /* @@ -277,52 +281,45 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) * such start at 2. */ for (i = 2; i < drvdata->nr_resource * 2; i++) - writel_relaxed(config->res_ctrl[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { /* always clear status bit on restart if using single-shot */ if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) config->ss_status[i] &= ~BIT(31); - writel_relaxed(config->ss_ctrl[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(config->ss_status[i], - drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); if (etm4x_sspcicrn_present(drvdata, i)) - writel_relaxed(config->ss_pe_cmp[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp; i++) { - writeq_relaxed(config->addr_val[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(config->addr_acc[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(config->ctxid_pid[i], - drvdata->base + TRCCIDCVRn(i)); - writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); + etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(config->vmid_val[i], - drvdata->base + TRCVMIDCVRn(i)); - writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); + etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); if (!drvdata->skip_power_up) { + u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); + /* * Request to keep the trace unit powered and also * emulation of powerdown */ - writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | - TRCPDCR_PU, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); } /* Enable the trace unit */ - writel_relaxed(1, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); /* wait for TRCSTATR.IDLE to go back down to '0' */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) @@ -599,12 +596,12 @@ static void etm4_disable_hw(void *info) if (!drvdata->skip_power_up) { /* power can be removed from the trace unit now */ - control = readl_relaxed(drvdata->base + TRCPDCR); + control = etm4x_relaxed_read32(csa, TRCPDCR); control &= ~TRCPDCR_PU; - writel_relaxed(control, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, control, TRCPDCR); } - control = readl_relaxed(drvdata->base + TRCPRGCTLR); + control = etm4x_relaxed_read32(csa, TRCPRGCTLR); /* EN, bit[0] Trace unit enable bit */ control &= ~0x1; @@ -616,7 +613,7 @@ static void etm4_disable_hw(void *info) */ dsb(sy); isb(); - writel_relaxed(control, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, control, TRCPRGCTLR); /* wait for TRCSTATR.PMSTABLE to go to '1' */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) @@ -626,13 +623,13 @@ static void etm4_disable_hw(void *info) /* read the status of the single shot comparators */ for (i = 0; i < drvdata->nr_ss_cmp; i++) { config->ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* read back the current counter values */ for (i = 0; i < drvdata->nr_cntr; i++) { config->cntr_val[i] = - readl_relaxed(drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_read32(csa, TRCCNTVRn(i)); } coresight_disclaim_device_unlocked(csdev); @@ -661,7 +658,7 @@ static int etm4_disable_perf(struct coresight_device *csdev, * scheduled again. Configuration of the start/stop logic happens in * function etm4_set_event_filters(). */ - control = readl_relaxed(drvdata->base + TRCVICTLR); + control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); /* TRCVICTLR::SSSTATUS, bit[9] */ filters->ssstatus = (control & BIT(9)); @@ -741,15 +738,17 @@ static void etm4_init_arch_data(void *info) u32 etmidr4; u32 etmidr5; struct etmv4_drvdata *drvdata = info; + struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); + struct csdev_access *csa = &tmp_csa; int i; /* Make sure all registers are accessible */ - etm4_os_unlock(drvdata); + etm4_os_unlock_csa(drvdata, csa); CS_UNLOCK(drvdata->base); /* find all capabilities of the tracing unit */ - etmidr0 = readl_relaxed(drvdata->base + TRCIDR0); + etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); /* INSTP0, bits[2:1] P0 tracing support field */ if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2)) @@ -789,7 +788,7 @@ static void etm4_init_arch_data(void *info) drvdata->ts_size = BMVAL(etmidr0, 24, 28); /* base architecture of trace unit */ - etmidr1 = readl_relaxed(drvdata->base + TRCIDR1); + etmidr1 = etm4x_relaxed_read32(csa, TRCIDR1); /* * TRCARCHMIN, bits[7:4] architecture the minor version number * TRCARCHMAJ, bits[11:8] architecture major versin number @@ -798,7 +797,7 @@ static void etm4_init_arch_data(void *info) drvdata->config.arch = drvdata->arch; /* maximum size of resources */ - etmidr2 = readl_relaxed(drvdata->base + TRCIDR2); + etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); /* CIDSIZE, bits[9:5] Indicates the Context ID size */ drvdata->ctxid_size = BMVAL(etmidr2, 5, 9); /* VMIDSIZE, bits[14:10] Indicates the VMID size */ @@ -806,7 +805,7 @@ static void etm4_init_arch_data(void *info) /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ drvdata->ccsize = BMVAL(etmidr2, 25, 28); - etmidr3 = readl_relaxed(drvdata->base + TRCIDR3); + etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ drvdata->ccitmin = BMVAL(etmidr3, 0, 11); /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ @@ -856,7 +855,7 @@ static void etm4_init_arch_data(void *info) drvdata->nooverflow = false; /* number of resources trace unit supports */ - etmidr4 = readl_relaxed(drvdata->base + TRCIDR4); + etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3); /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ @@ -882,14 +881,14 @@ static void etm4_init_arch_data(void *info) drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23); for (i = 0; i < drvdata->nr_ss_cmp; i++) { drvdata->config.ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ drvdata->numcidc = BMVAL(etmidr4, 24, 27); /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ drvdata->numvmidc = BMVAL(etmidr4, 28, 31); - etmidr5 = readl_relaxed(drvdata->base + TRCIDR5); + etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); /* NUMEXTIN, bits[8:0] number of external inputs implemented */ drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8); /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ @@ -1308,56 +1307,56 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state = drvdata->save_state; - state->trcprgctlr = readl(drvdata->base + TRCPRGCTLR); + state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); if (drvdata->nr_pe) - state->trcprocselr = readl(drvdata->base + TRCPROCSELR); - state->trcconfigr = readl(drvdata->base + TRCCONFIGR); - state->trcauxctlr = readl(drvdata->base + TRCAUXCTLR); - state->trceventctl0r = readl(drvdata->base + TRCEVENTCTL0R); - state->trceventctl1r = readl(drvdata->base + TRCEVENTCTL1R); - state->trcstallctlr = readl(drvdata->base + TRCSTALLCTLR); - state->trctsctlr = readl(drvdata->base + TRCTSCTLR); - state->trcsyncpr = readl(drvdata->base + TRCSYNCPR); - state->trcccctlr = readl(drvdata->base + TRCCCCTLR); - state->trcbbctlr = readl(drvdata->base + TRCBBCTLR); - state->trctraceidr = readl(drvdata->base + TRCTRACEIDR); - state->trcqctlr = readl(drvdata->base + TRCQCTLR); + state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); + state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); + state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); + state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); + state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); + state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); + state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); + state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); + state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); + state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); + state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); + state->trcqctlr = etm4x_read32(csa, TRCQCTLR); - state->trcvictlr = readl(drvdata->base + TRCVICTLR); - state->trcviiectlr = readl(drvdata->base + TRCVIIECTLR); - state->trcvissctlr = readl(drvdata->base + TRCVISSCTLR); + state->trcvictlr = etm4x_read32(csa, TRCVICTLR); + state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); + state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - state->trcvipcssctlr = readl(drvdata->base + TRCVIPCSSCTLR); - state->trcvdctlr = readl(drvdata->base + TRCVDCTLR); - state->trcvdsacctlr = readl(drvdata->base + TRCVDSACCTLR); - state->trcvdarcctlr = readl(drvdata->base + TRCVDARCCTLR); + state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); + state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); + state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); + state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - state->trcseqevr[i] = readl(drvdata->base + TRCSEQEVRn(i)); + state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); - state->trcseqrstevr = readl(drvdata->base + TRCSEQRSTEVR); - state->trcseqstr = readl(drvdata->base + TRCSEQSTR); - state->trcextinselr = readl(drvdata->base + TRCEXTINSELR); + state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); + state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); + state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - state->trccntrldvr[i] = readl(drvdata->base + TRCCNTRLDVRn(i)); - state->trccntctlr[i] = readl(drvdata->base + TRCCNTCTLRn(i)); - state->trccntvr[i] = readl(drvdata->base + TRCCNTVRn(i)); + state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); + state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); + state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - state->trcrsctlr[i] = readl(drvdata->base + TRCRSCTLRn(i)); + state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - state->trcssccr[i] = readl(drvdata->base + TRCSSCCRn(i)); - state->trcsscsr[i] = readl(drvdata->base + TRCSSCSRn(i)); + state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); + state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); if (etm4x_sspcicrn_present(drvdata, i)) - state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i)); + state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - state->trcacvr[i] = readq(drvdata->base + TRCACVRn(i)); - state->trcacatr[i] = readq(drvdata->base + TRCACATRn(i)); + state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); + state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); } /* @@ -1368,23 +1367,23 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) */ for (i = 0; i < drvdata->numcidc; i++) - state->trccidcvr[i] = readq(drvdata->base + TRCCIDCVRn(i)); + state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - state->trcvmidcvr[i] = readq(drvdata->base + TRCVMIDCVRn(i)); + state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); - state->trccidcctlr0 = readl(drvdata->base + TRCCIDCCTLR0); + state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - state->trccidcctlr1 = readl(drvdata->base + TRCCIDCCTLR1); + state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); - state->trcvmidcctlr0 = readl(drvdata->base + TRCVMIDCCTLR0); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - state->trcvmidcctlr1 = readl(drvdata->base + TRCVMIDCCTLR1); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); - state->trcclaimset = readl(drvdata->base + TRCCLAIMCLR); + state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); if (!drvdata->skip_power_up) - state->trcpdcr = readl(drvdata->base + TRCPDCR); + state->trcpdcr = etm4x_read32(csa, TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { @@ -1403,8 +1402,8 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * despite requesting software to save/restore state. */ if (!drvdata->skip_power_up) - writel_relaxed((state->trcpdcr & ~TRCPDCR_PU), - drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), + TRCPDCR); out: CS_LOCK(drvdata->base); return ret; @@ -1414,93 +1413,83 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) { int i; struct etmv4_save_state *state = drvdata->save_state; + struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); + struct csdev_access *csa = &tmp_csa; CS_UNLOCK(drvdata->base); - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); - writel_relaxed(state->trcprgctlr, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); if (drvdata->nr_pe) - writel_relaxed(state->trcprocselr, drvdata->base + TRCPROCSELR); - writel_relaxed(state->trcconfigr, drvdata->base + TRCCONFIGR); - writel_relaxed(state->trcauxctlr, drvdata->base + TRCAUXCTLR); - writel_relaxed(state->trceventctl0r, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(state->trceventctl1r, drvdata->base + TRCEVENTCTL1R); - writel_relaxed(state->trcstallctlr, drvdata->base + TRCSTALLCTLR); - writel_relaxed(state->trctsctlr, drvdata->base + TRCTSCTLR); - writel_relaxed(state->trcsyncpr, drvdata->base + TRCSYNCPR); - writel_relaxed(state->trcccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(state->trcbbctlr, drvdata->base + TRCBBCTLR); - writel_relaxed(state->trctraceidr, drvdata->base + TRCTRACEIDR); - writel_relaxed(state->trcqctlr, drvdata->base + TRCQCTLR); + etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); + etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); + etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); + etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); + etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); + etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); + etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); + etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); + etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); - writel_relaxed(state->trcvictlr, drvdata->base + TRCVICTLR); - writel_relaxed(state->trcviiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(state->trcvissctlr, drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); + etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(state->trcvipcssctlr, drvdata->base + TRCVIPCSSCTLR); - writel_relaxed(state->trcvdctlr, drvdata->base + TRCVDCTLR); - writel_relaxed(state->trcvdsacctlr, drvdata->base + TRCVDSACCTLR); - writel_relaxed(state->trcvdarcctlr, drvdata->base + TRCVDARCCTLR); + etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); + etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); + etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(state->trcseqevr[i], - drvdata->base + TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); - writel_relaxed(state->trcseqrstevr, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(state->trcseqstr, drvdata->base + TRCSEQSTR); - writel_relaxed(state->trcextinselr, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); + etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(state->trccntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(state->trccntctlr[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(state->trccntvr[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - writel_relaxed(state->trcrsctlr[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - writel_relaxed(state->trcssccr[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(state->trcsscsr[i], - drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); if (etm4x_sspcicrn_present(drvdata, i)) - writel_relaxed(state->trcsspcicr[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - writeq_relaxed(state->trcacvr[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(state->trcacatr[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(state->trccidcvr[i], - drvdata->base + TRCCIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(state->trcvmidcvr[i], - drvdata->base + TRCVMIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); - writel_relaxed(state->trccidcctlr0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(state->trccidcctlr1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); - writel_relaxed(state->trcvmidcctlr0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(state->trcvmidcctlr1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); if (!drvdata->skip_power_up) - writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); drvdata->state_needs_restore = false; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index c4781d4e5886..fce9df16bfb5 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2319,7 +2319,8 @@ static struct attribute *coresight_etmv4_attrs[] = { }; struct etmv4_reg { - void __iomem *addr; + struct coresight_device *csdev; + u32 offset; u32 data; }; @@ -2327,7 +2328,7 @@ static void do_smp_cross_read(void *data) { struct etmv4_reg *reg = data; - reg->data = readl_relaxed(reg->addr); + reg->data = etm4x_relaxed_read32(®->csdev->access, reg->offset); } static u32 etmv4_cross_read(const struct device *dev, u32 offset) @@ -2335,7 +2336,9 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset) struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); struct etmv4_reg reg; - reg.addr = drvdata->base + offset; + reg.offset = offset; + reg.csdev = drvdata->csdev; + /* * smp cross call ensures the CPU will be powered up before * accessing the ETMv4 trace core registers diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 80e480c7fe5c..b6854f6fd666 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -121,6 +121,30 @@ #define TRCCIDR2 0xFF8 #define TRCCIDR3 0xFFC +#define etm4x_relaxed_read32(csa, offset) \ + readl_relaxed((csa)->base + (offset)) + +#define etm4x_read32(csa, offset) \ + readl((csa)->base + (offset)) + +#define etm4x_relaxed_write32(csa, val, offset) \ + writel_relaxed((val), (csa)->base + (offset)) + +#define etm4x_write32(csa, val, offset) \ + writel((val), (csa)->base + (offset)) + +#define etm4x_relaxed_read64(csa, offset) \ + readq_relaxed((csa)->base + (offset)) + +#define etm4x_read64(csa, offset) \ + readq((csa)->base + (offset)) + +#define etm4x_relaxed_write64(csa, val, offset) \ + writeq_relaxed((val), (csa)->base + (offset)) + +#define etm4x_write64(csa, val, offset) \ + writeq((val), (csa)->base + (offset)) + /* ETMv4 resources */ #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 From c03ceec116ceaa5d3369d6df4849bd187e6b78b5 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:31 -0700 Subject: [PATCH 249/633] coresight: etm4x: Make offset available for sysfs attributes Some of the ETM management registers are not accessible via system instructions. Thus we need to filter accesses to these registers depending on the access mechanism for the ETM at runtime. The driver can cope with this for normal operation, by regular checks. But the driver also exposes them via sysfs, which now needs to be removed. So far, we have used the generic coresight sysfs helper macros to export a given device register, defining a "show" operation per register. This is not helpful to filter the files at runtime, based on the access. In order to do this dynamically, we need to filter the attributes by offsets and hard coded "show" functions doesn't make this easy. Thus, switch to extended attributes, storing the offset in the scratch space. This allows us to implement filtering based on the offset and also saves us some text size. This will be later used for determining a given attribute must be "visible" via sysfs. Link: https://lore.kernel.org/r/20210110224850.1880240-10-suzuki.poulose@arm.com Cc: Mathieu Poirier Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-12-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-sysfs.c | 113 +++++++++--------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index fce9df16bfb5..ddbfeb24fc3f 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2331,9 +2331,8 @@ static void do_smp_cross_read(void *data) reg->data = etm4x_relaxed_read32(®->csdev->access, reg->offset); } -static u32 etmv4_cross_read(const struct device *dev, u32 offset) +static u32 etmv4_cross_read(const struct etmv4_drvdata *drvdata, u32 offset) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); struct etmv4_reg reg; reg.offset = offset; @@ -2347,69 +2346,69 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset) return reg.data; } -#define coresight_etm4x_cross_read(name, offset) \ - coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read, \ - name, offset) +static inline u32 coresight_etm4x_attr_to_offset(struct device_attribute *attr) +{ + struct dev_ext_attribute *eattr; -coresight_etm4x_cross_read(trcpdcr, TRCPDCR); -coresight_etm4x_cross_read(trcpdsr, TRCPDSR); -coresight_etm4x_cross_read(trclsr, TRCLSR); -coresight_etm4x_cross_read(trcauthstatus, TRCAUTHSTATUS); -coresight_etm4x_cross_read(trcdevid, TRCDEVID); -coresight_etm4x_cross_read(trcdevtype, TRCDEVTYPE); -coresight_etm4x_cross_read(trcpidr0, TRCPIDR0); -coresight_etm4x_cross_read(trcpidr1, TRCPIDR1); -coresight_etm4x_cross_read(trcpidr2, TRCPIDR2); -coresight_etm4x_cross_read(trcpidr3, TRCPIDR3); -coresight_etm4x_cross_read(trcoslsr, TRCOSLSR); -coresight_etm4x_cross_read(trcconfig, TRCCONFIGR); -coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR); + eattr = container_of(attr, struct dev_ext_attribute, attr); + return (u32)(unsigned long)eattr->var; +} + +static ssize_t coresight_etm4x_reg_show(struct device *dev, + struct device_attribute *d_attr, + char *buf) +{ + u32 val, offset; + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + + offset = coresight_etm4x_attr_to_offset(d_attr); + + pm_runtime_get_sync(dev->parent); + val = etmv4_cross_read(drvdata, offset); + pm_runtime_put_sync(dev->parent); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); +} + +#define coresight_etm4x_reg(name, offset) \ + &((struct dev_ext_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_etm4x_reg_show, NULL), \ + (void *)(unsigned long)offset \ + } \ + })[0].attr.attr static struct attribute *coresight_etmv4_mgmt_attrs[] = { - &dev_attr_trcoslsr.attr, - &dev_attr_trcpdcr.attr, - &dev_attr_trcpdsr.attr, - &dev_attr_trclsr.attr, - &dev_attr_trcconfig.attr, - &dev_attr_trctraceid.attr, - &dev_attr_trcauthstatus.attr, - &dev_attr_trcdevid.attr, - &dev_attr_trcdevtype.attr, - &dev_attr_trcpidr0.attr, - &dev_attr_trcpidr1.attr, - &dev_attr_trcpidr2.attr, - &dev_attr_trcpidr3.attr, + coresight_etm4x_reg(trcpdcr, TRCPDCR), + coresight_etm4x_reg(trcpdsr, TRCPDSR), + coresight_etm4x_reg(trclsr, TRCLSR), + coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS), + coresight_etm4x_reg(trcdevid, TRCDEVID), + coresight_etm4x_reg(trcdevtype, TRCDEVTYPE), + coresight_etm4x_reg(trcpidr0, TRCPIDR0), + coresight_etm4x_reg(trcpidr1, TRCPIDR1), + coresight_etm4x_reg(trcpidr2, TRCPIDR2), + coresight_etm4x_reg(trcpidr3, TRCPIDR3), + coresight_etm4x_reg(trcoslsr, TRCOSLSR), + coresight_etm4x_reg(trcconfig, TRCCONFIGR), + coresight_etm4x_reg(trctraceid, TRCTRACEIDR), NULL, }; -coresight_etm4x_cross_read(trcidr0, TRCIDR0); -coresight_etm4x_cross_read(trcidr1, TRCIDR1); -coresight_etm4x_cross_read(trcidr2, TRCIDR2); -coresight_etm4x_cross_read(trcidr3, TRCIDR3); -coresight_etm4x_cross_read(trcidr4, TRCIDR4); -coresight_etm4x_cross_read(trcidr5, TRCIDR5); -/* trcidr[6,7] are reserved */ -coresight_etm4x_cross_read(trcidr8, TRCIDR8); -coresight_etm4x_cross_read(trcidr9, TRCIDR9); -coresight_etm4x_cross_read(trcidr10, TRCIDR10); -coresight_etm4x_cross_read(trcidr11, TRCIDR11); -coresight_etm4x_cross_read(trcidr12, TRCIDR12); -coresight_etm4x_cross_read(trcidr13, TRCIDR13); - static struct attribute *coresight_etmv4_trcidr_attrs[] = { - &dev_attr_trcidr0.attr, - &dev_attr_trcidr1.attr, - &dev_attr_trcidr2.attr, - &dev_attr_trcidr3.attr, - &dev_attr_trcidr4.attr, - &dev_attr_trcidr5.attr, + coresight_etm4x_reg(trcidr0, TRCIDR0), + coresight_etm4x_reg(trcidr1, TRCIDR1), + coresight_etm4x_reg(trcidr2, TRCIDR2), + coresight_etm4x_reg(trcidr3, TRCIDR3), + coresight_etm4x_reg(trcidr4, TRCIDR4), + coresight_etm4x_reg(trcidr5, TRCIDR5), /* trcidr[6,7] are reserved */ - &dev_attr_trcidr8.attr, - &dev_attr_trcidr9.attr, - &dev_attr_trcidr10.attr, - &dev_attr_trcidr11.attr, - &dev_attr_trcidr12.attr, - &dev_attr_trcidr13.attr, + coresight_etm4x_reg(trcidr8, TRCIDR8), + coresight_etm4x_reg(trcidr9, TRCIDR9), + coresight_etm4x_reg(trcidr10, TRCIDR10), + coresight_etm4x_reg(trcidr11, TRCIDR11), + coresight_etm4x_reg(trcidr12, TRCIDR12), + coresight_etm4x_reg(trcidr13, TRCIDR13), NULL, }; From 4f2a67266ab3dd29d3087b3297014a17367f2b26 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:32 -0700 Subject: [PATCH 250/633] coresight: etm4x: Add commentary on the registers As we are about define a switch..case table for individual register access by offset for implementing the system instruction support, document the possible set of registers for each group to make it easier to correlate. Link: https://lore.kernel.org/r/20210110224850.1880240-11-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-13-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x.h | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index b6854f6fd666..3c2b49ffabc8 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -45,13 +45,13 @@ #define TRCVDSACCTLR 0x0A4 #define TRCVDARCCTLR 0x0A8 /* Derived resources registers */ -#define TRCSEQEVRn(n) (0x100 + (n * 4)) +#define TRCSEQEVRn(n) (0x100 + (n * 4)) /* n = 0-2 */ #define TRCSEQRSTEVR 0x118 #define TRCSEQSTR 0x11C #define TRCEXTINSELR 0x120 -#define TRCCNTRLDVRn(n) (0x140 + (n * 4)) -#define TRCCNTCTLRn(n) (0x150 + (n * 4)) -#define TRCCNTVRn(n) (0x160 + (n * 4)) +#define TRCCNTRLDVRn(n) (0x140 + (n * 4)) /* n = 0-3 */ +#define TRCCNTCTLRn(n) (0x150 + (n * 4)) /* n = 0-3 */ +#define TRCCNTVRn(n) (0x160 + (n * 4)) /* n = 0-3 */ /* ID registers */ #define TRCIDR8 0x180 #define TRCIDR9 0x184 @@ -60,7 +60,7 @@ #define TRCIDR12 0x190 #define TRCIDR13 0x194 #define TRCIMSPEC0 0x1C0 -#define TRCIMSPECn(n) (0x1C0 + (n * 4)) +#define TRCIMSPECn(n) (0x1C0 + (n * 4)) /* n = 1-7 */ #define TRCIDR0 0x1E0 #define TRCIDR1 0x1E4 #define TRCIDR2 0x1E8 @@ -69,9 +69,12 @@ #define TRCIDR5 0x1F4 #define TRCIDR6 0x1F8 #define TRCIDR7 0x1FC -/* Resource selection registers */ +/* + * Resource selection registers, n = 2-31. + * First pair (regs 0, 1) is always present and is reserved. + */ #define TRCRSCTLRn(n) (0x200 + (n * 4)) -/* Single-shot comparator registers */ +/* Single-shot comparator registers, n = 0-7 */ #define TRCSSCCRn(n) (0x280 + (n * 4)) #define TRCSSCSRn(n) (0x2A0 + (n * 4)) #define TRCSSPCICRn(n) (0x2C0 + (n * 4)) @@ -81,11 +84,13 @@ #define TRCPDCR 0x310 #define TRCPDSR 0x314 /* Trace registers (0x318-0xEFC) */ -/* Comparator registers */ +/* Address Comparator registers n = 0-15 */ #define TRCACVRn(n) (0x400 + (n * 8)) #define TRCACATRn(n) (0x480 + (n * 8)) +/* Data Value Comparator Value registers, n = 0-7 */ #define TRCDVCVRn(n) (0x500 + (n * 16)) #define TRCDVCMRn(n) (0x580 + (n * 16)) +/* ContextID/Virtual ContextID comparators, n = 0-7 */ #define TRCCIDCVRn(n) (0x600 + (n * 8)) #define TRCVMIDCVRn(n) (0x640 + (n * 8)) #define TRCCIDCCTLR0 0x680 From 03336d0f4d0d74e7a5f899bf9d4933e0d4397c94 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:33 -0700 Subject: [PATCH 251/633] coresight: etm4x: Add sysreg access helpers ETM architecture defines the system instructions for accessing via register accesses. Add basic support for accessing a given register via system instructions. We split the list of registers as : 1) Accessible only from memory mapped interface 2) Accessible from system register instructions. All registers are accessible via the memory-mapped interface. However, some registers are not accessible via the system instructions. This list is then used to further filter out the files we expose via sysfs. Link: https://lore.kernel.org/r/20210110224850.1880240-12-suzuki.poulose@arm.com Cc: Mike Leach Cc: Mathieu Poirier Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-14-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 32 ++ drivers/hwtracing/coresight/coresight-etm4x.h | 331 +++++++++++++++++- 2 files changed, 347 insertions(+), 16 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 9331281b931f..37f90d380436 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -75,6 +75,38 @@ static inline bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) (drvdata->config.ss_status[n] & TRCSSCSRn_PC); } +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) +{ + u64 res = 0; + + switch (offset) { + ETM4x_READ_SYSREG_CASES(res) + default : + pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", + offset); + } + + if (!_relaxed) + __iormb(res); /* Imitate the !relaxed I/O helpers */ + + return res; +} + +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) +{ + if (!_relaxed) + __iowmb(); /* Imitate the !relaxed I/O helpers */ + if (!_64bit) + val &= GENMASK(31, 0); + + switch (offset) { + ETM4x_WRITE_SYSREG_CASES(val) + default : + pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", + offset); + } +} + static void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { /* Writing 0 to TRCOSLAR unlocks the trace registers */ diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 3c2b49ffabc8..24ba0da5b096 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -126,29 +126,325 @@ #define TRCCIDR2 0xFF8 #define TRCCIDR3 0xFFC -#define etm4x_relaxed_read32(csa, offset) \ - readl_relaxed((csa)->base + (offset)) +/* + * System instructions to access ETM registers. + * See ETMv4.4 spec ARM IHI0064F section 4.3.6 System instructions + */ +#define ETM4x_OFFSET_TO_REG(x) ((x) >> 2) -#define etm4x_read32(csa, offset) \ - readl((csa)->base + (offset)) +#define ETM4x_CRn(n) (((n) >> 7) & 0x7) +#define ETM4x_Op2(n) (((n) >> 4) & 0x7) +#define ETM4x_CRm(n) ((n) & 0xf) -#define etm4x_relaxed_write32(csa, val, offset) \ - writel_relaxed((val), (csa)->base + (offset)) +#include +#define ETM4x_REG_NUM_TO_SYSREG(n) \ + sys_reg(2, 1, ETM4x_CRn(n), ETM4x_CRm(n), ETM4x_Op2(n)) -#define etm4x_write32(csa, val, offset) \ - writel((val), (csa)->base + (offset)) +#define READ_ETM4x_REG(reg) \ + read_sysreg_s(ETM4x_REG_NUM_TO_SYSREG((reg))) +#define WRITE_ETM4x_REG(val, reg) \ + write_sysreg_s(val, ETM4x_REG_NUM_TO_SYSREG((reg))) -#define etm4x_relaxed_read64(csa, offset) \ - readq_relaxed((csa)->base + (offset)) +#define read_etm4x_sysreg_const_offset(offset) \ + READ_ETM4x_REG(ETM4x_OFFSET_TO_REG(offset)) -#define etm4x_read64(csa, offset) \ - readq((csa)->base + (offset)) +#define write_etm4x_sysreg_const_offset(val, offset) \ + WRITE_ETM4x_REG(val, ETM4x_OFFSET_TO_REG(offset)) -#define etm4x_relaxed_write64(csa, val, offset) \ - writeq_relaxed((val), (csa)->base + (offset)) +#define CASE_READ(res, x) \ + case (x): { (res) = read_etm4x_sysreg_const_offset((x)); break; } + +#define CASE_WRITE(val, x) \ + case (x): { write_etm4x_sysreg_const_offset((val), (x)); break; } + +/* List of registers accessible via System instructions */ +#define ETM_SYSREG_LIST(op, val) \ + CASE_##op((val), TRCPRGCTLR) \ + CASE_##op((val), TRCPROCSELR) \ + CASE_##op((val), TRCSTATR) \ + CASE_##op((val), TRCCONFIGR) \ + CASE_##op((val), TRCAUXCTLR) \ + CASE_##op((val), TRCEVENTCTL0R) \ + CASE_##op((val), TRCEVENTCTL1R) \ + CASE_##op((val), TRCSTALLCTLR) \ + CASE_##op((val), TRCTSCTLR) \ + CASE_##op((val), TRCSYNCPR) \ + CASE_##op((val), TRCCCCTLR) \ + CASE_##op((val), TRCBBCTLR) \ + CASE_##op((val), TRCTRACEIDR) \ + CASE_##op((val), TRCQCTLR) \ + CASE_##op((val), TRCVICTLR) \ + CASE_##op((val), TRCVIIECTLR) \ + CASE_##op((val), TRCVISSCTLR) \ + CASE_##op((val), TRCVIPCSSCTLR) \ + CASE_##op((val), TRCVDCTLR) \ + CASE_##op((val), TRCVDSACCTLR) \ + CASE_##op((val), TRCVDARCCTLR) \ + CASE_##op((val), TRCSEQEVRn(0)) \ + CASE_##op((val), TRCSEQEVRn(1)) \ + CASE_##op((val), TRCSEQEVRn(2)) \ + CASE_##op((val), TRCSEQRSTEVR) \ + CASE_##op((val), TRCSEQSTR) \ + CASE_##op((val), TRCEXTINSELR) \ + CASE_##op((val), TRCCNTRLDVRn(0)) \ + CASE_##op((val), TRCCNTRLDVRn(1)) \ + CASE_##op((val), TRCCNTRLDVRn(2)) \ + CASE_##op((val), TRCCNTRLDVRn(3)) \ + CASE_##op((val), TRCCNTCTLRn(0)) \ + CASE_##op((val), TRCCNTCTLRn(1)) \ + CASE_##op((val), TRCCNTCTLRn(2)) \ + CASE_##op((val), TRCCNTCTLRn(3)) \ + CASE_##op((val), TRCCNTVRn(0)) \ + CASE_##op((val), TRCCNTVRn(1)) \ + CASE_##op((val), TRCCNTVRn(2)) \ + CASE_##op((val), TRCCNTVRn(3)) \ + CASE_##op((val), TRCIDR8) \ + CASE_##op((val), TRCIDR9) \ + CASE_##op((val), TRCIDR10) \ + CASE_##op((val), TRCIDR11) \ + CASE_##op((val), TRCIDR12) \ + CASE_##op((val), TRCIDR13) \ + CASE_##op((val), TRCIMSPECn(0)) \ + CASE_##op((val), TRCIMSPECn(1)) \ + CASE_##op((val), TRCIMSPECn(2)) \ + CASE_##op((val), TRCIMSPECn(3)) \ + CASE_##op((val), TRCIMSPECn(4)) \ + CASE_##op((val), TRCIMSPECn(5)) \ + CASE_##op((val), TRCIMSPECn(6)) \ + CASE_##op((val), TRCIMSPECn(7)) \ + CASE_##op((val), TRCIDR0) \ + CASE_##op((val), TRCIDR1) \ + CASE_##op((val), TRCIDR2) \ + CASE_##op((val), TRCIDR3) \ + CASE_##op((val), TRCIDR4) \ + CASE_##op((val), TRCIDR5) \ + CASE_##op((val), TRCIDR6) \ + CASE_##op((val), TRCIDR7) \ + CASE_##op((val), TRCRSCTLRn(2)) \ + CASE_##op((val), TRCRSCTLRn(3)) \ + CASE_##op((val), TRCRSCTLRn(4)) \ + CASE_##op((val), TRCRSCTLRn(5)) \ + CASE_##op((val), TRCRSCTLRn(6)) \ + CASE_##op((val), TRCRSCTLRn(7)) \ + CASE_##op((val), TRCRSCTLRn(8)) \ + CASE_##op((val), TRCRSCTLRn(9)) \ + CASE_##op((val), TRCRSCTLRn(10)) \ + CASE_##op((val), TRCRSCTLRn(11)) \ + CASE_##op((val), TRCRSCTLRn(12)) \ + CASE_##op((val), TRCRSCTLRn(13)) \ + CASE_##op((val), TRCRSCTLRn(14)) \ + CASE_##op((val), TRCRSCTLRn(15)) \ + CASE_##op((val), TRCRSCTLRn(16)) \ + CASE_##op((val), TRCRSCTLRn(17)) \ + CASE_##op((val), TRCRSCTLRn(18)) \ + CASE_##op((val), TRCRSCTLRn(19)) \ + CASE_##op((val), TRCRSCTLRn(20)) \ + CASE_##op((val), TRCRSCTLRn(21)) \ + CASE_##op((val), TRCRSCTLRn(22)) \ + CASE_##op((val), TRCRSCTLRn(23)) \ + CASE_##op((val), TRCRSCTLRn(24)) \ + CASE_##op((val), TRCRSCTLRn(25)) \ + CASE_##op((val), TRCRSCTLRn(26)) \ + CASE_##op((val), TRCRSCTLRn(27)) \ + CASE_##op((val), TRCRSCTLRn(28)) \ + CASE_##op((val), TRCRSCTLRn(29)) \ + CASE_##op((val), TRCRSCTLRn(30)) \ + CASE_##op((val), TRCRSCTLRn(31)) \ + CASE_##op((val), TRCSSCCRn(0)) \ + CASE_##op((val), TRCSSCCRn(1)) \ + CASE_##op((val), TRCSSCCRn(2)) \ + CASE_##op((val), TRCSSCCRn(3)) \ + CASE_##op((val), TRCSSCCRn(4)) \ + CASE_##op((val), TRCSSCCRn(5)) \ + CASE_##op((val), TRCSSCCRn(6)) \ + CASE_##op((val), TRCSSCCRn(7)) \ + CASE_##op((val), TRCSSCSRn(0)) \ + CASE_##op((val), TRCSSCSRn(1)) \ + CASE_##op((val), TRCSSCSRn(2)) \ + CASE_##op((val), TRCSSCSRn(3)) \ + CASE_##op((val), TRCSSCSRn(4)) \ + CASE_##op((val), TRCSSCSRn(5)) \ + CASE_##op((val), TRCSSCSRn(6)) \ + CASE_##op((val), TRCSSCSRn(7)) \ + CASE_##op((val), TRCSSPCICRn(0)) \ + CASE_##op((val), TRCSSPCICRn(1)) \ + CASE_##op((val), TRCSSPCICRn(2)) \ + CASE_##op((val), TRCSSPCICRn(3)) \ + CASE_##op((val), TRCSSPCICRn(4)) \ + CASE_##op((val), TRCSSPCICRn(5)) \ + CASE_##op((val), TRCSSPCICRn(6)) \ + CASE_##op((val), TRCSSPCICRn(7)) \ + CASE_##op((val), TRCOSLAR) \ + CASE_##op((val), TRCOSLSR) \ + CASE_##op((val), TRCACVRn(0)) \ + CASE_##op((val), TRCACVRn(1)) \ + CASE_##op((val), TRCACVRn(2)) \ + CASE_##op((val), TRCACVRn(3)) \ + CASE_##op((val), TRCACVRn(4)) \ + CASE_##op((val), TRCACVRn(5)) \ + CASE_##op((val), TRCACVRn(6)) \ + CASE_##op((val), TRCACVRn(7)) \ + CASE_##op((val), TRCACVRn(8)) \ + CASE_##op((val), TRCACVRn(9)) \ + CASE_##op((val), TRCACVRn(10)) \ + CASE_##op((val), TRCACVRn(11)) \ + CASE_##op((val), TRCACVRn(12)) \ + CASE_##op((val), TRCACVRn(13)) \ + CASE_##op((val), TRCACVRn(14)) \ + CASE_##op((val), TRCACVRn(15)) \ + CASE_##op((val), TRCACATRn(0)) \ + CASE_##op((val), TRCACATRn(1)) \ + CASE_##op((val), TRCACATRn(2)) \ + CASE_##op((val), TRCACATRn(3)) \ + CASE_##op((val), TRCACATRn(4)) \ + CASE_##op((val), TRCACATRn(5)) \ + CASE_##op((val), TRCACATRn(6)) \ + CASE_##op((val), TRCACATRn(7)) \ + CASE_##op((val), TRCACATRn(8)) \ + CASE_##op((val), TRCACATRn(9)) \ + CASE_##op((val), TRCACATRn(10)) \ + CASE_##op((val), TRCACATRn(11)) \ + CASE_##op((val), TRCACATRn(12)) \ + CASE_##op((val), TRCACATRn(13)) \ + CASE_##op((val), TRCACATRn(14)) \ + CASE_##op((val), TRCACATRn(15)) \ + CASE_##op((val), TRCDVCVRn(0)) \ + CASE_##op((val), TRCDVCVRn(1)) \ + CASE_##op((val), TRCDVCVRn(2)) \ + CASE_##op((val), TRCDVCVRn(3)) \ + CASE_##op((val), TRCDVCVRn(4)) \ + CASE_##op((val), TRCDVCVRn(5)) \ + CASE_##op((val), TRCDVCVRn(6)) \ + CASE_##op((val), TRCDVCVRn(7)) \ + CASE_##op((val), TRCDVCMRn(0)) \ + CASE_##op((val), TRCDVCMRn(1)) \ + CASE_##op((val), TRCDVCMRn(2)) \ + CASE_##op((val), TRCDVCMRn(3)) \ + CASE_##op((val), TRCDVCMRn(4)) \ + CASE_##op((val), TRCDVCMRn(5)) \ + CASE_##op((val), TRCDVCMRn(6)) \ + CASE_##op((val), TRCDVCMRn(7)) \ + CASE_##op((val), TRCCIDCVRn(0)) \ + CASE_##op((val), TRCCIDCVRn(1)) \ + CASE_##op((val), TRCCIDCVRn(2)) \ + CASE_##op((val), TRCCIDCVRn(3)) \ + CASE_##op((val), TRCCIDCVRn(4)) \ + CASE_##op((val), TRCCIDCVRn(5)) \ + CASE_##op((val), TRCCIDCVRn(6)) \ + CASE_##op((val), TRCCIDCVRn(7)) \ + CASE_##op((val), TRCVMIDCVRn(0)) \ + CASE_##op((val), TRCVMIDCVRn(1)) \ + CASE_##op((val), TRCVMIDCVRn(2)) \ + CASE_##op((val), TRCVMIDCVRn(3)) \ + CASE_##op((val), TRCVMIDCVRn(4)) \ + CASE_##op((val), TRCVMIDCVRn(5)) \ + CASE_##op((val), TRCVMIDCVRn(6)) \ + CASE_##op((val), TRCVMIDCVRn(7)) \ + CASE_##op((val), TRCCIDCCTLR0) \ + CASE_##op((val), TRCCIDCCTLR1) \ + CASE_##op((val), TRCVMIDCCTLR0) \ + CASE_##op((val), TRCVMIDCCTLR1) \ + CASE_##op((val), TRCCLAIMSET) \ + CASE_##op((val), TRCCLAIMCLR) \ + CASE_##op((val), TRCAUTHSTATUS) \ + CASE_##op((val), TRCDEVARCH) \ + CASE_##op((val), TRCDEVID) + +/* List of registers only accessible via memory-mapped interface */ +#define ETM_MMAP_LIST(op, val) \ + CASE_##op((val), TRCDEVTYPE) \ + CASE_##op((val), TRCPDCR) \ + CASE_##op((val), TRCPDSR) \ + CASE_##op((val), TRCDEVAFF0) \ + CASE_##op((val), TRCDEVAFF1) \ + CASE_##op((val), TRCLAR) \ + CASE_##op((val), TRCLSR) \ + CASE_##op((val), TRCITCTRL) \ + CASE_##op((val), TRCPIDR4) \ + CASE_##op((val), TRCPIDR0) \ + CASE_##op((val), TRCPIDR1) \ + CASE_##op((val), TRCPIDR2) \ + CASE_##op((val), TRCPIDR3) + +#define ETM4x_READ_SYSREG_CASES(res) ETM_SYSREG_LIST(READ, (res)) +#define ETM4x_WRITE_SYSREG_CASES(val) ETM_SYSREG_LIST(WRITE, (val)) + +#define read_etm4x_sysreg_offset(offset, _64bit) \ + ({ \ + u64 __val; \ + \ + if (__builtin_constant_p((offset))) \ + __val = read_etm4x_sysreg_const_offset((offset)); \ + else \ + __val = etm4x_sysreg_read((offset), true, (_64bit)); \ + __val; \ + }) + +#define write_etm4x_sysreg_offset(val, offset, _64bit) \ + do { \ + if (__builtin_constant_p((offset))) \ + write_etm4x_sysreg_const_offset((val), \ + (offset)); \ + else \ + etm4x_sysreg_write((val), (offset), true, \ + (_64bit)); \ + } while (0) + + +#define etm4x_relaxed_read32(csa, offset) \ + ((u32)((csa)->io_mem ? \ + readl_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), false))) + +#define etm4x_relaxed_read64(csa, offset) \ + ((u64)((csa)->io_mem ? \ + readq_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), true))) + +#define etm4x_read32(csa, offset) \ + ({ \ + u32 __val = etm4x_relaxed_read32((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_read64(csa, offset) \ + ({ \ + u64 __val = etm4x_relaxed_read64((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_relaxed_write32(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writel_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + false); \ + } while (0) + +#define etm4x_relaxed_write64(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writeq_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + true); \ + } while (0) + +#define etm4x_write32(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write32((csa), (val), (offset)); \ + } while (0) + +#define etm4x_write64(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write64((csa), (val), (offset)); \ + } while (0) -#define etm4x_write64(csa, val, offset) \ - writeq((val), (csa)->base + (offset)) /* ETMv4 resources */ #define ETM_MAX_NR_PE 8 @@ -522,4 +818,7 @@ enum etm_addr_ctxtype { extern const struct attribute_group *coresight_etmv4_groups[]; void etm4_config_trace_mode(struct etmv4_config *config); + +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit); +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit); #endif From 91b9f018548747dfa6d543d74dfe59a8c6e9be7e Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:34 -0700 Subject: [PATCH 252/633] coresight: etm4x: Hide sysfs attributes for unavailable registers Some of the management registers in ETMv4.x are not accessible via system register instructions. Thus we must hide the sysfs files exposing them to the userspace, to prevent system crashes. This patch adds an is_visible() routine to control the visibility at runtime for the registers that may not be accessed. Link: https://lore.kernel.org/r/20210110224850.1880240-13-suzuki.poulose@arm.com Cc: Mathieu Poirier Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-15-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-sysfs.c | 51 +++++++++++++++++++ drivers/hwtracing/coresight/coresight-etm4x.h | 6 +++ 2 files changed, 57 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index ddbfeb24fc3f..e8fdda45ffca 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2370,6 +2370,56 @@ static ssize_t coresight_etm4x_reg_show(struct device *dev, return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); } +static inline bool +etm4x_register_implemented(struct etmv4_drvdata *drvdata, u32 offset) +{ + switch (offset) { + ETM4x_SYSREG_LIST_CASES + /* + * Registers accessible via system instructions are always + * implemented. + */ + return true; + ETM4x_MMAP_LIST_CASES + /* + * Registers accessible only via memory-mapped registers + * must not be accessed via system instructions. + * We cannot access the drvdata->csdev here, as this + * function is called during the device creation, via + * coresight_register() and the csdev is not initialized + * until that is done. So rely on the drvdata->base to + * detect if we have a memory mapped access. + */ + return !!drvdata->base; + } + + return false; +} + +/* + * Hide the ETM4x registers that may not be available on the + * hardware. + * There are certain management registers unavailable via system + * instructions. Make those sysfs attributes hidden on such + * systems. + */ +static umode_t +coresight_etm4x_attr_reg_implemented(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct device *dev = kobj_to_dev(kobj); + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct device_attribute *d_attr; + u32 offset; + + d_attr = container_of(attr, struct device_attribute, attr); + offset = coresight_etm4x_attr_to_offset(d_attr); + + if (etm4x_register_implemented(drvdata, offset)) + return attr->mode; + return 0; +} + #define coresight_etm4x_reg(name, offset) \ &((struct dev_ext_attribute[]) { \ { \ @@ -2417,6 +2467,7 @@ static const struct attribute_group coresight_etmv4_group = { }; static const struct attribute_group coresight_etmv4_mgmt_group = { + .is_visible = coresight_etm4x_attr_reg_implemented, .attrs = coresight_etmv4_mgmt_attrs, .name = "mgmt", }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 24ba0da5b096..193d2819afa7 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -157,6 +157,9 @@ #define CASE_WRITE(val, x) \ case (x): { write_etm4x_sysreg_const_offset((val), (x)); break; } +#define CASE_NOP(__unused, x) \ + case (x): /* fall through */ + /* List of registers accessible via System instructions */ #define ETM_SYSREG_LIST(op, val) \ CASE_##op((val), TRCPRGCTLR) \ @@ -369,6 +372,9 @@ #define ETM4x_READ_SYSREG_CASES(res) ETM_SYSREG_LIST(READ, (res)) #define ETM4x_WRITE_SYSREG_CASES(val) ETM_SYSREG_LIST(WRITE, (val)) +#define ETM4x_SYSREG_LIST_CASES ETM_SYSREG_LIST(NOP, __unused) +#define ETM4x_MMAP_LIST_CASES ETM_MMAP_LIST(NOP, __unused) + #define read_etm4x_sysreg_offset(offset, _64bit) \ ({ \ u64 __val; \ From d02dfac3431f3889ef05190709fa83c2e4ebb229 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:35 -0700 Subject: [PATCH 253/633] coresight: etm4x: Define DEVARCH register fields Define the fields of the DEVARCH register for identifying a component as an ETMv4.x unit. Going forward, we use the DEVARCH register for the component identification, rather than the TRCIDR3. Link: https://lore.kernel.org/r/20210110224850.1880240-14-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-16-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 4 +- drivers/hwtracing/coresight/coresight-etm4x.h | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 37f90d380436..04ec13ae22d0 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1720,8 +1720,8 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) static struct amba_cs_uci_id uci_id_etm4[] = { { /* ETMv4 UCI data */ - .devarch = 0x47704a13, - .devarch_mask = 0xfff0ffff, + .devarch = ETM_DEVARCH_ETMv4x_ARCH, + .devarch_mask = ETM_DEVARCH_ID_MASK, .devtype = 0x00000013, } }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 193d2819afa7..fba3c02eea0b 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -506,6 +506,48 @@ ETM_MODE_EXCL_KERN | \ ETM_MODE_EXCL_USER) +/* + * TRCDEVARCH Bit field definitions + * Bits[31:21] - ARCHITECT = Always Arm Ltd. + * * Bits[31:28] = 0x4 + * * Bits[27:21] = 0b0111011 + * Bit[20] - PRESENT, Indicates the presence of this register. + * + * Bit[19:16] - REVISION, Revision of the architecture. + * + * Bit[15:0] - ARCHID, Identifies this component as an ETM + * * Bits[15:12] - architecture version of ETM + * * = 4 for ETMv4 + * * Bits[11:0] = 0xA13, architecture part number for ETM. + */ +#define ETM_DEVARCH_ARCHITECT_MASK GENMASK(31, 21) +#define ETM_DEVARCH_ARCHITECT_ARM ((0x4 << 28) | (0b0111011 << 21)) +#define ETM_DEVARCH_PRESENT BIT(20) +#define ETM_DEVARCH_REVISION_SHIFT 16 +#define ETM_DEVARCH_REVISION_MASK GENMASK(19, 16) +#define ETM_DEVARCH_REVISION(x) \ + (((x) & ETM_DEVARCH_REVISION_MASK) >> ETM_DEVARCH_REVISION_SHIFT) +#define ETM_DEVARCH_ARCHID_MASK GENMASK(15, 0) +#define ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT 12 +#define ETM_DEVARCH_ARCHID_ARCH_VER_MASK GENMASK(15, 12) +#define ETM_DEVARCH_ARCHID_ARCH_VER(x) \ + (((x) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) >> ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) + +#define ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(ver) \ + (((ver) << ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) + +#define ETM_DEVARCH_ARCHID_ARCH_PART(x) ((x) & 0xfffUL) + +#define ETM_DEVARCH_MAKE_ARCHID(major) \ + ((ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(major)) | ETM_DEVARCH_ARCHID_ARCH_PART(0xA13)) + +#define ETM_DEVARCH_ARCHID_ETMv4x ETM_DEVARCH_MAKE_ARCHID(0x4) + +#define ETM_DEVARCH_ID_MASK \ + (ETM_DEVARCH_ARCHITECT_MASK | ETM_DEVARCH_ARCHID_MASK | ETM_DEVARCH_PRESENT) +#define ETM_DEVARCH_ETMv4x_ARCH \ + (ETM_DEVARCH_ARCHITECT_ARM | ETM_DEVARCH_ARCHID_ETMv4x | ETM_DEVARCH_PRESENT) + #define TRCSTATR_IDLE_BIT 0 #define TRCSTATR_PMSTABLE_BIT 1 #define ETM_DEFAULT_ADDR_COMP 0 From 33d5573a15c2a6f91ca0ef2ab28076be6a2a4a2d Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:36 -0700 Subject: [PATCH 254/633] coresight: etm4x: Check for Software Lock The Software lock is not implemented for system instructions based accesses. So, skip the lock register access in such cases. Link: https://lore.kernel.org/r/20210110224850.1880240-15-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-17-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 04ec13ae22d0..f095ab9949d9 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -133,6 +133,21 @@ static void etm4_os_lock(struct etmv4_drvdata *drvdata) isb(); } +static void etm4_cs_lock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + /* Software Lock is only accessible via memory mapped interface */ + if (csa->io_mem) + CS_LOCK(csa->base); +} + +static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + if (csa->io_mem) + CS_UNLOCK(csa->base); +} + static bool etm4_arch_supported(u8 arch) { /* Mask out the minor version number */ @@ -263,7 +278,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) struct device *etm_dev = &csdev->dev; struct csdev_access *csa = &csdev->access; - CS_UNLOCK(drvdata->base); + + etm4_cs_unlock(drvdata, csa); etm4_enable_arch_specific(drvdata); etm4_os_unlock(drvdata); @@ -366,7 +382,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) isb(); done: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", drvdata->cpu, rc); @@ -623,7 +639,7 @@ static void etm4_disable_hw(void *info) struct csdev_access *csa = &csdev->access; int i; - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); etm4_disable_arch_specific(drvdata); if (!drvdata->skip_power_up) { @@ -665,8 +681,7 @@ static void etm4_disable_hw(void *info) } coresight_disclaim_device_unlocked(csdev); - - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); dev_dbg(&drvdata->csdev->dev, "cpu: %d disable smp call done\n", drvdata->cpu); @@ -776,8 +791,7 @@ static void etm4_init_arch_data(void *info) /* Make sure all registers are accessible */ etm4_os_unlock_csa(drvdata, csa); - - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); /* find all capabilities of the tracing unit */ etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); @@ -942,7 +956,7 @@ static void etm4_init_arch_data(void *info) drvdata->nrseqstate = BMVAL(etmidr5, 25, 27); /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = BMVAL(etmidr5, 28, 30); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); } /* Set ELx trace filter access in the TRCVICTLR register */ @@ -1323,8 +1337,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) dsb(sy); isb(); - CS_UNLOCK(drvdata->base); - + etm4_cs_unlock(drvdata, csa); /* Lock the OS lock to disable trace and external debugger access */ etm4_os_lock(drvdata); @@ -1437,7 +1450,7 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), TRCPDCR); out: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); return ret; } @@ -1448,8 +1461,7 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); struct csdev_access *csa = &tmp_csa; - CS_UNLOCK(drvdata->base); - + etm4_cs_unlock(drvdata, csa); etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); @@ -1534,7 +1546,7 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) /* Unlock the OS lock to re-enable trace and external debug access */ etm4_os_unlock(drvdata); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); } static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, From 1d3eead7e9fba77c310e07d5e296d044abd704eb Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:37 -0700 Subject: [PATCH 255/633] coresight: etm4x: Cleanup secure exception level masks We rely on the ETM architecture version to decide whether Secure EL2 is available on the CPU for excluding the level for address comparators and viewinst main control register. We must instead use the TRCDIDR3.EXLEVEL_S field to detect the supported levels. Link: https://lore.kernel.org/r/20210110224850.1880240-16-suzuki.poulose@arm.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-18-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 13 +++---------- drivers/hwtracing/coresight/coresight-etm4x.h | 6 ++++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index f095ab9949d9..d91b259a4334 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -840,7 +840,6 @@ static void etm4_init_arch_data(void *info) * TRCARCHMAJ, bits[11:8] architecture major versin number */ drvdata->arch = BMVAL(etmidr1, 4, 11); - drvdata->config.arch = drvdata->arch; /* maximum size of resources */ etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); @@ -856,6 +855,7 @@ static void etm4_init_arch_data(void *info) drvdata->ccitmin = BMVAL(etmidr3, 0, 11); /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ drvdata->s_ex_level = BMVAL(etmidr3, 16, 19); + drvdata->config.s_ex_level = drvdata->s_ex_level; /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */ drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23); @@ -1027,16 +1027,9 @@ static u64 etm4_get_ns_access_type(struct etmv4_config *config) static u64 etm4_get_access_type(struct etmv4_config *config) { u64 access_type = etm4_get_ns_access_type(config); - u64 s_hyp = (config->arch & 0x0f) >= 0x4 ? ETM_EXLEVEL_S_HYP : 0; - /* - * EXLEVEL_S, bits[11:8], don't trace anything happening - * in secure state. - */ - access_type |= (ETM_EXLEVEL_S_APP | - ETM_EXLEVEL_S_OS | - s_hyp | - ETM_EXLEVEL_S_MON); + /* All supported secure ELs are excluded */ + access_type |= (u64)config->s_ex_level << TRCACATR_EXLEVEL_SHIFT; return access_type; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index fba3c02eea0b..29cd27f53e72 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -557,6 +557,8 @@ /* PowerDown Control Register bits */ #define TRCPDCR_PU BIT(3) +#define TRCACATR_EXLEVEL_SHIFT 8 + /* secure state access levels - TRCACATRn */ #define ETM_EXLEVEL_S_APP BIT(8) #define ETM_EXLEVEL_S_OS BIT(9) @@ -631,7 +633,7 @@ enum etm_impdef_type { * @vmid_mask0: VM ID comparator mask for comparator 0-3. * @vmid_mask1: VM ID comparator mask for comparator 4-7. * @ext_inp: External input selection. - * @arch: ETM architecture version (for arch dependent config). + * @s_ex_level: Secure ELs where tracing is supported. */ struct etmv4_config { u32 mode; @@ -675,7 +677,7 @@ struct etmv4_config { u32 vmid_mask0; u32 vmid_mask1; u32 ext_inp; - u8 arch; + u8 s_ex_level; }; /** From 4d1b1fd72908b3deceb1a2e1cfcf173cf7f03f3a Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:38 -0700 Subject: [PATCH 256/633] coresight: etm4x: Clean up exception level masks etm4_get_access_type() calculates the exception level bits for use in address comparator registers. This is also used by the TRCVICTLR register by shifting to the required position. This patch cleans up the logic to make etm4_get_access_type() calculate a generic mask which can be used by all users by shifting to their field. No functional changes intended. Link: https://lore.kernel.org/r/20210110224850.1880240-17-suzuki.poulose@arm.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-19-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 47 +++++++++---------- .../coresight/coresight-etm4x-sysfs.c | 12 ++--- drivers/hwtracing/coresight/coresight-etm4x.h | 45 ++++++++++++------ 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d91b259a4334..c9fcb17968a0 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -959,20 +959,16 @@ static void etm4_init_arch_data(void *info) etm4_cs_lock(drvdata, csa); } +static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) +{ + return etm4_get_access_type(config) << TRCVICTLR_EXLEVEL_SHIFT; +} + /* Set ELx trace filter access in the TRCVICTLR register */ static void etm4_set_victlr_access(struct etmv4_config *config) { - u64 access_type; - - config->vinst_ctrl &= ~(ETM_EXLEVEL_S_VICTLR_MASK | ETM_EXLEVEL_NS_VICTLR_MASK); - - /* - * TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering - * bits in vinst_ctrl, same bit pattern as TRCACATRn values returned by - * etm4_get_access_type() but with a relative shift in this register. - */ - access_type = etm4_get_access_type(config) << ETM_EXLEVEL_LSHIFT_TRCVICTLR; - config->vinst_ctrl |= (u32)access_type; + config->vinst_ctrl &= ~TRCVICTLR_EXLEVEL_MASK; + config->vinst_ctrl |= etm4_get_victlr_access_type(config); } static void etm4_set_default_config(struct etmv4_config *config) @@ -1002,12 +998,9 @@ static u64 etm4_get_ns_access_type(struct etmv4_config *config) u64 access_type = 0; /* - * EXLEVEL_NS, bits[15:12] - * The Exception levels are: - * Bit[12] Exception level 0 - Application - * Bit[13] Exception level 1 - OS - * Bit[14] Exception level 2 - Hypervisor - * Bit[15] Never implemented + * EXLEVEL_NS, for NonSecure Exception levels. + * The mask here is a generic value and must be + * shifted to the corresponding field for the registers */ if (!is_kernel_in_hyp_mode()) { /* Stay away from hypervisor mode for non-VHE */ @@ -1024,20 +1017,26 @@ static u64 etm4_get_ns_access_type(struct etmv4_config *config) return access_type; } +/* + * Construct the exception level masks for a given config. + * This must be shifted to the corresponding register field + * for usage. + */ static u64 etm4_get_access_type(struct etmv4_config *config) { - u64 access_type = etm4_get_ns_access_type(config); + /* All Secure exception levels are excluded from the trace */ + return etm4_get_ns_access_type(config) | (u64)config->s_ex_level; +} - /* All supported secure ELs are excluded */ - access_type |= (u64)config->s_ex_level << TRCACATR_EXLEVEL_SHIFT; - - return access_type; +static u64 etm4_get_comparator_access_type(struct etmv4_config *config) +{ + return etm4_get_access_type(config) << TRCACATR_EXLEVEL_SHIFT; } static void etm4_set_comparator_filter(struct etmv4_config *config, u64 start, u64 stop, int comparator) { - u64 access_type = etm4_get_access_type(config); + u64 access_type = etm4_get_comparator_access_type(config); /* First half of default address comparator */ config->addr_val[comparator] = start; @@ -1072,7 +1071,7 @@ static void etm4_set_start_stop_filter(struct etmv4_config *config, enum etm_addr_type type) { int shift; - u64 access_type = etm4_get_access_type(config); + u64 access_type = etm4_get_comparator_access_type(config); /* Configure the comparator */ config->addr_val[comparator] = address; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index e8fdda45ffca..45aeeac2f50e 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -743,7 +743,7 @@ static ssize_t s_exlevel_vinst_show(struct device *dev, struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etmv4_config *config = &drvdata->config; - val = (config->vinst_ctrl & ETM_EXLEVEL_S_VICTLR_MASK) >> 16; + val = (config->vinst_ctrl & TRCVICTLR_EXLEVEL_S_MASK) >> TRCVICTLR_EXLEVEL_S_SHIFT; return scnprintf(buf, PAGE_SIZE, "%#lx\n", val); } @@ -760,10 +760,10 @@ static ssize_t s_exlevel_vinst_store(struct device *dev, spin_lock(&drvdata->spinlock); /* clear all EXLEVEL_S bits */ - config->vinst_ctrl &= ~(ETM_EXLEVEL_S_VICTLR_MASK); + config->vinst_ctrl &= ~(TRCVICTLR_EXLEVEL_S_MASK); /* enable instruction tracing for corresponding exception level */ val &= drvdata->s_ex_level; - config->vinst_ctrl |= (val << 16); + config->vinst_ctrl |= (val << TRCVICTLR_EXLEVEL_S_SHIFT); spin_unlock(&drvdata->spinlock); return size; } @@ -778,7 +778,7 @@ static ssize_t ns_exlevel_vinst_show(struct device *dev, struct etmv4_config *config = &drvdata->config; /* EXLEVEL_NS, bits[23:20] */ - val = (config->vinst_ctrl & ETM_EXLEVEL_NS_VICTLR_MASK) >> 20; + val = (config->vinst_ctrl & TRCVICTLR_EXLEVEL_NS_MASK) >> TRCVICTLR_EXLEVEL_NS_SHIFT; return scnprintf(buf, PAGE_SIZE, "%#lx\n", val); } @@ -795,10 +795,10 @@ static ssize_t ns_exlevel_vinst_store(struct device *dev, spin_lock(&drvdata->spinlock); /* clear EXLEVEL_NS bits */ - config->vinst_ctrl &= ~(ETM_EXLEVEL_NS_VICTLR_MASK); + config->vinst_ctrl &= ~(TRCVICTLR_EXLEVEL_NS_MASK); /* enable instruction tracing for corresponding exception level */ val &= drvdata->ns_ex_level; - config->vinst_ctrl |= (val << 20); + config->vinst_ctrl |= (val << TRCVICTLR_EXLEVEL_NS_SHIFT); spin_unlock(&drvdata->spinlock); return size; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 29cd27f53e72..91b82002e260 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -559,24 +559,39 @@ #define TRCACATR_EXLEVEL_SHIFT 8 -/* secure state access levels - TRCACATRn */ -#define ETM_EXLEVEL_S_APP BIT(8) -#define ETM_EXLEVEL_S_OS BIT(9) -#define ETM_EXLEVEL_S_HYP BIT(10) -#define ETM_EXLEVEL_S_MON BIT(11) -/* non-secure state access levels - TRCACATRn */ -#define ETM_EXLEVEL_NS_APP BIT(12) -#define ETM_EXLEVEL_NS_OS BIT(13) -#define ETM_EXLEVEL_NS_HYP BIT(14) -#define ETM_EXLEVEL_NS_NA BIT(15) +/* + * Exception level mask for Secure and Non-Secure ELs. + * ETM defines the bits for EL control (e.g, TRVICTLR, TRCACTRn). + * The Secure and Non-Secure ELs are always to gether. + * Non-secure EL3 is never implemented. + * We use the following generic mask as they appear in different + * registers and this can be shifted for the appropriate + * fields. + */ +#define ETM_EXLEVEL_S_APP BIT(0) /* Secure EL0 */ +#define ETM_EXLEVEL_S_OS BIT(1) /* Secure EL1 */ +#define ETM_EXLEVEL_S_HYP BIT(2) /* Secure EL2 */ +#define ETM_EXLEVEL_S_MON BIT(3) /* Secure EL3/Monitor */ +#define ETM_EXLEVEL_NS_APP BIT(4) /* NonSecure EL0 */ +#define ETM_EXLEVEL_NS_OS BIT(5) /* NonSecure EL1 */ +#define ETM_EXLEVEL_NS_HYP BIT(6) /* NonSecure EL2 */ -/* access level control in TRCVICTLR - same bits as TRCACATRn but shifted */ -#define ETM_EXLEVEL_LSHIFT_TRCVICTLR 8 +#define ETM_EXLEVEL_MASK (GENMASK(6, 0)) +#define ETM_EXLEVEL_S_MASK (GENMASK(3, 0)) +#define ETM_EXLEVEL_NS_MASK (GENMASK(6, 4)) + +/* access level controls in TRCACATRn */ +#define TRCACATR_EXLEVEL_SHIFT 8 + +/* access level control in TRCVICTLR */ +#define TRCVICTLR_EXLEVEL_SHIFT 16 +#define TRCVICTLR_EXLEVEL_S_SHIFT 16 +#define TRCVICTLR_EXLEVEL_NS_SHIFT 20 /* secure / non secure masks - TRCVICTLR, IDR3 */ -#define ETM_EXLEVEL_S_VICTLR_MASK GENMASK(19, 16) -/* NS MON (EL3) mode never implemented */ -#define ETM_EXLEVEL_NS_VICTLR_MASK GENMASK(22, 20) +#define TRCVICTLR_EXLEVEL_MASK (ETM_EXLEVEL_MASK << TRCVICTLR_EXLEVEL_SHIFT) +#define TRCVICTLR_EXLEVEL_S_MASK (ETM_EXLEVEL_S_MASK << TRCVICTLR_EXLEVEL_SHIFT) +#define TRCVICTLR_EXLEVEL_NS_MASK (ETM_EXLEVEL_NS_MASK << TRCVICTLR_EXLEVEL_SHIFT) /* Interpretation of resource numbers change at ETM v4.3 architecture */ #define ETM4X_ARCH_4V3 0x43 From e49516e2df5b03344a54b3c670890816978b500a Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:39 -0700 Subject: [PATCH 257/633] coresight: etm4x: Handle ETM architecture version We are about to rely on TRCDEVARCH for detecting the ETM and its architecture version, falling back to TRCIDR1 if the former is not implemented (in older broken implementations). Also, we use the architecture version information to make some decisions. Streamline the architecture version handling by adding helpers. Link: https://lore.kernel.org/r/20210110224850.1880240-18-suzuki.poulose@arm.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-20-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 2 +- drivers/hwtracing/coresight/coresight-etm4x.h | 60 ++++++++++++++++++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c9fcb17968a0..59da9efae9c2 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -917,7 +917,7 @@ static void etm4_init_arch_data(void *info) * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. */ drvdata->nr_resource = BMVAL(etmidr4, 16, 19); - if ((drvdata->arch < ETM4X_ARCH_4V3) || (drvdata->nr_resource > 0)) + if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) drvdata->nr_resource += 1; /* * NUMSSCC, bits[23:20] the number of single-shot diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 91b82002e260..0af60571aa23 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -468,7 +468,6 @@ #define ETM_MAX_RES_SEL 32 #define ETM_MAX_SS_CMP 8 -#define ETM_ARCH_V4 0x40 #define ETMv4_SYNC_MASK 0x1F #define ETM_CYC_THRESHOLD_MASK 0xFFF #define ETM_CYC_THRESHOLD_DEFAULT 0x100 @@ -593,8 +592,63 @@ #define TRCVICTLR_EXLEVEL_S_MASK (ETM_EXLEVEL_S_MASK << TRCVICTLR_EXLEVEL_SHIFT) #define TRCVICTLR_EXLEVEL_NS_MASK (ETM_EXLEVEL_NS_MASK << TRCVICTLR_EXLEVEL_SHIFT) +#define ETM_TRCIDR1_ARCH_MAJOR_SHIFT 8 +#define ETM_TRCIDR1_ARCH_MAJOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MAJOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MAJOR_MASK) >> ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR_SHIFT 4 +#define ETM_TRCIDR1_ARCH_MINOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MINOR_MASK) >> ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_SHIFT ETM_TRCIDR1_ARCH_MINOR_SHIFT +#define ETM_TRCIDR1_ARCH_MASK \ + (ETM_TRCIDR1_ARCH_MAJOR_MASK | ETM_TRCIDR1_ARCH_MINOR_MASK) + +#define ETM_TRCIDR1_ARCH_ETMv4 0x4 + +/* + * Driver representation of the ETM architecture. + * The version of an ETM component can be detected from + * + * TRCDEVARCH - CoreSight architected register + * - Bits[15:12] - Major version + * - Bits[19:16] - Minor version + * TRCIDR1 - ETM architected register + * - Bits[11:8] - Major version + * - Bits[7:4] - Minor version + * We must rely on TRCDEVARCH for the version information, + * however we don't want to break the support for potential + * old implementations which might not implement it. Thus + * we fall back to TRCIDR1 if TRCDEVARCH is not implemented + * for memory mapped components. + * Now to make certain decisions easier based on the version + * we use an internal representation of the version in the + * driver, as follows : + * + * ETM_ARCH_VERSION[7:0], where : + * Bits[7:4] - Major version + * Bits[3:0] - Minro version + */ +#define ETM_ARCH_VERSION(major, minor) \ + ((((major) & 0xfU) << 4) | (((minor) & 0xfU))) +#define ETM_ARCH_MAJOR_VERSION(arch) (((arch) >> 4) & 0xfU) +#define ETM_ARCH_MINOR_VERSION(arch) ((arch) & 0xfU) + +#define ETM_ARCH_V4 ETM_ARCH_VERSION(4, 0) /* Interpretation of resource numbers change at ETM v4.3 architecture */ -#define ETM4X_ARCH_4V3 0x43 +#define ETM_ARCH_V4_3 ETM_ARCH_VERSION(4, 3) + +static inline u8 etm_devarch_to_arch(u32 devarch) +{ + return ETM_ARCH_VERSION(ETM_DEVARCH_ARCHID_ARCH_VER(devarch), + ETM_DEVARCH_REVISION(devarch)); +} + +static inline u8 etm_trcidr_to_arch(u32 trcidr1) +{ + return ETM_ARCH_VERSION(ETM_TRCIDR1_ARCH_MAJOR(trcidr1), + ETM_TRCIDR1_ARCH_MINOR(trcidr1)); +} enum etm_impdef_type { ETM4_IMPDEF_HISI_CORE_COMMIT, @@ -761,7 +815,7 @@ struct etmv4_save_state { * @spinlock: Only one at a time pls. * @mode: This tracer's mode, i.e sysFS, Perf or disabled. * @cpu: The cpu this component is affined to. - * @arch: ETM version number. + * @arch: ETM architecture version. * @nr_pe: The number of processing entity available for tracing. * @nr_pe_cmp: The number of processing entity comparator inputs that are * available for tracing. From fd6e79050066be087aa1db46116438dc923675b6 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:40 -0700 Subject: [PATCH 258/633] coresight: etm4x: Detect access early on the target CPU In preparation to detect the support for system instruction support, move the detection of the device access to the target CPU. Link: https://lore.kernel.org/r/20210110224850.1880240-19-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-21-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 59da9efae9c2..bfe3b8224d9a 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -59,6 +59,11 @@ static u64 etm4_get_access_type(struct etmv4_config *config); static enum cpuhp_state hp_online; +struct etm4_init_arg { + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; +}; + /* * Check if TRCSSPCICRn(i) is implemented for a given instance. * @@ -776,6 +781,22 @@ static const struct coresight_ops etm4_cs_ops = { .source_ops = &etm4_source_ops, }; +static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + *csa = CSDEV_ACCESS_IOMEM(drvdata->base); + return true; +} + +static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + if (drvdata->base) + return etm4_init_iomem_access(drvdata, csa); + + return false; +} + static void etm4_init_arch_data(void *info) { u32 etmidr0; @@ -784,11 +805,22 @@ static void etm4_init_arch_data(void *info) u32 etmidr3; u32 etmidr4; u32 etmidr5; - struct etmv4_drvdata *drvdata = info; - struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); - struct csdev_access *csa = &tmp_csa; + struct etm4_init_arg *init_arg = info; + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; int i; + drvdata = init_arg->drvdata; + csa = init_arg->csa; + + /* + * If we are unable to detect the access mechanism, + * or unable to detect the trace unit type, fail + * early. + */ + if (!etm4_init_csdev_access(drvdata, csa)) + return; + /* Make sure all registers are accessible */ etm4_os_unlock_csa(drvdata, csa); etm4_cs_unlock(drvdata, csa); @@ -1634,6 +1666,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) struct etmv4_drvdata *drvdata; struct resource *res = &adev->res; struct coresight_desc desc = { 0 }; + struct etm4_init_arg init_arg = { 0 }; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -1661,7 +1694,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; - desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); @@ -1673,8 +1705,11 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (!desc.name) return -ENOMEM; + init_arg.drvdata = drvdata; + init_arg.csa = &desc.access; + if (smp_call_function_single(drvdata->cpu, - etm4_init_arch_data, drvdata, 1)) + etm4_init_arch_data, &init_arg, 1)) dev_err(dev, "ETM arch init failed\n"); if (etm4_arch_supported(drvdata->arch) == false) From 8b94db1edaee7b6cf8f39d2ea600258a74351404 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:41 -0700 Subject: [PATCH 259/633] coresight: etm4x: Use TRCDEVARCH for component discovery We have been using TRCIDR1 for detecting the ETM version. This is in preparation for the future IP support. Link: https://lore.kernel.org/r/20210110224850.1880240-20-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-22-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index bfe3b8224d9a..a09a653fc5b0 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -153,18 +153,6 @@ static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, CS_UNLOCK(csa->base); } -static bool etm4_arch_supported(u8 arch) -{ - /* Mask out the minor version number */ - switch (arch & 0xf0) { - case ETM_ARCH_V4: - break; - default: - return false; - } - return true; -} - static int etm4_cpu_id(struct coresight_device *csdev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); @@ -784,6 +772,26 @@ static const struct coresight_ops etm4_cs_ops = { static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { + u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); + u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1); + + /* + * All ETMs must implement TRCDEVARCH to indicate that + * the component is an ETMv4. To support any broken + * implementations we fall back to TRCIDR1 check, which + * is not really reliable. + */ + if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) { + drvdata->arch = etm_devarch_to_arch(devarch); + } else { + pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n", + smp_processor_id(), devarch); + + if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4) + return false; + drvdata->arch = etm_trcidr_to_arch(idr1); + } + *csa = CSDEV_ACCESS_IOMEM(drvdata->base); return true; } @@ -800,7 +808,6 @@ static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, static void etm4_init_arch_data(void *info) { u32 etmidr0; - u32 etmidr1; u32 etmidr2; u32 etmidr3; u32 etmidr4; @@ -865,14 +872,6 @@ static void etm4_init_arch_data(void *info) /* TSSIZE, bits[28:24] Global timestamp size field */ drvdata->ts_size = BMVAL(etmidr0, 24, 28); - /* base architecture of trace unit */ - etmidr1 = etm4x_relaxed_read32(csa, TRCIDR1); - /* - * TRCARCHMIN, bits[7:4] architecture the minor version number - * TRCARCHMAJ, bits[11:8] architecture major versin number - */ - drvdata->arch = BMVAL(etmidr1, 4, 11); - /* maximum size of resources */ etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); /* CIDSIZE, bits[9:5] Indicates the Context ID size */ @@ -1712,7 +1711,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) etm4_init_arch_data, &init_arg, 1)) dev_err(dev, "ETM arch init failed\n"); - if (etm4_arch_supported(drvdata->arch) == false) + if (!drvdata->arch) return -EINVAL; etm4_init_trace_id(drvdata); @@ -1744,7 +1743,8 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) pm_runtime_put(&adev->dev); dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n", - drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf); + drvdata->cpu, ETM_ARCH_MAJOR_VERSION(drvdata->arch), + ETM_ARCH_MINOR_VERSION(drvdata->arch)); if (boot_enable) { coresight_enable(drvdata->csdev); From 4211bfce1eb9962f2ae1972837c032ad6d48f292 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:42 -0700 Subject: [PATCH 260/633] coresight: etm4x: Expose trcdevarch via sysfs Expose the TRCDEVARCH register via the sysfs for component detection. Given that the TRCIDR1 may not completely identify the ETM component and instead need to use TRCDEVARCH, expose this via sysfs for tools to use it for identification. Link: https://lore.kernel.org/r/20210110224850.1880240-21-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-23-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../ABI/testing/sysfs-bus-coresight-devices-etm4x | 8 ++++++++ drivers/hwtracing/coresight/coresight-etm4x-sysfs.c | 1 + 2 files changed, 9 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x index 881f0cd99ce4..8e53a32f8150 100644 --- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x @@ -371,6 +371,14 @@ Contact: Mathieu Poirier Description: (Read) Print the content of the Device ID Register (0xFC8). The value is taken directly from the HW. +What: /sys/bus/coresight/devices/etm/mgmt/trcdevarch +Date: January 2021 +KernelVersion: 5.12 +Contact: Mathieu Poirier +Description: (Read) Print the content of the Device Architecture Register + (offset 0xFBC). The value is taken directly read + from the HW. + What: /sys/bus/coresight/devices/etm/mgmt/trcdevtype Date: April 2015 KernelVersion: 4.01 diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index 45aeeac2f50e..b646d53a3133 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2442,6 +2442,7 @@ static struct attribute *coresight_etmv4_mgmt_attrs[] = { coresight_etm4x_reg(trcoslsr, TRCOSLSR), coresight_etm4x_reg(trcconfig, TRCCONFIGR), coresight_etm4x_reg(trctraceid, TRCTRACEIDR), + coresight_etm4x_reg(trcdevarch, TRCDEVARCH), NULL, }; From 1ab3bb9df5e35183fee8da2b3fb30feda9a53ce9 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:43 -0700 Subject: [PATCH 261/633] coresight: etm4x: Add necessary synchronization for sysreg access As per the specification any update to the TRCPRGCTLR must be synchronized by a context synchronization event (in our case an explicist ISB) before the TRCSTATR is checked. Link: https://lore.kernel.org/r/20210110224850.1880240-22-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-24-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index a09a653fc5b0..8d644e93de51 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -284,6 +284,15 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) /* Disable the trace unit before programming trace registers */ etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); + /* + * If we use system instructions, we need to synchronize the + * write to the TRCPRGCTLR, before accessing the TRCSTATR. + * See ARM IHI0064F, section + * "4.3.7 Synchronization of register updates" + */ + if (!csa->io_mem) + isb(); + /* wait for TRCSTATR.IDLE to go up */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) dev_err(etm_dev, @@ -362,6 +371,10 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) /* Enable the trace unit */ etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); + /* Synchronize the register updates for sysreg access */ + if (!csa->io_mem) + isb(); + /* wait for TRCSTATR.IDLE to go back down to '0' */ if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) dev_err(etm_dev, From dc1747a716fe91b88691cc8bd35f986a6774fc47 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:44 -0700 Subject: [PATCH 262/633] coresight: etm4x: Detect system instructions support ETM v4.4 onwards adds support for system instruction access to the ETM. Detect the support on an ETM and switch to using the mode when available. Link: https://lore.kernel.org/r/20210110224850.1880240-23-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-25-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 8d644e93de51..48d8e99e31eb 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -782,6 +782,37 @@ static const struct coresight_ops etm4_cs_ops = { .source_ops = &etm4_source_ops, }; +static inline bool cpu_supports_sysreg_trace(void) +{ + u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); + + return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0; +} + +static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 devarch; + + if (!cpu_supports_sysreg_trace()) + return false; + + /* + * ETMs implementing sysreg access must implement TRCDEVARCH. + */ + devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); + if ((devarch & ETM_DEVARCH_ID_MASK) != ETM_DEVARCH_ETMv4x_ARCH) + return false; + *csa = (struct csdev_access) { + .io_mem = false, + .read = etm4x_sysreg_read, + .write = etm4x_sysreg_write, + }; + + drvdata->arch = etm_devarch_to_arch(devarch); + return true; +} + static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { @@ -812,9 +843,17 @@ static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, struct csdev_access *csa) { + /* + * Always choose the memory mapped io, if there is + * a memory map to prevent sysreg access on broken + * systems. + */ if (drvdata->base) return etm4_init_iomem_access(drvdata, csa); + if (etm4_init_sysreg_access(drvdata, csa)) + return true; + return false; } From c23bc382ef0ec9e91ef7bb689755bddbddb0fb25 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:45 -0700 Subject: [PATCH 263/633] coresight: etm4x: Refactor probing routine CoreSight ETM with system register access may not have a memory mapped i/o access. Refactor the ETM specific probing into a common routine to allow reusing the code for such ETMs. Link: https://lore.kernel.org/r/20210110224850.1880240-24-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-26-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 62 ++++++++++++------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 48d8e99e31eb..c3e458af618a 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1708,14 +1708,11 @@ static void etm4_pm_clear(void) } } -static int etm4_probe(struct amba_device *adev, const struct amba_id *id) +static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) { int ret; - void __iomem *base; - struct device *dev = &adev->dev; struct coresight_platform_data *pdata = NULL; struct etmv4_drvdata *drvdata; - struct resource *res = &adev->res; struct coresight_desc desc = { 0 }; struct etm4_init_arg init_arg = { 0 }; @@ -1739,11 +1736,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) drvdata->skip_power_up = true; - /* Validity for the resource is already checked by the AMBA core */ - base = devm_ioremap_resource(dev, res); - if (IS_ERR(base)) - return PTR_ERR(base); - drvdata->base = base; spin_lock_init(&drvdata->spinlock); @@ -1773,7 +1765,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(pdata)) return PTR_ERR(pdata); - adev->dev.platform_data = pdata; + dev->platform_data = pdata; desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; @@ -1793,7 +1785,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) etmdrvdata[drvdata->cpu] = drvdata; - pm_runtime_put(&adev->dev); dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n", drvdata->cpu, ETM_ARCH_MAJOR_VERSION(drvdata->arch), ETM_ARCH_MINOR_VERSION(drvdata->arch)); @@ -1803,11 +1794,30 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) drvdata->boot_enable = true; } - etm4_check_arch_features(drvdata, id->id); + etm4_check_arch_features(drvdata, etm_pid); return 0; } +static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) +{ + void __iomem *base; + struct device *dev = &adev->dev; + struct resource *res = &adev->res; + int ret; + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + ret = etm4_probe(dev, base, id->id); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + static struct amba_cs_uci_id uci_id_etm4[] = { { /* ETMv4 UCI data */ @@ -1824,15 +1834,12 @@ static void clear_etmdrvdata(void *info) etmdrvdata[cpu] = NULL; } -static int etm4_remove(struct amba_device *adev) +static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); - etm_perf_symlink(drvdata->csdev, false); - /* - * Taking hotplug lock here to avoid racing between etm4_remove and - * CPU hotplug call backs. + * Taking hotplug lock here to avoid racing between etm4_remove_dev() + * and CPU hotplug call backs. */ cpus_read_lock(); /* @@ -1851,6 +1858,15 @@ static int etm4_remove(struct amba_device *adev) return 0; } +static int __exit etm4_remove_amba(struct amba_device *adev) +{ + struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); + + if (drvdata) + return etm4_remove_dev(drvdata); + return 0; +} + static const struct amba_id etm4_ids[] = { CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ @@ -1874,14 +1890,14 @@ static const struct amba_id etm4_ids[] = { MODULE_DEVICE_TABLE(amba, etm4_ids); -static struct amba_driver etm4x_driver = { +static struct amba_driver etm4x_amba_driver = { .drv = { .name = "coresight-etm4x", .owner = THIS_MODULE, .suppress_bind_attrs = true, }, - .probe = etm4_probe, - .remove = etm4_remove, + .probe = etm4_probe_amba, + .remove = etm4_remove_amba, .id_table = etm4_ids, }; @@ -1895,7 +1911,7 @@ static int __init etm4x_init(void) if (ret) return ret; - ret = amba_driver_register(&etm4x_driver); + ret = amba_driver_register(&etm4x_amba_driver); if (ret) { pr_err("Error registering etm4x driver\n"); etm4_pm_clear(); @@ -1906,7 +1922,7 @@ static int __init etm4x_init(void) static void __exit etm4x_exit(void) { - amba_driver_unregister(&etm4x_driver); + amba_driver_unregister(&etm4x_amba_driver); etm4_pm_clear(); } From e97db2cf091ac152f20b79419e1f186331e5bd90 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:46 -0700 Subject: [PATCH 264/633] coresight: etm4x: Run arch feature detection on the CPU As we are about to add support for system register based devices, we don't get an AMBA pid. So, the detection code could check the system registers running on the CPU to check for the architecture specific features. Thus we move the arch feature detection to run on the CPU. We cannot always read the PID from the HW, as the PID could be overridden by DT for broken devices. So, use the PID from AMBA layer if available. Link: https://lore.kernel.org/r/20210110224850.1880240-25-suzuki.poulose@arm.com Cc: Mathieu Poirier Cc: Mike Leach Cc: liuqi115@huawei.com Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-27-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c3e458af618a..fc26ecbc2d87 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -60,6 +60,7 @@ static u64 etm4_get_access_type(struct etmv4_config *config); static enum cpuhp_state hp_online; struct etm4_init_arg { + unsigned int pid; struct etmv4_drvdata *drvdata; struct csdev_access *csa; }; @@ -884,6 +885,8 @@ static void etm4_init_arch_data(void *info) etm4_os_unlock_csa(drvdata, csa); etm4_cs_unlock(drvdata, csa); + etm4_check_arch_features(drvdata, init_arg->pid); + /* find all capabilities of the tracing unit */ etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); @@ -1750,6 +1753,7 @@ static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) init_arg.drvdata = drvdata; init_arg.csa = &desc.access; + init_arg.pid = etm_pid; if (smp_call_function_single(drvdata->cpu, etm4_init_arch_data, &init_arg, 1)) @@ -1794,8 +1798,6 @@ static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) drvdata->boot_enable = true; } - etm4_check_arch_features(drvdata, etm_pid); - return 0; } From 5214b563588e8414193bd7a174c52350256942a6 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:47 -0700 Subject: [PATCH 265/633] coresight: etm4x: Add support for sysreg only devices Add support for devices with system instruction access only. They don't have a memory mapped interface and thus are not AMBA devices. System register access is not permitted to TRCPDCR and thus skip access to them. Link: https://lore.kernel.org/r/20210110224850.1880240-26-suzuki.poulose@arm.com Cc: Mike Leach Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-28-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index fc26ecbc2d87..c6238e208685 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -1736,9 +1737,6 @@ static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) return -ENOMEM; } - if (fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) - drvdata->skip_power_up = true; - drvdata->base = base; spin_lock_init(&drvdata->spinlock); @@ -1762,6 +1760,11 @@ static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) if (!drvdata->arch) return -EINVAL; + /* TRCPDCR is not accessible with system instructions. */ + if (!desc.access.io_mem || + fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) + drvdata->skip_power_up = true; + etm4_init_trace_id(drvdata); etm4_set_default(&drvdata->config); @@ -1820,6 +1823,25 @@ static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) return ret; } +static int etm4_probe_platform_dev(struct platform_device *pdev) +{ + int ret; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + /* + * System register based devices could match the + * HW by reading appropriate registers on the HW + * and thus we could skip the PID. + */ + ret = etm4_probe(&pdev->dev, NULL, 0); + + pm_runtime_put(&pdev->dev); + return ret; +} + static struct amba_cs_uci_id uci_id_etm4[] = { { /* ETMv4 UCI data */ @@ -1869,6 +1891,17 @@ static int __exit etm4_remove_amba(struct amba_device *adev) return 0; } +static int __exit etm4_remove_platform_dev(struct platform_device *pdev) +{ + int ret = 0; + struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (drvdata) + ret = etm4_remove_dev(drvdata); + pm_runtime_disable(&pdev->dev); + return ret; +} + static const struct amba_id etm4_ids[] = { CS_AMBA_ID(0x000bb95d), /* Cortex-A53 */ CS_AMBA_ID(0x000bb95e), /* Cortex-A57 */ @@ -1903,6 +1936,21 @@ static struct amba_driver etm4x_amba_driver = { .id_table = etm4_ids, }; +static const struct of_device_id etm4_sysreg_match[] = { + { .compatible = "arm,coresight-etm4x-sysreg" }, + {} +}; + +static struct platform_driver etm4_platform_driver = { + .probe = etm4_probe_platform_dev, + .remove = etm4_remove_platform_dev, + .driver = { + .name = "coresight-etm4x", + .of_match_table = etm4_sysreg_match, + .suppress_bind_attrs = true, + }, +}; + static int __init etm4x_init(void) { int ret; @@ -1915,16 +1963,26 @@ static int __init etm4x_init(void) ret = amba_driver_register(&etm4x_amba_driver); if (ret) { - pr_err("Error registering etm4x driver\n"); - etm4_pm_clear(); + pr_err("Error registering etm4x AMBA driver\n"); + goto clear_pm; } + ret = platform_driver_register(&etm4_platform_driver); + if (!ret) + return 0; + + pr_err("Error registering etm4x platform driver\n"); + amba_driver_unregister(&etm4x_amba_driver); + +clear_pm: + etm4_pm_clear(); return ret; } static void __exit etm4x_exit(void) { amba_driver_unregister(&etm4x_amba_driver); + platform_driver_unregister(&etm4_platform_driver); etm4_pm_clear(); } From 61c68c68b826182d93f5e4fe77eb92676a28d513 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:48 -0700 Subject: [PATCH 266/633] dts: bindings: coresight: ETM system register access only units Document the bindings for ETMs with system register accesses. Link: https://lore.kernel.org/r/20210110224850.1880240-27-suzuki.poulose@arm.com Cc: devicetree@vger.kernel.org Cc: Mathieu Poirier Cc: Mike Leach Acked-by: Rob Herring Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-29-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- Documentation/devicetree/bindings/arm/coresight.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/arm/coresight.txt b/Documentation/devicetree/bindings/arm/coresight.txt index d711676b4a51..7f9c1ca87487 100644 --- a/Documentation/devicetree/bindings/arm/coresight.txt +++ b/Documentation/devicetree/bindings/arm/coresight.txt @@ -34,9 +34,12 @@ its hardware characteristcs. Program Flow Trace Macrocell: "arm,coresight-etm3x", "arm,primecell"; - - Embedded Trace Macrocell (version 4.x): + - Embedded Trace Macrocell (version 4.x), with memory mapped access. "arm,coresight-etm4x", "arm,primecell"; + - Embedded Trace Macrocell (version 4.x), with system register access only. + "arm,coresight-etm4x-sysreg"; + - Coresight programmable Replicator : "arm,coresight-dynamic-replicator", "arm,primecell"; From 4b6929f50d79cd674e4315a5dad5284b98534942 Mon Sep 17 00:00:00 2001 From: Jonathan Zhou Date: Mon, 1 Feb 2021 11:13:49 -0700 Subject: [PATCH 267/633] arm64: Add TRFCR_ELx definitions Add definitions for the Arm v8.4 SelfHosted trace extensions registers. [ split the register definitions to separate patch rename some of the symbols ] Link: https://lore.kernel.org/r/20210110224850.1880240-28-suzuki.poulose@arm.com Cc: Will Deacon Acked-by: Catalin Marinas Signed-off-by: Jonathan Zhou Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-30-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- arch/arm64/include/asm/sysreg.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 8b5e7e5c3cc8..4acff97519b9 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -191,6 +191,7 @@ #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) #define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0) +#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) #define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0) #define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1) @@ -471,6 +472,7 @@ #define SYS_SCTLR_EL2 sys_reg(3, 4, 1, 0, 0) #define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0) +#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0) #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) @@ -829,6 +831,7 @@ #define ID_AA64MMFR2_CNP_SHIFT 0 /* id_aa64dfr0 */ +#define ID_AA64DFR0_TRACE_FILT_SHIFT 40 #define ID_AA64DFR0_DOUBLELOCK_SHIFT 36 #define ID_AA64DFR0_PMSVER_SHIFT 32 #define ID_AA64DFR0_CTX_CMPS_SHIFT 28 @@ -1003,6 +1006,14 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) +#define TRFCR_ELx_TS_SHIFT 5 +#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_EL2_CX BIT(3) +#define TRFCR_ELx_ExTRE BIT(1) +#define TRFCR_ELx_E0TRE BIT(0) + #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 From e5d51fbe9bf5f13e72e3c742705928c74d8ba8bf Mon Sep 17 00:00:00 2001 From: Jonathan Zhou Date: Mon, 1 Feb 2021 11:13:50 -0700 Subject: [PATCH 268/633] coresight: Add support for v8.4 SelfHosted tracing v8.4 tracing extensions added support for trace filtering controlled by TRFCR_ELx. This must be programmed to allow tracing at EL1/EL2 and EL0. The timestamp used is the virtual time. Also enable CONTEXIDR_EL2 tracing if we are running the kernel at EL2. Link: https://lore.kernel.org/r/20210110224850.1880240-29-suzuki.poulose@arm.com Cc: Catalin Marinas Cc: Mike Leach Cc: Will Deacon Reviewed-by: Mathieu Poirier Signed-off-by: Jonathan Zhou [ Move the trace filtering setup etm_init_arch_data() and clean ups] Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-31-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../coresight/coresight-etm4x-core.c | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c6238e208685..473ab7480a36 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -859,6 +859,30 @@ static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, return false; } +static void cpu_enable_tracing(void) +{ + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + u64 trfcr; + + if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT)) + return; + + /* + * If the CPU supports v8.4 SelfHosted Tracing, enable + * tracing at the kernel EL and EL0, forcing to use the + * virtual time as the timestamp. + */ + trfcr = (TRFCR_ELx_TS_VIRTUAL | + TRFCR_ELx_ExTRE | + TRFCR_ELx_E0TRE); + + /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ + if (is_kernel_in_hyp_mode()) + trfcr |= TRFCR_EL2_CX; + + write_sysreg_s(trfcr, SYS_TRFCR_EL1); +} + static void etm4_init_arch_data(void *info) { u32 etmidr0; @@ -1044,6 +1068,7 @@ static void etm4_init_arch_data(void *info) /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = BMVAL(etmidr5, 28, 30); etm4_cs_lock(drvdata, csa); + cpu_enable_tracing(); } static inline u32 etm4_get_victlr_access_type(struct etmv4_config *config) From f72896063396b0cb205cbf0fd76ec6ab3ca11c8a Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Mon, 1 Feb 2021 11:13:51 -0700 Subject: [PATCH 269/633] coresight: etm4x: Handle accesses to TRCSTALLCTLR TRCSTALLCTLR register is only implemented if TRCIDR3.STALLCTL == 0b1 Make sure the driver touches the register only it is implemented. Link: https://lore.kernel.org/r/20210127184617.3684379-1-suzuki.poulose@arm.com Cc: stable@vger.kernel.org Cc: Mathieu Poirier Cc: Mike Leach Cc: Leo Yan Signed-off-by: Suzuki K Poulose Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210201181351.1475223-32-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 9 ++++++--- drivers/hwtracing/coresight/coresight-etm4x-sysfs.c | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 473ab7480a36..5017d33ba4f5 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -306,7 +306,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); - etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); + if (drvdata->stallctl) + etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); @@ -1463,7 +1464,8 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); - state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); + if (drvdata->stallctl) + state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); @@ -1575,7 +1577,8 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); - etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); + if (drvdata->stallctl) + etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index b646d53a3133..0995a10790f4 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -389,7 +389,7 @@ static ssize_t mode_store(struct device *dev, config->eventctrl1 &= ~BIT(12); /* bit[8], Instruction stall bit */ - if (config->mode & ETM_MODE_ISTALL_EN) + if ((config->mode & ETM_MODE_ISTALL_EN) && (drvdata->stallctl == true)) config->stall_ctrl |= BIT(8); else config->stall_ctrl &= ~BIT(8); From 76ec1ec8fc7c4bda24af8c3a73c5f56fa8d6e460 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 29 Jan 2021 16:25:05 +0800 Subject: [PATCH 270/633] cxl: Simplify bool conversion Fix the following coccicheck warning: ./drivers/misc/cxl/sysfs.c:181:48-53: WARNING: conversion to bool not needed here Reported-by: Abaci Robot Reviewed-by: Andrew Donnellan Acked-by: Frederic Barrat Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1611908705-98507-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cxl/sysfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/cxl/sysfs.c b/drivers/misc/cxl/sysfs.c index d97a243ad30c..c173a5e88c91 100644 --- a/drivers/misc/cxl/sysfs.c +++ b/drivers/misc/cxl/sysfs.c @@ -178,7 +178,7 @@ static ssize_t perst_reloads_same_image_store(struct device *device, if ((rc != 1) || !(val == 1 || val == 0)) return -EINVAL; - adapter->perst_same_image = (val == 1 ? true : false); + adapter->perst_same_image = (val == 1); return count; } From 72e008ce307fa2f35f6783997378b32e83122839 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 29 Jan 2021 17:14:26 +0000 Subject: [PATCH 271/633] nvmem: core: Fix a resource leak on error in nvmem_add_cells_from_of() This doesn't call of_node_put() on the error path so it leads to a memory leak. Fixes: 0749aa25af82 ("nvmem: core: fix regression in of_nvmem_cell_get()") Signed-off-by: Dan Carpenter Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210129171430.11328-2-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 177f5bf27c6d..68ae6f24b57f 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -713,6 +713,7 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) cell->name, nvmem->stride); /* Cells already added will be freed later. */ kfree_const(cell->name); + of_node_put(cell->np); kfree(cell); return -EINVAL; } From 579db09c6106977c0496f2cca48606b289df4bdf Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Fri, 29 Jan 2021 17:14:27 +0000 Subject: [PATCH 272/633] nvmem: imx-iim: Use of_device_get_match_data() The retrieval of driver data via of_device_get_match_data() can make the code simpler. Use of_device_get_match_data() to simplify the code. Signed-off-by: Fabio Estevam Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210129171430.11328-3-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/imx-iim.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/nvmem/imx-iim.c b/drivers/nvmem/imx-iim.c index 701704b87dc9..c86339a7f583 100644 --- a/drivers/nvmem/imx-iim.c +++ b/drivers/nvmem/imx-iim.c @@ -96,7 +96,6 @@ MODULE_DEVICE_TABLE(of, imx_iim_dt_ids); static int imx_iim_probe(struct platform_device *pdev) { - const struct of_device_id *of_id; struct device *dev = &pdev->dev; struct iim_priv *iim; struct nvmem_device *nvmem; @@ -111,11 +110,7 @@ static int imx_iim_probe(struct platform_device *pdev) if (IS_ERR(iim->base)) return PTR_ERR(iim->base); - of_id = of_match_device(imx_iim_dt_ids, dev); - if (!of_id) - return -ENODEV; - - drvdata = of_id->data; + drvdata = of_device_get_match_data(&pdev->dev); iim->clk = devm_clk_get(dev, NULL); if (IS_ERR(iim->clk)) From f90714e56cb6c5d1e57fb4385b107932f76ef94f Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Fri, 29 Jan 2021 17:14:28 +0000 Subject: [PATCH 273/633] dt-bindings: nvmem: Add bindings for rmem driver Firmware/co-processors might use reserved memory areas in order to pass data stemming from an nvmem device otherwise non accessible to Linux. For example an EEPROM memory only physically accessible to firmware, or data only accessible early at boot time. Introduce the dt-bindings to nvmem's rmem. Reviewed-by: Rob Herring Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210129171430.11328-4-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../devicetree/bindings/nvmem/rmem.yaml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Documentation/devicetree/bindings/nvmem/rmem.yaml diff --git a/Documentation/devicetree/bindings/nvmem/rmem.yaml b/Documentation/devicetree/bindings/nvmem/rmem.yaml new file mode 100644 index 000000000000..1d85a0a30846 --- /dev/null +++ b/Documentation/devicetree/bindings/nvmem/rmem.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvmem/rmem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Reserved Memory Based nvmem Device + +maintainers: + - Nicolas Saenz Julienne + +allOf: + - $ref: "nvmem.yaml#" + +properties: + compatible: + items: + - enum: + - raspberrypi,bootloader-config + - const: nvmem-rmem + + no-map: + $ref: /schemas/types.yaml#/definitions/flag + description: + Avoid creating a virtual mapping of the region as part of the OS' + standard mapping of system memory. + +required: + - compatible + - no-map + +unevaluatedProperties: false + +examples: + - | + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + + blconfig: nvram@10000000 { + compatible = "raspberrypi,bootloader-config", "nvmem-rmem"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x10000000 0x1000>; + no-map; + }; + }; + +... From 5a3fa75a4d9cb6bcfc9081ef224a4cdcd4b3eafe Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Fri, 29 Jan 2021 17:14:29 +0000 Subject: [PATCH 274/633] nvmem: Add driver to expose reserved memory as nvmem Firmware/co-processors might use reserved memory areas in order to pass data stemming from an nvmem device otherwise non accessible to Linux. For example an EEPROM memory only physically accessible to firmware, or data only accessible early at boot time. In order to expose this data to other drivers and user-space, the driver models the reserved memory area as an nvmem device. Tested-by: Tim Gover Reviewed-by: Rob Herring Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210129171430.11328-5-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/Kconfig | 8 ++++ drivers/nvmem/Makefile | 2 + drivers/nvmem/rmem.c | 97 ++++++++++++++++++++++++++++++++++++++++++ drivers/of/platform.c | 1 + 4 files changed, 108 insertions(+) create mode 100644 drivers/nvmem/rmem.c diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index 954d3b4a52ab..fecc19b884bf 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -270,4 +270,12 @@ config SPRD_EFUSE This driver can also be built as a module. If so, the module will be called nvmem-sprd-efuse. +config NVMEM_RMEM + tristate "Reserved Memory Based Driver Support" + help + This drivers maps reserved memory into an nvmem device. It might be + useful to expose information left by firmware in memory. + + This driver can also be built as a module. If so, the module + will be called nvmem-rmem. endif diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile index a7c377218341..5376b8e0dae5 100644 --- a/drivers/nvmem/Makefile +++ b/drivers/nvmem/Makefile @@ -55,3 +55,5 @@ obj-$(CONFIG_NVMEM_ZYNQMP) += nvmem_zynqmp_nvmem.o nvmem_zynqmp_nvmem-y := zynqmp_nvmem.o obj-$(CONFIG_SPRD_EFUSE) += nvmem_sprd_efuse.o nvmem_sprd_efuse-y := sprd-efuse.o +obj-$(CONFIG_NVMEM_RMEM) += nvmem-rmem.o +nvmem-rmem-y := rmem.o diff --git a/drivers/nvmem/rmem.c b/drivers/nvmem/rmem.c new file mode 100644 index 000000000000..b11c3c974b3d --- /dev/null +++ b/drivers/nvmem/rmem.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2020 Nicolas Saenz Julienne + */ + +#include +#include +#include +#include +#include + +struct rmem { + struct device *dev; + struct nvmem_device *nvmem; + struct reserved_mem *mem; + + phys_addr_t size; +}; + +static int rmem_read(void *context, unsigned int offset, + void *val, size_t bytes) +{ + struct rmem *priv = context; + size_t available = priv->mem->size; + loff_t off = offset; + void *addr; + int count; + + /* + * Only map the reserved memory at this point to avoid potential rogue + * kernel threads inadvertently modifying it. Based on the current + * uses-cases for this driver, the performance hit isn't a concern. + * Nor is likely to be, given the nature of the subsystem. Most nvmem + * devices operate over slow buses to begin with. + * + * An alternative would be setting the memory as RO, set_memory_ro(), + * but as of Dec 2020 this isn't possible on arm64. + */ + addr = memremap(priv->mem->base, available, MEMREMAP_WB); + if (IS_ERR(addr)) { + dev_err(priv->dev, "Failed to remap memory region\n"); + return PTR_ERR(addr); + } + + count = memory_read_from_buffer(val, bytes, &off, addr, available); + + memunmap(addr); + + return count; +} + +static int rmem_probe(struct platform_device *pdev) +{ + struct nvmem_config config = { }; + struct device *dev = &pdev->dev; + struct reserved_mem *mem; + struct rmem *priv; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + + mem = of_reserved_mem_lookup(dev->of_node); + if (!mem) { + dev_err(dev, "Failed to lookup reserved memory\n"); + return -EINVAL; + } + priv->mem = mem; + + config.dev = dev; + config.priv = priv; + config.name = "rmem"; + config.size = mem->size; + config.reg_read = rmem_read; + + return PTR_ERR_OR_ZERO(devm_nvmem_register(dev, &config)); +} + +static const struct of_device_id rmem_match[] = { + { .compatible = "nvmem-rmem", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, rmem_match); + +static struct platform_driver rmem_driver = { + .probe = rmem_probe, + .driver = { + .name = "rmem", + .of_match_table = rmem_match, + }, +}; +module_platform_driver(rmem_driver); + +MODULE_AUTHOR("Nicolas Saenz Julienne "); +MODULE_DESCRIPTION("Reserved Memory Based nvmem Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/of/platform.c b/drivers/of/platform.c index 79bd5f5a1bf1..6699cdbe58b6 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -511,6 +511,7 @@ static const struct of_device_id reserved_mem_matches[] = { { .compatible = "qcom,rmtfs-mem" }, { .compatible = "qcom,cmd-db" }, { .compatible = "ramoops" }, + { .compatible = "nvmem-rmem" }, {} }; From 0445efacec75b85c2a3c176957ee050ba9be53f0 Mon Sep 17 00:00:00 2001 From: Ahmad Fatoum Date: Fri, 29 Jan 2021 17:14:30 +0000 Subject: [PATCH 275/633] nvmem: core: skip child nodes not matching binding The nvmem cell binding applies to all eeprom child nodes matching "^.*@[0-9a-f]+$" without taking a compatible into account. Linux drivers, like at24, are even more extensive and assume _all_ at24 eeprom child nodes to be nvmem cells since e888d445ac33 ("nvmem: resolve cells from DT at registration time"). Since df5f3b6f5357 ("dt-bindings: nvmem: stm32: new property for data access"), the additionalProperties: True means it's Ok to have other properties as long as they don't match "^.*@[0-9a-f]+$". The barebox bootloader extends the MTD partitions binding to EEPROM and can fix up following device tree node: &eeprom { partitions { compatible = "fixed-partitions"; }; }; This is allowed binding-wise, but drivers using nvmem_register() like at24 will fail to parse because the function expects all child nodes to have a reg property present. This results in the whole EEPROM driver probe failing despite the device tree being correct. Fix this by skipping nodes lacking a reg property instead of returning an error. This effectively makes the drivers adhere to the binding because all nodes with a unit address must have a reg property and vice versa. Fixes: e888d445ac33 ("nvmem: resolve cells from DT at registration time"). Signed-off-by: Ahmad Fatoum Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210129171430.11328-6-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 68ae6f24b57f..a5ab1e0c74cf 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -682,7 +682,9 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) for_each_child_of_node(parent, child) { addr = of_get_property(child, "reg", &len); - if (!addr || (len < 2 * sizeof(u32))) { + if (!addr) + continue; + if (len < 2 * sizeof(u32)) { dev_err(dev, "nvmem: invalid reg on %pOF\n", child); return -EINVAL; } From 49f259eff82e688d83a9dd3be392fcfcc4f7cde5 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Thu, 28 Jan 2021 19:01:15 +0100 Subject: [PATCH 276/633] speakup ABI: Advertise synth parameters for all synths The parameters were advertised for the "soft" synth, but they are available for all synths. Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210128180116.1848120-2-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/stable/sysfs-driver-speakup | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-driver-speakup b/Documentation/ABI/stable/sysfs-driver-speakup index 792f58ba327d..47f5c8a9843a 100644 --- a/Documentation/ABI/stable/sysfs-driver-speakup +++ b/Documentation/ABI/stable/sysfs-driver-speakup @@ -273,7 +273,7 @@ Description: In `/sys/accessibility/speakup` is a directory corresponding to Below is a description of values and parameters for soft synthesizer, which is currently the most commonly used. -What: /sys/accessibility/speakup/soft/caps_start +What: /sys/accessibility/speakup//caps_start KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string that is sent to the synthesizer to cause it @@ -281,7 +281,7 @@ Description: This is the string that is sent to the synthesizer to cause it and most others, this causes the pitch of the voice to rise above the currently set pitch. -What: /sys/accessibility/speakup/soft/caps_stop +What: /sys/accessibility/speakup//caps_stop KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string sent to the synthesizer to cause it to stop @@ -290,12 +290,12 @@ Description: This is the string sent to the synthesizer to cause it to stop down to the currently set pitch. -What: /sys/accessibility/speakup/soft/delay_time +What: /sys/accessibility/speakup//delay_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/direct +What: /sys/accessibility/speakup//direct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Controls if punctuation is spoken by speakup, or by the @@ -306,36 +306,36 @@ Description: Controls if punctuation is spoken by speakup, or by the than". Zero lets speakup speak the punctuation. One lets the synthesizer itself speak punctuation. -What: /sys/accessibility/speakup/soft/freq +What: /sys/accessibility/speakup//freq KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the frequency of the speech synthesizer. Range is 0-9. -What: /sys/accessibility/speakup/soft/full_time +What: /sys/accessibility/speakup//full_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/jiffy_delta +What: /sys/accessibility/speakup//jiffy_delta KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This controls how many jiffys the kernel gives to the synthesizer. Setting this too high can make a system unstable, or even crash it. -What: /sys/accessibility/speakup/soft/pitch +What: /sys/accessibility/speakup//pitch KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the pitch of the synthesizer. The range is 0-9. -What: /sys/accessibility/speakup/soft/inflection +What: /sys/accessibility/speakup//inflection KernelVersion: 5.8 Contact: speakup@linux-speakup.org Description: Gets or sets the inflection of the synthesizer, i.e. the pitch range. The range is 0-9. -What: /sys/accessibility/speakup/soft/punct +What: /sys/accessibility/speakup//punct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the amount of punctuation spoken by the @@ -343,13 +343,13 @@ Description: Gets or sets the amount of punctuation spoken by the TODO: How is this related to speakup's punc_level, or reading_punc. -What: /sys/accessibility/speakup/soft/rate +What: /sys/accessibility/speakup//rate KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the rate of the synthesizer. Range is from zero slowest, to nine fastest. -What: /sys/accessibility/speakup/soft/tone +What: /sys/accessibility/speakup//tone KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the tone of the speech synthesizer. The range for @@ -357,12 +357,12 @@ Description: Gets or sets the tone of the speech synthesizer. The range for difference if using espeak and the espeakup connector. TODO: does espeakup support different tonalities? -What: /sys/accessibility/speakup/soft/trigger_time +What: /sys/accessibility/speakup//trigger_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/voice +What: /sys/accessibility/speakup//voice KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the voice used by the synthesizer if the @@ -371,7 +371,7 @@ Description: Gets or sets the voice used by the synthesizer if the voices, this parameter will not set the voice when the espeakup connector is used between speakup and espeak. -What: /sys/accessibility/speakup/soft/vol +What: /sys/accessibility/speakup//vol KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the volume of the speech synthesizer. Range is 0-9, From 1f7c14afd4ad5aae5220dfc878f29770239911b1 Mon Sep 17 00:00:00 2001 From: Samuel Thibault Date: Thu, 28 Jan 2021 19:01:16 +0100 Subject: [PATCH 277/633] speakup: Make dectlk flush timeout configurable In case the serial port or cable got faulty, we may not be getting acknowledgements any more. The driver then currently waits for 4s to avoid jamming the device. This makes this delay configurable. Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20210128180116.1848120-3-samuel.thibault@ens-lyon.org Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/stable/sysfs-driver-speakup | 7 +++++++ drivers/accessibility/speakup/speakup_dectlk.c | 11 ++++++++++- drivers/accessibility/speakup/spk_types.h | 3 ++- drivers/accessibility/speakup/synth.c | 3 +++ drivers/accessibility/speakup/varhandlers.c | 1 + 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-driver-speakup b/Documentation/ABI/stable/sysfs-driver-speakup index 47f5c8a9843a..dc2a6ba1674b 100644 --- a/Documentation/ABI/stable/sysfs-driver-speakup +++ b/Documentation/ABI/stable/sysfs-driver-speakup @@ -312,6 +312,13 @@ Contact: speakup@linux-speakup.org Description: Gets or sets the frequency of the speech synthesizer. Range is 0-9. +What: /sys/accessibility/speakup//flush_time +KernelVersion: 5.12 +Contact: speakup@linux-speakup.org +Description: Gets or sets the timeout to wait for the synthesizer flush to + complete. This can be used when the cable gets faulty and flush + notifications are getting lost. + What: /sys/accessibility/speakup//full_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index d75de36e96c3..580ec796816b 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -78,6 +78,8 @@ static struct kobj_attribute direct_attribute = __ATTR(direct, 0644, spk_var_show, spk_var_store); static struct kobj_attribute full_time_attribute = __ATTR(full_time, 0644, spk_var_show, spk_var_store); +static struct kobj_attribute flush_time_attribute = + __ATTR(flush_time, 0644, spk_var_show, spk_var_store); static struct kobj_attribute jiffy_delta_attribute = __ATTR(jiffy_delta, 0644, spk_var_show, spk_var_store); static struct kobj_attribute trigger_time_attribute = @@ -99,6 +101,7 @@ static struct attribute *synth_attrs[] = { &delay_time_attribute.attr, &direct_attribute.attr, &full_time_attribute.attr, + &flush_time_attribute.attr, &jiffy_delta_attribute.attr, &trigger_time_attribute.attr, NULL, /* need to NULL terminate the list of attributes */ @@ -118,6 +121,7 @@ static struct spk_synth synth_dectlk = { .trigger = 50, .jiffies = 50, .full = 40000, + .flush_time = 4000, .dev_name = SYNTH_DEFAULT_DEV, .startup = SYNTH_START, .checkval = SYNTH_CHECK, @@ -200,18 +204,23 @@ static void do_catch_up(struct spk_synth *synth) static u_char last = '\0'; unsigned long flags; unsigned long jiff_max; - unsigned long timeout = msecs_to_jiffies(4000); + unsigned long timeout; DEFINE_WAIT(wait); struct var_t *jiffy_delta; struct var_t *delay_time; + struct var_t *flush_time; int jiffy_delta_val; int delay_time_val; + int timeout_val; jiffy_delta = spk_get_var(JIFFY); delay_time = spk_get_var(DELAY); + flush_time = spk_get_var(FLUSH); spin_lock_irqsave(&speakup_info.spinlock, flags); jiffy_delta_val = jiffy_delta->u.n.value; + timeout_val = flush_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); + timeout = msecs_to_jiffies(timeout_val); jiff_max = jiffies + jiffy_delta_val; while (!kthread_should_stop()) { diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 7789f5dca62a..6a96ad94bc3f 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -48,7 +48,7 @@ enum var_id_t { ATTRIB_BLEEP, BLEEPS, RATE, PITCH, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG, DIRECT, PAUSE, - CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, + CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, FLUSH, MAXVARS }; @@ -178,6 +178,7 @@ struct spk_synth { int trigger; int jiffies; int full; + int flush_time; int ser; char *dev_name; short flags; diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index 6c14b682da13..2b8699673bac 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -348,6 +348,7 @@ struct var_t synth_time_vars[] = { { TRIGGER, .u.n = {NULL, 20, 10, 2000, 0, 0, NULL } }, { JIFFY, .u.n = {NULL, 50, 20, 200, 0, 0, NULL } }, { FULL, .u.n = {NULL, 400, 200, 60000, 0, 0, NULL } }, + { FLUSH, .u.n = {NULL, 4000, 100, 4000, 0, 0, NULL } }, V_LAST_VAR }; @@ -408,6 +409,8 @@ static int do_synth_init(struct spk_synth *in_synth) synth_time_vars[2].u.n.default_val = synth->jiffies; synth_time_vars[3].u.n.value = synth_time_vars[3].u.n.default_val = synth->full; + synth_time_vars[4].u.n.value = + synth_time_vars[4].u.n.default_val = synth->flush_time; synth_printf("%s", synth->init); for (var = synth->vars; (var->var_id >= 0) && (var->var_id < MAXVARS); var++) diff --git a/drivers/accessibility/speakup/varhandlers.c b/drivers/accessibility/speakup/varhandlers.c index d7f6bec7ff06..067c0da97dcb 100644 --- a/drivers/accessibility/speakup/varhandlers.c +++ b/drivers/accessibility/speakup/varhandlers.c @@ -23,6 +23,7 @@ static struct st_var_header var_headers[] = { { "trigger_time", TRIGGER, VAR_TIME, NULL, NULL }, { "jiffy_delta", JIFFY, VAR_TIME, NULL, NULL }, { "full_time", FULL, VAR_TIME, NULL, NULL }, + { "flush_time", FLUSH, VAR_TIME, NULL, NULL }, { "spell_delay", SPELL_DELAY, VAR_NUM, &spk_spell_delay, NULL }, { "bleeps", BLEEPS, VAR_NUM, &spk_bleeps, NULL }, { "attrib_bleep", ATTRIB_BLEEP, VAR_NUM, &spk_attrib_bleep, NULL }, From 920fd8a70619074eac7687352c8f1c6f3c2a64a5 Mon Sep 17 00:00:00 2001 From: Ricky Wu Date: Thu, 4 Feb 2021 16:31:15 +0800 Subject: [PATCH 278/633] misc: rtsx: init of rts522a add OCP power off when no card is present Power down OCP for power consumption when no SD/MMC card is present Cc: stable@vger.kernel.org Signed-off-by: Ricky Wu Link: https://lore.kernel.org/r/20210204083115.9471-1-ricky_wu@realtek.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cardreader/rts5227.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/misc/cardreader/rts5227.c b/drivers/misc/cardreader/rts5227.c index 8859011672cb..8200af22b529 100644 --- a/drivers/misc/cardreader/rts5227.c +++ b/drivers/misc/cardreader/rts5227.c @@ -398,6 +398,11 @@ static int rts522a_extra_init_hw(struct rtsx_pcr *pcr) { rts5227_extra_init_hw(pcr); + /* Power down OCP for power consumption */ + if (!pcr->card_exist) + rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN, + OC_POWER_DOWN); + rtsx_pci_write_register(pcr, FUNC_FORCE_CTL, FUNC_FORCE_UPME_XMT_DBG, FUNC_FORCE_UPME_XMT_DBG); rtsx_pci_write_register(pcr, PCLK_CTL, 0x04, 0x04); From b31f1eb41c140d7979f855df73064b3a3ae8055a Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Fri, 5 Feb 2021 10:08:52 +0000 Subject: [PATCH 279/633] nvmem: Kconfig: Correct typo in NVMEM_RMEM s/drivers/driver/ as the configuration selects a single driver. Suggested-by: Randy Dunlap Acked-by: Randy Dunlap Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210205100853.32372-2-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index fecc19b884bf..75d2594c16e1 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -273,7 +273,7 @@ config SPRD_EFUSE config NVMEM_RMEM tristate "Reserved Memory Based Driver Support" help - This drivers maps reserved memory into an nvmem device. It might be + This driver maps reserved memory into an nvmem device. It might be useful to expose information left by firmware in memory. This driver can also be built as a module. If so, the module From e2057ee29973b9741d43d3f475a6b02fb46a0e61 Mon Sep 17 00:00:00 2001 From: Subbaraman Narayanamurthy Date: Fri, 5 Feb 2021 10:08:53 +0000 Subject: [PATCH 280/633] nvmem: qcom-spmi-sdam: Fix uninitialized pdev pointer "sdam->pdev" is uninitialized and it is used to print error logs. Fix it. Since device pointer can be used from sdam_config, use it directly thereby removing pdev pointer. Fixes: 40ce9798794f ("nvmem: add QTI SDAM driver") Cc: stable@vger.kernel.org Signed-off-by: Subbaraman Narayanamurthy Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20210205100853.32372-3-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/qcom-spmi-sdam.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/nvmem/qcom-spmi-sdam.c b/drivers/nvmem/qcom-spmi-sdam.c index a72704cd0468..f6e9f96933ca 100644 --- a/drivers/nvmem/qcom-spmi-sdam.c +++ b/drivers/nvmem/qcom-spmi-sdam.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2017, 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2017, 2020-2021, The Linux Foundation. All rights reserved. */ #include @@ -18,7 +18,6 @@ #define SDAM_PBS_TRIG_CLR 0xE6 struct sdam_chip { - struct platform_device *pdev; struct regmap *regmap; struct nvmem_config sdam_config; unsigned int base; @@ -65,7 +64,7 @@ static int sdam_read(void *priv, unsigned int offset, void *val, size_t bytes) { struct sdam_chip *sdam = priv; - struct device *dev = &sdam->pdev->dev; + struct device *dev = sdam->sdam_config.dev; int rc; if (!sdam_is_valid(sdam, offset, bytes)) { @@ -86,7 +85,7 @@ static int sdam_write(void *priv, unsigned int offset, void *val, size_t bytes) { struct sdam_chip *sdam = priv; - struct device *dev = &sdam->pdev->dev; + struct device *dev = sdam->sdam_config.dev; int rc; if (!sdam_is_valid(sdam, offset, bytes)) { From ca338fed2a5fce66660904d7ab50bec061d2c8a2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 4 Feb 2021 17:05:08 +0200 Subject: [PATCH 281/633] platform/x86: intel_scu_wdt: Drop mistakenly added const Neither original structure nor platform_data is declared with const. Drop mistakenly added const when assing platform_data. Fixes: a507e5d90f3d ("platform/x86: intel_scu_wdt: Get rid of custom x86 model comparison") Reported-by: Stephen Rothwell Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20210204150508.62659-1-andriy.shevchenko@linux.intel.com Signed-off-by: Hans de Goede --- drivers/platform/x86/intel_scu_wdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/platform/x86/intel_scu_wdt.c b/drivers/platform/x86/intel_scu_wdt.c index 85ee85ca2215..c2479777a1d6 100644 --- a/drivers/platform/x86/intel_scu_wdt.c +++ b/drivers/platform/x86/intel_scu_wdt.c @@ -63,7 +63,7 @@ static int __init register_mid_wdt(void) if (!id) return -ENODEV; - wdt_dev.dev.platform_data = (const struct intel_mid_wdt_pdata *)id->driver_data; + wdt_dev.dev.platform_data = (struct intel_mid_wdt_pdata *)id->driver_data; return platform_device_register(&wdt_dev); } arch_initcall(register_mid_wdt); From 1e2f29ba83c55b9778bdb60e77216b08dbd69bf5 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Wed, 3 Feb 2021 17:39:42 +0100 Subject: [PATCH 282/633] mhi: pci_generic: Print warning in case of firmware crash Print warning when MHI detects sys error. Signed-off-by: Loic Poulain Reviewed-by: Hemant Kumar Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/1612370382-21643-1-git-send-email-loic.poulain@linaro.org Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 444693e3fd72..2476041f8d56 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -217,7 +217,17 @@ static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl, static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, enum mhi_callback cb) { + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + /* Nothing to do for now */ + switch (cb) { + case MHI_CB_FATAL_ERROR: + case MHI_CB_SYS_ERROR: + dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb); + break; + default: + break; + } } static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) From 026c5b1ec29cb9904406c7b3090eaf54e345e7f2 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Fri, 5 Feb 2021 09:36:35 +0100 Subject: [PATCH 283/633] bus: mhi: pci_generic: Increase num of elements in hw event ring We met some sporadic modem crashes during high throughput testing, this has been root caused to a lack of elements in the event ring. Indeed, the modem is simply crashing when event ring becomes empty. It appears that the total number event ring elements is too low given the performances of the modem (IPA hardware accelerator). This change increases the number of elements in the hardware event ring to 2048, which is aligned with what is defined in downstream version: https://source.codeaurora.org/quic/la/kernel/msm-4.14/tree/arch/arm64/boot/dts/qcom/sm8150-mhi.dtsi?h=msm-4.14#n482 With this change, modem coes not crash anymore. Note: An event ring element is 16-Byte, so the total memory usage of a hardware event ring is now 32KB. Signed-off-by: Loic Poulain Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/1612514195-8257-1-git-send-email-loic.poulain@linaro.org Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/pci_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index 2476041f8d56..20673a4b4a3c 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -129,7 +129,7 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ - .num_elements = 256, \ + .num_elements = 2048, \ .irq_moderation_ms = 1, \ .irq = (ev_ring) + 1, \ .priority = 1, \ From 1609faa9e675c8168b640697c82c7b31befd2658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 5 Feb 2021 14:08:47 +0100 Subject: [PATCH 284/633] coresight: etm4x: Fix merge resolution for amba rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was non-trivial to get right because commits c23bc382ef0e ("coresight: etm4x: Refactor probing routine") and 5214b563588e ("coresight: etm4x: Add support for sysreg only devices") changed the code flow considerably. With this change the driver can be built again. Fixes: 0573d3fa4864 ("Merge branch 'devel-stable' of git://git.armlinux.org.uk/~rmk/linux-arm into char-misc-next") Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210205130848.20009-1-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index bc55b261af23..c8ecd91e289e 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1906,15 +1906,16 @@ static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) cpus_read_unlock(); coresight_unregister(drvdata->csdev); + + return 0; } -static int __exit etm4_remove_amba(struct amba_device *adev) +static void __exit etm4_remove_amba(struct amba_device *adev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); if (drvdata) - return etm4_remove_dev(drvdata); - return 0; + etm4_remove_dev(drvdata); } static int __exit etm4_remove_platform_dev(struct platform_device *pdev) From d0858167492b59297c5c2aac10cdc9904c5a1cc6 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Thu, 4 Feb 2021 22:28:03 +0530 Subject: [PATCH 285/633] dt-bindings: phy: qcom,qmp: Add SM8350 UFS PHY bindings Add the compatible strings for the UFS PHY found on SM8350 SoC. Signed-off-by: Vinod Koul Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210204165805.62235-2-vkoul@kernel.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index c38061de242a..626447fee092 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -40,6 +40,7 @@ properties: - qcom,sm8250-qmp-modem-pcie-phy - qcom,sm8250-qmp-usb3-phy - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-ufs-phy - qcom,sm8350-qmp-usb3-phy - qcom,sm8350-qmp-usb3-uni-phy - qcom,sdx55-qmp-usb3-uni-phy From 920abc105b5de6489d61bd8c5d0d44463665ae3f Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Thu, 4 Feb 2021 22:28:04 +0530 Subject: [PATCH 286/633] phy: qcom-qmp: Add UFS V5 registers found in SM8350 Add the registers for UFS found in SM8350. The UFS phy used in SM8350 seems to have same offsets as V5 phy, although Documentation for that is lacking. Signed-off-by: Vinod Koul Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210204165805.62235-3-vkoul@kernel.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.h | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index dff7be5a1cc1..71ce3aa174ae 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -824,6 +824,32 @@ #define QPHY_V4_PCS_PCIE_PRESET_P10_PRE 0xbc #define QPHY_V4_PCS_PCIE_PRESET_P10_POST 0xe0 +/* Only for QMP V5 PHY - QSERDES COM registers */ +#define QSERDES_V5_COM_PLL_IVCO 0x058 +#define QSERDES_V5_COM_CP_CTRL_MODE0 0x074 +#define QSERDES_V5_COM_CP_CTRL_MODE1 0x078 +#define QSERDES_V5_COM_PLL_RCTRL_MODE0 0x07c +#define QSERDES_V5_COM_PLL_RCTRL_MODE1 0x080 +#define QSERDES_V5_COM_PLL_CCTRL_MODE0 0x084 +#define QSERDES_V5_COM_PLL_CCTRL_MODE1 0x088 +#define QSERDES_V5_COM_SYSCLK_EN_SEL 0x094 +#define QSERDES_V5_COM_LOCK_CMP_EN 0x0a4 +#define QSERDES_V5_COM_LOCK_CMP1_MODE0 0x0ac +#define QSERDES_V5_COM_LOCK_CMP2_MODE0 0x0b0 +#define QSERDES_V5_COM_LOCK_CMP1_MODE1 0x0b4 +#define QSERDES_V5_COM_DEC_START_MODE0 0x0bc +#define QSERDES_V5_COM_LOCK_CMP2_MODE1 0x0b8 +#define QSERDES_V5_COM_DEC_START_MODE1 0x0c4 +#define QSERDES_V5_COM_VCO_TUNE_MAP 0x10c +#define QSERDES_V5_COM_VCO_TUNE_INITVAL2 0x124 +#define QSERDES_V5_COM_HSCLK_SEL 0x158 +#define QSERDES_V5_COM_HSCLK_HS_SWITCH_SEL 0x15c +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE0 0x1ac +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE0 0x1b0 +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE1 0x1b4 +#define QSERDES_V5_COM_BIN_VCOCAL_HSCLK_SEL 0x1bc +#define QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE1 0x1b8 + /* Only for QMP V5 PHY - TX registers */ #define QSERDES_V5_TX_RES_CODE_LANE_TX 0x34 #define QSERDES_V5_TX_RES_CODE_LANE_RX 0x38 @@ -837,6 +863,10 @@ #define QSERDES_V5_TX_RCV_DETECT_LVL_2 0xa4 #define QSERDES_V5_TX_TRAN_DRVR_EMP_EN 0xc0 #define QSERDES_V5_TX_PI_QEC_CTRL 0xe4 +#define QSERDES_V5_TX_PWM_GEAR_1_DIVIDER_BAND0_1 0x178 +#define QSERDES_V5_TX_PWM_GEAR_2_DIVIDER_BAND0_1 0x17c +#define QSERDES_V5_TX_PWM_GEAR_3_DIVIDER_BAND0_1 0x180 +#define QSERDES_V5_TX_PWM_GEAR_4_DIVIDER_BAND0_1 0x184 /* Only for QMP V5 PHY - RX registers */ #define QSERDES_V5_RX_UCDR_FO_GAIN 0x008 @@ -893,6 +923,23 @@ #define QSERDES_V5_RX_DCC_CTRL1 0x1a8 #define QSERDES_V5_RX_VTH_CODE 0x1b0 +/* Only for QMP V5 PHY - UFS PCS registers */ +#define QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB 0x00c +#define QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB 0x010 +#define QPHY_V5_PCS_UFS_PLL_CNTL 0x02c +#define QPHY_V5_PCS_UFS_TX_LARGE_AMP_DRV_LVL 0x030 +#define QPHY_V5_PCS_UFS_TX_SMALL_AMP_DRV_LVL 0x038 +#define QPHY_V5_PCS_UFS_TX_HSGEAR_CAPABILITY 0x074 +#define QPHY_V5_PCS_UFS_RX_HSGEAR_CAPABILITY 0x0b4 +#define QPHY_V5_PCS_UFS_DEBUG_BUS_CLKSEL 0x124 +#define QPHY_V5_PCS_UFS_RX_MIN_HIBERN8_TIME 0x150 +#define QPHY_V5_PCS_UFS_RX_SIGDET_CTRL1 0x154 +#define QPHY_V5_PCS_UFS_RX_SIGDET_CTRL2 0x158 +#define QPHY_V5_PCS_UFS_TX_PWM_GEAR_BAND 0x160 +#define QPHY_V5_PCS_UFS_TX_HS_GEAR_BAND 0x168 +#define QPHY_V5_PCS_UFS_TX_MID_TERM_CTRL1 0x1d8 +#define QPHY_V5_PCS_UFS_MULTI_LANE_CTRL1 0x1e0 + /* Only for QMP V5 PHY - USB3 have different offsets than V4 */ #define QPHY_V5_PCS_USB3_POWER_STATE_CONFIG1 0x300 #define QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_STATUS 0x304 From 0e43fdb94a8363cfd78e8d14580ea2f5b82789a8 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Thu, 4 Feb 2021 22:28:05 +0530 Subject: [PATCH 287/633] phy: qcom-qmp: Add support for SM8350 UFS phy Add the tables for init sequences for UFS QMP phy found in SM8350 SoC. Signed-off-by: Vinod Koul Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210204165805.62235-4-vkoul@kernel.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 127 ++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index 5eaf6dee1e45..d2c8e1dbee09 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -2030,6 +2030,106 @@ static const struct qmp_phy_init_tbl sdx55_usb3_uniphy_rx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V4_RX_GM_CAL, 0x1f), }; +static const struct qmp_phy_init_tbl sm8350_ufsphy_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_EN_SEL, 0xd9), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_SEL, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_HS_SWITCH_SEL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP_EN, 0x42), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_INITVAL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_HSCLK_SEL, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE0, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE0, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE0, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE0, 0x19), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0xac), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE1, 0x98), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE1, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE1, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE1, 0x65), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE1, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE1, 0xdd), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE1, 0x23), + + /* Rate B */ + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x06), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_1_DIVIDER_BAND0_1, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_2_DIVIDER_BAND0_1, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_3_DIVIDER_BAND0_1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PWM_GEAR_4_DIVIDER_BAND0_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0xf5), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_TRAN_DRVR_EMP_EN, 0x0c), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_LVL, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_BAND, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x5a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0xf1), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FO_GAIN, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_TERM_BW, 0x1b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL3, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x17), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_MEASURE_TIME, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_LOW, 0xc0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_IDAC_TSETTLE_HIGH, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0x6d), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x6d), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xed), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0x3c), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0xe0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xb7), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_LOW, 0xe0), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH2, 0xc8), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH3, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_10_HIGH4, 0xb7), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DCC_CTRL1, 0x0c), +}; + +static const struct qmp_phy_init_tbl sm8350_ufsphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_SIGDET_CTRL2, 0x6d), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_SMALL_AMP_DRV_LVL, 0x02), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_MID_TERM_CTRL1, 0x43), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_DEBUG_BUS_CLKSEL, 0x1f), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_MIN_HIBERN8_TIME, 0xff), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_PLL_CNTL, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB, 0x16), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB, 0xd8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_PWM_GEAR_BAND, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_HS_GEAR_BAND, 0x06), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_RX_SIGDET_CTRL1, 0x0e), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_MULTI_LANE_CTRL1, 0x02), +}; + static const struct qmp_phy_init_tbl sm8350_usb3_tx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_TX, 0x00), QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_RX, 0x00), @@ -3051,6 +3151,30 @@ static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; +static const struct qmp_phy_cfg sm8350_ufsphy_cfg = { + .type = PHY_TYPE_UFS, + .nlanes = 2, + + .serdes_tbl = sm8350_ufsphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sm8350_ufsphy_serdes_tbl), + .tx_tbl = sm8350_ufsphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sm8350_ufsphy_tx_tbl), + .rx_tbl = sm8350_ufsphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sm8350_ufsphy_rx_tbl), + .pcs_tbl = sm8350_ufsphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sm8350_ufsphy_pcs_tbl), + .clk_list = sdm845_ufs_phy_clk_l, + .num_clks = ARRAY_SIZE(sdm845_ufs_phy_clk_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = sm8150_ufsphy_regs_layout, + + .start_ctrl = SERDES_START, + .pwrdn_ctrl = SW_PWRDN, + + .is_dual_lane_phy = true, +}; + static const struct qmp_phy_cfg sm8350_usb3phy_cfg = { .type = PHY_TYPE_USB3, .nlanes = 1, @@ -4462,6 +4586,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = { }, { .compatible = "qcom,sm8250-qmp-gen3x2-pcie-phy", .data = &sm8250_qmp_gen3x2_pciephy_cfg, + }, { + .compatible = "qcom,sm8350-qmp-ufs-phy", + .data = &sm8350_ufsphy_cfg, }, { .compatible = "qcom,sm8250-qmp-modem-pcie-phy", .data = &sm8250_qmp_gen3x2_pciephy_cfg, From 3dbbc8e97cb267ef2af03e5545211f260291f830 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Thu, 4 Feb 2021 14:11:42 +0100 Subject: [PATCH 288/633] phy: zynqmp: Simplify code by using dev_err_probe() Use already prepared dev_err_probe() introduced by commit a787e5400a1c ("driver core: add device probe log helper"). It simplifies EPROBE_DEFER handling. Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/0df30548f721b10475a6cc5659beda102fec3c87.1612444300.git.michal.simek@xilinx.com Signed-off-by: Vinod Koul --- drivers/phy/xilinx/phy-zynqmp.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c index 2b0f921b6ee3..2b65f84a5f89 100644 --- a/drivers/phy/xilinx/phy-zynqmp.c +++ b/drivers/phy/xilinx/phy-zynqmp.c @@ -874,13 +874,10 @@ static int xpsgtr_get_ref_clocks(struct xpsgtr_dev *gtr_dev) snprintf(name, sizeof(name), "ref%u", refclk); clk = devm_clk_get_optional(gtr_dev->dev, name); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(gtr_dev->dev, - "Failed to get reference clock %u: %ld\n", - refclk, PTR_ERR(clk)); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(gtr_dev->dev, PTR_ERR(clk), + "Failed to get reference clock %u\n", + refclk); if (!clk) continue; From 43851904cbd7c7bb18c84cc553b6232ace23747b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 4 Feb 2021 18:03:13 +0000 Subject: [PATCH 289/633] phy: qcom-qmp: make a const array static, makes object smaller Don't populate the const array cfg1_settings on the stack but instead make it static. Makes the object code smaller by 24 bytes: Before: text data bss dec hex filename 73585 20240 64 93889 16ec1 drivers/phy/qualcomm/phy-qcom-qmp.o After: text data bss dec hex filename 73465 20336 64 93865 16ea9 drivers/phy/qualcomm/phy-qcom-qmp.o (gcc version 10.2.0) Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20210204180313.108876-1-colin.king@canonical.com Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c index d2c8e1dbee09..9cdebe7f26cb 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp.c @@ -3545,7 +3545,7 @@ static int qcom_qmp_phy_configure_dp_phy(struct qmp_phy *qphy) static int qcom_qmp_dp_phy_calibrate(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); - const u8 cfg1_settings[] = { 0x13, 0x23, 0x1d }; + static const u8 cfg1_settings[] = { 0x13, 0x23, 0x1d }; u8 val; qphy->dp_aux_cfg++; From d68f2cb095979dff1c99e7803c7730ec2b53fc83 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 29 Jan 2021 15:38:35 +0800 Subject: [PATCH 290/633] phy: cpcap-usb: Simplify bool conversion Fix the following coccicheck warning: ./drivers/phy/motorola/phy-cpcap-usb.c:146:31-36: WARNING: conversion to bool not needed here Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1611905915-50394-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Vinod Koul --- drivers/phy/motorola/phy-cpcap-usb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index 442522ba487f..c276f7ce2ea9 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -143,7 +143,7 @@ static bool cpcap_usb_vbus_valid(struct cpcap_phy_ddata *ddata) error = iio_read_channel_processed(ddata->vbus, &value); if (error >= 0) - return value > 3900 ? true : false; + return value > 3900; dev_err(ddata->dev, "error reading VBUS: %i\n", error); From a1d9fda296b86842014f6799a108f71726696fe2 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Sat, 6 Feb 2021 15:51:22 +0530 Subject: [PATCH 291/633] soundwire: Revert "soundwire: debugfs: use controller id instead of link_id" This reverts commit 6d5e7af1f6f5 ("soundwire: debugfs: use controller id instead of link_id") for now while we arrive at a better way for this. Signed-off-by: Vinod Koul --- drivers/soundwire/debugfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soundwire/debugfs.c b/drivers/soundwire/debugfs.c index 5f9efa42bb25..b6cad0d59b7b 100644 --- a/drivers/soundwire/debugfs.c +++ b/drivers/soundwire/debugfs.c @@ -19,7 +19,7 @@ void sdw_bus_debugfs_init(struct sdw_bus *bus) return; /* create the debugfs master-N */ - snprintf(name, sizeof(name), "master-%d", bus->id); + snprintf(name, sizeof(name), "master-%d", bus->link_id); bus->debugfs = debugfs_create_dir(name, sdw_debugfs_root); } From b04c975e654cfdea6d691cd403b5a81cce7e593d Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 22 Jan 2021 15:06:26 +0800 Subject: [PATCH 292/633] soundwire: bus: use sdw_update_no_pm when initializing a device When a Slave device is resumed, it may resume the bus and restart the enumeration. During that process, we absolutely don't want to call regular read/write routines which will wait for the resume to complete, otherwise a deadlock occurs. Fixes: 60ee9be25571 ('soundwire: bus: add PM/no-PM versions of read/write functions') Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-2-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index d9deafdcf495..6933edbbeff6 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -491,6 +491,18 @@ sdw_read_no_pm(struct sdw_slave *slave, u32 addr) return buf; } +static int sdw_update_no_pm(struct sdw_slave *slave, u32 addr, u8 mask, u8 val) +{ + int tmp; + + tmp = sdw_read_no_pm(slave, addr); + if (tmp < 0) + return tmp; + + tmp = (tmp & ~mask) | val; + return sdw_write_no_pm(slave, addr, tmp); +} + /** * sdw_nread() - Read "n" contiguous SDW Slave registers * @slave: SDW Slave @@ -1254,7 +1266,7 @@ static int sdw_initialize_slave(struct sdw_slave *slave) val = slave->prop.scp_int1_mask; /* Enable SCP interrupts */ - ret = sdw_update(slave, SDW_SCP_INTMASK1, val, val); + ret = sdw_update_no_pm(slave, SDW_SCP_INTMASK1, val, val); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INTMASK1 write failed:%d\n", ret); @@ -1269,7 +1281,7 @@ static int sdw_initialize_slave(struct sdw_slave *slave) val = prop->dp0_prop->imp_def_interrupts; val |= SDW_DP0_INT_PORT_READY | SDW_DP0_INT_BRA_FAILURE; - ret = sdw_update(slave, SDW_DP0_INTMASK, val, val); + ret = sdw_update_no_pm(slave, SDW_DP0_INTMASK, val, val); if (ret < 0) dev_err(slave->bus->dev, "SDW_DP0_INTMASK read failed:%d\n", ret); From 299e9780b9196bcb15b26dfdccd3244eb072d560 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 22 Jan 2021 15:06:27 +0800 Subject: [PATCH 293/633] soundwire: bus: use sdw_write_no_pm when setting the bus scale registers When a Slave device is resumed, it may resume the bus and restart the enumeration. During that process, we absolutely don't want to call regular read/write routines which will wait for the resume to complete, otherwise a deadlock occurs. This patch fixes the same problem as the previous one, but is split to make the life of linux-stable maintainers less painful. Fixes: 29d158f90690 ('soundwire: bus: initialize bus clock base and scale registers') Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-3-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 6933edbbeff6..b8b578191a3b 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -1220,7 +1220,7 @@ static int sdw_slave_set_frequency(struct sdw_slave *slave) } scale_index++; - ret = sdw_write(slave, SDW_SCP_BUS_CLOCK_BASE, base); + ret = sdw_write_no_pm(slave, SDW_SCP_BUS_CLOCK_BASE, base); if (ret < 0) { dev_err(&slave->dev, "SDW_SCP_BUS_CLOCK_BASE write failed:%d\n", ret); @@ -1228,13 +1228,13 @@ static int sdw_slave_set_frequency(struct sdw_slave *slave) } /* initialize scale for both banks */ - ret = sdw_write(slave, SDW_SCP_BUSCLOCK_SCALE_B0, scale_index); + ret = sdw_write_no_pm(slave, SDW_SCP_BUSCLOCK_SCALE_B0, scale_index); if (ret < 0) { dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE_B0 write failed:%d\n", ret); return ret; } - ret = sdw_write(slave, SDW_SCP_BUSCLOCK_SCALE_B1, scale_index); + ret = sdw_write_no_pm(slave, SDW_SCP_BUSCLOCK_SCALE_B1, scale_index); if (ret < 0) dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE_B1 write failed:%d\n", ret); From c30b63ef0d84e0c9ae2d55073fdfeddb7edeb660 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 22 Jan 2021 15:06:28 +0800 Subject: [PATCH 294/633] soundwire: bus: use no_pm IO routines for all interrupt handling There is no need to play with pm_runtime reference counts, if needed the codec drivers are already explicitly resumed. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-4-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index b8b578191a3b..675d028e923d 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -1293,7 +1293,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) u8 clear, impl_int_mask; int status, status2, ret, count = 0; - status = sdw_read(slave, SDW_DP0_INT); + status = sdw_read_no_pm(slave, SDW_DP0_INT); if (status < 0) { dev_err(slave->bus->dev, "SDW_DP0_INT read failed:%d\n", status); @@ -1332,7 +1332,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } /* clear the interrupts but don't touch reserved and SDCA_CASCADE fields */ - ret = sdw_write(slave, SDW_DP0_INT, clear); + ret = sdw_write_no_pm(slave, SDW_DP0_INT, clear); if (ret < 0) { dev_err(slave->bus->dev, "SDW_DP0_INT write failed:%d\n", ret); @@ -1340,7 +1340,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } /* Read DP0 interrupt again */ - status2 = sdw_read(slave, SDW_DP0_INT); + status2 = sdw_read_no_pm(slave, SDW_DP0_INT); if (status2 < 0) { dev_err(slave->bus->dev, "SDW_DP0_INT read failed:%d\n", status2); @@ -1371,7 +1371,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, return sdw_handle_dp0_interrupt(slave, slave_status); addr = SDW_DPN_INT(port); - status = sdw_read(slave, addr); + status = sdw_read_no_pm(slave, addr); if (status < 0) { dev_err(slave->bus->dev, "SDW_DPN_INT read failed:%d\n", status); @@ -1405,7 +1405,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, } /* clear the interrupt but don't touch reserved fields */ - ret = sdw_write(slave, addr, clear); + ret = sdw_write_no_pm(slave, addr, clear); if (ret < 0) { dev_err(slave->bus->dev, "SDW_DPN_INT write failed:%d\n", ret); @@ -1413,7 +1413,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, } /* Read DPN interrupt again */ - status2 = sdw_read(slave, addr); + status2 = sdw_read_no_pm(slave, addr); if (status2 < 0) { dev_err(slave->bus->dev, "SDW_DPN_INT read failed:%d\n", status2); @@ -1455,7 +1455,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } /* Read Intstat 1, Intstat 2 and Intstat 3 registers */ - ret = sdw_read(slave, SDW_SCP_INT1); + ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INT1 read failed:%d\n", ret); @@ -1463,7 +1463,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } buf = ret; - ret = sdw_nread(slave, SDW_SCP_INTSTAT2, 2, buf2); + ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, buf2); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); @@ -1471,7 +1471,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } if (slave->prop.is_sdca) { - ret = sdw_read(slave, SDW_DP0_INT); + ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { dev_err(slave->bus->dev, "SDW_DP0_INT read failed:%d\n", ret); @@ -1568,7 +1568,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } /* Ack interrupt */ - ret = sdw_write(slave, SDW_SCP_INT1, clear); + ret = sdw_write_no_pm(slave, SDW_SCP_INT1, clear); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INT1 write failed:%d\n", ret); @@ -1582,7 +1582,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) * Read status again to ensure no new interrupts arrived * while servicing interrupts. */ - ret = sdw_read(slave, SDW_SCP_INT1); + ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INT1 read failed:%d\n", ret); @@ -1590,7 +1590,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } _buf = ret; - ret = sdw_nread(slave, SDW_SCP_INTSTAT2, 2, _buf2); + ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, _buf2); if (ret < 0) { dev_err(slave->bus->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); @@ -1598,7 +1598,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } if (slave->prop.is_sdca) { - ret = sdw_read(slave, SDW_DP0_INT); + ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { dev_err(slave->bus->dev, "SDW_DP0_INT read failed:%d\n", ret); From 36edb1407c3c042240b5dbc2530d6e7119535baa Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Sat, 6 Feb 2021 16:43:21 +0200 Subject: [PATCH 295/633] mei: allow clients on bus to communicate in remove callback Introduce new intermediate state to allow the clients on the bus to communicate with the firmware from the remove handler. This is to enable to perform a clean shutdown. Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210206144325.25682-2-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/bus.c | 6 ++++-- drivers/misc/mei/client.c | 3 ++- drivers/misc/mei/init.c | 5 ++++- drivers/misc/mei/mei_dev.h | 1 + 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 2907db260fba..34fb5e541fe5 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -44,7 +44,8 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, bus = cl->dev; mutex_lock(&bus->device_lock); - if (bus->dev_state != MEI_DEV_ENABLED) { + if (bus->dev_state != MEI_DEV_ENABLED && + bus->dev_state != MEI_DEV_POWERING_DOWN) { rets = -ENODEV; goto out; } @@ -128,7 +129,8 @@ ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, u8 *vtag, bus = cl->dev; mutex_lock(&bus->device_lock); - if (bus->dev_state != MEI_DEV_ENABLED) { + if (bus->dev_state != MEI_DEV_ENABLED && + bus->dev_state != MEI_DEV_POWERING_DOWN) { rets = -ENODEV; goto out; } diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index beb6cdff20fb..d3f060dce4a6 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -990,7 +990,8 @@ int mei_cl_disconnect(struct mei_cl *cl) return 0; } - if (dev->dev_state == MEI_DEV_POWER_DOWN) { + if (dev->dev_state == MEI_DEV_POWERING_DOWN || + dev->dev_state == MEI_DEV_POWER_DOWN) { cl_dbg(dev, cl, "Device is powering down, don't bother with disconnection\n"); mei_cl_set_disconnected(cl); return 0; diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c index bcee77768b91..5c8cb679b997 100644 --- a/drivers/misc/mei/init.c +++ b/drivers/misc/mei/init.c @@ -303,9 +303,12 @@ void mei_stop(struct mei_device *dev) dev_dbg(dev->dev, "stopping the device.\n"); mutex_lock(&dev->device_lock); - mei_set_devstate(dev, MEI_DEV_POWER_DOWN); + mei_set_devstate(dev, MEI_DEV_POWERING_DOWN); mutex_unlock(&dev->device_lock); mei_cl_bus_remove_devices(dev); + mutex_lock(&dev->device_lock); + mei_set_devstate(dev, MEI_DEV_POWER_DOWN); + mutex_unlock(&dev->device_lock); mei_cancel_work(dev); diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 8c395bfdf6f3..585a6f615bf8 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -57,6 +57,7 @@ enum mei_dev_state { MEI_DEV_ENABLED, MEI_DEV_RESETTING, MEI_DEV_DISABLED, + MEI_DEV_POWERING_DOWN, MEI_DEV_POWER_DOWN, MEI_DEV_POWER_UP }; From b7a4804129c7adb3ee9caf5505e3f91fb1467ec3 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Sat, 6 Feb 2021 16:43:22 +0200 Subject: [PATCH 296/633] mei: add support for client dma capability Client DMA capability indicates whether the firmware supports setting up a direct DMA channel between the host and me client. The DMA capabilities are supported from firmware HBM version 2.2 and newer. Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210206144325.25682-3-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/debugfs.c | 1 + drivers/misc/mei/hbm.c | 13 ++++++++++++- drivers/misc/mei/hw.h | 8 ++++++++ drivers/misc/mei/mei_dev.h | 2 ++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c index 3ab1a431d810..1ce61e9e24fc 100644 --- a/drivers/misc/mei/debugfs.c +++ b/drivers/misc/mei/debugfs.c @@ -106,6 +106,7 @@ static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused) seq_printf(m, "\tDR: %01d\n", dev->hbm_f_dr_supported); seq_printf(m, "\tVT: %01d\n", dev->hbm_f_vt_supported); seq_printf(m, "\tCAP: %01d\n", dev->hbm_f_cap_supported); + seq_printf(m, "\tCD: %01d\n", dev->hbm_f_cd_supported); } seq_printf(m, "pg: %s, %s\n", diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index df0f62de3dca..6e748da7e55d 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -339,7 +339,9 @@ static int mei_hbm_capabilities_req(struct mei_device *dev) memset(&req, 0, sizeof(req)); req.hbm_cmd = MEI_HBM_CAPABILITIES_REQ_CMD; if (dev->hbm_f_vt_supported) - req.capability_requested[0] = HBM_CAP_VT; + req.capability_requested[0] |= HBM_CAP_VT; + if (dev->hbm_f_cd_supported) + req.capability_requested[0] |= HBM_CAP_CD; ret = mei_hbm_write_message(dev, &mei_hdr, &req); if (ret) { @@ -1085,6 +1087,13 @@ static void mei_hbm_config_features(struct mei_device *dev) (dev->version.major_version == HBM_MAJOR_VERSION_CAP && dev->version.minor_version >= HBM_MINOR_VERSION_CAP)) dev->hbm_f_cap_supported = 1; + + /* Client DMA Support */ + dev->hbm_f_cd_supported = 0; + if (dev->version.major_version > HBM_MAJOR_VERSION_CD || + (dev->version.major_version == HBM_MAJOR_VERSION_CD && + dev->version.minor_version >= HBM_MINOR_VERSION_CD)) + dev->hbm_f_cd_supported = 1; } /** @@ -1233,6 +1242,8 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) capability_res = (struct hbm_capability_response *)mei_msg; if (!(capability_res->capability_granted[0] & HBM_CAP_VT)) dev->hbm_f_vt_supported = 0; + if (!(capability_res->capability_granted[0] & HBM_CAP_CD)) + dev->hbm_f_cd_supported = 0; if (dev->hbm_f_dr_supported) { if (mei_dmam_ring_alloc(dev)) diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index df2fb9520dd8..9577a2cc0733 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -88,6 +88,12 @@ #define HBM_MINOR_VERSION_CAP 2 #define HBM_MAJOR_VERSION_CAP 2 +/* + * MEI version with client DMA support + */ +#define HBM_MINOR_VERSION_CD 2 +#define HBM_MAJOR_VERSION_CD 2 + /* Host bus message command opcode */ #define MEI_HBM_CMD_OP_MSK 0x7f /* Host bus message command RESPONSE */ @@ -648,6 +654,8 @@ struct hbm_dma_ring_ctrl { /* virtual tag supported */ #define HBM_CAP_VT BIT(0) +/* client dma supported */ +#define HBM_CAP_CD BIT(2) /** * struct hbm_capability_request - capability request from host to fw diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 585a6f615bf8..8ebd32cb2075 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -451,6 +451,7 @@ struct mei_fw_version { * @hbm_f_dr_supported : hbm feature dma ring supported * @hbm_f_vt_supported : hbm feature vtag supported * @hbm_f_cap_supported : hbm feature capabilities message supported + * @hbm_f_cd_supported : hbm feature client dma supported * * @fw_ver : FW versions * @@ -538,6 +539,7 @@ struct mei_device { unsigned int hbm_f_dr_supported:1; unsigned int hbm_f_vt_supported:1; unsigned int hbm_f_cap_supported:1; + unsigned int hbm_f_cd_supported:1; struct mei_fw_version fw_ver[MEI_MAX_FW_VER_BLOCKS]; From dfad8742a328598d52d8472ce34fac312f4a5acb Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Sat, 6 Feb 2021 16:43:23 +0200 Subject: [PATCH 297/633] mei: hbm: add client dma hbm messages Define structures for client DMA HBM protocol. The protocol requires passing dma buffer address and the buffer id. Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210206144325.25682-4-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/hw.h | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index 9577a2cc0733..b10606550613 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -142,6 +142,12 @@ #define MEI_HBM_CAPABILITIES_REQ_CMD 0x13 #define MEI_HBM_CAPABILITIES_RES_CMD 0x93 +#define MEI_HBM_CLIENT_DMA_MAP_REQ_CMD 0x14 +#define MEI_HBM_CLIENT_DMA_MAP_RES_CMD 0x94 + +#define MEI_HBM_CLIENT_DMA_UNMAP_REQ_CMD 0x15 +#define MEI_HBM_CLIENT_DMA_UNMAP_RES_CMD 0x95 + /* * MEI Stop Reason * used by hbm_host_stop_request.reason @@ -679,4 +685,51 @@ struct hbm_capability_response { u8 capability_granted[3]; } __packed; +/** + * struct hbm_client_dma_map_request - client dma map request from host to fw + * + * @hbm_cmd: bus message command header + * @client_buffer_id: client buffer id + * @reserved: reserved + * @address_lsb: DMA address LSB + * @address_msb: DMA address MSB + * @size: DMA size + */ +struct hbm_client_dma_map_request { + u8 hbm_cmd; + u8 client_buffer_id; + u8 reserved[2]; + u32 address_lsb; + u32 address_msb; + u32 size; +} __packed; + +/** + * struct hbm_client_dma_unmap_request + * client dma unmap request from the host to the firmware + * + * @hbm_cmd: bus message command header + * @status: unmap status + * @client_buffer_id: client buffer id + * @reserved: reserved + */ +struct hbm_client_dma_unmap_request { + u8 hbm_cmd; + u8 status; + u8 client_buffer_id; + u8 reserved; +} __packed; + +/** + * struct hbm_client_dma_response + * client dma unmap response from the firmware to the host + * + * @hbm_cmd: bus message command header + * @status: command status + */ +struct hbm_client_dma_response { + u8 hbm_cmd; + u8 status; +} __packed; + #endif From 369aea84595189200a2e6b028f556a7efa0ec489 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Sat, 6 Feb 2021 16:43:24 +0200 Subject: [PATCH 298/633] mei: implement client dma setup. Implement HBM message protocol to setup and tear down DMA buffer on behalf of an client. On top there DMA buffer allocation and its life time management. Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210206144325.25682-5-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/client.c | 286 +++++++++++++++++++++++++++++++++++ drivers/misc/mei/client.h | 8 + drivers/misc/mei/hbm.c | 124 +++++++++++++++ drivers/misc/mei/hbm.h | 4 +- drivers/misc/mei/interrupt.c | 10 ++ drivers/misc/mei/mei_dev.h | 15 ++ 6 files changed, 446 insertions(+), 1 deletion(-) diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index d3f060dce4a6..4378a9b25848 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -2114,6 +2115,8 @@ void mei_cl_complete(struct mei_cl *cl, struct mei_cl_cb *cb) case MEI_FOP_DISCONNECT: case MEI_FOP_NOTIFY_STOP: case MEI_FOP_NOTIFY_START: + case MEI_FOP_DMA_MAP: + case MEI_FOP_DMA_UNMAP: if (waitqueue_active(&cl->wait)) wake_up(&cl->wait); @@ -2140,3 +2143,286 @@ void mei_cl_all_disconnect(struct mei_device *dev) list_for_each_entry(cl, &dev->file_list, link) mei_cl_set_disconnected(cl); } + +static struct mei_cl *mei_cl_dma_map_find(struct mei_device *dev, u8 buffer_id) +{ + struct mei_cl *cl; + + list_for_each_entry(cl, &dev->file_list, link) + if (cl->dma.buffer_id == buffer_id) + return cl; + return NULL; +} + +/** + * mei_cl_irq_dma_map - send client dma map request in irq_thread context + * + * @cl: client + * @cb: callback block. + * @cmpl_list: complete list. + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_irq_dma_map(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list) +{ + struct mei_device *dev = cl->dev; + u32 msg_slots; + int slots; + int ret; + + msg_slots = mei_hbm2slots(sizeof(struct hbm_client_dma_map_request)); + slots = mei_hbuf_empty_slots(dev); + if (slots < 0) + return -EOVERFLOW; + + if ((u32)slots < msg_slots) + return -EMSGSIZE; + + ret = mei_hbm_cl_dma_map_req(dev, cl); + if (ret) { + cl->status = ret; + list_move_tail(&cb->list, cmpl_list); + return ret; + } + + list_move_tail(&cb->list, &dev->ctrl_rd_list); + return 0; +} + +/** + * mei_cl_irq_dma_unmap - send client dma unmap request in irq_thread context + * + * @cl: client + * @cb: callback block. + * @cmpl_list: complete list. + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_irq_dma_unmap(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list) +{ + struct mei_device *dev = cl->dev; + u32 msg_slots; + int slots; + int ret; + + msg_slots = mei_hbm2slots(sizeof(struct hbm_client_dma_unmap_request)); + slots = mei_hbuf_empty_slots(dev); + if (slots < 0) + return -EOVERFLOW; + + if ((u32)slots < msg_slots) + return -EMSGSIZE; + + ret = mei_hbm_cl_dma_unmap_req(dev, cl); + if (ret) { + cl->status = ret; + list_move_tail(&cb->list, cmpl_list); + return ret; + } + + list_move_tail(&cb->list, &dev->ctrl_rd_list); + return 0; +} + +static int mei_cl_dma_alloc(struct mei_cl *cl, u8 buf_id, size_t size) +{ + cl->dma.vaddr = dmam_alloc_coherent(cl->dev->dev, size, + &cl->dma.daddr, GFP_KERNEL); + if (!cl->dma.vaddr) + return -ENOMEM; + + cl->dma.buffer_id = buf_id; + cl->dma.size = size; + + return 0; +} + +static void mei_cl_dma_free(struct mei_cl *cl) +{ + cl->dma.buffer_id = 0; + dmam_free_coherent(cl->dev->dev, + cl->dma.size, cl->dma.vaddr, cl->dma.daddr); + cl->dma.size = 0; + cl->dma.vaddr = NULL; + cl->dma.daddr = 0; +} + +/** + * mei_cl_alloc_and_map - send client dma map request + * + * @cl: host client + * @fp: pointer to file structure + * @buffer_id: id of the mapped buffer + * @size: size of the buffer + * + * Locking: called under "dev->device_lock" lock + * + * Return: + * * -ENODEV + * * -EINVAL + * * -EOPNOTSUPP + * * -EPROTO + * * -ENOMEM; + */ +int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp, + u8 buffer_id, size_t size) +{ + struct mei_device *dev; + struct mei_cl_cb *cb; + int rets; + + if (WARN_ON(!cl || !cl->dev)) + return -ENODEV; + + dev = cl->dev; + + if (!dev->hbm_f_cd_supported) { + cl_dbg(dev, cl, "client dma is not supported\n"); + return -EOPNOTSUPP; + } + + if (buffer_id == 0) + return -EINVAL; + + if (!mei_cl_is_connected(cl)) + return -ENODEV; + + if (cl->dma_mapped) + return -EPROTO; + + if (mei_cl_dma_map_find(dev, buffer_id)) { + cl_dbg(dev, cl, "client dma with id %d is already allocated\n", + cl->dma.buffer_id); + return -EPROTO; + } + + rets = pm_runtime_get(dev->dev); + if (rets < 0 && rets != -EINPROGRESS) { + pm_runtime_put_noidle(dev->dev); + cl_err(dev, cl, "rpm: get failed %d\n", rets); + return rets; + } + + rets = mei_cl_dma_alloc(cl, buffer_id, size); + if (rets) { + pm_runtime_put_noidle(dev->dev); + return rets; + } + + cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_MAP, fp); + if (!cb) { + rets = -ENOMEM; + goto out; + } + + if (mei_hbuf_acquire(dev)) { + if (mei_hbm_cl_dma_map_req(dev, cl)) { + rets = -ENODEV; + goto out; + } + list_move_tail(&cb->list, &dev->ctrl_rd_list); + } + + mutex_unlock(&dev->device_lock); + wait_event_timeout(cl->wait, + cl->dma_mapped || + cl->status || + !mei_cl_is_connected(cl), + mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + mutex_lock(&dev->device_lock); + + if (!cl->dma_mapped && !cl->status) + cl->status = -EFAULT; + + rets = cl->status; + +out: + if (rets) + mei_cl_dma_free(cl); + + cl_dbg(dev, cl, "rpm: autosuspend\n"); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + mei_io_cb_free(cb); + return rets; +} + +/** + * mei_cl_unmap_and_free - send client dma unmap request + * + * @cl: host client + * @fp: pointer to file structure + * + * Locking: called under "dev->device_lock" lock + * + * Return: 0 on such and error otherwise. + */ +int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp) +{ + struct mei_device *dev; + struct mei_cl_cb *cb; + int rets; + + if (WARN_ON(!cl || !cl->dev)) + return -ENODEV; + + dev = cl->dev; + + if (!dev->hbm_f_cd_supported) { + cl_dbg(dev, cl, "client dma is not supported\n"); + return -EOPNOTSUPP; + } + + if (!mei_cl_is_connected(cl)) + return -ENODEV; + + if (!cl->dma_mapped) + return -EPROTO; + + rets = pm_runtime_get(dev->dev); + if (rets < 0 && rets != -EINPROGRESS) { + pm_runtime_put_noidle(dev->dev); + cl_err(dev, cl, "rpm: get failed %d\n", rets); + return rets; + } + + cb = mei_cl_enqueue_ctrl_wr_cb(cl, 0, MEI_FOP_DMA_UNMAP, fp); + if (!cb) { + rets = -ENOMEM; + goto out; + } + + if (mei_hbuf_acquire(dev)) { + if (mei_hbm_cl_dma_unmap_req(dev, cl)) { + rets = -ENODEV; + goto out; + } + list_move_tail(&cb->list, &dev->ctrl_rd_list); + } + + mutex_unlock(&dev->device_lock); + wait_event_timeout(cl->wait, + !cl->dma_mapped || + cl->status || + !mei_cl_is_connected(cl), + mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + mutex_lock(&dev->device_lock); + + if (cl->dma_mapped && !cl->status) + cl->status = -EFAULT; + + rets = cl->status; + + if (!rets) + mei_cl_dma_free(cl); +out: + cl_dbg(dev, cl, "rpm: autosuspend\n"); + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + mei_io_cb_free(cb); + return rets; +} diff --git a/drivers/misc/mei/client.h b/drivers/misc/mei/client.h index 9e08a9843bba..b12cdcde9436 100644 --- a/drivers/misc/mei/client.h +++ b/drivers/misc/mei/client.h @@ -265,6 +265,14 @@ void mei_cl_notify(struct mei_cl *cl); void mei_cl_all_disconnect(struct mei_device *dev); +int mei_cl_irq_dma_map(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list); +int mei_cl_irq_dma_unmap(struct mei_cl *cl, struct mei_cl_cb *cb, + struct list_head *cmpl_list); +int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp, + u8 buffer_id, size_t size); +int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp); + #define MEI_CL_FMT "cl:host=%02d me=%02d " #define MEI_CL_PRM(cl) (cl)->host_client_id, mei_cl_me_id(cl) diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index 6e748da7e55d..d0277c7fed10 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -594,6 +594,117 @@ static void mei_hbm_cl_notify(struct mei_device *dev, mei_cl_notify(cl); } +/** + * mei_hbm_cl_dma_map_req - send client dma map request + * + * @dev: the device structure + * @cl: mei host client + * + * Return: 0 on success and -EIO on write failure + */ +int mei_hbm_cl_dma_map_req(struct mei_device *dev, struct mei_cl *cl) +{ + struct mei_msg_hdr mei_hdr; + struct hbm_client_dma_map_request req; + int ret; + + mei_hbm_hdr(&mei_hdr, sizeof(req)); + + memset(&req, 0, sizeof(req)); + + req.hbm_cmd = MEI_HBM_CLIENT_DMA_MAP_REQ_CMD; + req.client_buffer_id = cl->dma.buffer_id; + req.address_lsb = lower_32_bits(cl->dma.daddr); + req.address_msb = upper_32_bits(cl->dma.daddr); + req.size = cl->dma.size; + + ret = mei_hbm_write_message(dev, &mei_hdr, &req); + if (ret) + dev_err(dev->dev, "dma map request failed: ret = %d\n", ret); + + return ret; +} + +/** + * mei_hbm_cl_dma_unmap_req - send client dma unmap request + * + * @dev: the device structure + * @cl: mei host client + * + * Return: 0 on success and -EIO on write failure + */ +int mei_hbm_cl_dma_unmap_req(struct mei_device *dev, struct mei_cl *cl) +{ + struct mei_msg_hdr mei_hdr; + struct hbm_client_dma_unmap_request req; + int ret; + + mei_hbm_hdr(&mei_hdr, sizeof(req)); + + memset(&req, 0, sizeof(req)); + + req.hbm_cmd = MEI_HBM_CLIENT_DMA_UNMAP_REQ_CMD; + req.client_buffer_id = cl->dma.buffer_id; + + ret = mei_hbm_write_message(dev, &mei_hdr, &req); + if (ret) + dev_err(dev->dev, "dma unmap request failed: ret = %d\n", ret); + + return ret; +} + +static void mei_hbm_cl_dma_map_res(struct mei_device *dev, + struct hbm_client_dma_response *res) +{ + struct mei_cl *cl; + struct mei_cl_cb *cb, *next; + + cl = NULL; + list_for_each_entry_safe(cb, next, &dev->ctrl_rd_list, list) { + if (cb->fop_type != MEI_FOP_DMA_MAP) + continue; + if (!cb->cl->dma.buffer_id || cb->cl->dma_mapped) + continue; + + cl = cb->cl; + break; + } + if (!cl) + return; + + dev_dbg(dev->dev, "cl dma map result = %d\n", res->status); + cl->status = res->status; + if (!cl->status) + cl->dma_mapped = 1; + wake_up(&cl->wait); +} + +static void mei_hbm_cl_dma_unmap_res(struct mei_device *dev, + struct hbm_client_dma_response *res) +{ + struct mei_cl *cl; + struct mei_cl_cb *cb, *next; + + cl = NULL; + list_for_each_entry_safe(cb, next, &dev->ctrl_rd_list, list) { + if (cb->fop_type != MEI_FOP_DMA_UNMAP) + continue; + if (!cb->cl->dma.buffer_id || !cb->cl->dma_mapped) + continue; + + cl = cb->cl; + break; + } + if (!cl) + return; + + dev_dbg(dev->dev, "cl dma unmap result = %d\n", res->status); + cl->status = res->status; + if (!cl->status) + cl->dma_mapped = 0; + wake_up(&cl->wait); +} + /** * mei_hbm_prop_req - request property for a single client * @@ -1133,6 +1244,7 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) struct mei_hbm_cl_cmd *cl_cmd; struct hbm_client_connect_request *disconnect_req; struct hbm_flow_control *fctrl; + struct hbm_client_dma_response *client_dma_res; /* read the message to our buffer */ BUG_ON(hdr->length >= sizeof(dev->rd_msg_buf)); @@ -1459,6 +1571,18 @@ int mei_hbm_dispatch(struct mei_device *dev, struct mei_msg_hdr *hdr) mei_hbm_cl_notify(dev, cl_cmd); break; + case MEI_HBM_CLIENT_DMA_MAP_RES_CMD: + dev_dbg(dev->dev, "hbm: client dma map response: message received.\n"); + client_dma_res = (struct hbm_client_dma_response *)mei_msg; + mei_hbm_cl_dma_map_res(dev, client_dma_res); + break; + + case MEI_HBM_CLIENT_DMA_UNMAP_RES_CMD: + dev_dbg(dev->dev, "hbm: client dma unmap response: message received.\n"); + client_dma_res = (struct hbm_client_dma_response *)mei_msg; + mei_hbm_cl_dma_unmap_res(dev, client_dma_res); + break; + default: WARN(1, "hbm: wrong command %d\n", mei_msg->hbm_cmd); return -EPROTO; diff --git a/drivers/misc/mei/hbm.h b/drivers/misc/mei/hbm.h index 4d95e38e4ddf..cd5b08ca34b6 100644 --- a/drivers/misc/mei/hbm.h +++ b/drivers/misc/mei/hbm.h @@ -10,6 +10,7 @@ struct mei_device; struct mei_msg_hdr; struct mei_cl; +struct mei_dma_data; /** * enum mei_hbm_state - host bus message protocol state @@ -51,6 +52,7 @@ int mei_hbm_pg(struct mei_device *dev, u8 pg_cmd); void mei_hbm_pg_resume(struct mei_device *dev); int mei_hbm_cl_notify_req(struct mei_device *dev, struct mei_cl *cl, u8 request); - +int mei_hbm_cl_dma_map_req(struct mei_device *dev, struct mei_cl *cl); +int mei_hbm_cl_dma_unmap_req(struct mei_device *dev, struct mei_cl *cl); #endif /* _MEI_HBM_H_ */ diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c index 2161c1234ad7..a98f6b895af7 100644 --- a/drivers/misc/mei/interrupt.c +++ b/drivers/misc/mei/interrupt.c @@ -547,6 +547,16 @@ int mei_irq_write_handler(struct mei_device *dev, struct list_head *cmpl_list) if (ret) return ret; break; + case MEI_FOP_DMA_MAP: + ret = mei_cl_irq_dma_map(cl, cb, cmpl_list); + if (ret) + return ret; + break; + case MEI_FOP_DMA_UNMAP: + ret = mei_cl_irq_dma_unmap(cl, cb, cmpl_list); + if (ret) + return ret; + break; default: BUG(); } diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 8ebd32cb2075..b7b6ef344e80 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -79,6 +79,8 @@ enum mei_file_transaction_states { * @MEI_FOP_DISCONNECT_RSP: disconnect response * @MEI_FOP_NOTIFY_START: start notification * @MEI_FOP_NOTIFY_STOP: stop notification + * @MEI_FOP_DMA_MAP: request client dma map + * @MEI_FOP_DMA_UNMAP: request client dma unmap */ enum mei_cb_file_ops { MEI_FOP_READ = 0, @@ -88,6 +90,8 @@ enum mei_cb_file_ops { MEI_FOP_DISCONNECT_RSP, MEI_FOP_NOTIFY_START, MEI_FOP_NOTIFY_STOP, + MEI_FOP_DMA_MAP, + MEI_FOP_DMA_UNMAP, }; /** @@ -113,6 +117,13 @@ struct mei_msg_data { unsigned char *data; }; +struct mei_dma_data { + u8 buffer_id; + void *vaddr; + dma_addr_t daddr; + size_t size; +}; + /** * struct mei_dma_dscr - dma address descriptor * @@ -236,6 +247,8 @@ struct mei_cl_vtag { * @rd_pending: pending read credits * @rd_completed_lock: protects rd_completed queue * @rd_completed: completed read + * @dma: dma settings + * @dma_mapped: dma buffer is currently mapped. * * @cldev: device on the mei client bus */ @@ -263,6 +276,8 @@ struct mei_cl { struct list_head rd_pending; spinlock_t rd_completed_lock; /* protects rd_completed queue */ struct list_head rd_completed; + struct mei_dma_data dma; + u8 dma_mapped; struct mei_cl_device *cldev; }; From 167790abb90fa073d8341ee0e408ccad3d2109cd Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 22 Jan 2021 15:06:29 +0800 Subject: [PATCH 299/633] soundwire: export sdw_write/read_no_pm functions sdw_write_no_pm and sdw_read_no_pm are useful when we want to do IO without touching PM. Fixes: 0231453bc08f ('soundwire: bus: add clock stop helpers') Fixes: 60ee9be25571 ('soundwire: bus: add PM/no-PM versions of read/write functions') Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-5-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 7 ++++--- include/linux/soundwire/sdw.h | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 675d028e923d..45e9fc9f472a 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -407,10 +407,11 @@ sdw_nwrite_no_pm(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) return sdw_transfer(slave->bus, &msg); } -static int sdw_write_no_pm(struct sdw_slave *slave, u32 addr, u8 value) +int sdw_write_no_pm(struct sdw_slave *slave, u32 addr, u8 value) { return sdw_nwrite_no_pm(slave, addr, 1, &value); } +EXPORT_SYMBOL(sdw_write_no_pm); static int sdw_bread_no_pm(struct sdw_bus *bus, u16 dev_num, u32 addr) @@ -478,8 +479,7 @@ int sdw_bwrite_no_pm_unlocked(struct sdw_bus *bus, u16 dev_num, u32 addr, u8 val } EXPORT_SYMBOL(sdw_bwrite_no_pm_unlocked); -static int -sdw_read_no_pm(struct sdw_slave *slave, u32 addr) +int sdw_read_no_pm(struct sdw_slave *slave, u32 addr) { u8 buf; int ret; @@ -490,6 +490,7 @@ sdw_read_no_pm(struct sdw_slave *slave, u32 addr) else return buf; } +EXPORT_SYMBOL(sdw_read_no_pm); static int sdw_update_no_pm(struct sdw_slave *slave, u32 addr, u8 mask, u8 val) { diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index f0b01b728640..d08039d65825 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -1005,6 +1005,8 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus); int sdw_read(struct sdw_slave *slave, u32 addr); int sdw_write(struct sdw_slave *slave, u32 addr, u8 value); +int sdw_write_no_pm(struct sdw_slave *slave, u32 addr, u8 value); +int sdw_read_no_pm(struct sdw_slave *slave, u32 addr); int sdw_nread(struct sdw_slave *slave, u32 addr, size_t count, u8 *val); int sdw_nwrite(struct sdw_slave *slave, u32 addr, size_t count, u8 *val); From 973794e85610d9a716a897baa9007ff56e192826 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 22 Jan 2021 15:06:33 +0800 Subject: [PATCH 300/633] soundwire: bus: fix confusion on device used by pm_runtime Intel stress-tests routinely report IO timeouts and invalid power management transitions. Upon further analysis, we seem to be using the wrong devices in pm_runtime calls. Before reading and writing registers, we first need to make sure the Slave is fully resumed. The existing code attempts to do such that, however because of a confusion dating from 2017 and copy/paste, we end-up resuming the parent only instead of resuming the codec device. This can lead to accesses to the Slave registers while the bus is still being configured and the Slave not enumerated, and as a result IO errors occur. This is a classic problem, similar confusions happened for HDaudio between bus and codec device, leading to power management issues. Fix by using the relevant device for all uses of pm_runtime functions. Fixes: 60ee9be255712 ('soundwire: bus: add PM/no-PM versions of read/write functions') Fixes: aa79293517b39 ('soundwire: bus: fix io error when processing alert event') Fixes: 9d715fa005ebc ('soundwire: Add IO transfer') Reported-by: Bard Liao Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-9-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 45e9fc9f472a..ca12b6d6434d 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -515,16 +515,16 @@ int sdw_nread(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) { int ret; - ret = pm_runtime_get_sync(slave->bus->dev); + ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } ret = sdw_nread_no_pm(slave, addr, count, val); - pm_runtime_mark_last_busy(slave->bus->dev); - pm_runtime_put(slave->bus->dev); + pm_runtime_mark_last_busy(&slave->dev); + pm_runtime_put(&slave->dev); return ret; } @@ -541,16 +541,16 @@ int sdw_nwrite(struct sdw_slave *slave, u32 addr, size_t count, u8 *val) { int ret; - ret = pm_runtime_get_sync(slave->bus->dev); + ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } ret = sdw_nwrite_no_pm(slave, addr, count, val); - pm_runtime_mark_last_busy(slave->bus->dev); - pm_runtime_put(slave->bus->dev); + pm_runtime_mark_last_busy(&slave->dev); + pm_runtime_put(&slave->dev); return ret; } @@ -1451,7 +1451,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) ret = pm_runtime_get_sync(&slave->dev); if (ret < 0 && ret != -EACCES) { dev_err(&slave->dev, "Failed to resume device: %d\n", ret); - pm_runtime_put_noidle(slave->bus->dev); + pm_runtime_put_noidle(&slave->dev); return ret; } From 6d7a1ff71cbb326fadfbedb7f75c1fc8f5c84d84 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Fri, 22 Jan 2021 15:06:34 +0800 Subject: [PATCH 301/633] soundwire: bus: clarify dev_err/dbg device references The SoundWire bus code confuses bus and Slave device levels for dev_err/dbg logs. That's not impacting functionality but the accuracy of kernel logs. We should only use bus->dev for bus-level operations and handling of Device0. For all other logs where the device number is not zero, we should use &slave->dev to provide more precisions to the user/integrator. Reported-by: Rander Wang Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20210122070634.12825-10-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/bus.c | 63 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index ca12b6d6434d..46885429928a 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -638,6 +638,7 @@ static int sdw_get_device_num(struct sdw_slave *slave) static int sdw_assign_device_num(struct sdw_slave *slave) { + struct sdw_bus *bus = slave->bus; int ret, dev_num; bool new_device = false; @@ -648,7 +649,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) dev_num = sdw_get_device_num(slave); mutex_unlock(&slave->bus->bus_lock); if (dev_num < 0) { - dev_err(slave->bus->dev, "Get dev_num failed: %d\n", + dev_err(bus->dev, "Get dev_num failed: %d\n", dev_num); return dev_num; } @@ -661,7 +662,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) } if (!new_device) - dev_dbg(slave->bus->dev, + dev_dbg(bus->dev, "Slave already registered, reusing dev_num:%d\n", slave->dev_num); @@ -671,7 +672,7 @@ static int sdw_assign_device_num(struct sdw_slave *slave) ret = sdw_write_no_pm(slave, SDW_SCP_DEVNUMBER, dev_num); if (ret < 0) { - dev_err(&slave->dev, "Program device_num %d failed: %d\n", + dev_err(bus->dev, "Program device_num %d failed: %d\n", dev_num, ret); return ret; } @@ -749,7 +750,7 @@ static int sdw_program_device_num(struct sdw_bus *bus) */ ret = sdw_assign_device_num(slave); if (ret) { - dev_err(slave->bus->dev, + dev_err(bus->dev, "Assign dev_num failed:%d\n", ret); return ret; @@ -789,9 +790,11 @@ static int sdw_program_device_num(struct sdw_bus *bus) static void sdw_modify_slave_status(struct sdw_slave *slave, enum sdw_slave_status status) { - mutex_lock(&slave->bus->bus_lock); + struct sdw_bus *bus = slave->bus; - dev_vdbg(&slave->dev, + mutex_lock(&bus->bus_lock); + + dev_vdbg(bus->dev, "%s: changing status slave %d status %d new status %d\n", __func__, slave->dev_num, slave->status, status); @@ -812,7 +815,7 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, complete(&slave->enumeration_complete); } slave->status = status; - mutex_unlock(&slave->bus->bus_lock); + mutex_unlock(&bus->bus_lock); } static enum sdw_clk_stop_mode sdw_get_clk_stop_mode(struct sdw_slave *slave) @@ -1138,7 +1141,7 @@ int sdw_configure_dpn_intr(struct sdw_slave *slave, ret = sdw_update(slave, addr, (mask | SDW_DPN_INT_PORT_READY), val); if (ret < 0) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INTMASK write failed:%d\n", val); return ret; @@ -1269,7 +1272,7 @@ static int sdw_initialize_slave(struct sdw_slave *slave) /* Enable SCP interrupts */ ret = sdw_update_no_pm(slave, SDW_SCP_INTMASK1, val, val); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INTMASK1 write failed:%d\n", ret); return ret; } @@ -1284,7 +1287,7 @@ static int sdw_initialize_slave(struct sdw_slave *slave) ret = sdw_update_no_pm(slave, SDW_DP0_INTMASK, val, val); if (ret < 0) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INTMASK read failed:%d\n", ret); return ret; } @@ -1296,7 +1299,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) status = sdw_read_no_pm(slave, SDW_DP0_INT); if (status < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", status); return status; } @@ -1335,7 +1338,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) /* clear the interrupts but don't touch reserved and SDCA_CASCADE fields */ ret = sdw_write_no_pm(slave, SDW_DP0_INT, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT write failed:%d\n", ret); return ret; } @@ -1343,7 +1346,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) /* Read DP0 interrupt again */ status2 = sdw_read_no_pm(slave, SDW_DP0_INT); if (status2 < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", status2); return status2; } @@ -1356,7 +1359,7 @@ static int sdw_handle_dp0_interrupt(struct sdw_slave *slave, u8 *slave_status) } while ((status & SDW_DP0_INTERRUPTS) && (count < SDW_READ_INTR_CLEAR_RETRY)); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on DP0 read\n"); + dev_warn(&slave->dev, "Reached MAX_RETRY on DP0 read\n"); return ret; } @@ -1374,7 +1377,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, addr = SDW_DPN_INT(port); status = sdw_read_no_pm(slave, addr); if (status < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT read failed:%d\n", status); return status; @@ -1408,7 +1411,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, /* clear the interrupt but don't touch reserved fields */ ret = sdw_write_no_pm(slave, addr, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT write failed:%d\n", ret); return ret; } @@ -1416,7 +1419,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, /* Read DPN interrupt again */ status2 = sdw_read_no_pm(slave, addr); if (status2 < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DPN_INT read failed:%d\n", status2); return status2; } @@ -1429,7 +1432,7 @@ static int sdw_handle_port_interrupt(struct sdw_slave *slave, } while ((status & SDW_DPN_INTERRUPTS) && (count < SDW_READ_INTR_CLEAR_RETRY)); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on port read"); + dev_warn(&slave->dev, "Reached MAX_RETRY on port read"); return ret; } @@ -1458,7 +1461,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) /* Read Intstat 1, Intstat 2 and Intstat 3 registers */ ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 read failed:%d\n", ret); goto io_err; } @@ -1466,7 +1469,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, buf2); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); goto io_err; } @@ -1474,7 +1477,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) if (slave->prop.is_sdca) { ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", ret); goto io_err; } @@ -1571,7 +1574,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) /* Ack interrupt */ ret = sdw_write_no_pm(slave, SDW_SCP_INT1, clear); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 write failed:%d\n", ret); goto io_err; } @@ -1585,7 +1588,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) */ ret = sdw_read_no_pm(slave, SDW_SCP_INT1); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT1 read failed:%d\n", ret); goto io_err; } @@ -1593,7 +1596,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) ret = sdw_nread_no_pm(slave, SDW_SCP_INTSTAT2, 2, _buf2); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_SCP_INT2/3 read failed:%d\n", ret); goto io_err; } @@ -1601,7 +1604,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) if (slave->prop.is_sdca) { ret = sdw_read_no_pm(slave, SDW_DP0_INT); if (ret < 0) { - dev_err(slave->bus->dev, + dev_err(&slave->dev, "SDW_DP0_INT read failed:%d\n", ret); goto io_err; } @@ -1627,7 +1630,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) } while (stat != 0 && count < SDW_READ_INTR_CLEAR_RETRY); if (count == SDW_READ_INTR_CLEAR_RETRY) - dev_warn(slave->bus->dev, "Reached MAX_RETRY on alert read\n"); + dev_warn(&slave->dev, "Reached MAX_RETRY on alert read\n"); io_err: pm_runtime_mark_last_busy(&slave->dev); @@ -1733,7 +1736,7 @@ int sdw_handle_slave_status(struct sdw_bus *bus, case SDW_SLAVE_ALERT: ret = sdw_handle_slave_alerts(slave); if (ret) - dev_err(bus->dev, + dev_err(&slave->dev, "Slave %d alert handling failed: %d\n", i, ret); break; @@ -1752,21 +1755,21 @@ int sdw_handle_slave_status(struct sdw_bus *bus, ret = sdw_initialize_slave(slave); if (ret) - dev_err(bus->dev, + dev_err(&slave->dev, "Slave %d initialization failed: %d\n", i, ret); break; default: - dev_err(bus->dev, "Invalid slave %d status:%d\n", + dev_err(&slave->dev, "Invalid slave %d status:%d\n", i, status[i]); break; } ret = sdw_update_slave_status(slave, status[i]); if (ret) - dev_err(slave->bus->dev, + dev_err(&slave->dev, "Update Slave status failed:%d\n", ret); if (attached_initializing) { dev_dbg(&slave->dev, From 6c1e3f92f9f1dfc7f14b43fd432c8ec95b1a188f Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 28 Jan 2021 08:50:25 +0200 Subject: [PATCH 302/633] habanalabs: fix integer handling issue Need to add ull suffix to constant when doing shift of constant into 64-bit variables Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/mmu/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index 27dc0d116db5..71703a32350f 100644 --- a/drivers/misc/habanalabs/common/mmu/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -507,7 +507,7 @@ static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, p = (char *)p + hop0_shift_off; p = (char *)p + ((hops->used_hops - 1) * sizeof(u64)); hop_shift = *(u64 *)p; - offset_mask = (1 << hop_shift) - 1; + offset_mask = (1ull << hop_shift) - 1; addr_mask = ~(offset_mask); *phys_addr = (tmp_phys_addr & addr_mask) | (virt_addr & offset_mask); From 5dbd7b4de6ef84321cc1378eccdd92d4730c2e56 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 28 Jan 2021 16:30:25 +0200 Subject: [PATCH 303/633] habanalabs: improve communication protocol with cpucp Current messaging communictaion protocol with cpucp can get out of sync due to coherency issues. In order to improve the protocol reliability, we modify the protocol to expect a different acknowledgment for every packet sent to cpucp. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 17 +++++++++++++++-- drivers/misc/habanalabs/common/habanalabs.h | 3 +++ drivers/misc/habanalabs/gaudi/gaudi.c | 6 +++++- drivers/misc/habanalabs/goya/goya.c | 6 +++++- .../misc/habanalabs/include/common/hl_boot_if.h | 5 +++++ 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index ba6920f2b4ab..31b52a223f02 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -90,9 +90,10 @@ int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode) int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, u16 len, u32 timeout, u64 *result) { + struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id]; struct cpucp_packet *pkt; dma_addr_t pkt_dma_addr; - u32 tmp; + u32 tmp, expected_ack_val; int rc = 0; pkt = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, len, @@ -115,14 +116,22 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } + /* set fence to a non valid value */ + pkt->fence = UINT_MAX; + rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr); if (rc) { dev_err(hdev->dev, "Failed to send CB on CPU PQ (%d)\n", rc); goto out; } + if (hdev->asic_prop.fw_cpucp_ack_with_pi) + expected_ack_val = queue->pi; + else + expected_ack_val = CPUCP_PACKET_FENCE_VAL; + rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp, - (tmp == CPUCP_PACKET_FENCE_VAL), 1000, + (tmp == expected_ack_val), 1000, timeout, true); hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); @@ -777,6 +786,10 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; + if (prop->fw_boot_cpu_security_map & + CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) + prop->fw_cpucp_ack_with_pi = true; + dev_dbg(hdev->dev, "Firmware boot CPU security status %#x\n", prop->fw_boot_cpu_security_map); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 30f32f2edb8a..3c54010f7ab9 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -419,6 +419,8 @@ struct hl_mmu_properties { * from BOOT_DEV_STS0 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow + * @fw_cpucp_ack_with_pi: true if cpucp is acking messages with the PQ PI + * instead of a magic number * @num_functional_hbms: number of functional HBMs in each DCORE. */ struct asic_fixed_properties { @@ -479,6 +481,7 @@ struct asic_fixed_properties { u8 fw_security_status_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; + u8 fw_cpucp_ack_with_pi; u8 num_functional_hbms; }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 52fcaf25531a..006c34ae35c2 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -533,6 +533,7 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->fw_security_disabled = true; prop->fw_security_status_valid = false; prop->hard_reset_done_by_fw = false; + prop->fw_cpucp_ack_with_pi = false; return 0; } @@ -4438,9 +4439,12 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) /* ring the doorbell */ WREG32(db_reg_offset, db_value); - if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) + if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) { + /* make sure device CPU will read latest data from host */ + mb(); WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE); + } } static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe, diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index a954e7c02375..53db7e966866 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -461,6 +461,7 @@ int goya_get_fixed_properties(struct hl_device *hdev) prop->fw_security_disabled = true; prop->fw_security_status_valid = false; prop->hard_reset_done_by_fw = false; + prop->fw_cpucp_ack_with_pi = false; return 0; } @@ -2806,9 +2807,12 @@ void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) /* ring the doorbell */ WREG32(db_reg_offset, db_value); - if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) + if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ) { + /* make sure device CPU will read latest data from host */ + mb(); WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GOYA_ASYNC_EVENT_ID_PI_UPDATE); + } } void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index 57785478a4ef..e87f5a98e193 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -166,6 +166,10 @@ * FW handles HBM ECC indications. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN Packets ack value used in the armcpd + * is set to the PI counter. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -190,6 +194,7 @@ #define CPU_BOOT_DEV_STS0_SP_SRAM_EN (1 << 12) #define CPU_BOOT_DEV_STS0_CLK_GATE_EN (1 << 13) #define CPU_BOOT_DEV_STS0_HBM_ECC_EN (1 << 14) +#define CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN (1 << 15) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) enum cpu_boot_status { From e52606d2f5363f4900cfe8419e391644b0229c6f Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Wed, 27 Jan 2021 16:34:37 +0200 Subject: [PATCH 304/633] habanalabs: support fetching first available user CQ User must be aware of the available CQs when it needs to use them. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs.h | 2 ++ drivers/misc/habanalabs/common/habanalabs_ioctl.c | 3 ++- drivers/misc/habanalabs/gaudi/gaudi.c | 3 +++ drivers/misc/habanalabs/goya/goya.c | 3 +++ include/uapi/misc/habanalabs.h | 3 +++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 3c54010f7ab9..98163317ec43 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -411,6 +411,7 @@ struct hl_mmu_properties { * @first_available_user_mon: first monitor available for the user * @first_available_user_msix_interrupt: first available msix interrupt * reserved for the user + * @first_available_cq: first available CQ for the user. * @tpc_enabled_mask: which TPCs are enabled. * @completion_queues_count: number of completion queues. * @fw_security_disabled: true if security measures are disabled in firmware, @@ -475,6 +476,7 @@ struct asic_fixed_properties { u16 first_available_user_sob[HL_MAX_DCORES]; u16 first_available_user_mon[HL_MAX_DCORES]; u16 first_available_user_msix_interrupt; + u16 first_available_cq[HL_MAX_DCORES]; u8 tpc_enabled_mask; u8 completion_queues_count; u8 fw_security_disabled; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index e86f46d4b613..03af61cecd37 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -397,7 +397,8 @@ static int sync_manager_info(struct hl_fpriv *hpriv, struct hl_info_args *args) prop->first_available_user_sob[args->dcore_id]; sm_info.first_available_monitor = prop->first_available_user_mon[args->dcore_id]; - + sm_info.first_available_cq = + prop->first_available_cq[args->dcore_id]; return copy_to_user(out, &sm_info, min_t(size_t, (size_t) max_size, sizeof(sm_info))) ? -EFAULT : 0; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 006c34ae35c2..8fc0de3cf3a9 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -529,6 +529,9 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->first_available_user_msix_interrupt = USHRT_MAX; + for (i = 0 ; i < HL_MAX_DCORES ; i++) + prop->first_available_cq[i] = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 53db7e966866..d26b405f0c17 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -457,6 +457,9 @@ int goya_get_fixed_properties(struct hl_device *hdev) prop->first_available_user_msix_interrupt = USHRT_MAX; + for (i = 0 ; i < HL_MAX_DCORES ; i++) + prop->first_available_cq[i] = USHRT_MAX; + /* disable fw security for now, set it in a later stage */ prop->fw_security_disabled = true; prop->fw_security_status_valid = false; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index ebde42b37b43..64ae83b5f8e5 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -414,10 +414,13 @@ struct hl_pll_frequency_info { * struct hl_info_sync_manager - sync manager information * @first_available_sync_object: first available sob * @first_available_monitor: first available monitor + * @first_available_cq: first available cq */ struct hl_info_sync_manager { __u32 first_available_sync_object; __u32 first_available_monitor; + __u32 first_available_cq; + __u32 reserved; }; /** From b520ca5d82f69ac28ca3d57f001430c203487cb3 Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Wed, 27 Jan 2021 15:42:53 +0200 Subject: [PATCH 305/633] habanalabs/gaudi: use HBM_ECC_EN bit for ECC ERR driver should use ECC info from FW only if HBM ECC CAP is set. otherwise, try to fetch the data from MC regs only if security is disabled. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 8fc0de3cf3a9..b929e602fa3d 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7105,7 +7105,9 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch; int err = 0; - if (!hdev->asic_prop.fw_security_disabled) { + if (hdev->asic_prop.fw_security_status_valid && + (hdev->asic_prop.fw_app_security_map & + CPU_BOOT_DEV_STS0_HBM_ECC_EN)) { if (!hbm_ecc_data) { dev_err(hdev->dev, "No FW ECC data"); return 0; @@ -7127,14 +7129,24 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device, le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); dev_err(hdev->dev, - "HBM%d pc%d ECC: TYPE=%d, WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", - device, ch, type, wr_par, rd_par, ca_par, serr, derr); + "HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n", + device, ch, wr_par, rd_par, ca_par, serr, derr); + dev_err(hdev->dev, + "HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%u, SEC_CNT=%d, DEC_CNT=%d\n", + device, ch, hbm_ecc_data->first_addr, type, + hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, + hbm_ecc_data->dec_cnt); err = 1; return 0; } + if (!hdev->asic_prop.fw_security_disabled) { + dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); + return 0; + } + base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET; for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) { val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF); From 28bcf1fdc408cd2f7393ae5dcd71c756e1163cdb Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 1 Feb 2021 21:23:43 +0200 Subject: [PATCH 306/633] habanalabs: enable F/W events after init done Only after the initialization of the device is done, the driver is ready to receive events from the F/W. The driver can't handle events before that because of races so it will ignore events. In case of a fatal event, the driver won't know about it and the device will be operational although it shouldn't be. Same logic should be applied after hard-reset. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/device.c | 25 ++++++++++++++++----- drivers/misc/habanalabs/common/habanalabs.h | 9 ++++++-- drivers/misc/habanalabs/gaudi/gaudi.c | 10 ++++++--- drivers/misc/habanalabs/goya/goya.c | 12 ++++++---- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 59219c862ca0..15fcb5c31c4b 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -1159,12 +1159,20 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset, atomic_set(&hdev->in_reset, 0); hdev->needs_reset = false; - if (hard_reset) - hdev->hard_reset_cnt++; - else - hdev->soft_reset_cnt++; + dev_notice(hdev->dev, "Successfully finished resetting the device\n"); - dev_warn(hdev->dev, "Successfully finished resetting the device\n"); + if (hard_reset) { + hdev->hard_reset_cnt++; + + /* After reset is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events + * and if those events are fatal, we won't know about it and + * the device will be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + } else { + hdev->soft_reset_cnt++; + } return 0; @@ -1415,6 +1423,13 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) hdev->init_done = true; + /* After initialization is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events and if + * those events are fatal, we won't know about it and the device will + * be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + return 0; release_ctx: diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 98163317ec43..18ed3a6000b0 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -860,12 +860,16 @@ enum div_select_defs { * and place them in the relevant cs jobs * @collective_wait_create_jobs: allocate collective wait cs jobs * @scramble_addr: Routine to scramble the address prior of mapping it - * in the MMU. + * in the MMU. * @descramble_addr: Routine to de-scramble the address prior of - * showing it to users. + * showing it to users. * @ack_protection_bits_errors: ack and dump all security violations * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. * @hw_block_mmap: mmap a HW block with a given id. + * @enable_events_from_fw: send interrupt to firmware to notify them the + * driver is ready to receive asynchronous events. This + * function should be called during the first init and + * after every hard-reset of the device */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -982,6 +986,7 @@ struct hl_asic_funcs { u32 *block_id); int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size); + void (*enable_events_from_fw)(struct hl_device *hdev); }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index b929e602fa3d..6905857b363b 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -1383,8 +1383,6 @@ static int gaudi_late_init(struct hl_device *hdev) return rc; } - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); - rc = gaudi_fetch_psoc_frequency(hdev); if (rc) { dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); @@ -8500,6 +8498,11 @@ static int gaudi_block_mmap(struct hl_device *hdev, return -EPERM; } +static void gaudi_enable_events_from_fw(struct hl_device *hdev) +{ + WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER); +} + static const struct hl_asic_funcs gaudi_funcs = { .early_init = gaudi_early_init, .early_fini = gaudi_early_fini, @@ -8581,7 +8584,8 @@ static const struct hl_asic_funcs gaudi_funcs = { .descramble_addr = hl_mmu_descramble_addr, .ack_protection_bits_errors = gaudi_ack_protection_bits_errors, .get_hw_block_id = gaudi_get_hw_block_id, - .hw_block_mmap = gaudi_block_mmap + .hw_block_mmap = gaudi_block_mmap, + .enable_events_from_fw = gaudi_enable_events_from_fw }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index d26b405f0c17..af6a5760924c 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -798,9 +798,6 @@ int goya_late_init(struct hl_device *hdev) return rc; } - WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, - GOYA_ASYNC_EVENT_ID_INTS_REGISTER); - return 0; } @@ -5400,6 +5397,12 @@ static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, return -EPERM; } +static void goya_enable_events_from_fw(struct hl_device *hdev) +{ + WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, + GOYA_ASYNC_EVENT_ID_INTS_REGISTER); +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5481,7 +5484,8 @@ static const struct hl_asic_funcs goya_funcs = { .descramble_addr = hl_mmu_descramble_addr, .ack_protection_bits_errors = goya_ack_protection_bits_errors, .get_hw_block_id = goya_get_hw_block_id, - .hw_block_mmap = goya_block_mmap + .hw_block_mmap = goya_block_mmap, + .enable_events_from_fw = goya_enable_events_from_fw }; /* From 5b6b780660ad9e3ce60a1f04cfa1f4d5013e267a Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Tue, 2 Feb 2021 13:33:34 +0200 Subject: [PATCH 307/633] habanalabs: update security map after init CPU Qs when reading CPU_BOOT_DEV_STS0 reg after FW reports SRAM AVAILABLE the value in the register might not yet be updated by FW. to overcome this issue another "up-to-date" read of this register is done at the end of CPU queues init. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 7 ++----- drivers/misc/habanalabs/common/habanalabs.h | 3 --- drivers/misc/habanalabs/gaudi/gaudi.c | 6 +++++- drivers/misc/habanalabs/goya/goya.c | 6 +++++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 31b52a223f02..09706c571e95 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -125,7 +125,8 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, goto out; } - if (hdev->asic_prop.fw_cpucp_ack_with_pi) + if (hdev->asic_prop.fw_app_security_map & + CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) expected_ack_val = queue->pi; else expected_ack_val = CPUCP_PACKET_FENCE_VAL; @@ -786,10 +787,6 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, CPU_BOOT_DEV_STS0_FW_HARD_RST_EN) prop->hard_reset_done_by_fw = true; - if (prop->fw_boot_cpu_security_map & - CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN) - prop->fw_cpucp_ack_with_pi = true; - dev_dbg(hdev->dev, "Firmware boot CPU security status %#x\n", prop->fw_boot_cpu_security_map); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 18ed3a6000b0..46c37b0c704a 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -420,8 +420,6 @@ struct hl_mmu_properties { * from BOOT_DEV_STS0 * @dram_supports_virtual_memory: is there an MMU towards the DRAM * @hard_reset_done_by_fw: true if firmware is handling hard reset flow - * @fw_cpucp_ack_with_pi: true if cpucp is acking messages with the PQ PI - * instead of a magic number * @num_functional_hbms: number of functional HBMs in each DCORE. */ struct asic_fixed_properties { @@ -483,7 +481,6 @@ struct asic_fixed_properties { u8 fw_security_status_valid; u8 dram_supports_virtual_memory; u8 hard_reset_done_by_fw; - u8 fw_cpucp_ack_with_pi; u8 num_functional_hbms; }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 6905857b363b..f937d90786b2 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -536,7 +536,6 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->fw_security_disabled = true; prop->fw_security_status_valid = false; prop->hard_reset_done_by_fw = false; - prop->fw_cpucp_ack_with_pi = false; return 0; } @@ -3727,6 +3726,7 @@ static int gaudi_init_cpu(struct hl_device *hdev) static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) { struct gaudi_device *gaudi = hdev->asic_specific; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_eq *eq; u32 status; struct hl_hw_queue *cpu_pq = @@ -3783,6 +3783,10 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout) return -EIO; } + /* update FW application security bits */ + if (prop->fw_security_status_valid) + prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index af6a5760924c..c30524660761 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -464,7 +464,6 @@ int goya_get_fixed_properties(struct hl_device *hdev) prop->fw_security_disabled = true; prop->fw_security_status_valid = false; prop->hard_reset_done_by_fw = false; - prop->fw_cpucp_ack_with_pi = false; return 0; } @@ -1189,6 +1188,7 @@ static int goya_stop_external_queues(struct hl_device *hdev) int goya_init_cpu_queues(struct hl_device *hdev) { struct goya_device *goya = hdev->asic_specific; + struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_eq *eq; u32 status; struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; @@ -1241,6 +1241,10 @@ int goya_init_cpu_queues(struct hl_device *hdev) return -EIO; } + /* update FW application security bits */ + if (prop->fw_security_status_valid) + prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0); + goya->hw_cap_initialized |= HW_CAP_CPU_Q; return 0; } From 6df50d274363aa189a31435024339b781a6e32a9 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Fri, 5 Feb 2021 16:04:34 +0200 Subject: [PATCH 308/633] habanalabs: return block size + block ID When user gives us a block address to get its ID to mmap it, he also needs to get from us the block size to pass to the driver in the mmap function. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs.h | 4 ++- drivers/misc/habanalabs/common/memory.c | 19 +++++++++------ drivers/misc/habanalabs/gaudi/gaudi.c | 2 +- drivers/misc/habanalabs/goya/goya.c | 2 +- include/uapi/misc/habanalabs.h | 27 ++++++++++++++++----- 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 46c37b0c704a..dc9f5a83dfc9 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -862,6 +862,8 @@ enum div_select_defs { * showing it to users. * @ack_protection_bits_errors: ack and dump all security violations * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. + * also returns the size of the block if caller supplies + * a valid pointer for it * @hw_block_mmap: mmap a HW block with a given id. * @enable_events_from_fw: send interrupt to firmware to notify them the * driver is ready to receive asynchronous events. This @@ -980,7 +982,7 @@ struct hl_asic_funcs { u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); void (*ack_protection_bits_errors)(struct hl_device *hdev); int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr, - u32 *block_id); + u32 *block_size, u32 *block_id); int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, u32 block_id, u32 block_size); void (*enable_events_from_fw)(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 7171e8820a2d..7cadf75ebb81 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1289,12 +1289,13 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, return rc; } -static int map_block(struct hl_device *hdev, u64 address, u64 *handle) +static int map_block(struct hl_device *hdev, u64 address, u64 *handle, + u32 *size) { u32 block_id = 0; int rc; - rc = hdev->asic_funcs->get_hw_block_id(hdev, address, &block_id); + rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id); *handle = block_id | HL_MMAP_TYPE_BLOCK; *handle <<= PAGE_SHIFT; @@ -1371,7 +1372,7 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; u64 block_handle, device_addr = 0; - u32 handle = 0; + u32 handle = 0, block_size; int rc; switch (args->in.op) { @@ -1416,8 +1417,9 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) case HL_MEM_OP_MAP_BLOCK: rc = map_block(hdev, args->in.map_block.block_addr, - &block_handle); - args->out.handle = block_handle; + &block_handle, &block_size); + args->out.block_handle = block_handle; + args->out.block_size = block_size; break; default: @@ -1437,7 +1439,7 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; u64 block_handle, device_addr = 0; - u32 handle = 0; + u32 handle = 0, block_size; int rc; if (!hl_device_operational(hdev, &status)) { @@ -1524,8 +1526,9 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) case HL_MEM_OP_MAP_BLOCK: rc = map_block(hdev, args->in.map_block.block_addr, - &block_handle); - args->out.handle = block_handle; + &block_handle, &block_size); + args->out.block_handle = block_handle; + args->out.block_size = block_size; break; default: diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index f937d90786b2..35342edd4a02 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -8490,7 +8490,7 @@ static u64 gaudi_get_device_time(struct hl_device *hdev) } static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr, - u32 *block_id) + u32 *block_size, u32 *block_id) { return -EPERM; } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index c30524660761..ed566c52ccaa 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5390,7 +5390,7 @@ static void goya_ctx_fini(struct hl_ctx *ctx) } static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, - u32 *block_id) + u32 *block_size, u32 *block_id) { return -EPERM; } diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 64ae83b5f8e5..5a86b521a450 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -782,10 +782,10 @@ struct hl_mem_in { /* HL_MEM_OP_MAP_BLOCK - map a hw block */ struct { /* - * HW block address to map, a handle will be returned - * to the user and will be used to mmap the relevant - * block. Only addresses from configuration space are - * allowed. + * HW block address to map, a handle and size will be + * returned to the user and will be used to mmap the + * relevant block. Only addresses from configuration + * space are allowed. */ __u64 block_addr; } map_block; @@ -816,11 +816,26 @@ struct hl_mem_out { __u64 device_virt_addr; /* - * Used for HL_MEM_OP_ALLOC and HL_MEM_OP_MAP_BLOCK. + * Used in HL_MEM_OP_ALLOC * This is the assigned handle for the allocated memory - * or mapped block */ __u64 handle; + + struct { + /* + * Used in HL_MEM_OP_MAP_BLOCK. + * This is the assigned handle for the mapped block + */ + __u64 block_handle; + + /* + * Used in HL_MEM_OP_MAP_BLOCK + * This is the size of the mapped block + */ + __u32 block_size; + + __u32 pad; + }; }; }; From da5dfbb97a82ff698e1dc7b229d4d4f5759dad2b Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sat, 6 Feb 2021 19:34:59 +0200 Subject: [PATCH 309/633] habanalabs/gaudi: don't enable clock gating on DMA5 Graph Compiler uses DMA5 in a non-standard way and it requires the driver to disable clock gating on that DMA. Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 35342edd4a02..9152242778f5 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -3460,6 +3460,12 @@ static void gaudi_set_clock_gating(struct hl_device *hdev) enable = !!(hdev->clock_gating_mask & (BIT_ULL(gaudi_dma_assignment[i]))); + /* GC sends work to DMA engine through Upper CP in DMA5 so + * we need to not enable clock gating in that DMA + */ + if (i == GAUDI_HBM_DMA_4) + enable = 0; + qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET; WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, enable ? QMAN_CGM1_PWR_GATE_EN : 0); From f320ff0387a8a2d3123c4f7d6d61eecc740d0466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 8 Feb 2021 08:37:04 +0100 Subject: [PATCH 310/633] mei: bus: simplify mei_cl_device_remove() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver core only calls a bus' remove function when there is actually a driver and a device. So drop the needless check and assign cldrv earlier. (Side note: The check for cldev being non-NULL is broken anyhow, because to_mei_cl_device() is a wrapper around container_of() for a member that is not the first one. So cldev only can become NULL if dev is (void *)0xc (for archs with 32 bit pointers) or (void *)0x18 (for archs with 64 bit pointers).) Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210208073705.428185-2-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/bus.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 34fb5e541fe5..393a41330a6b 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -880,13 +880,9 @@ static int mei_cl_device_probe(struct device *dev) static int mei_cl_device_remove(struct device *dev) { struct mei_cl_device *cldev = to_mei_cl_device(dev); - struct mei_cl_driver *cldrv; + struct mei_cl_driver *cldrv = to_mei_cl_driver(dev->driver); int ret = 0; - if (!cldev || !dev->driver) - return 0; - - cldrv = to_mei_cl_driver(dev->driver); if (cldrv->remove) ret = cldrv->remove(cldev); From bf5c9cc8ad7fffd1f72df3baa5870449e4c16d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 8 Feb 2021 08:37:05 +0100 Subject: [PATCH 311/633] mei: bus: change remove callback to return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver core ignores the return value of mei_cl_device_remove() so passing an error value doesn't solve any problem. As most mei drivers' remove callbacks return 0 unconditionally and returning a different value doesn't have any effect, change this prototype to return void and return 0 unconditionally in mei_cl_device_remove(). The only driver that could return an error value is modified to emit an explicit warning in the error case. Acked-by: Guenter Roeck Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210208073705.428185-3-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/bus.c | 5 ++--- drivers/misc/mei/hdcp/mei_hdcp.c | 7 +++++-- drivers/nfc/microread/mei.c | 4 +--- drivers/nfc/pn544/mei.c | 4 +--- drivers/watchdog/mei_wdt.c | 4 +--- include/linux/mei_cl_bus.h | 2 +- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 393a41330a6b..580074e32599 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -881,17 +881,16 @@ static int mei_cl_device_remove(struct device *dev) { struct mei_cl_device *cldev = to_mei_cl_device(dev); struct mei_cl_driver *cldrv = to_mei_cl_driver(dev->driver); - int ret = 0; if (cldrv->remove) - ret = cldrv->remove(cldev); + cldrv->remove(cldev); mei_cldev_unregister_callbacks(cldev); mei_cl_bus_module_put(cldev); module_put(THIS_MODULE); - return ret; + return 0; } static ssize_t name_show(struct device *dev, struct device_attribute *a, diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c index 9ae9669e46ea..8447ad4b7d47 100644 --- a/drivers/misc/mei/hdcp/mei_hdcp.c +++ b/drivers/misc/mei/hdcp/mei_hdcp.c @@ -845,16 +845,19 @@ static int mei_hdcp_probe(struct mei_cl_device *cldev, return ret; } -static int mei_hdcp_remove(struct mei_cl_device *cldev) +static void mei_hdcp_remove(struct mei_cl_device *cldev) { struct i915_hdcp_comp_master *comp_master = mei_cldev_get_drvdata(cldev); + int ret; component_master_del(&cldev->dev, &mei_component_master_ops); kfree(comp_master); mei_cldev_set_drvdata(cldev, NULL); - return mei_cldev_disable(cldev); + ret = mei_cldev_disable(cldev); + if (ret) + dev_warn(&cldev->dev, "mei_cldev_disable() failed\n"); } #define MEI_UUID_HDCP GUID_INIT(0xB638AB7E, 0x94E2, 0x4EA2, 0xA5, \ diff --git a/drivers/nfc/microread/mei.c b/drivers/nfc/microread/mei.c index 5dad8847a9b3..8fa7771085eb 100644 --- a/drivers/nfc/microread/mei.c +++ b/drivers/nfc/microread/mei.c @@ -44,15 +44,13 @@ static int microread_mei_probe(struct mei_cl_device *cldev, return 0; } -static int microread_mei_remove(struct mei_cl_device *cldev) +static void microread_mei_remove(struct mei_cl_device *cldev) { struct nfc_mei_phy *phy = mei_cldev_get_drvdata(cldev); microread_remove(phy->hdev); nfc_mei_phy_free(phy); - - return 0; } static struct mei_cl_device_id microread_mei_tbl[] = { diff --git a/drivers/nfc/pn544/mei.c b/drivers/nfc/pn544/mei.c index 579bc599f545..5c10aac085a4 100644 --- a/drivers/nfc/pn544/mei.c +++ b/drivers/nfc/pn544/mei.c @@ -42,7 +42,7 @@ static int pn544_mei_probe(struct mei_cl_device *cldev, return 0; } -static int pn544_mei_remove(struct mei_cl_device *cldev) +static void pn544_mei_remove(struct mei_cl_device *cldev) { struct nfc_mei_phy *phy = mei_cldev_get_drvdata(cldev); @@ -51,8 +51,6 @@ static int pn544_mei_remove(struct mei_cl_device *cldev) pn544_hci_remove(phy->hdev); nfc_mei_phy_free(phy); - - return 0; } static struct mei_cl_device_id pn544_mei_tbl[] = { diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c index 5391bf3e6b11..53165e49c298 100644 --- a/drivers/watchdog/mei_wdt.c +++ b/drivers/watchdog/mei_wdt.c @@ -619,7 +619,7 @@ static int mei_wdt_probe(struct mei_cl_device *cldev, return ret; } -static int mei_wdt_remove(struct mei_cl_device *cldev) +static void mei_wdt_remove(struct mei_cl_device *cldev) { struct mei_wdt *wdt = mei_cldev_get_drvdata(cldev); @@ -636,8 +636,6 @@ static int mei_wdt_remove(struct mei_cl_device *cldev) dbgfs_unregister(wdt); kfree(wdt); - - return 0; } #define MEI_UUID_WD UUID_LE(0x05B79A6F, 0x4628, 0x4D7F, \ diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index 959ad7d850b4..07f5ef8fc456 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -68,7 +68,7 @@ struct mei_cl_driver { int (*probe)(struct mei_cl_device *cldev, const struct mei_cl_device_id *id); - int (*remove)(struct mei_cl_device *cldev); + void (*remove)(struct mei_cl_device *cldev); }; int __mei_cldev_driver_register(struct mei_cl_driver *cldrv, From b398d53cd421454d64850f8b1f6d609ede9042d9 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Mon, 8 Feb 2021 17:06:48 +0200 Subject: [PATCH 312/633] mei: bus: block send with vtag on non-conformat FW Block data send with vtag if either transport layer or FW client are not supporting vtags. Cc: # v5.10+ Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210208150649.141358-1-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/bus.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 580074e32599..935acc6bbf3c 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -61,6 +61,13 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, goto out; } + if (vtag) { + /* Check if vtag is supported by client */ + rets = mei_cl_vt_support_check(cl); + if (rets) + goto out; + } + if (length > mei_cl_mtu(cl)) { rets = -EFBIG; goto out; From e666b79e22958564fc23e32bb67ef57b21729067 Mon Sep 17 00:00:00 2001 From: Tomas Winkler Date: Mon, 8 Feb 2021 17:06:49 +0200 Subject: [PATCH 313/633] mei: use sysfs_emit() in tx_queue_limit_show sysfs Using of snprintf is discouraged in sysfs use the new sysfs_emit() API. Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20210208150649.141358-2-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 9f6682033ed7..28937b6e7e0c 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -1026,7 +1026,7 @@ static ssize_t tx_queue_limit_show(struct device *device, size = dev->tx_queue_limit; mutex_unlock(&dev->device_lock); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t tx_queue_limit_store(struct device *device, From c31d32ba580d2f361e6169c093fdf624e5851925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Sun, 7 Feb 2021 22:55:55 +0100 Subject: [PATCH 314/633] ipack: Fail earlier for drivers without probe function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A driver without a probe function isn't useful as it can never be used. Let registering such a driver fail already instead of failing every binding. This is only cosmetic as there is no ipack driver without a probe function. Signed-off-by: Uwe Kleine-König Acked-by: Samuel Iglesias Gonsalvez Link: https://lore.kernel.org/r/20210207215556.96371-1-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/ipack/ipack.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/ipack/ipack.c b/drivers/ipack/ipack.c index 9267a85fee18..eabc4d08792a 100644 --- a/drivers/ipack/ipack.c +++ b/drivers/ipack/ipack.c @@ -64,9 +64,6 @@ static int ipack_bus_probe(struct device *device) struct ipack_device *dev = to_ipack_dev(device); struct ipack_driver *drv = to_ipack_driver(device->driver); - if (!drv->ops->probe) - return -EINVAL; - return drv->ops->probe(dev); } @@ -252,6 +249,9 @@ EXPORT_SYMBOL_GPL(ipack_bus_unregister); int ipack_driver_register(struct ipack_driver *edrv, struct module *owner, const char *name) { + if (!edrv->ops->probe) + return -EINVAL; + edrv->driver.owner = owner; edrv->driver.name = name; edrv->driver.bus = &ipack_bus_type; From 609cf09c56802f7997497b1fdc89209db52d4449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Sun, 7 Feb 2021 22:55:56 +0100 Subject: [PATCH 315/633] ipack: Handle a driver without remove callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A driver that only consumes devm-managed resources might well have no remove callback. Additionally given that the device core ignores the return value of ipack_bus_remove() stop returning an error code. Signed-off-by: Uwe Kleine-König Acked-by: Samuel Iglesias Gonsalvez Link: https://lore.kernel.org/r/20210207215556.96371-2-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/ipack/ipack.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/ipack/ipack.c b/drivers/ipack/ipack.c index eabc4d08792a..7de9605cac4f 100644 --- a/drivers/ipack/ipack.c +++ b/drivers/ipack/ipack.c @@ -72,10 +72,9 @@ static int ipack_bus_remove(struct device *device) struct ipack_device *dev = to_ipack_dev(device); struct ipack_driver *drv = to_ipack_driver(device->driver); - if (!drv->ops->remove) - return -EINVAL; + if (drv->ops->remove) + drv->ops->remove(dev); - drv->ops->remove(dev); return 0; } From ef0fec22146ba874a1a64d4dd0303183e26a5b6f Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 8 Feb 2021 19:09:47 +0200 Subject: [PATCH 316/633] bus: fsl-mc: Fix test for end of loop The "desc" pointer can't possibly be NULL here. If we can't find the correct "desc" then tt points to the last element of the fsl_mc_accepted_cmds[] array. Fix this by testing if "i == FSL_MC_NUM_ACCEPTED_CMDS" instead. Fixes: 2cf1e703f066 ("bus: fsl-mc: add fsl-mc userspace support") Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20210208170949.3070898-2-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/bus/fsl-mc/fsl-mc-uapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c index eeb988c9f4bb..bdcd9d983a78 100644 --- a/drivers/bus/fsl-mc/fsl-mc-uapi.c +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -338,7 +338,7 @@ static int fsl_mc_command_check(struct fsl_mc_device *mc_dev, if ((cmdid & desc->cmdid_mask) == desc->cmdid_value) break; } - if (!desc) { + if (i == FSL_MC_NUM_ACCEPTED_CMDS) { dev_err(&mc_dev->dev, "MC command 0x%04x: cmdid not accepted\n", cmdid); return -EACCES; } From e70ba1b06c26cddccb6caf5d6fb18b4eee61f67d Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Mon, 8 Feb 2021 19:09:48 +0200 Subject: [PATCH 317/633] bus: fsl-mc: add the dpdbg device type A new object type was recently added in MC. This has to be added in the fsl-mc bus device type list so that it can be properly listed. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210208170949.3070898-3-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/bus/fsl-mc/fsl-mc-bus.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 70f4b69556f5..0c8bf020ba43 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -388,6 +388,11 @@ struct device_type fsl_mc_bus_dpdmai_type = { }; EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdmai_type); +struct device_type fsl_mc_bus_dpdbg_type = { + .name = "fsl_mc_bus_dpdbg" +}; +EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdbg_type); + static struct device_type *fsl_mc_get_device_type(const char *type) { static const struct { @@ -409,6 +414,7 @@ static struct device_type *fsl_mc_get_device_type(const char *type) { &fsl_mc_bus_dpaiop_type, "dpaiop" }, { &fsl_mc_bus_dpci_type, "dpci" }, { &fsl_mc_bus_dpdmai_type, "dpdmai" }, + { &fsl_mc_bus_dpdbg_type, "dpdbg" }, { NULL, NULL } }; int i; From d67cc29e6d1f9d0d6a03d81595ce9d7f6dd5500e Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Mon, 8 Feb 2021 19:09:49 +0200 Subject: [PATCH 318/633] bus: fsl-mc: list more commands as accepted through the ioctl Add some new MC firmware commands that can be received through the userspace ioctl interface - *get_max_frame_length and *_get_counter. Acked-by: Laurentiu Tudor Signed-off-by: Ioana Ciornei Link: https://lore.kernel.org/r/20210208170949.3070898-4-ciorneiioana@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/bus/fsl-mc/fsl-mc-uapi.c | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c index bdcd9d983a78..9c4c1395fcdb 100644 --- a/drivers/bus/fsl-mc/fsl-mc-uapi.c +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -60,6 +60,13 @@ enum fsl_mc_cmd_index { DPNI_GET_PRIM_MAC, DPNI_GET_STATISTICS, DPNI_GET_LINK_STATE, + DPNI_GET_MAX_FRAME_LENGTH, + DPSW_GET_TAILDROP, + DPSW_SET_TAILDROP, + DPSW_IF_GET_COUNTER, + DPSW_IF_GET_MAX_FRAME_LENGTH, + DPDMUX_GET_COUNTER, + DPDMUX_IF_GET_MAX_FRAME_LENGTH, GET_ATTR, GET_IRQ_MASK, GET_IRQ_STATUS, @@ -261,6 +268,49 @@ static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { .token = true, .size = 8, }, + [DPNI_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x2170, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPSW_GET_TAILDROP] = { + .cmdid_value = 0x0A80, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPSW_SET_TAILDROP] = { + .cmdid_value = 0x0A90, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 24, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPSW_IF_GET_COUNTER] = { + .cmdid_value = 0x0340, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPSW_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0450, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPDMUX_GET_COUNTER] = { + .cmdid_value = 0x0b20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPDMUX_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0a20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, [GET_ATTR] = { .cmdid_value = 0x0040, .cmdid_mask = 0xFFF0, From b212658aebda82f92967bcbd4c7380d607c3d803 Mon Sep 17 00:00:00 2001 From: Jonathan Marek Date: Mon, 8 Feb 2021 15:04:01 -0500 Subject: [PATCH 319/633] misc: fastrpc: fix incorrect usage of dma_map_sgtable dma_map_sgtable() returns 0 on success, which is the opposite of what this code was doing. Fixes: 7cd7edb89437 ("misc: fastrpc: fix common struct sg_table related issues") Acked-by: Marek Szyprowski Signed-off-by: Jonathan Marek Link: https://lore.kernel.org/r/20210208200401.31100-1-jonathan@marek.ca Signed-off-by: Greg Kroah-Hartman --- drivers/misc/fastrpc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 70eb5ed942d0..f12e909034ac 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -520,12 +520,13 @@ fastrpc_map_dma_buf(struct dma_buf_attachment *attachment, { struct fastrpc_dma_buf_attachment *a = attachment->priv; struct sg_table *table; + int ret; table = &a->sgt; - if (!dma_map_sgtable(attachment->dev, table, dir, 0)) - return ERR_PTR(-ENOMEM); - + ret = dma_map_sgtable(attachment->dev, table, dir, 0); + if (ret) + table = ERR_PTR(ret); return table; } From c619a804cc43345be3a1a1c4b46f72a3525cf1af Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:23 +0800 Subject: [PATCH 320/633] docs: acrn: Introduce ACRN Add documentation on the following aspects of ACRN: 1) A brief introduction on the architecture of ACRN. 2) I/O request handling in ACRN. 3) CPUID functions of ACRN. To learn more about ACRN, please go to ACRN project website https://projectacrn.org, or the documentation page https://projectacrn.github.io/. Cc: Dave Hansen Cc: Sen Christopherson Cc: Dan Williams Cc: Fengwei Yin Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Cc: Randy Dunlap Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Reviewed-by: Borislav Petkov Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-2-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- Documentation/virt/acrn/cpuid.rst | 46 +++++++++++ Documentation/virt/acrn/index.rst | 12 +++ Documentation/virt/acrn/introduction.rst | 43 +++++++++++ Documentation/virt/acrn/io-request.rst | 97 ++++++++++++++++++++++++ Documentation/virt/index.rst | 1 + MAINTAINERS | 7 ++ 6 files changed, 206 insertions(+) create mode 100644 Documentation/virt/acrn/cpuid.rst create mode 100644 Documentation/virt/acrn/index.rst create mode 100644 Documentation/virt/acrn/introduction.rst create mode 100644 Documentation/virt/acrn/io-request.rst diff --git a/Documentation/virt/acrn/cpuid.rst b/Documentation/virt/acrn/cpuid.rst new file mode 100644 index 000000000000..65fa4b9c1798 --- /dev/null +++ b/Documentation/virt/acrn/cpuid.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN CPUID bits +=============== + +A guest VM running on an ACRN hypervisor can check some of its features using +CPUID. + +ACRN cpuid functions are: + +function: 0x40000000 + +returns:: + + eax = 0x40000010 + ebx = 0x4e524341 + ecx = 0x4e524341 + edx = 0x4e524341 + +Note that this value in ebx, ecx and edx corresponds to the string +"ACRNACRNACRN". The value in eax corresponds to the maximum cpuid function +present in this leaf, and will be updated if more functions are added in the +future. + +function: define ACRN_CPUID_FEATURES (0x40000001) + +returns:: + + ebx, ecx, edx + eax = an OR'ed group of (1 << flag) + +where ``flag`` is defined as below: + +================================= =========== ================================ +flag value meaning +================================= =========== ================================ +ACRN_FEATURE_PRIVILEGED_VM 0 guest VM is a privileged VM +================================= =========== ================================ + +function: 0x40000010 + +returns:: + + ebx, ecx, edx + eax = (Virtual) TSC frequency in kHz. diff --git a/Documentation/virt/acrn/index.rst b/Documentation/virt/acrn/index.rst new file mode 100644 index 000000000000..b5f793e73df5 --- /dev/null +++ b/Documentation/virt/acrn/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN Hypervisor +=============== + +.. toctree:: + :maxdepth: 1 + + introduction + io-request + cpuid diff --git a/Documentation/virt/acrn/introduction.rst b/Documentation/virt/acrn/introduction.rst new file mode 100644 index 000000000000..f8d081bc084d --- /dev/null +++ b/Documentation/virt/acrn/introduction.rst @@ -0,0 +1,43 @@ +.. SPDX-License-Identifier: GPL-2.0 + +ACRN Hypervisor Introduction +============================ + +The ACRN Hypervisor is a Type 1 hypervisor, running directly on bare-metal +hardware. It has a privileged management VM, called Service VM, to manage User +VMs and do I/O emulation. + +ACRN userspace is an application running in the Service VM that emulates +devices for a User VM based on command line configurations. ACRN Hypervisor +Service Module (HSM) is a kernel module in the Service VM which provides +hypervisor services to the ACRN userspace. + +Below figure shows the architecture. + +:: + + Service VM User VM + +----------------------------+ | +------------------+ + | +--------------+ | | | | + | |ACRN userspace| | | | | + | +--------------+ | | | | + |-----------------ioctl------| | | | ... + |kernel space +----------+ | | | | + | | HSM | | | | Drivers | + | +----------+ | | | | + +--------------------|-------+ | +------------------+ + +---------------------hypercall----------------------------------------+ + | ACRN Hypervisor | + +----------------------------------------------------------------------+ + | Hardware | + +----------------------------------------------------------------------+ + +ACRN userspace allocates memory for the User VM, configures and initializes the +devices used by the User VM, loads the virtual bootloader, initializes the +virtual CPU state and handles I/O request accesses from the User VM. It uses +ioctls to communicate with the HSM. HSM implements hypervisor services by +interacting with the ACRN Hypervisor via hypercalls. HSM exports a char device +interface (/dev/acrn_hsm) to userspace. + +The ACRN hypervisor is open for contribution from anyone. The source repo is +available at https://github.com/projectacrn/acrn-hypervisor. diff --git a/Documentation/virt/acrn/io-request.rst b/Documentation/virt/acrn/io-request.rst new file mode 100644 index 000000000000..6cc3ea0fa1f5 --- /dev/null +++ b/Documentation/virt/acrn/io-request.rst @@ -0,0 +1,97 @@ +.. SPDX-License-Identifier: GPL-2.0 + +I/O request handling +==================== + +An I/O request of a User VM, which is constructed by the hypervisor, is +distributed by the ACRN Hypervisor Service Module to an I/O client +corresponding to the address range of the I/O request. Details of I/O request +handling are described in the following sections. + +1. I/O request +-------------- + +For each User VM, there is a shared 4-KByte memory region used for I/O requests +communication between the hypervisor and Service VM. An I/O request is a +256-byte structure buffer, which is 'struct acrn_io_request', that is filled by +an I/O handler of the hypervisor when a trapped I/O access happens in a User +VM. ACRN userspace in the Service VM first allocates a 4-KByte page and passes +the GPA (Guest Physical Address) of the buffer to the hypervisor. The buffer is +used as an array of 16 I/O request slots with each I/O request slot being 256 +bytes. This array is indexed by vCPU ID. + +2. I/O clients +-------------- + +An I/O client is responsible for handling User VM I/O requests whose accessed +GPA falls in a certain range. Multiple I/O clients can be associated with each +User VM. There is a special client associated with each User VM, called the +default client, that handles all I/O requests that do not fit into the range of +any other clients. The ACRN userspace acts as the default client for each User +VM. + +Below illustration shows the relationship between I/O requests shared buffer, +I/O requests and I/O clients. + +:: + + +------------------------------------------------------+ + | Service VM | + |+--------------------------------------------------+ | + || +----------------------------------------+ | | + || | shared page ACRN userspace | | | + || | +-----------------+ +------------+ | | | + || +----+->| acrn_io_request |<-+ default | | | | + || | | | +-----------------+ | I/O client | | | | + || | | | | ... | +------------+ | | | + || | | | +-----------------+ | | | + || | +-|--------------------------------------+ | | + ||---|----|-----------------------------------------| | + || | | kernel | | + || | | +----------------------+ | | + || | | | +-------------+ HSM | | | + || | +--------------+ | | | | + || | | | I/O clients | | | | + || | | | | | | | + || | | +-------------+ | | | + || | +----------------------+ | | + |+---|----------------------------------------------+ | + +----|-------------------------------------------------+ + | + +----|-------------------------------------------------+ + | +-+-----------+ | + | | I/O handler | ACRN Hypervisor | + | +-------------+ | + +------------------------------------------------------+ + +3. I/O request state transition +------------------------------- + +The state transitions of an ACRN I/O request are as follows. + +:: + + FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... + +- FREE: this I/O request slot is empty +- PENDING: a valid I/O request is pending in this slot +- PROCESSING: the I/O request is being processed +- COMPLETE: the I/O request has been processed + +An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and +ACRN userspace are in charge of processing the others. + +4. Processing flow of I/O requests +---------------------------------- + +a. The I/O handler of the hypervisor will fill an I/O request with PENDING + state when a trapped I/O access happens in a User VM. +b. The hypervisor makes an upcall, which is a notification interrupt, to + the Service VM. +c. The upcall handler schedules a worker to dispatch I/O requests. +d. The worker looks for the PENDING I/O requests, assigns them to different + registered clients based on the address of the I/O accesses, updates + their state to PROCESSING, and notifies the corresponding client to handle. +e. The notified client handles the assigned I/O requests. +f. The HSM updates I/O requests states to COMPLETE and notifies the hypervisor + of the completion via hypercalls. diff --git a/Documentation/virt/index.rst b/Documentation/virt/index.rst index 350f5c869b56..edea7fea95a8 100644 --- a/Documentation/virt/index.rst +++ b/Documentation/virt/index.rst @@ -12,6 +12,7 @@ Linux Virtualization Support paravirt_ops guest-halt-polling ne_overview + acrn/index .. only:: html and subproject diff --git a/MAINTAINERS b/MAINTAINERS index a6f1f5889346..6a3d11233d3c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -436,6 +436,13 @@ S: Orphan F: drivers/platform/x86/wmi.c F: include/uapi/linux/wmi.h +ACRN HYPERVISOR SERVICE MODULE +M: Shuo Liu +L: acrn-dev@lists.projectacrn.org +S: Supported +W: https://projectacrn.org +F: Documentation/virt/acrn/ + AD1889 ALSA SOUND DRIVER L: linux-parisc@vger.kernel.org S: Maintained From 7995700e65f1d5c97ee56b7e9c8f68d2b0fac253 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:24 +0800 Subject: [PATCH 321/633] x86/acrn: Introduce acrn_{setup, remove}_intr_handler() The ACRN Hypervisor builds an I/O request when a trapped I/O access happens in User VM. Then, ACRN Hypervisor issues an upcall by sending a notification interrupt to the Service VM. HSM in the Service VM needs to hook the notification interrupt to handle I/O requests. Notification interrupts from ACRN Hypervisor are already supported and a, currently uninitialized, callback called. Export two APIs for HSM to setup/remove its callback. Cc: Dave Hansen Cc: Sean Christopherson Cc: Dan Williams Cc: Fengwei Yin Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Originally-by: Yakui Zhao Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Acked-by: Borislav Petkov Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-3-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- arch/x86/include/asm/acrn.h | 8 ++++++++ arch/x86/kernel/cpu/acrn.c | 14 ++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 arch/x86/include/asm/acrn.h diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h new file mode 100644 index 000000000000..ff259b69cde7 --- /dev/null +++ b/arch/x86/include/asm/acrn.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ACRN_H +#define _ASM_X86_ACRN_H + +void acrn_setup_intr_handler(void (*handler)(void)); +void acrn_remove_intr_handler(void); + +#endif /* _ASM_X86_ACRN_H */ diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 0b2c03943ac6..e0c181781905 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -10,6 +10,8 @@ */ #include + +#include #include #include #include @@ -55,6 +57,18 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) set_irq_regs(old_regs); } +void acrn_setup_intr_handler(void (*handler)(void)) +{ + acrn_intr_handler = handler; +} +EXPORT_SYMBOL_GPL(acrn_setup_intr_handler); + +void acrn_remove_intr_handler(void) +{ + acrn_intr_handler = NULL; +} +EXPORT_SYMBOL_GPL(acrn_remove_intr_handler); + const __initconst struct hypervisor_x86 x86_hyper_acrn = { .name = "ACRN", .detect = acrn_detect, From ebbfc978f37e2b33dc15ba00b26eea10c6d02425 Mon Sep 17 00:00:00 2001 From: Yin Fengwei Date: Sun, 7 Feb 2021 11:10:25 +0800 Subject: [PATCH 322/633] x86/acrn: Introduce acrn_cpuid_base() and hypervisor feature bits ACRN Hypervisor reports hypervisor features via CPUID leaf 0x40000001 which is similar to KVM. A VM can check if it's the privileged VM using the feature bits. The Service VM is the only privileged VM by design. Cc: Dave Hansen Cc: Sean Christopherson Cc: Dan Williams Cc: Fengwei Yin Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Reinette Chatre Acked-by: Borislav Petkov Signed-off-by: Yin Fengwei Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-4-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- arch/x86/include/asm/acrn.h | 16 ++++++++++++++++ arch/x86/kernel/cpu/acrn.c | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h index ff259b69cde7..127f20672c5d 100644 --- a/arch/x86/include/asm/acrn.h +++ b/arch/x86/include/asm/acrn.h @@ -2,7 +2,23 @@ #ifndef _ASM_X86_ACRN_H #define _ASM_X86_ACRN_H +/* + * This CPUID returns feature bitmaps in EAX. + * Guest VM uses this to detect the appropriate feature bit. + */ +#define ACRN_CPUID_FEATURES 0x40000001 +/* Bit 0 indicates whether guest VM is privileged */ +#define ACRN_FEATURE_PRIVILEGED_VM BIT(0) + void acrn_setup_intr_handler(void (*handler)(void)); void acrn_remove_intr_handler(void); +static inline u32 acrn_cpuid_base(void) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return hypervisor_cpuid_base("ACRNACRNACRN", 0); + + return 0; +} + #endif /* _ASM_X86_ACRN_H */ diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index e0c181781905..23f5f27b5a02 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -21,7 +21,7 @@ static u32 __init acrn_detect(void) { - return hypervisor_cpuid_base("ACRNACRNACRN", 0); + return acrn_cpuid_base(); } static void __init acrn_init_platform(void) From 8a0a87198a584202616868f9c82d9611bb675c90 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:26 +0800 Subject: [PATCH 323/633] x86/acrn: Introduce hypercall interfaces The Service VM communicates with the hypervisor via conventional hypercalls. VMCALL instruction is used to make the hypercalls. ACRN hypercall ABI: * Hypercall number is in R8 register. * Up to 2 parameters are in RDI and RSI registers. * Return value is in RAX register. Introduce the ACRN hypercall interfaces. Because GCC doesn't support R8 register as direct register constraints, use supported constraint as input with a explicit MOV to R8 in beginning of asm. Cc: Dave Hansen Cc: Sean Christopherson Cc: Dan Williams Cc: Fengwei Yin Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Cc: Borislav Petkov Cc: Arvind Sankar Cc: Peter Zijlstra Cc: Nick Desaulniers Cc: Segher Boessenkool Originally-by: Yakui Zhao Reviewed-by: Reinette Chatre Reviewed-by: Nick Desaulniers Acked-by: Borislav Petkov Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-5-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- arch/x86/include/asm/acrn.h | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h index 127f20672c5d..e003a01b7c67 100644 --- a/arch/x86/include/asm/acrn.h +++ b/arch/x86/include/asm/acrn.h @@ -21,4 +21,58 @@ static inline u32 acrn_cpuid_base(void) return 0; } +/* + * Hypercalls for ACRN + * + * - VMCALL instruction is used to implement ACRN hypercalls. + * - ACRN hypercall ABI: + * - Hypercall number is passed in R8 register. + * - Up to 2 arguments are passed in RDI, RSI. + * - Return value will be placed in RAX. + * + * Because GCC doesn't support R8 register as direct register constraints, use + * supported constraint as input with a explicit MOV to R8 in beginning of asm. + */ +static inline long acrn_hypercall0(unsigned long hcall_id) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall1(unsigned long hcall_id, + unsigned long param1) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall2(unsigned long hcall_id, + unsigned long param1, + unsigned long param2) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1), "S" (param2) + : "r8", "memory"); + + return result; +} + #endif /* _ASM_X86_ACRN_H */ From 666834c47d3b41da550bbcbc709148e5fc14879c Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:27 +0800 Subject: [PATCH 324/633] virt: acrn: Introduce ACRN HSM basic driver ACRN Hypervisor Service Module (HSM) is a kernel module in Service VM which communicates with ACRN userspace through ioctls and talks to ACRN Hypervisor through hypercalls. Add a basic HSM driver which allows Service VM userspace to communicate with ACRN. The following patches will add more ioctls, guest VM memory mapping caching, I/O request processing, ioeventfd and irqfd into this module. HSM exports a char device interface (/dev/acrn_hsm) to userspace. Cc: Dave Hansen Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-6-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 1 + drivers/virt/Kconfig | 2 + drivers/virt/Makefile | 1 + drivers/virt/acrn/Kconfig | 14 ++++++ drivers/virt/acrn/Makefile | 3 ++ drivers/virt/acrn/acrn_drv.h | 18 ++++++++ drivers/virt/acrn/hsm.c | 87 ++++++++++++++++++++++++++++++++++++ 7 files changed, 126 insertions(+) create mode 100644 drivers/virt/acrn/Kconfig create mode 100644 drivers/virt/acrn/Makefile create mode 100644 drivers/virt/acrn/acrn_drv.h create mode 100644 drivers/virt/acrn/hsm.c diff --git a/MAINTAINERS b/MAINTAINERS index 6a3d11233d3c..4ac781da514f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -442,6 +442,7 @@ L: acrn-dev@lists.projectacrn.org S: Supported W: https://projectacrn.org F: Documentation/virt/acrn/ +F: drivers/virt/acrn/ AD1889 ALSA SOUND DRIVER L: linux-parisc@vger.kernel.org diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index 80c5f9c16ec1..8061e8ef449f 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -34,4 +34,6 @@ config FSL_HV_MANAGER source "drivers/virt/vboxguest/Kconfig" source "drivers/virt/nitro_enclaves/Kconfig" + +source "drivers/virt/acrn/Kconfig" endif diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile index f28425ce4b39..3e272ea60cd9 100644 --- a/drivers/virt/Makefile +++ b/drivers/virt/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_FSL_HV_MANAGER) += fsl_hypervisor.o obj-y += vboxguest/ obj-$(CONFIG_NITRO_ENCLAVES) += nitro_enclaves/ +obj-$(CONFIG_ACRN_HSM) += acrn/ diff --git a/drivers/virt/acrn/Kconfig b/drivers/virt/acrn/Kconfig new file mode 100644 index 000000000000..36c80378c30c --- /dev/null +++ b/drivers/virt/acrn/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +config ACRN_HSM + tristate "ACRN Hypervisor Service Module" + depends on ACRN_GUEST + help + ACRN Hypervisor Service Module (HSM) is a kernel module which + communicates with ACRN userspace through ioctls and talks to + the ACRN Hypervisor through hypercalls. HSM will only run in + a privileged management VM, called Service VM, to manage User + VMs and do I/O emulation. Not required for simply running + under ACRN as a User VM. + + To compile as a module, choose M, the module will be called + acrn. If unsure, say N. diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile new file mode 100644 index 000000000000..6920ed798aaf --- /dev/null +++ b/drivers/virt/acrn/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ACRN_HSM) := acrn.o +acrn-y := hsm.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h new file mode 100644 index 000000000000..29eedd696327 --- /dev/null +++ b/drivers/virt/acrn/acrn_drv.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ACRN_HSM_DRV_H +#define __ACRN_HSM_DRV_H + +#include + +#define ACRN_INVALID_VMID (0xffffU) + +/** + * struct acrn_vm - Properties of ACRN User VM. + * @vmid: User VM ID + */ +struct acrn_vm { + u16 vmid; +}; + +#endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c new file mode 100644 index 000000000000..a8dcb250649d --- /dev/null +++ b/drivers/virt/acrn/hsm.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN Hypervisor Service Module (HSM) + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Fengwei Yin + * Yakui Zhao + */ + +#include +#include +#include +#include + +#include +#include + +#include "acrn_drv.h" + +/* + * When /dev/acrn_hsm is opened, a 'struct acrn_vm' object is created to + * represent a VM instance and continues to be associated with the opened file + * descriptor. All ioctl operations on this file descriptor will be targeted to + * the VM instance. Release of this file descriptor will destroy the object. + */ +static int acrn_dev_open(struct inode *inode, struct file *filp) +{ + struct acrn_vm *vm; + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + vm->vmid = ACRN_INVALID_VMID; + filp->private_data = vm; + return 0; +} + +static int acrn_dev_release(struct inode *inode, struct file *filp) +{ + struct acrn_vm *vm = filp->private_data; + + kfree(vm); + return 0; +} + +static const struct file_operations acrn_fops = { + .owner = THIS_MODULE, + .open = acrn_dev_open, + .release = acrn_dev_release, +}; + +static struct miscdevice acrn_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "acrn_hsm", + .fops = &acrn_fops, +}; + +static int __init hsm_init(void) +{ + int ret; + + if (x86_hyper_type != X86_HYPER_ACRN) + return -ENODEV; + + if (!(cpuid_eax(ACRN_CPUID_FEATURES) & ACRN_FEATURE_PRIVILEGED_VM)) + return -EPERM; + + ret = misc_register(&acrn_dev); + if (ret) + pr_err("Create misc dev failed!\n"); + + return ret; +} + +static void __exit hsm_exit(void) +{ + misc_deregister(&acrn_dev); +} +module_init(hsm_init); +module_exit(hsm_exit); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ACRN Hypervisor Service Module (HSM)"); From 9c5137aedd112f78a968bdd2325de2ea06df46c0 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:28 +0800 Subject: [PATCH 325/633] virt: acrn: Introduce VM management interfaces The VM management interfaces expose several VM operations to ACRN userspace via ioctls. For example, creating VM, starting VM, destroying VM and so on. The ACRN Hypervisor needs to exchange data with the ACRN userspace during the VM operations. HSM provides VM operation ioctls to the ACRN userspace and communicates with the ACRN Hypervisor for VM operations via hypercalls. HSM maintains a list of User VM. Each User VM will be bound to an existing file descriptor of /dev/acrn_hsm. The User VM will be destroyed when the file descriptor is closed. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-7-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- .../userspace-api/ioctl/ioctl-number.rst | 1 + MAINTAINERS | 1 + drivers/virt/acrn/Makefile | 2 +- drivers/virt/acrn/acrn_drv.h | 21 ++++- drivers/virt/acrn/hsm.c | 76 +++++++++++++++++- drivers/virt/acrn/hypercall.h | 78 +++++++++++++++++++ drivers/virt/acrn/vm.c | 68 ++++++++++++++++ include/uapi/linux/acrn.h | 58 ++++++++++++++ 8 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 drivers/virt/acrn/hypercall.h create mode 100644 drivers/virt/acrn/vm.c create mode 100644 include/uapi/linux/acrn.h diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 19d730f9d136..76c27dbae96b 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -320,6 +320,7 @@ Code Seq# Include File Comments 0xA0 all linux/sdp/sdp.h Industrial Device Project 0xA1 0 linux/vtpm_proxy.h TPM Emulator Proxy Driver +0xA2 all uapi/linux/acrn.h ACRN hypervisor 0xA3 80-8F Port ACL in development: 0xA3 90-9F linux/dtlk.h diff --git a/MAINTAINERS b/MAINTAINERS index 4ac781da514f..6b5990a28b44 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -443,6 +443,7 @@ S: Supported W: https://projectacrn.org F: Documentation/virt/acrn/ F: drivers/virt/acrn/ +F: include/uapi/linux/acrn.h AD1889 ALSA SOUND DRIVER L: linux-parisc@vger.kernel.org diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile index 6920ed798aaf..cf8b4ed5e74e 100644 --- a/drivers/virt/acrn/Makefile +++ b/drivers/virt/acrn/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACRN_HSM) := acrn.o -acrn-y := hsm.o +acrn-y := hsm.o vm.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index 29eedd696327..e5aba86cad8c 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -3,16 +3,35 @@ #ifndef __ACRN_HSM_DRV_H #define __ACRN_HSM_DRV_H +#include +#include +#include #include +#include "hypercall.h" + +extern struct miscdevice acrn_dev; + #define ACRN_INVALID_VMID (0xffffU) +#define ACRN_VM_FLAG_DESTROYED 0U /** * struct acrn_vm - Properties of ACRN User VM. + * @list: Entry within global list of all VMs * @vmid: User VM ID + * @vcpu_num: Number of virtual CPUs in the VM + * @flags: Flags (ACRN_VM_FLAG_*) of the VM. This is VM flag management + * in HSM which is different from the &acrn_vm_creation.vm_flag. */ struct acrn_vm { - u16 vmid; + struct list_head list; + u16 vmid; + int vcpu_num; + unsigned long flags; }; +struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, + struct acrn_vm_creation *vm_param); +int acrn_vm_destroy(struct acrn_vm *vm); + #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index a8dcb250649d..5fd933471683 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -9,7 +9,6 @@ * Yakui Zhao */ -#include #include #include #include @@ -38,10 +37,82 @@ static int acrn_dev_open(struct inode *inode, struct file *filp) return 0; } +/* + * HSM relies on hypercall layer of the ACRN hypervisor to do the + * sanity check against the input parameters. + */ +static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, + unsigned long ioctl_param) +{ + struct acrn_vm *vm = filp->private_data; + struct acrn_vm_creation *vm_param; + int ret = 0; + + if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { + dev_dbg(acrn_dev.this_device, + "ioctl 0x%x: Invalid VM state!\n", cmd); + return -EINVAL; + } + + switch (cmd) { + case ACRN_IOCTL_CREATE_VM: + vm_param = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_vm_creation)); + if (IS_ERR(vm_param)) + return PTR_ERR(vm_param); + + if ((vm_param->reserved0 | vm_param->reserved1) != 0) + return -EINVAL; + + vm = acrn_vm_create(vm, vm_param); + if (!vm) { + ret = -EINVAL; + kfree(vm_param); + break; + } + + if (copy_to_user((void __user *)ioctl_param, vm_param, + sizeof(struct acrn_vm_creation))) { + acrn_vm_destroy(vm); + ret = -EFAULT; + } + + kfree(vm_param); + break; + case ACRN_IOCTL_START_VM: + ret = hcall_start_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to start VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_PAUSE_VM: + ret = hcall_pause_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to pause VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_RESET_VM: + ret = hcall_reset_vm(vm->vmid); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to restart VM %u!\n", vm->vmid); + break; + case ACRN_IOCTL_DESTROY_VM: + ret = acrn_vm_destroy(vm); + break; + default: + dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); + ret = -ENOTTY; + } + + return ret; +} + static int acrn_dev_release(struct inode *inode, struct file *filp) { struct acrn_vm *vm = filp->private_data; + acrn_vm_destroy(vm); kfree(vm); return 0; } @@ -50,9 +121,10 @@ static const struct file_operations acrn_fops = { .owner = THIS_MODULE, .open = acrn_dev_open, .release = acrn_dev_release, + .unlocked_ioctl = acrn_dev_ioctl, }; -static struct miscdevice acrn_dev = { +struct miscdevice acrn_dev = { .minor = MISC_DYNAMIC_MINOR, .name = "acrn_hsm", .fops = &acrn_fops, diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h new file mode 100644 index 000000000000..426b66cadb1f --- /dev/null +++ b/drivers/virt/acrn/hypercall.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ACRN HSM: hypercalls of ACRN Hypervisor + */ +#ifndef __ACRN_HSM_HYPERCALL_H +#define __ACRN_HSM_HYPERCALL_H +#include + +/* + * Hypercall IDs of the ACRN Hypervisor + */ +#define _HC_ID(x, y) (((x) << 24) | (y)) + +#define HC_ID 0x80UL + +#define HC_ID_VM_BASE 0x10UL +#define HC_CREATE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x00) +#define HC_DESTROY_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x01) +#define HC_START_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x02) +#define HC_PAUSE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x03) +#define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) + +/** + * hcall_create_vm() - Create a User VM + * @vminfo: Service VM GPA of info of User VM creation + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_create_vm(u64 vminfo) +{ + return acrn_hypercall1(HC_CREATE_VM, vminfo); +} + +/** + * hcall_start_vm() - Start a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_start_vm(u64 vmid) +{ + return acrn_hypercall1(HC_START_VM, vmid); +} + +/** + * hcall_pause_vm() - Pause a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_pause_vm(u64 vmid) +{ + return acrn_hypercall1(HC_PAUSE_VM, vmid); +} + +/** + * hcall_destroy_vm() - Destroy a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_destroy_vm(u64 vmid) +{ + return acrn_hypercall1(HC_DESTROY_VM, vmid); +} + +/** + * hcall_reset_vm() - Reset a User VM + * @vmid: User VM ID + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_reset_vm(u64 vmid) +{ + return acrn_hypercall1(HC_RESET_VM, vmid); +} + +#endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c new file mode 100644 index 000000000000..3f667ac8ac1e --- /dev/null +++ b/drivers/virt/acrn/vm.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN_HSM: Virtual Machine management + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Jason Chen CJ + * Yakui Zhao + */ +#include +#include +#include + +#include "acrn_drv.h" + +/* List of VMs */ +static LIST_HEAD(acrn_vm_list); +/* To protect acrn_vm_list */ +static DEFINE_MUTEX(acrn_vm_list_lock); + +struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, + struct acrn_vm_creation *vm_param) +{ + int ret; + + ret = hcall_create_vm(virt_to_phys(vm_param)); + if (ret < 0 || vm_param->vmid == ACRN_INVALID_VMID) { + dev_err(acrn_dev.this_device, + "Failed to create VM! Error: %d\n", ret); + return NULL; + } + + vm->vmid = vm_param->vmid; + vm->vcpu_num = vm_param->vcpu_num; + + mutex_lock(&acrn_vm_list_lock); + list_add(&vm->list, &acrn_vm_list); + mutex_unlock(&acrn_vm_list_lock); + + dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid); + return vm; +} + +int acrn_vm_destroy(struct acrn_vm *vm) +{ + int ret; + + if (vm->vmid == ACRN_INVALID_VMID || + test_and_set_bit(ACRN_VM_FLAG_DESTROYED, &vm->flags)) + return 0; + + /* Remove from global VM list */ + mutex_lock(&acrn_vm_list_lock); + list_del_init(&vm->list); + mutex_unlock(&acrn_vm_list_lock); + + ret = hcall_destroy_vm(vm->vmid); + if (ret < 0) { + dev_err(acrn_dev.this_device, + "Failed to destroy VM %u\n", vm->vmid); + clear_bit(ACRN_VM_FLAG_DESTROYED, &vm->flags); + return ret; + } + dev_dbg(acrn_dev.this_device, "VM %u destroyed.\n", vm->vmid); + vm->vmid = ACRN_INVALID_VMID; + return 0; +} diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h new file mode 100644 index 000000000000..32521168075f --- /dev/null +++ b/include/uapi/linux/acrn.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Userspace interface for /dev/acrn_hsm - ACRN Hypervisor Service Module + * + * This file can be used by applications that need to communicate with the HSM + * via the ioctl interface. + * + * Copyright (C) 2021 Intel Corporation. All rights reserved. + */ + +#ifndef _UAPI_ACRN_H +#define _UAPI_ACRN_H + +#include +#include + +/** + * struct acrn_vm_creation - Info to create a User VM + * @vmid: User VM ID returned from the hypervisor + * @reserved0: Reserved and must be 0 + * @vcpu_num: Number of vCPU in the VM. Return from hypervisor. + * @reserved1: Reserved and must be 0 + * @uuid: UUID of the VM. Pass to hypervisor directly. + * @vm_flag: Flag of the VM creating. Pass to hypervisor directly. + * @ioreq_buf: Service VM GPA of I/O request buffer. Pass to + * hypervisor directly. + * @cpu_affinity: CPU affinity of the VM. Pass to hypervisor directly. + * It's a bitmap which indicates CPUs used by the VM. + */ +struct acrn_vm_creation { + __u16 vmid; + __u16 reserved0; + __u16 vcpu_num; + __u16 reserved1; + guid_t uuid; + __u64 vm_flag; + __u64 ioreq_buf; + __u64 cpu_affinity; +}; + +/* The ioctl type, documented in ioctl-number.rst */ +#define ACRN_IOCTL_TYPE 0xA2 + +/* + * Common IOCTL IDs definition for ACRN userspace + */ +#define ACRN_IOCTL_CREATE_VM \ + _IOWR(ACRN_IOCTL_TYPE, 0x10, struct acrn_vm_creation) +#define ACRN_IOCTL_DESTROY_VM \ + _IO(ACRN_IOCTL_TYPE, 0x11) +#define ACRN_IOCTL_START_VM \ + _IO(ACRN_IOCTL_TYPE, 0x12) +#define ACRN_IOCTL_PAUSE_VM \ + _IO(ACRN_IOCTL_TYPE, 0x13) +#define ACRN_IOCTL_RESET_VM \ + _IO(ACRN_IOCTL_TYPE, 0x15) + +#endif /* _UAPI_ACRN_H */ From 2ad2aaee1bc9568d0c146463483d2c926ef20055 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:29 +0800 Subject: [PATCH 326/633] virt: acrn: Introduce an ioctl to set vCPU registers state A virtual CPU of User VM has different context due to the different registers state. ACRN userspace needs to set the virtual CPU registers state (e.g. giving a initial registers state to a virtual BSP of a User VM). HSM provides an ioctl ACRN_IOCTL_SET_VCPU_REGS to do the virtual CPU registers state setting. The ioctl passes the registers state from ACRN userspace to the hypervisor directly. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-8-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/hsm.c | 34 +++++++++- drivers/virt/acrn/hypercall.h | 13 ++++ include/uapi/linux/acrn.h | 119 ++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 1 deletion(-) diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 5fd933471683..ee5cc7413239 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -9,6 +9,7 @@ * Yakui Zhao */ +#include #include #include #include @@ -46,7 +47,8 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, { struct acrn_vm *vm = filp->private_data; struct acrn_vm_creation *vm_param; - int ret = 0; + struct acrn_vcpu_regs *cpu_regs; + int i, ret = 0; if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { dev_dbg(acrn_dev.this_device, @@ -100,6 +102,36 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, case ACRN_IOCTL_DESTROY_VM: ret = acrn_vm_destroy(vm); break; + case ACRN_IOCTL_SET_VCPU_REGS: + cpu_regs = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_vcpu_regs)); + if (IS_ERR(cpu_regs)) + return PTR_ERR(cpu_regs); + + for (i = 0; i < ARRAY_SIZE(cpu_regs->reserved); i++) + if (cpu_regs->reserved[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_32); i++) + if (cpu_regs->vcpu_regs.reserved_32[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.reserved_64); i++) + if (cpu_regs->vcpu_regs.reserved_64[i]) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(cpu_regs->vcpu_regs.gdt.reserved); i++) + if (cpu_regs->vcpu_regs.gdt.reserved[i] | + cpu_regs->vcpu_regs.idt.reserved[i]) + return -EINVAL; + + ret = hcall_set_vcpu_regs(vm->vmid, virt_to_phys(cpu_regs)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set regs state of VM%u!\n", + vm->vmid); + kfree(cpu_regs); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index 426b66cadb1f..f29cfae08862 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -19,6 +19,7 @@ #define HC_START_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x02) #define HC_PAUSE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x03) #define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) +#define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06) /** * hcall_create_vm() - Create a User VM @@ -75,4 +76,16 @@ static inline long hcall_reset_vm(u64 vmid) return acrn_hypercall1(HC_RESET_VM, vmid); } +/** + * hcall_set_vcpu_regs() - Set up registers of virtual CPU of a User VM + * @vmid: User VM ID + * @regs_state: Service VM GPA of registers state + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) +{ + return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state); +} + #endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index 32521168075f..775c58bad026 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -38,6 +38,123 @@ struct acrn_vm_creation { __u64 cpu_affinity; }; +/** + * struct acrn_gp_regs - General registers of a User VM + * @rax: Value of register RAX + * @rcx: Value of register RCX + * @rdx: Value of register RDX + * @rbx: Value of register RBX + * @rsp: Value of register RSP + * @rbp: Value of register RBP + * @rsi: Value of register RSI + * @rdi: Value of register RDI + * @r8: Value of register R8 + * @r9: Value of register R9 + * @r10: Value of register R10 + * @r11: Value of register R11 + * @r12: Value of register R12 + * @r13: Value of register R13 + * @r14: Value of register R14 + * @r15: Value of register R15 + */ +struct acrn_gp_regs { + __le64 rax; + __le64 rcx; + __le64 rdx; + __le64 rbx; + __le64 rsp; + __le64 rbp; + __le64 rsi; + __le64 rdi; + __le64 r8; + __le64 r9; + __le64 r10; + __le64 r11; + __le64 r12; + __le64 r13; + __le64 r14; + __le64 r15; +}; + +/** + * struct acrn_descriptor_ptr - Segment descriptor table of a User VM. + * @limit: Limit field. + * @base: Base field. + * @reserved: Reserved and must be 0. + */ +struct acrn_descriptor_ptr { + __le16 limit; + __le64 base; + __le16 reserved[3]; +} __attribute__ ((__packed__)); + +/** + * struct acrn_regs - Registers structure of a User VM + * @gprs: General registers + * @gdt: Global Descriptor Table + * @idt: Interrupt Descriptor Table + * @rip: Value of register RIP + * @cs_base: Base of code segment selector + * @cr0: Value of register CR0 + * @cr4: Value of register CR4 + * @cr3: Value of register CR3 + * @ia32_efer: Value of IA32_EFER MSR + * @rflags: Value of regsiter RFLAGS + * @reserved_64: Reserved and must be 0 + * @cs_ar: Attribute field of code segment selector + * @cs_limit: Limit field of code segment selector + * @reserved_32: Reserved and must be 0 + * @cs_sel: Value of code segment selector + * @ss_sel: Value of stack segment selector + * @ds_sel: Value of data segment selector + * @es_sel: Value of extra segment selector + * @fs_sel: Value of FS selector + * @gs_sel: Value of GS selector + * @ldt_sel: Value of LDT descriptor selector + * @tr_sel: Value of TSS descriptor selector + */ +struct acrn_regs { + struct acrn_gp_regs gprs; + struct acrn_descriptor_ptr gdt; + struct acrn_descriptor_ptr idt; + + __le64 rip; + __le64 cs_base; + __le64 cr0; + __le64 cr4; + __le64 cr3; + __le64 ia32_efer; + __le64 rflags; + __le64 reserved_64[4]; + + __le32 cs_ar; + __le32 cs_limit; + __le32 reserved_32[3]; + + __le16 cs_sel; + __le16 ss_sel; + __le16 ds_sel; + __le16 es_sel; + __le16 fs_sel; + __le16 gs_sel; + __le16 ldt_sel; + __le16 tr_sel; +}; + +/** + * struct acrn_vcpu_regs - Info of vCPU registers state + * @vcpu_id: vCPU ID + * @reserved: Reserved and must be 0 + * @vcpu_regs: vCPU registers state + * + * This structure will be passed to hypervisor directly. + */ +struct acrn_vcpu_regs { + __u16 vcpu_id; + __u16 reserved[3]; + struct acrn_regs vcpu_regs; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -54,5 +171,7 @@ struct acrn_vm_creation { _IO(ACRN_IOCTL_TYPE, 0x13) #define ACRN_IOCTL_RESET_VM \ _IO(ACRN_IOCTL_TYPE, 0x15) +#define ACRN_IOCTL_SET_VCPU_REGS \ + _IOW(ACRN_IOCTL_TYPE, 0x16, struct acrn_vcpu_regs) #endif /* _UAPI_ACRN_H */ From 88f537d5e8ddc89c2622f4a2bc1eb28455e8339c Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:30 +0800 Subject: [PATCH 327/633] virt: acrn: Introduce EPT mapping management The HSM provides hypervisor services to the ACRN userspace. While launching a User VM, ACRN userspace needs to allocate memory and request the ACRN Hypervisor to set up the EPT mapping for the VM. A mapping cache is introduced for accelerating the translation between the Service VM kernel virtual address and User VM physical address. >From the perspective of the hypervisor, the types of GPA of User VM can be listed as following: 1) RAM region, which is used by User VM as system ram. 2) MMIO region, which is recognized by User VM as MMIO. MMIO region is used to be utilized for devices emulation. Generally, User VM RAM regions mapping is set up before VM started and is released in the User VM destruction. MMIO regions mapping may be set and unset dynamically during User VM running. To achieve this, ioctls ACRN_IOCTL_SET_MEMSEG and ACRN_IOCTL_UNSET_MEMSEG are introduced in HSM. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-9-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/Makefile | 2 +- drivers/virt/acrn/acrn_drv.h | 96 ++++++++++- drivers/virt/acrn/hsm.c | 15 ++ drivers/virt/acrn/hypercall.h | 14 ++ drivers/virt/acrn/mm.c | 306 ++++++++++++++++++++++++++++++++++ drivers/virt/acrn/vm.c | 4 + include/uapi/linux/acrn.h | 49 ++++++ 7 files changed, 476 insertions(+), 10 deletions(-) create mode 100644 drivers/virt/acrn/mm.c diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile index cf8b4ed5e74e..38bc44b6edcd 100644 --- a/drivers/virt/acrn/Makefile +++ b/drivers/virt/acrn/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACRN_HSM) := acrn.o -acrn-y := hsm.o vm.o +acrn-y := hsm.o vm.o mm.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index e5aba86cad8c..e47a45280eea 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -12,26 +12,104 @@ extern struct miscdevice acrn_dev; +#define ACRN_MEM_MAPPING_MAX 256 + +#define ACRN_MEM_REGION_ADD 0 +#define ACRN_MEM_REGION_DEL 2 +/** + * struct vm_memory_region_op - Hypervisor memory operation + * @type: Operation type (ACRN_MEM_REGION_*) + * @attr: Memory attribute (ACRN_MEM_TYPE_* | ACRN_MEM_ACCESS_*) + * @user_vm_pa: Physical address of User VM to be mapped. + * @service_vm_pa: Physical address of Service VM to be mapped. + * @size: Size of this region. + * + * Structure containing needed information that is provided to ACRN Hypervisor + * to manage the EPT mappings of a single memory region of the User VM. Several + * &struct vm_memory_region_op can be batched to ACRN Hypervisor, see &struct + * vm_memory_region_batch. + */ +struct vm_memory_region_op { + u32 type; + u32 attr; + u64 user_vm_pa; + u64 service_vm_pa; + u64 size; +}; + +/** + * struct vm_memory_region_batch - A batch of vm_memory_region_op. + * @vmid: A User VM ID. + * @reserved: Reserved. + * @regions_num: The number of vm_memory_region_op. + * @regions_gpa: Physical address of a vm_memory_region_op array. + * + * HC_VM_SET_MEMORY_REGIONS uses this structure to manage EPT mappings of + * multiple memory regions of a User VM. A &struct vm_memory_region_batch + * contains multiple &struct vm_memory_region_op for batch processing in the + * ACRN Hypervisor. + */ +struct vm_memory_region_batch { + u16 vmid; + u16 reserved[3]; + u32 regions_num; + u64 regions_gpa; +}; + +/** + * struct vm_memory_mapping - Memory map between a User VM and the Service VM + * @pages: Pages in Service VM kernel. + * @npages: Number of pages. + * @service_vm_va: Virtual address in Service VM kernel. + * @user_vm_pa: Physical address in User VM. + * @size: Size of this memory region. + * + * HSM maintains memory mappings between a User VM GPA and the Service VM + * kernel VA for accelerating the User VM GPA translation. + */ +struct vm_memory_mapping { + struct page **pages; + int npages; + void *service_vm_va; + u64 user_vm_pa; + size_t size; +}; + #define ACRN_INVALID_VMID (0xffffU) #define ACRN_VM_FLAG_DESTROYED 0U /** * struct acrn_vm - Properties of ACRN User VM. - * @list: Entry within global list of all VMs - * @vmid: User VM ID - * @vcpu_num: Number of virtual CPUs in the VM - * @flags: Flags (ACRN_VM_FLAG_*) of the VM. This is VM flag management - * in HSM which is different from the &acrn_vm_creation.vm_flag. + * @list: Entry within global list of all VMs. + * @vmid: User VM ID. + * @vcpu_num: Number of virtual CPUs in the VM. + * @flags: Flags (ACRN_VM_FLAG_*) of the VM. This is VM + * flag management in HSM which is different + * from the &acrn_vm_creation.vm_flag. + * @regions_mapping_lock: Lock to protect &acrn_vm.regions_mapping and + * &acrn_vm.regions_mapping_count. + * @regions_mapping: Memory mappings of this VM. + * @regions_mapping_count: Number of memory mapping of this VM. */ struct acrn_vm { - struct list_head list; - u16 vmid; - int vcpu_num; - unsigned long flags; + struct list_head list; + u16 vmid; + int vcpu_num; + unsigned long flags; + struct mutex regions_mapping_lock; + struct vm_memory_mapping regions_mapping[ACRN_MEM_MAPPING_MAX]; + int regions_mapping_count; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, struct acrn_vm_creation *vm_param); int acrn_vm_destroy(struct acrn_vm *vm); +int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, + u64 size, u32 mem_type, u32 mem_access_right); +int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size); +int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); +void acrn_vm_all_ram_unmap(struct acrn_vm *vm); #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index ee5cc7413239..2c40d3dc5e94 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -48,6 +48,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_vm *vm = filp->private_data; struct acrn_vm_creation *vm_param; struct acrn_vcpu_regs *cpu_regs; + struct acrn_vm_memmap memmap; int i, ret = 0; if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { @@ -132,6 +133,20 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, vm->vmid); kfree(cpu_regs); break; + case ACRN_IOCTL_SET_MEMSEG: + if (copy_from_user(&memmap, (void __user *)ioctl_param, + sizeof(memmap))) + return -EFAULT; + + ret = acrn_vm_memseg_map(vm, &memmap); + break; + case ACRN_IOCTL_UNSET_MEMSEG: + if (copy_from_user(&memmap, (void __user *)ioctl_param, + sizeof(memmap))) + return -EFAULT; + + ret = acrn_vm_memseg_unmap(vm, &memmap); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index f29cfae08862..a1a70a071713 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -21,6 +21,9 @@ #define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) #define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06) +#define HC_ID_MEM_BASE 0x40UL +#define HC_VM_SET_MEMORY_REGIONS _HC_ID(HC_ID, HC_ID_MEM_BASE + 0x02) + /** * hcall_create_vm() - Create a User VM * @vminfo: Service VM GPA of info of User VM creation @@ -88,4 +91,15 @@ static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state); } +/** + * hcall_set_memory_regions() - Inform the hypervisor to set up EPT mappings + * @regions_pa: Service VM GPA of &struct vm_memory_region_batch + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_memory_regions(u64 regions_pa) +{ + return acrn_hypercall1(HC_VM_SET_MEMORY_REGIONS, regions_pa); +} + #endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/drivers/virt/acrn/mm.c b/drivers/virt/acrn/mm.c new file mode 100644 index 000000000000..c4f2e15c8a2b --- /dev/null +++ b/drivers/virt/acrn/mm.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN: Memory mapping management + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Fei Li + * Shuo Liu + */ + +#include +#include +#include + +#include "acrn_drv.h" + +static int modify_region(struct acrn_vm *vm, struct vm_memory_region_op *region) +{ + struct vm_memory_region_batch *regions; + int ret; + + regions = kzalloc(sizeof(*regions), GFP_KERNEL); + if (!regions) + return -ENOMEM; + + regions->vmid = vm->vmid; + regions->regions_num = 1; + regions->regions_gpa = virt_to_phys(region); + + ret = hcall_set_memory_regions(virt_to_phys(regions)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set memory region for VM[%u]!\n", vm->vmid); + + kfree(regions); + return ret; +} + +/** + * acrn_mm_region_add() - Set up the EPT mapping of a memory region. + * @vm: User VM. + * @user_gpa: A GPA of User VM. + * @service_gpa: A GPA of Service VM. + * @size: Size of the region. + * @mem_type: Combination of ACRN_MEM_TYPE_*. + * @mem_access_right: Combination of ACRN_MEM_ACCESS_*. + * + * Return: 0 on success, <0 on error. + */ +int acrn_mm_region_add(struct acrn_vm *vm, u64 user_gpa, u64 service_gpa, + u64 size, u32 mem_type, u32 mem_access_right) +{ + struct vm_memory_region_op *region; + int ret = 0; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->type = ACRN_MEM_REGION_ADD; + region->user_vm_pa = user_gpa; + region->service_vm_pa = service_gpa; + region->size = size; + region->attr = ((mem_type & ACRN_MEM_TYPE_MASK) | + (mem_access_right & ACRN_MEM_ACCESS_RIGHT_MASK)); + ret = modify_region(vm, region); + + dev_dbg(acrn_dev.this_device, + "%s: user-GPA[%pK] service-GPA[%pK] size[0x%llx].\n", + __func__, (void *)user_gpa, (void *)service_gpa, size); + kfree(region); + return ret; +} + +/** + * acrn_mm_region_del() - Del the EPT mapping of a memory region. + * @vm: User VM. + * @user_gpa: A GPA of the User VM. + * @size: Size of the region. + * + * Return: 0 on success, <0 for error. + */ +int acrn_mm_region_del(struct acrn_vm *vm, u64 user_gpa, u64 size) +{ + struct vm_memory_region_op *region; + int ret = 0; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->type = ACRN_MEM_REGION_DEL; + region->user_vm_pa = user_gpa; + region->service_vm_pa = 0UL; + region->size = size; + region->attr = 0U; + + ret = modify_region(vm, region); + + dev_dbg(acrn_dev.this_device, "%s: user-GPA[%pK] size[0x%llx].\n", + __func__, (void *)user_gpa, size); + kfree(region); + return ret; +} + +int acrn_vm_memseg_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + int ret; + + if (memmap->type == ACRN_MEMMAP_RAM) + return acrn_vm_ram_map(vm, memmap); + + if (memmap->type != ACRN_MEMMAP_MMIO) { + dev_dbg(acrn_dev.this_device, + "Invalid memmap type: %u\n", memmap->type); + return -EINVAL; + } + + ret = acrn_mm_region_add(vm, memmap->user_vm_pa, + memmap->service_vm_pa, memmap->len, + ACRN_MEM_TYPE_UC, memmap->attr); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Add memory region failed, VM[%u]!\n", vm->vmid); + + return ret; +} + +int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + int ret; + + if (memmap->type != ACRN_MEMMAP_MMIO) { + dev_dbg(acrn_dev.this_device, + "Invalid memmap type: %u\n", memmap->type); + return -EINVAL; + } + + ret = acrn_mm_region_del(vm, memmap->user_vm_pa, memmap->len); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Del memory region failed, VM[%u]!\n", vm->vmid); + + return ret; +} + +/** + * acrn_vm_ram_map() - Create a RAM EPT mapping of User VM. + * @vm: The User VM pointer + * @memmap: Info of the EPT mapping + * + * Return: 0 on success, <0 for error. + */ +int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap) +{ + struct vm_memory_region_batch *regions_info; + int nr_pages, i = 0, order, nr_regions = 0; + struct vm_memory_mapping *region_mapping; + struct vm_memory_region_op *vm_region; + struct page **pages = NULL, *page; + void *remap_vaddr; + int ret, pinned; + u64 user_vm_pa; + + if (!vm || !memmap) + return -EINVAL; + + /* Get the page number of the map region */ + nr_pages = memmap->len >> PAGE_SHIFT; + pages = vzalloc(nr_pages * sizeof(struct page *)); + if (!pages) + return -ENOMEM; + + /* Lock the pages of user memory map region */ + pinned = pin_user_pages_fast(memmap->vma_base, + nr_pages, FOLL_WRITE | FOLL_LONGTERM, + pages); + if (pinned < 0) { + ret = pinned; + goto free_pages; + } else if (pinned != nr_pages) { + ret = -EFAULT; + goto put_pages; + } + + /* Create a kernel map for the map region */ + remap_vaddr = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); + if (!remap_vaddr) { + ret = -ENOMEM; + goto put_pages; + } + + /* Record Service VM va <-> User VM pa mapping */ + mutex_lock(&vm->regions_mapping_lock); + region_mapping = &vm->regions_mapping[vm->regions_mapping_count]; + if (vm->regions_mapping_count < ACRN_MEM_MAPPING_MAX) { + region_mapping->pages = pages; + region_mapping->npages = nr_pages; + region_mapping->size = memmap->len; + region_mapping->service_vm_va = remap_vaddr; + region_mapping->user_vm_pa = memmap->user_vm_pa; + vm->regions_mapping_count++; + } else { + dev_warn(acrn_dev.this_device, + "Run out of memory mapping slots!\n"); + ret = -ENOMEM; + mutex_unlock(&vm->regions_mapping_lock); + goto unmap_no_count; + } + mutex_unlock(&vm->regions_mapping_lock); + + /* Calculate count of vm_memory_region_op */ + while (i < nr_pages) { + page = pages[i]; + VM_BUG_ON_PAGE(PageTail(page), page); + order = compound_order(page); + nr_regions++; + i += 1 << order; + } + + /* Prepare the vm_memory_region_batch */ + regions_info = kzalloc(sizeof(*regions_info) + + sizeof(*vm_region) * nr_regions, + GFP_KERNEL); + if (!regions_info) { + ret = -ENOMEM; + goto unmap_kernel_map; + } + + /* Fill each vm_memory_region_op */ + vm_region = (struct vm_memory_region_op *)(regions_info + 1); + regions_info->vmid = vm->vmid; + regions_info->regions_num = nr_regions; + regions_info->regions_gpa = virt_to_phys(vm_region); + user_vm_pa = memmap->user_vm_pa; + i = 0; + while (i < nr_pages) { + u32 region_size; + + page = pages[i]; + VM_BUG_ON_PAGE(PageTail(page), page); + order = compound_order(page); + region_size = PAGE_SIZE << order; + vm_region->type = ACRN_MEM_REGION_ADD; + vm_region->user_vm_pa = user_vm_pa; + vm_region->service_vm_pa = page_to_phys(page); + vm_region->size = region_size; + vm_region->attr = (ACRN_MEM_TYPE_WB & ACRN_MEM_TYPE_MASK) | + (memmap->attr & ACRN_MEM_ACCESS_RIGHT_MASK); + + vm_region++; + user_vm_pa += region_size; + i += 1 << order; + } + + /* Inform the ACRN Hypervisor to set up EPT mappings */ + ret = hcall_set_memory_regions(virt_to_phys(regions_info)); + if (ret < 0) { + dev_dbg(acrn_dev.this_device, + "Failed to set regions, VM[%u]!\n", vm->vmid); + goto unset_region; + } + kfree(regions_info); + + dev_dbg(acrn_dev.this_device, + "%s: VM[%u] service-GVA[%pK] user-GPA[%pK] size[0x%llx]\n", + __func__, vm->vmid, + remap_vaddr, (void *)memmap->user_vm_pa, memmap->len); + return ret; + +unset_region: + kfree(regions_info); +unmap_kernel_map: + mutex_lock(&vm->regions_mapping_lock); + vm->regions_mapping_count--; + mutex_unlock(&vm->regions_mapping_lock); +unmap_no_count: + vunmap(remap_vaddr); +put_pages: + for (i = 0; i < pinned; i++) + unpin_user_page(pages[i]); +free_pages: + vfree(pages); + return ret; +} + +/** + * acrn_vm_all_ram_unmap() - Destroy a RAM EPT mapping of User VM. + * @vm: The User VM + */ +void acrn_vm_all_ram_unmap(struct acrn_vm *vm) +{ + struct vm_memory_mapping *region_mapping; + int i, j; + + mutex_lock(&vm->regions_mapping_lock); + for (i = 0; i < vm->regions_mapping_count; i++) { + region_mapping = &vm->regions_mapping[i]; + vunmap(region_mapping->service_vm_va); + for (j = 0; j < region_mapping->npages; j++) + unpin_user_page(region_mapping->pages[j]); + vfree(region_mapping->pages); + } + mutex_unlock(&vm->regions_mapping_lock); +} diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c index 3f667ac8ac1e..ff5df7acb551 100644 --- a/drivers/virt/acrn/vm.c +++ b/drivers/virt/acrn/vm.c @@ -31,6 +31,7 @@ struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, return NULL; } + mutex_init(&vm->regions_mapping_lock); vm->vmid = vm_param->vmid; vm->vcpu_num = vm_param->vcpu_num; @@ -62,6 +63,9 @@ int acrn_vm_destroy(struct acrn_vm *vm) clear_bit(ACRN_VM_FLAG_DESTROYED, &vm->flags); return ret; } + + acrn_vm_all_ram_unmap(vm); + dev_dbg(acrn_dev.this_device, "VM %u destroyed.\n", vm->vmid); vm->vmid = ACRN_INVALID_VMID; return 0; diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index 775c58bad026..ec4c61e7c170 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -155,6 +155,50 @@ struct acrn_vcpu_regs { struct acrn_regs vcpu_regs; }; +#define ACRN_MEM_ACCESS_RIGHT_MASK 0x00000007U +#define ACRN_MEM_ACCESS_READ 0x00000001U +#define ACRN_MEM_ACCESS_WRITE 0x00000002U +#define ACRN_MEM_ACCESS_EXEC 0x00000004U +#define ACRN_MEM_ACCESS_RWX (ACRN_MEM_ACCESS_READ | \ + ACRN_MEM_ACCESS_WRITE | \ + ACRN_MEM_ACCESS_EXEC) + +#define ACRN_MEM_TYPE_MASK 0x000007C0U +#define ACRN_MEM_TYPE_WB 0x00000040U +#define ACRN_MEM_TYPE_WT 0x00000080U +#define ACRN_MEM_TYPE_UC 0x00000100U +#define ACRN_MEM_TYPE_WC 0x00000200U +#define ACRN_MEM_TYPE_WP 0x00000400U + +/* Memory mapping types */ +#define ACRN_MEMMAP_RAM 0 +#define ACRN_MEMMAP_MMIO 1 + +/** + * struct acrn_vm_memmap - A EPT memory mapping info for a User VM. + * @type: Type of the memory mapping (ACRM_MEMMAP_*). + * Pass to hypervisor directly. + * @attr: Attribute of the memory mapping. + * Pass to hypervisor directly. + * @user_vm_pa: Physical address of User VM. + * Pass to hypervisor directly. + * @service_vm_pa: Physical address of Service VM. + * Pass to hypervisor directly. + * @vma_base: VMA address of Service VM. Pass to hypervisor directly. + * @len: Length of the memory mapping. + * Pass to hypervisor directly. + */ +struct acrn_vm_memmap { + __u32 type; + __u32 attr; + __u64 user_vm_pa; + union { + __u64 service_vm_pa; + __u64 vma_base; + }; + __u64 len; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -174,4 +218,9 @@ struct acrn_vcpu_regs { #define ACRN_IOCTL_SET_VCPU_REGS \ _IOW(ACRN_IOCTL_TYPE, 0x16, struct acrn_vcpu_regs) +#define ACRN_IOCTL_SET_MEMSEG \ + _IOW(ACRN_IOCTL_TYPE, 0x41, struct acrn_vm_memmap) +#define ACRN_IOCTL_UNSET_MEMSEG \ + _IOW(ACRN_IOCTL_TYPE, 0x42, struct acrn_vm_memmap) + #endif /* _UAPI_ACRN_H */ From 72f293de3ff40b57db573c1bf623f494f3446f74 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:31 +0800 Subject: [PATCH 328/633] virt: acrn: Introduce I/O request management An I/O request of a User VM, which is constructed by the hypervisor, is distributed by the ACRN Hypervisor Service Module to an I/O client corresponding to the address range of the I/O request. For each User VM, there is a shared 4-KByte memory region used for I/O requests communication between the hypervisor and Service VM. An I/O request is a 256-byte structure buffer, which is 'struct acrn_io_request', that is filled by an I/O handler of the hypervisor when a trapped I/O access happens in a User VM. ACRN userspace in the Service VM first allocates a 4-KByte page and passes the GPA (Guest Physical Address) of the buffer to the hypervisor. The buffer is used as an array of 16 I/O request slots with each I/O request slot being 256 bytes. This array is indexed by vCPU ID. An I/O client, which is 'struct acrn_ioreq_client', is responsible for handling User VM I/O requests whose accessed GPA falls in a certain range. Multiple I/O clients can be associated with each User VM. There is a special client associated with each User VM, called the default client, that handles all I/O requests that do not fit into the range of any other I/O clients. The ACRN userspace acts as the default client for each User VM. The state transitions of a ACRN I/O request are as follows. FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... FREE: this I/O request slot is empty PENDING: a valid I/O request is pending in this slot PROCESSING: the I/O request is being processed COMPLETE: the I/O request has been processed An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and ACRN userspace are in charge of processing the others. The processing flow of I/O requests are listed as following: a) The I/O handler of the hypervisor will fill an I/O request with PENDING state when a trapped I/O access happens in a User VM. b) The hypervisor makes an upcall, which is a notification interrupt, to the Service VM. c) The upcall handler schedules a worker to dispatch I/O requests. d) The worker looks for the PENDING I/O requests, assigns them to different registered clients based on the address of the I/O accesses, updates their state to PROCESSING, and notifies the corresponding client to handle. e) The notified client handles the assigned I/O requests. f) The HSM updates I/O requests states to COMPLETE and notifies the hypervisor of the completion via hypercalls. Cc: Davidlohr Bueso Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Acked-by: Davidlohr Bueso Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-10-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/Makefile | 2 +- drivers/virt/acrn/acrn_drv.h | 82 ++++++ drivers/virt/acrn/hsm.c | 43 ++- drivers/virt/acrn/hypercall.h | 28 ++ drivers/virt/acrn/ioreq.c | 521 ++++++++++++++++++++++++++++++++++ drivers/virt/acrn/vm.c | 27 +- include/uapi/linux/acrn.h | 150 ++++++++++ 7 files changed, 843 insertions(+), 10 deletions(-) create mode 100644 drivers/virt/acrn/ioreq.c diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile index 38bc44b6edcd..21721cbf6a80 100644 --- a/drivers/virt/acrn/Makefile +++ b/drivers/virt/acrn/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACRN_HSM) := acrn.o -acrn-y := hsm.o vm.o mm.o +acrn-y := hsm.o vm.o mm.o ioreq.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index e47a45280eea..68bd8db6c8be 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -12,10 +12,15 @@ extern struct miscdevice acrn_dev; +#define ACRN_NAME_LEN 16 #define ACRN_MEM_MAPPING_MAX 256 #define ACRN_MEM_REGION_ADD 0 #define ACRN_MEM_REGION_DEL 2 + +struct acrn_vm; +struct acrn_ioreq_client; + /** * struct vm_memory_region_op - Hypervisor memory operation * @type: Operation type (ACRN_MEM_REGION_*) @@ -75,9 +80,63 @@ struct vm_memory_mapping { size_t size; }; +/** + * struct acrn_ioreq_buffer - Data for setting the ioreq buffer of User VM + * @ioreq_buf: The GPA of the IO request shared buffer of a VM + * + * The parameter for the HC_SET_IOREQ_BUFFER hypercall used to set up + * the shared I/O request buffer between Service VM and ACRN hypervisor. + */ +struct acrn_ioreq_buffer { + u64 ioreq_buf; +}; + +struct acrn_ioreq_range { + struct list_head list; + u32 type; + u64 start; + u64 end; +}; + +#define ACRN_IOREQ_CLIENT_DESTROYING 0U +typedef int (*ioreq_handler_t)(struct acrn_ioreq_client *client, + struct acrn_io_request *req); +/** + * struct acrn_ioreq_client - Structure of I/O client. + * @name: Client name + * @vm: The VM that the client belongs to + * @list: List node for this acrn_ioreq_client + * @is_default: If this client is the default one + * @flags: Flags (ACRN_IOREQ_CLIENT_*) + * @range_list: I/O ranges + * @range_lock: Lock to protect range_list + * @ioreqs_map: The pending I/O requests bitmap. + * @handler: I/O requests handler of this client + * @thread: The thread which executes the handler + * @wq: The wait queue for the handler thread parking + * @priv: Data for the thread + */ +struct acrn_ioreq_client { + char name[ACRN_NAME_LEN]; + struct acrn_vm *vm; + struct list_head list; + bool is_default; + unsigned long flags; + struct list_head range_list; + rwlock_t range_lock; + DECLARE_BITMAP(ioreqs_map, ACRN_IO_REQUEST_MAX); + ioreq_handler_t handler; + struct task_struct *thread; + wait_queue_head_t wq; + void *priv; +}; + #define ACRN_INVALID_VMID (0xffffU) #define ACRN_VM_FLAG_DESTROYED 0U +#define ACRN_VM_FLAG_CLEARING_IOREQ 1U +extern struct list_head acrn_vm_list; +extern rwlock_t acrn_vm_list_lock; /** * struct acrn_vm - Properties of ACRN User VM. * @list: Entry within global list of all VMs. @@ -90,6 +149,11 @@ struct vm_memory_mapping { * &acrn_vm.regions_mapping_count. * @regions_mapping: Memory mappings of this VM. * @regions_mapping_count: Number of memory mapping of this VM. + * @ioreq_clients_lock: Lock to protect ioreq_clients and default_client + * @ioreq_clients: The I/O request clients list of this VM + * @default_client: The default I/O request client + * @ioreq_buf: I/O request shared buffer + * @ioreq_page: The page of the I/O request shared buffer */ struct acrn_vm { struct list_head list; @@ -99,6 +163,11 @@ struct acrn_vm { struct mutex regions_mapping_lock; struct vm_memory_mapping regions_mapping[ACRN_MEM_MAPPING_MAX]; int regions_mapping_count; + spinlock_t ioreq_clients_lock; + struct list_head ioreq_clients; + struct acrn_ioreq_client *default_client; + struct acrn_io_request_buffer *ioreq_buf; + struct page *ioreq_page; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, @@ -112,4 +181,17 @@ int acrn_vm_memseg_unmap(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); int acrn_vm_ram_map(struct acrn_vm *vm, struct acrn_vm_memmap *memmap); void acrn_vm_all_ram_unmap(struct acrn_vm *vm); +int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma); +void acrn_ioreq_deinit(struct acrn_vm *vm); +int acrn_ioreq_intr_setup(void); +void acrn_ioreq_intr_remove(void); +void acrn_ioreq_request_clear(struct acrn_vm *vm); +int acrn_ioreq_client_wait(struct acrn_ioreq_client *client); +int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu); +struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, + ioreq_handler_t handler, + void *data, bool is_default, + const char *name); +void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client); + #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 2c40d3dc5e94..1cc0c612dc09 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -48,6 +48,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_vm *vm = filp->private_data; struct acrn_vm_creation *vm_param; struct acrn_vcpu_regs *cpu_regs; + struct acrn_ioreq_notify notify; struct acrn_vm_memmap memmap; int i, ret = 0; @@ -147,6 +148,35 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, ret = acrn_vm_memseg_unmap(vm, &memmap); break; + case ACRN_IOCTL_CREATE_IOREQ_CLIENT: + if (vm->default_client) + return -EEXIST; + if (!acrn_ioreq_client_create(vm, NULL, NULL, true, "acrndm")) + ret = -EINVAL; + break; + case ACRN_IOCTL_DESTROY_IOREQ_CLIENT: + if (vm->default_client) + acrn_ioreq_client_destroy(vm->default_client); + break; + case ACRN_IOCTL_ATTACH_IOREQ_CLIENT: + if (vm->default_client) + ret = acrn_ioreq_client_wait(vm->default_client); + else + ret = -ENODEV; + break; + case ACRN_IOCTL_NOTIFY_REQUEST_FINISH: + if (copy_from_user(¬ify, (void __user *)ioctl_param, + sizeof(struct acrn_ioreq_notify))) + return -EFAULT; + + if (notify.reserved != 0) + return -EINVAL; + + ret = acrn_ioreq_request_default_complete(vm, notify.vcpu); + break; + case ACRN_IOCTL_CLEAR_VM_IOREQ: + acrn_ioreq_request_clear(vm); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; @@ -188,14 +218,23 @@ static int __init hsm_init(void) return -EPERM; ret = misc_register(&acrn_dev); - if (ret) + if (ret) { pr_err("Create misc dev failed!\n"); + return ret; + } - return ret; + ret = acrn_ioreq_intr_setup(); + if (ret) { + pr_err("Setup I/O request handler failed!\n"); + misc_deregister(&acrn_dev); + return ret; + } + return 0; } static void __exit hsm_exit(void) { + acrn_ioreq_intr_remove(); misc_deregister(&acrn_dev); } module_init(hsm_init); diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index a1a70a071713..5eba29e3ed38 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -21,6 +21,10 @@ #define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) #define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06) +#define HC_ID_IOREQ_BASE 0x30UL +#define HC_SET_IOREQ_BUFFER _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x00) +#define HC_NOTIFY_REQUEST_FINISH _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x01) + #define HC_ID_MEM_BASE 0x40UL #define HC_VM_SET_MEMORY_REGIONS _HC_ID(HC_ID, HC_ID_MEM_BASE + 0x02) @@ -91,6 +95,30 @@ static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state); } +/** + * hcall_set_ioreq_buffer() - Set up the shared buffer for I/O Requests. + * @vmid: User VM ID + * @buffer: Service VM GPA of the shared buffer + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_ioreq_buffer(u64 vmid, u64 buffer) +{ + return acrn_hypercall2(HC_SET_IOREQ_BUFFER, vmid, buffer); +} + +/** + * hcall_notify_req_finish() - Notify ACRN Hypervisor of I/O request completion. + * @vmid: User VM ID + * @vcpu: The vCPU which initiated the I/O request + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_notify_req_finish(u64 vmid, u64 vcpu) +{ + return acrn_hypercall2(HC_NOTIFY_REQUEST_FINISH, vmid, vcpu); +} + /** * hcall_set_memory_regions() - Inform the hypervisor to set up EPT mappings * @regions_pa: Service VM GPA of &struct vm_memory_region_batch diff --git a/drivers/virt/acrn/ioreq.c b/drivers/virt/acrn/ioreq.c new file mode 100644 index 000000000000..51cb41ef7c72 --- /dev/null +++ b/drivers/virt/acrn/ioreq.c @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN_HSM: Handle I/O requests + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Jason Chen CJ + * Fengwei Yin + */ + +#include +#include +#include +#include +#include + +#include + +#include "acrn_drv.h" + +static void ioreq_pause(void); +static void ioreq_resume(void); + +static void ioreq_dispatcher(struct work_struct *work); +static struct workqueue_struct *ioreq_wq; +static DECLARE_WORK(ioreq_work, ioreq_dispatcher); + +static inline bool has_pending_request(struct acrn_ioreq_client *client) +{ + return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX); +} + +static inline bool is_destroying(struct acrn_ioreq_client *client) +{ + return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags); +} + +static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu, + struct acrn_io_request *acrn_req) +{ + bool polling_mode; + int ret = 0; + + polling_mode = acrn_req->completion_polling; + /* Add barrier() to make sure the writes are done before completion */ + smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE); + + /* + * To fulfill the requirement of real-time in several industry + * scenarios, like automotive, ACRN can run under the partition mode, + * in which User VMs and Service VM are bound to dedicated CPU cores. + * Polling mode of handling the I/O request is introduced to achieve a + * faster I/O request handling. In polling mode, the hypervisor polls + * I/O request's completion. Once an I/O request is marked as + * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point + * to continue the I/O request flow. Thus, the completion notification + * from HSM of I/O request is not needed. Please note, + * completion_polling needs to be read before the I/O request being + * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the + * hypervisor. + */ + if (!polling_mode) { + ret = hcall_notify_req_finish(vm->vmid, vcpu); + if (ret < 0) + dev_err(acrn_dev.this_device, + "Notify I/O request finished failed!\n"); + } + + return ret; +} + +static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client, + u16 vcpu, + struct acrn_io_request *acrn_req) +{ + int ret; + + if (vcpu >= client->vm->vcpu_num) + return -EINVAL; + + clear_bit(vcpu, client->ioreqs_map); + if (!acrn_req) { + acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf; + acrn_req += vcpu; + } + + ret = ioreq_complete_request(client->vm, vcpu, acrn_req); + + return ret; +} + +int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu) +{ + int ret = 0; + + spin_lock_bh(&vm->ioreq_clients_lock); + if (vm->default_client) + ret = acrn_ioreq_complete_request(vm->default_client, + vcpu, NULL); + spin_unlock_bh(&vm->ioreq_clients_lock); + + return ret; +} + +/* + * ioreq_task() is the execution entity of handler thread of an I/O client. + * The handler callback of the I/O client is called within the handler thread. + */ +static int ioreq_task(void *data) +{ + struct acrn_ioreq_client *client = data; + struct acrn_io_request *req; + unsigned long *ioreqs_map; + int vcpu, ret; + + /* + * Lockless access to ioreqs_map is safe, because + * 1) set_bit() and clear_bit() are atomic operations. + * 2) I/O requests arrives serialized. The access flow of ioreqs_map is: + * set_bit() - in ioreq_work handler + * Handler callback handles corresponding I/O request + * clear_bit() - in handler thread (include ACRN userspace) + * Mark corresponding I/O request completed + * Loop again if a new I/O request occurs + */ + ioreqs_map = client->ioreqs_map; + while (!kthread_should_stop()) { + acrn_ioreq_client_wait(client); + while (has_pending_request(client)) { + vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num); + req = client->vm->ioreq_buf->req_slot + vcpu; + ret = client->handler(client, req); + if (ret < 0) { + dev_err(acrn_dev.this_device, + "IO handle failure: %d\n", ret); + break; + } + acrn_ioreq_complete_request(client, vcpu, req); + } + } + + return 0; +} + +/* + * For the non-default I/O clients, give them chance to complete the current + * I/O requests if there are any. For the default I/O client, it is safe to + * clear all pending I/O requests because the clearing request is from ACRN + * userspace. + */ +void acrn_ioreq_request_clear(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client; + bool has_pending = false; + unsigned long vcpu; + int retry = 10; + + /* + * IO requests of this VM will be completed directly in + * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set. + */ + set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags); + + /* + * acrn_ioreq_request_clear is only called in VM reset case. Simply + * wait 100ms in total for the IO requests' completion. + */ + do { + spin_lock_bh(&vm->ioreq_clients_lock); + list_for_each_entry(client, &vm->ioreq_clients, list) { + has_pending = has_pending_request(client); + if (has_pending) + break; + } + spin_unlock_bh(&vm->ioreq_clients_lock); + + if (has_pending) + schedule_timeout_interruptible(HZ / 100); + } while (has_pending && --retry > 0); + if (retry == 0) + dev_warn(acrn_dev.this_device, + "%s cannot flush pending request!\n", client->name); + + /* Clear all ioreqs belonging to the default client */ + spin_lock_bh(&vm->ioreq_clients_lock); + client = vm->default_client; + if (client) { + vcpu = find_next_bit(client->ioreqs_map, + ACRN_IO_REQUEST_MAX, 0); + while (vcpu < ACRN_IO_REQUEST_MAX) { + acrn_ioreq_complete_request(client, vcpu, NULL); + vcpu = find_next_bit(client->ioreqs_map, + ACRN_IO_REQUEST_MAX, vcpu + 1); + } + } + spin_unlock_bh(&vm->ioreq_clients_lock); + + /* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */ + clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags); +} + +int acrn_ioreq_client_wait(struct acrn_ioreq_client *client) +{ + if (client->is_default) { + /* + * In the default client, a user space thread waits on the + * waitqueue. The is_destroying() check is used to notify user + * space the client is going to be destroyed. + */ + wait_event_interruptible(client->wq, + has_pending_request(client) || + is_destroying(client)); + if (is_destroying(client)) + return -ENODEV; + } else { + wait_event_interruptible(client->wq, + has_pending_request(client) || + kthread_should_stop()); + } + + return 0; +} + +static bool in_range(struct acrn_ioreq_range *range, + struct acrn_io_request *req) +{ + bool ret = false; + + if (range->type == req->type) { + switch (req->type) { + case ACRN_IOREQ_TYPE_MMIO: + if (req->reqs.mmio_request.address >= range->start && + (req->reqs.mmio_request.address + + req->reqs.mmio_request.size - 1) <= range->end) + ret = true; + break; + case ACRN_IOREQ_TYPE_PORTIO: + if (req->reqs.pio_request.address >= range->start && + (req->reqs.pio_request.address + + req->reqs.pio_request.size - 1) <= range->end) + ret = true; + break; + default: + break; + } + } + + return ret; +} + +static struct acrn_ioreq_client *find_ioreq_client(struct acrn_vm *vm, + struct acrn_io_request *req) +{ + struct acrn_ioreq_client *client, *found = NULL; + struct acrn_ioreq_range *range; + + lockdep_assert_held(&vm->ioreq_clients_lock); + + list_for_each_entry(client, &vm->ioreq_clients, list) { + read_lock_bh(&client->range_lock); + list_for_each_entry(range, &client->range_list, list) { + if (in_range(range, req)) { + found = client; + break; + } + } + read_unlock_bh(&client->range_lock); + if (found) + break; + } + return found ? found : vm->default_client; +} + +/** + * acrn_ioreq_client_create() - Create an ioreq client + * @vm: The VM that this client belongs to + * @handler: The ioreq_handler of ioreq client acrn_hsm will create a kernel + * thread and call the handler to handle I/O requests. + * @priv: Private data for the handler + * @is_default: If it is the default client + * @name: The name of ioreq client + * + * Return: acrn_ioreq_client pointer on success, NULL on error + */ +struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, + ioreq_handler_t handler, + void *priv, bool is_default, + const char *name) +{ + struct acrn_ioreq_client *client; + + if (!handler && !is_default) { + dev_dbg(acrn_dev.this_device, + "Cannot create non-default client w/o handler!\n"); + return NULL; + } + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return NULL; + + client->handler = handler; + client->vm = vm; + client->priv = priv; + client->is_default = is_default; + if (name) + strncpy(client->name, name, sizeof(client->name) - 1); + rwlock_init(&client->range_lock); + INIT_LIST_HEAD(&client->range_list); + init_waitqueue_head(&client->wq); + + if (client->handler) { + client->thread = kthread_run(ioreq_task, client, "VM%u-%s", + client->vm->vmid, client->name); + if (IS_ERR(client->thread)) { + kfree(client); + return NULL; + } + } + + spin_lock_bh(&vm->ioreq_clients_lock); + if (is_default) + vm->default_client = client; + else + list_add(&client->list, &vm->ioreq_clients); + spin_unlock_bh(&vm->ioreq_clients_lock); + + dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name); + return client; +} + +/** + * acrn_ioreq_client_destroy() - Destroy an ioreq client + * @client: The ioreq client + */ +void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client) +{ + struct acrn_ioreq_range *range, *next; + struct acrn_vm *vm = client->vm; + + dev_dbg(acrn_dev.this_device, + "Destroy ioreq client %s.\n", client->name); + ioreq_pause(); + set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags); + if (client->is_default) + wake_up_interruptible(&client->wq); + else + kthread_stop(client->thread); + + spin_lock_bh(&vm->ioreq_clients_lock); + if (client->is_default) + vm->default_client = NULL; + else + list_del(&client->list); + spin_unlock_bh(&vm->ioreq_clients_lock); + + write_lock_bh(&client->range_lock); + list_for_each_entry_safe(range, next, &client->range_list, list) { + list_del(&range->list); + kfree(range); + } + write_unlock_bh(&client->range_lock); + kfree(client); + + ioreq_resume(); +} + +static int acrn_ioreq_dispatch(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client; + struct acrn_io_request *req; + int i; + + for (i = 0; i < vm->vcpu_num; i++) { + req = vm->ioreq_buf->req_slot + i; + + /* barrier the read of processed of acrn_io_request */ + if (smp_load_acquire(&req->processed) == + ACRN_IOREQ_STATE_PENDING) { + /* Complete the IO request directly in clearing stage */ + if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) { + ioreq_complete_request(vm, i, req); + continue; + } + + spin_lock_bh(&vm->ioreq_clients_lock); + client = find_ioreq_client(vm, req); + if (!client) { + dev_err(acrn_dev.this_device, + "Failed to find ioreq client!\n"); + spin_unlock_bh(&vm->ioreq_clients_lock); + return -EINVAL; + } + if (!client->is_default) + req->kernel_handled = 1; + else + req->kernel_handled = 0; + /* + * Add barrier() to make sure the writes are done + * before setting ACRN_IOREQ_STATE_PROCESSING + */ + smp_store_release(&req->processed, + ACRN_IOREQ_STATE_PROCESSING); + set_bit(i, client->ioreqs_map); + wake_up_interruptible(&client->wq); + spin_unlock_bh(&vm->ioreq_clients_lock); + } + } + + return 0; +} + +static void ioreq_dispatcher(struct work_struct *work) +{ + struct acrn_vm *vm; + + read_lock(&acrn_vm_list_lock); + list_for_each_entry(vm, &acrn_vm_list, list) { + if (!vm->ioreq_buf) + break; + acrn_ioreq_dispatch(vm); + } + read_unlock(&acrn_vm_list_lock); +} + +static void ioreq_intr_handler(void) +{ + queue_work(ioreq_wq, &ioreq_work); +} + +static void ioreq_pause(void) +{ + /* Flush and unarm the handler to ensure no I/O requests pending */ + acrn_remove_intr_handler(); + drain_workqueue(ioreq_wq); +} + +static void ioreq_resume(void) +{ + /* Schedule after enabling in case other clients miss interrupt */ + acrn_setup_intr_handler(ioreq_intr_handler); + queue_work(ioreq_wq, &ioreq_work); +} + +int acrn_ioreq_intr_setup(void) +{ + acrn_setup_intr_handler(ioreq_intr_handler); + ioreq_wq = alloc_workqueue("ioreq_wq", + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, 1); + if (!ioreq_wq) { + dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n"); + acrn_remove_intr_handler(); + return -ENOMEM; + } + return 0; +} + +void acrn_ioreq_intr_remove(void) +{ + if (ioreq_wq) + destroy_workqueue(ioreq_wq); + acrn_remove_intr_handler(); +} + +int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma) +{ + struct acrn_ioreq_buffer *set_buffer; + struct page *page; + int ret; + + if (vm->ioreq_buf) + return -EEXIST; + + set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL); + if (!set_buffer) + return -ENOMEM; + + ret = pin_user_pages_fast(buf_vma, 1, + FOLL_WRITE | FOLL_LONGTERM, &page); + if (unlikely(ret != 1) || !page) { + dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n"); + ret = -EFAULT; + goto free_buf; + } + + vm->ioreq_buf = page_address(page); + vm->ioreq_page = page; + set_buffer->ioreq_buf = page_to_phys(page); + ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer)); + if (ret < 0) { + dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n"); + unpin_user_page(page); + vm->ioreq_buf = NULL; + goto free_buf; + } + + dev_dbg(acrn_dev.this_device, + "Init ioreq buffer %pK!\n", vm->ioreq_buf); + ret = 0; +free_buf: + kfree(set_buffer); + return ret; +} + +void acrn_ioreq_deinit(struct acrn_vm *vm) +{ + struct acrn_ioreq_client *client, *next; + + dev_dbg(acrn_dev.this_device, + "Deinit ioreq buffer %pK!\n", vm->ioreq_buf); + /* Destroy all clients belonging to this VM */ + list_for_each_entry_safe(client, next, &vm->ioreq_clients, list) + acrn_ioreq_client_destroy(client); + if (vm->default_client) + acrn_ioreq_client_destroy(vm->default_client); + + if (vm->ioreq_buf && vm->ioreq_page) { + unpin_user_page(vm->ioreq_page); + vm->ioreq_buf = NULL; + } +} diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c index ff5df7acb551..2bc649c7c1d3 100644 --- a/drivers/virt/acrn/vm.c +++ b/drivers/virt/acrn/vm.c @@ -15,9 +15,12 @@ #include "acrn_drv.h" /* List of VMs */ -static LIST_HEAD(acrn_vm_list); -/* To protect acrn_vm_list */ -static DEFINE_MUTEX(acrn_vm_list_lock); +LIST_HEAD(acrn_vm_list); +/* + * acrn_vm_list is read in a worker thread which dispatch I/O requests and + * is wrote in VM creation ioctl. Use the rwlock mechanism to protect it. + */ +DEFINE_RWLOCK(acrn_vm_list_lock); struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, struct acrn_vm_creation *vm_param) @@ -32,12 +35,20 @@ struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, } mutex_init(&vm->regions_mapping_lock); + INIT_LIST_HEAD(&vm->ioreq_clients); + spin_lock_init(&vm->ioreq_clients_lock); vm->vmid = vm_param->vmid; vm->vcpu_num = vm_param->vcpu_num; - mutex_lock(&acrn_vm_list_lock); + if (acrn_ioreq_init(vm, vm_param->ioreq_buf) < 0) { + hcall_destroy_vm(vm_param->vmid); + vm->vmid = ACRN_INVALID_VMID; + return NULL; + } + + write_lock_bh(&acrn_vm_list_lock); list_add(&vm->list, &acrn_vm_list); - mutex_unlock(&acrn_vm_list_lock); + write_unlock_bh(&acrn_vm_list_lock); dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid); return vm; @@ -52,9 +63,11 @@ int acrn_vm_destroy(struct acrn_vm *vm) return 0; /* Remove from global VM list */ - mutex_lock(&acrn_vm_list_lock); + write_lock_bh(&acrn_vm_list_lock); list_del_init(&vm->list); - mutex_unlock(&acrn_vm_list_lock); + write_unlock_bh(&acrn_vm_list_lock); + + acrn_ioreq_deinit(vm); ret = hcall_destroy_vm(vm->vmid); if (ret < 0) { diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index ec4c61e7c170..b0f73ab5e4e8 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -14,6 +14,145 @@ #include #include +#define ACRN_IO_REQUEST_MAX 16 + +#define ACRN_IOREQ_STATE_PENDING 0 +#define ACRN_IOREQ_STATE_COMPLETE 1 +#define ACRN_IOREQ_STATE_PROCESSING 2 +#define ACRN_IOREQ_STATE_FREE 3 + +#define ACRN_IOREQ_TYPE_PORTIO 0 +#define ACRN_IOREQ_TYPE_MMIO 1 + +#define ACRN_IOREQ_DIR_READ 0 +#define ACRN_IOREQ_DIR_WRITE 1 + +/** + * struct acrn_mmio_request - Info of a MMIO I/O request + * @direction: Access direction of this request (ACRN_IOREQ_DIR_*) + * @reserved: Reserved for alignment and should be 0 + * @address: Access address of this MMIO I/O request + * @size: Access size of this MMIO I/O request + * @value: Read/write value of this MMIO I/O request + */ +struct acrn_mmio_request { + __u32 direction; + __u32 reserved; + __u64 address; + __u64 size; + __u64 value; +}; + +/** + * struct acrn_pio_request - Info of a PIO I/O request + * @direction: Access direction of this request (ACRN_IOREQ_DIR_*) + * @reserved: Reserved for alignment and should be 0 + * @address: Access address of this PIO I/O request + * @size: Access size of this PIO I/O request + * @value: Read/write value of this PIO I/O request + */ +struct acrn_pio_request { + __u32 direction; + __u32 reserved; + __u64 address; + __u64 size; + __u32 value; +}; + +/** + * struct acrn_io_request - 256-byte ACRN I/O request + * @type: Type of this request (ACRN_IOREQ_TYPE_*). + * @completion_polling: Polling flag. Hypervisor will poll completion of the + * I/O request if this flag set. + * @reserved0: Reserved fields. + * @reqs: Union of different types of request. Byte offset: 64. + * @reqs.pio_request: PIO request data of the I/O request. + * @reqs.mmio_request: MMIO request data of the I/O request. + * @reqs.data: Raw data of the I/O request. + * @reserved1: Reserved fields. + * @kernel_handled: Flag indicates this request need be handled in kernel. + * @processed: The status of this request (ACRN_IOREQ_STATE_*). + * + * The state transitions of ACRN I/O request: + * + * FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... + * + * An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and + * ACRN userspace are in charge of processing the others. + * + * On basis of the states illustrated above, a typical lifecycle of ACRN IO + * request would look like: + * + * Flow (assume the initial state is FREE) + * | + * | Service VM vCPU 0 Service VM vCPU x User vCPU y + * | + * | hypervisor: + * | fills in type, addr, etc. + * | pauses the User VM vCPU y + * | sets the state to PENDING (a) + * | fires an upcall to Service VM + * | + * | HSM: + * | scans for PENDING requests + * | sets the states to PROCESSING (b) + * | assigns the requests to clients (c) + * V + * | client: + * | scans for the assigned requests + * | handles the requests (d) + * | HSM: + * | sets states to COMPLETE + * | notifies the hypervisor + * | + * | hypervisor: + * | resumes User VM vCPU y (e) + * | + * | hypervisor: + * | post handling (f) + * V sets states to FREE + * + * Note that the procedures (a) to (f) in the illustration above require to be + * strictly processed in the order. One vCPU cannot trigger another request of + * I/O emulation before completing the previous one. + * + * Atomic and barriers are required when HSM and hypervisor accessing the state + * of &struct acrn_io_request. + * + */ +struct acrn_io_request { + __u32 type; + __u32 completion_polling; + __u32 reserved0[14]; + union { + struct acrn_pio_request pio_request; + struct acrn_mmio_request mmio_request; + __u64 data[8]; + } reqs; + __u32 reserved1; + __u32 kernel_handled; + __u32 processed; +} __attribute__((aligned(256))); + +struct acrn_io_request_buffer { + union { + struct acrn_io_request req_slot[ACRN_IO_REQUEST_MAX]; + __u8 reserved[4096]; + }; +}; + +/** + * struct acrn_ioreq_notify - The structure of ioreq completion notification + * @vmid: User VM ID + * @reserved: Reserved and should be 0 + * @vcpu: vCPU ID + */ +struct acrn_ioreq_notify { + __u16 vmid; + __u16 reserved; + __u32 vcpu; +}; + /** * struct acrn_vm_creation - Info to create a User VM * @vmid: User VM ID returned from the hypervisor @@ -218,6 +357,17 @@ struct acrn_vm_memmap { #define ACRN_IOCTL_SET_VCPU_REGS \ _IOW(ACRN_IOCTL_TYPE, 0x16, struct acrn_vcpu_regs) +#define ACRN_IOCTL_NOTIFY_REQUEST_FINISH \ + _IOW(ACRN_IOCTL_TYPE, 0x31, struct acrn_ioreq_notify) +#define ACRN_IOCTL_CREATE_IOREQ_CLIENT \ + _IO(ACRN_IOCTL_TYPE, 0x32) +#define ACRN_IOCTL_ATTACH_IOREQ_CLIENT \ + _IO(ACRN_IOCTL_TYPE, 0x33) +#define ACRN_IOCTL_DESTROY_IOREQ_CLIENT \ + _IO(ACRN_IOCTL_TYPE, 0x34) +#define ACRN_IOCTL_CLEAR_VM_IOREQ \ + _IO(ACRN_IOCTL_TYPE, 0x35) + #define ACRN_IOCTL_SET_MEMSEG \ _IOW(ACRN_IOCTL_TYPE, 0x41, struct acrn_vm_memmap) #define ACRN_IOCTL_UNSET_MEMSEG \ From 3c4c331667d4d9f1b5f3fdff9c4db36776da30ae Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:32 +0800 Subject: [PATCH 329/633] virt: acrn: Introduce PCI configuration space PIO accesses combiner A User VM can access its virtual PCI configuration spaces via port IO approach, which has two following steps: 1) writes address into port 0xCF8 2) put/get data in/from port 0xCFC To distribute a complete PCI configuration space access one time, HSM need to combine such two accesses together. Combine two paired PIO I/O requests into one PCI I/O request and continue the I/O request distribution. Cc: Greg Kroah-Hartman Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-11-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/acrn_drv.h | 2 + drivers/virt/acrn/ioreq.c | 76 ++++++++++++++++++++++++++++++++++++ include/uapi/linux/acrn.h | 27 +++++++++++++ 3 files changed, 105 insertions(+) diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index 68bd8db6c8be..6c92a505fa20 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -154,6 +154,7 @@ extern rwlock_t acrn_vm_list_lock; * @default_client: The default I/O request client * @ioreq_buf: I/O request shared buffer * @ioreq_page: The page of the I/O request shared buffer + * @pci_conf_addr: Address of a PCI configuration access emulation */ struct acrn_vm { struct list_head list; @@ -168,6 +169,7 @@ struct acrn_vm { struct acrn_ioreq_client *default_client; struct acrn_io_request_buffer *ioreq_buf; struct page *ioreq_page; + u32 pci_conf_addr; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, diff --git a/drivers/virt/acrn/ioreq.c b/drivers/virt/acrn/ioreq.c index 51cb41ef7c72..d19c05582d38 100644 --- a/drivers/virt/acrn/ioreq.c +++ b/drivers/virt/acrn/ioreq.c @@ -222,6 +222,80 @@ int acrn_ioreq_client_wait(struct acrn_ioreq_client *client) return 0; } +static bool is_cfg_addr(struct acrn_io_request *req) +{ + return ((req->type == ACRN_IOREQ_TYPE_PORTIO) && + (req->reqs.pio_request.address == 0xcf8)); +} + +static bool is_cfg_data(struct acrn_io_request *req) +{ + return ((req->type == ACRN_IOREQ_TYPE_PORTIO) && + ((req->reqs.pio_request.address >= 0xcfc) && + (req->reqs.pio_request.address < (0xcfc + 4)))); +} + +/* The low 8-bit of supported pci_reg addr.*/ +#define PCI_LOWREG_MASK 0xFC +/* The high 4-bit of supported pci_reg addr */ +#define PCI_HIGHREG_MASK 0xF00 +/* Max number of supported functions */ +#define PCI_FUNCMAX 7 +/* Max number of supported slots */ +#define PCI_SLOTMAX 31 +/* Max number of supported buses */ +#define PCI_BUSMAX 255 +#define CONF1_ENABLE 0x80000000UL +/* + * A PCI configuration space access via PIO 0xCF8 and 0xCFC normally has two + * following steps: + * 1) writes address into 0xCF8 port + * 2) accesses data in/from 0xCFC + * This function combines such paired PCI configuration space I/O requests into + * one ACRN_IOREQ_TYPE_PCICFG type I/O request and continues the processing. + */ +static bool handle_cf8cfc(struct acrn_vm *vm, + struct acrn_io_request *req, u16 vcpu) +{ + int offset, pci_cfg_addr, pci_reg; + bool is_handled = false; + + if (is_cfg_addr(req)) { + WARN_ON(req->reqs.pio_request.size != 4); + if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_WRITE) + vm->pci_conf_addr = req->reqs.pio_request.value; + else + req->reqs.pio_request.value = vm->pci_conf_addr; + is_handled = true; + } else if (is_cfg_data(req)) { + if (!(vm->pci_conf_addr & CONF1_ENABLE)) { + if (req->reqs.pio_request.direction == + ACRN_IOREQ_DIR_READ) + req->reqs.pio_request.value = 0xffffffff; + is_handled = true; + } else { + offset = req->reqs.pio_request.address - 0xcfc; + + req->type = ACRN_IOREQ_TYPE_PCICFG; + pci_cfg_addr = vm->pci_conf_addr; + req->reqs.pci_request.bus = + (pci_cfg_addr >> 16) & PCI_BUSMAX; + req->reqs.pci_request.dev = + (pci_cfg_addr >> 11) & PCI_SLOTMAX; + req->reqs.pci_request.func = + (pci_cfg_addr >> 8) & PCI_FUNCMAX; + pci_reg = (pci_cfg_addr & PCI_LOWREG_MASK) + + ((pci_cfg_addr >> 16) & PCI_HIGHREG_MASK); + req->reqs.pci_request.reg = pci_reg + offset; + } + } + + if (is_handled) + ioreq_complete_request(vm, vcpu, req); + + return is_handled; +} + static bool in_range(struct acrn_ioreq_range *range, struct acrn_io_request *req) { @@ -382,6 +456,8 @@ static int acrn_ioreq_dispatch(struct acrn_vm *vm) ioreq_complete_request(vm, i, req); continue; } + if (handle_cf8cfc(vm, req, i)) + continue; spin_lock_bh(&vm->ioreq_clients_lock); client = find_ioreq_client(vm, req); diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index b0f73ab5e4e8..da40f7ad13d9 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -23,6 +23,7 @@ #define ACRN_IOREQ_TYPE_PORTIO 0 #define ACRN_IOREQ_TYPE_MMIO 1 +#define ACRN_IOREQ_TYPE_PCICFG 2 #define ACRN_IOREQ_DIR_READ 0 #define ACRN_IOREQ_DIR_WRITE 1 @@ -59,6 +60,30 @@ struct acrn_pio_request { __u32 value; }; +/** + * struct acrn_pci_request - Info of a PCI I/O request + * @direction: Access direction of this request (ACRN_IOREQ_DIR_*) + * @reserved: Reserved for alignment and should be 0 + * @size: Access size of this PCI I/O request + * @value: Read/write value of this PIO I/O request + * @bus: PCI bus value of this PCI I/O request + * @dev: PCI device value of this PCI I/O request + * @func: PCI function value of this PCI I/O request + * @reg: PCI config space offset of this PCI I/O request + * + * Need keep same header layout with &struct acrn_pio_request. + */ +struct acrn_pci_request { + __u32 direction; + __u32 reserved[3]; + __u64 size; + __u32 value; + __u32 bus; + __u32 dev; + __u32 func; + __u32 reg; +}; + /** * struct acrn_io_request - 256-byte ACRN I/O request * @type: Type of this request (ACRN_IOREQ_TYPE_*). @@ -67,6 +92,7 @@ struct acrn_pio_request { * @reserved0: Reserved fields. * @reqs: Union of different types of request. Byte offset: 64. * @reqs.pio_request: PIO request data of the I/O request. + * @reqs.pci_request: PCI configuration space request data of the I/O request. * @reqs.mmio_request: MMIO request data of the I/O request. * @reqs.data: Raw data of the I/O request. * @reserved1: Reserved fields. @@ -126,6 +152,7 @@ struct acrn_io_request { __u32 reserved0[14]; union { struct acrn_pio_request pio_request; + struct acrn_pci_request pci_request; struct acrn_mmio_request mmio_request; __u64 data[8]; } reqs; From ce011e1363a1fe43de0ca05abc394022ee4fefeb Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:33 +0800 Subject: [PATCH 330/633] virt: acrn: Introduce interfaces for PCI device passthrough PCI device passthrough enables an OS in a virtual machine to directly access a PCI device in the host. It promises almost the native performance, which is required in performance-critical scenarios of ACRN. HSM provides the following ioctls: - Assign - ACRN_IOCTL_ASSIGN_PCIDEV Pass data struct acrn_pcidev from userspace to the hypervisor, and inform the hypervisor to assign a PCI device to a User VM. - De-assign - ACRN_IOCTL_DEASSIGN_PCIDEV Pass data struct acrn_pcidev from userspace to the hypervisor, and inform the hypervisor to de-assign a PCI device from a User VM. - Set a interrupt of a passthrough device - ACRN_IOCTL_SET_PTDEV_INTR Pass data struct acrn_ptdev_irq from userspace to the hypervisor, and inform the hypervisor to map a INTx interrupt of passthrough device of User VM. - Reset passthrough device interrupt - ACRN_IOCTL_RESET_PTDEV_INTR Pass data struct acrn_ptdev_irq from userspace to the hypervisor, and inform the hypervisor to unmap a INTx interrupt of passthrough device of User VM. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-12-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/hsm.c | 50 ++++++++++++++++++++++++++++ drivers/virt/acrn/hypercall.h | 54 +++++++++++++++++++++++++++++++ include/uapi/linux/acrn.h | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 1cc0c612dc09..94d70b1c1e5c 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -49,7 +49,9 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_vm_creation *vm_param; struct acrn_vcpu_regs *cpu_regs; struct acrn_ioreq_notify notify; + struct acrn_ptdev_irq *irq_info; struct acrn_vm_memmap memmap; + struct acrn_pcidev *pcidev; int i, ret = 0; if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { @@ -148,6 +150,54 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, ret = acrn_vm_memseg_unmap(vm, &memmap); break; + case ACRN_IOCTL_ASSIGN_PCIDEV: + pcidev = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_pcidev)); + if (IS_ERR(pcidev)) + return PTR_ERR(pcidev); + + ret = hcall_assign_pcidev(vm->vmid, virt_to_phys(pcidev)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to assign pci device!\n"); + kfree(pcidev); + break; + case ACRN_IOCTL_DEASSIGN_PCIDEV: + pcidev = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_pcidev)); + if (IS_ERR(pcidev)) + return PTR_ERR(pcidev); + + ret = hcall_deassign_pcidev(vm->vmid, virt_to_phys(pcidev)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to deassign pci device!\n"); + kfree(pcidev); + break; + case ACRN_IOCTL_SET_PTDEV_INTR: + irq_info = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_ptdev_irq)); + if (IS_ERR(irq_info)) + return PTR_ERR(irq_info); + + ret = hcall_set_ptdev_intr(vm->vmid, virt_to_phys(irq_info)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to configure intr for ptdev!\n"); + kfree(irq_info); + break; + case ACRN_IOCTL_RESET_PTDEV_INTR: + irq_info = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_ptdev_irq)); + if (IS_ERR(irq_info)) + return PTR_ERR(irq_info); + + ret = hcall_reset_ptdev_intr(vm->vmid, virt_to_phys(irq_info)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to reset intr for ptdev!\n"); + kfree(irq_info); + break; case ACRN_IOCTL_CREATE_IOREQ_CLIENT: if (vm->default_client) return -EEXIST; diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index 5eba29e3ed38..f448301832cf 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -28,6 +28,12 @@ #define HC_ID_MEM_BASE 0x40UL #define HC_VM_SET_MEMORY_REGIONS _HC_ID(HC_ID, HC_ID_MEM_BASE + 0x02) +#define HC_ID_PCI_BASE 0x50UL +#define HC_SET_PTDEV_INTR _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x03) +#define HC_RESET_PTDEV_INTR _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x04) +#define HC_ASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x05) +#define HC_DEASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x06) + /** * hcall_create_vm() - Create a User VM * @vminfo: Service VM GPA of info of User VM creation @@ -130,4 +136,52 @@ static inline long hcall_set_memory_regions(u64 regions_pa) return acrn_hypercall1(HC_VM_SET_MEMORY_REGIONS, regions_pa); } +/** + * hcall_assign_pcidev() - Assign a PCI device to a User VM + * @vmid: User VM ID + * @addr: Service VM GPA of the &struct acrn_pcidev + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_assign_pcidev(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_ASSIGN_PCIDEV, vmid, addr); +} + +/** + * hcall_deassign_pcidev() - De-assign a PCI device from a User VM + * @vmid: User VM ID + * @addr: Service VM GPA of the &struct acrn_pcidev + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_deassign_pcidev(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_DEASSIGN_PCIDEV, vmid, addr); +} + +/** + * hcall_set_ptdev_intr() - Configure an interrupt for an assigned PCI device. + * @vmid: User VM ID + * @irq: Service VM GPA of the &struct acrn_ptdev_irq + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_ptdev_intr(u64 vmid, u64 irq) +{ + return acrn_hypercall2(HC_SET_PTDEV_INTR, vmid, irq); +} + +/** + * hcall_reset_ptdev_intr() - Reset an interrupt for an assigned PCI device. + * @vmid: User VM ID + * @irq: Service VM GPA of the &struct acrn_ptdev_irq + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_reset_ptdev_intr(u64 vmid, u64 irq) +{ + return acrn_hypercall2(HC_RESET_PTDEV_INTR, vmid, irq); +} + #endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index da40f7ad13d9..b25ca8c33b92 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -365,6 +365,58 @@ struct acrn_vm_memmap { __u64 len; }; +/* Type of interrupt of a passthrough device */ +#define ACRN_PTDEV_IRQ_INTX 0 +#define ACRN_PTDEV_IRQ_MSI 1 +#define ACRN_PTDEV_IRQ_MSIX 2 +/** + * struct acrn_ptdev_irq - Interrupt data of a passthrough device. + * @type: Type (ACRN_PTDEV_IRQ_*) + * @virt_bdf: Virtual Bus/Device/Function + * @phys_bdf: Physical Bus/Device/Function + * @intx: Info of interrupt + * @intx.virt_pin: Virtual IOAPIC pin + * @intx.phys_pin: Physical IOAPIC pin + * @intx.is_pic_pin: Is PIC pin or not + * + * This structure will be passed to hypervisor directly. + */ +struct acrn_ptdev_irq { + __u32 type; + __u16 virt_bdf; + __u16 phys_bdf; + + struct { + __u32 virt_pin; + __u32 phys_pin; + __u32 is_pic_pin; + } intx; +}; + +/* Type of PCI device assignment */ +#define ACRN_PTDEV_QUIRK_ASSIGN (1U << 0) + +#define ACRN_PCI_NUM_BARS 6 +/** + * struct acrn_pcidev - Info for assigning or de-assigning a PCI device + * @type: Type of the assignment + * @virt_bdf: Virtual Bus/Device/Function + * @phys_bdf: Physical Bus/Device/Function + * @intr_line: PCI interrupt line + * @intr_pin: PCI interrupt pin + * @bar: PCI BARs. + * + * This structure will be passed to hypervisor directly. + */ +struct acrn_pcidev { + __u32 type; + __u16 virt_bdf; + __u16 phys_bdf; + __u8 intr_line; + __u8 intr_pin; + __u32 bar[ACRN_PCI_NUM_BARS]; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -400,4 +452,13 @@ struct acrn_vm_memmap { #define ACRN_IOCTL_UNSET_MEMSEG \ _IOW(ACRN_IOCTL_TYPE, 0x42, struct acrn_vm_memmap) +#define ACRN_IOCTL_SET_PTDEV_INTR \ + _IOW(ACRN_IOCTL_TYPE, 0x53, struct acrn_ptdev_irq) +#define ACRN_IOCTL_RESET_PTDEV_INTR \ + _IOW(ACRN_IOCTL_TYPE, 0x54, struct acrn_ptdev_irq) +#define ACRN_IOCTL_ASSIGN_PCIDEV \ + _IOW(ACRN_IOCTL_TYPE, 0x55, struct acrn_pcidev) +#define ACRN_IOCTL_DEASSIGN_PCIDEV \ + _IOW(ACRN_IOCTL_TYPE, 0x56, struct acrn_pcidev) + #endif /* _UAPI_ACRN_H */ From c7cf8d27244f2ccdde30c79eb6314c943bbeac28 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:34 +0800 Subject: [PATCH 331/633] virt: acrn: Introduce interrupt injection interfaces ACRN userspace need to inject virtual interrupts into a User VM in devices emulation. HSM needs provide interfaces to do so. Introduce following interrupt injection interfaces: ioctl ACRN_IOCTL_SET_IRQLINE: Pass data from userspace to the hypervisor, and inform the hypervisor to inject a virtual IOAPIC GSI interrupt to a User VM. ioctl ACRN_IOCTL_INJECT_MSI: Pass data struct acrn_msi_entry from userspace to the hypervisor, and inform the hypervisor to inject a virtual MSI to a User VM. ioctl ACRN_IOCTL_VM_INTR_MONITOR: Set a 4-Kbyte aligned shared page for statistics information of interrupts of a User VM. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-13-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/acrn_drv.h | 4 ++++ drivers/virt/acrn/hsm.c | 40 ++++++++++++++++++++++++++++++++++ drivers/virt/acrn/hypercall.h | 41 +++++++++++++++++++++++++++++++++++ drivers/virt/acrn/vm.c | 36 ++++++++++++++++++++++++++++++ include/uapi/linux/acrn.h | 17 +++++++++++++++ 5 files changed, 138 insertions(+) diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index 6c92a505fa20..068f0be769f6 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -155,6 +155,7 @@ extern rwlock_t acrn_vm_list_lock; * @ioreq_buf: I/O request shared buffer * @ioreq_page: The page of the I/O request shared buffer * @pci_conf_addr: Address of a PCI configuration access emulation + * @monitor_page: Page of interrupt statistics of User VM */ struct acrn_vm { struct list_head list; @@ -170,6 +171,7 @@ struct acrn_vm { struct acrn_io_request_buffer *ioreq_buf; struct page *ioreq_page; u32 pci_conf_addr; + struct page *monitor_page; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, @@ -196,4 +198,6 @@ struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, const char *name); void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client); +int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data); + #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 94d70b1c1e5c..419271f32be8 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -51,7 +51,9 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_ioreq_notify notify; struct acrn_ptdev_irq *irq_info; struct acrn_vm_memmap memmap; + struct acrn_msi_entry *msi; struct acrn_pcidev *pcidev; + struct page *page; int i, ret = 0; if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { @@ -198,6 +200,44 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, "Failed to reset intr for ptdev!\n"); kfree(irq_info); break; + case ACRN_IOCTL_SET_IRQLINE: + ret = hcall_set_irqline(vm->vmid, ioctl_param); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to set interrupt line!\n"); + break; + case ACRN_IOCTL_INJECT_MSI: + msi = memdup_user((void __user *)ioctl_param, + sizeof(struct acrn_msi_entry)); + if (IS_ERR(msi)) + return PTR_ERR(msi); + + ret = hcall_inject_msi(vm->vmid, virt_to_phys(msi)); + if (ret < 0) + dev_dbg(acrn_dev.this_device, + "Failed to inject MSI!\n"); + kfree(msi); + break; + case ACRN_IOCTL_VM_INTR_MONITOR: + ret = pin_user_pages_fast(ioctl_param, 1, + FOLL_WRITE | FOLL_LONGTERM, &page); + if (unlikely(ret != 1)) { + dev_dbg(acrn_dev.this_device, + "Failed to pin intr hdr buffer!\n"); + return -EFAULT; + } + + ret = hcall_vm_intr_monitor(vm->vmid, page_to_phys(page)); + if (ret < 0) { + unpin_user_page(page); + dev_dbg(acrn_dev.this_device, + "Failed to monitor intr data!\n"); + return ret; + } + if (vm->monitor_page) + unpin_user_page(vm->monitor_page); + vm->monitor_page = page; + break; case ACRN_IOCTL_CREATE_IOREQ_CLIENT: if (vm->default_client) return -EEXIST; diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index f448301832cf..a8813397a3fe 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -21,6 +21,11 @@ #define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05) #define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06) +#define HC_ID_IRQ_BASE 0x20UL +#define HC_INJECT_MSI _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x03) +#define HC_VM_INTR_MONITOR _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x04) +#define HC_SET_IRQLINE _HC_ID(HC_ID, HC_ID_IRQ_BASE + 0x05) + #define HC_ID_IOREQ_BASE 0x30UL #define HC_SET_IOREQ_BUFFER _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x00) #define HC_NOTIFY_REQUEST_FINISH _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x01) @@ -101,6 +106,42 @@ static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state); } +/** + * hcall_inject_msi() - Deliver a MSI interrupt to a User VM + * @vmid: User VM ID + * @msi: Service VM GPA of MSI message + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_inject_msi(u64 vmid, u64 msi) +{ + return acrn_hypercall2(HC_INJECT_MSI, vmid, msi); +} + +/** + * hcall_vm_intr_monitor() - Set a shared page for User VM interrupt statistics + * @vmid: User VM ID + * @addr: Service VM GPA of the shared page + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_vm_intr_monitor(u64 vmid, u64 addr) +{ + return acrn_hypercall2(HC_VM_INTR_MONITOR, vmid, addr); +} + +/** + * hcall_set_irqline() - Set or clear an interrupt line + * @vmid: User VM ID + * @op: Service VM GPA of interrupt line operations + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_set_irqline(u64 vmid, u64 op) +{ + return acrn_hypercall2(HC_SET_IRQLINE, vmid, op); +} + /** * hcall_set_ioreq_buffer() - Set up the shared buffer for I/O Requests. * @vmid: User VM ID diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c index 2bc649c7c1d3..db597f27690a 100644 --- a/drivers/virt/acrn/vm.c +++ b/drivers/virt/acrn/vm.c @@ -68,6 +68,10 @@ int acrn_vm_destroy(struct acrn_vm *vm) write_unlock_bh(&acrn_vm_list_lock); acrn_ioreq_deinit(vm); + if (vm->monitor_page) { + put_page(vm->monitor_page); + vm->monitor_page = NULL; + } ret = hcall_destroy_vm(vm->vmid); if (ret < 0) { @@ -83,3 +87,35 @@ int acrn_vm_destroy(struct acrn_vm *vm) vm->vmid = ACRN_INVALID_VMID; return 0; } + +/** + * acrn_inject_msi() - Inject a MSI interrupt into a User VM + * @vm: User VM + * @msi_addr: The MSI address + * @msi_data: The MSI data + * + * Return: 0 on success, <0 on error + */ +int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data) +{ + struct acrn_msi_entry *msi; + int ret; + + /* might be used in interrupt context, so use GFP_ATOMIC */ + msi = kzalloc(sizeof(*msi), GFP_ATOMIC); + if (!msi) + return -ENOMEM; + + /* + * msi_addr: addr[19:12] with dest vcpu id + * msi_data: data[7:0] with vector + */ + msi->msi_addr = msi_addr; + msi->msi_data = msi_data; + ret = hcall_inject_msi(vm->vmid, virt_to_phys(msi)); + if (ret < 0) + dev_err(acrn_dev.this_device, + "Failed to inject MSI to VM %u!\n", vm->vmid); + kfree(msi); + return ret; +} diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index b25ca8c33b92..b1c06b28ebdc 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -417,6 +417,16 @@ struct acrn_pcidev { __u32 bar[ACRN_PCI_NUM_BARS]; }; +/** + * struct acrn_msi_entry - Info for injecting a MSI interrupt to a VM + * @msi_addr: MSI addr[19:12] with dest vCPU ID + * @msi_data: MSI data[7:0] with vector + */ +struct acrn_msi_entry { + __u64 msi_addr; + __u64 msi_data; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -436,6 +446,13 @@ struct acrn_pcidev { #define ACRN_IOCTL_SET_VCPU_REGS \ _IOW(ACRN_IOCTL_TYPE, 0x16, struct acrn_vcpu_regs) +#define ACRN_IOCTL_INJECT_MSI \ + _IOW(ACRN_IOCTL_TYPE, 0x23, struct acrn_msi_entry) +#define ACRN_IOCTL_VM_INTR_MONITOR \ + _IOW(ACRN_IOCTL_TYPE, 0x24, unsigned long) +#define ACRN_IOCTL_SET_IRQLINE \ + _IOW(ACRN_IOCTL_TYPE, 0x25, __u64) + #define ACRN_IOCTL_NOTIFY_REQUEST_FINISH \ _IOW(ACRN_IOCTL_TYPE, 0x31, struct acrn_ioreq_notify) #define ACRN_IOCTL_CREATE_IOREQ_CLIENT \ From 3d679d5aec648f50e645702929890b9611998a0b Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:35 +0800 Subject: [PATCH 332/633] virt: acrn: Introduce interfaces to query C-states and P-states allowed by hypervisor The C-states and P-states data are used to support CPU power management. The hypervisor controls C-states and P-states for a User VM. ACRN userspace need to query the data from the hypervisor to build ACPI tables for a User VM. HSM provides ioctls for ACRN userspace to query C-states and P-states data obtained from the hypervisor. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-14-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/hsm.c | 69 +++++++++++++++++++++++++++++++++++ drivers/virt/acrn/hypercall.h | 12 ++++++ include/uapi/linux/acrn.h | 55 ++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 419271f32be8..b7f2deddc3e7 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -38,6 +38,67 @@ static int acrn_dev_open(struct inode *inode, struct file *filp) return 0; } +static int pmcmd_ioctl(u64 cmd, void __user *uptr) +{ + struct acrn_pstate_data *px_data; + struct acrn_cstate_data *cx_data; + u64 *pm_info; + int ret = 0; + + switch (cmd & PMCMD_TYPE_MASK) { + case ACRN_PMCMD_GET_PX_CNT: + case ACRN_PMCMD_GET_CX_CNT: + pm_info = kmalloc(sizeof(u64), GFP_KERNEL); + if (!pm_info) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(pm_info)); + if (ret < 0) { + kfree(pm_info); + break; + } + + if (copy_to_user(uptr, pm_info, sizeof(u64))) + ret = -EFAULT; + kfree(pm_info); + break; + case ACRN_PMCMD_GET_PX_DATA: + px_data = kmalloc(sizeof(*px_data), GFP_KERNEL); + if (!px_data) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(px_data)); + if (ret < 0) { + kfree(px_data); + break; + } + + if (copy_to_user(uptr, px_data, sizeof(*px_data))) + ret = -EFAULT; + kfree(px_data); + break; + case ACRN_PMCMD_GET_CX_DATA: + cx_data = kmalloc(sizeof(*cx_data), GFP_KERNEL); + if (!cx_data) + return -ENOMEM; + + ret = hcall_get_cpu_state(cmd, virt_to_phys(cx_data)); + if (ret < 0) { + kfree(cx_data); + break; + } + + if (copy_to_user(uptr, cx_data, sizeof(*cx_data))) + ret = -EFAULT; + kfree(cx_data); + break; + default: + break; + } + + return ret; +} + /* * HSM relies on hypercall layer of the ACRN hypervisor to do the * sanity check against the input parameters. @@ -54,6 +115,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_msi_entry *msi; struct acrn_pcidev *pcidev; struct page *page; + u64 cstate_cmd; int i, ret = 0; if (vm->vmid == ACRN_INVALID_VMID && cmd != ACRN_IOCTL_CREATE_VM) { @@ -267,6 +329,13 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, case ACRN_IOCTL_CLEAR_VM_IOREQ: acrn_ioreq_request_clear(vm); break; + case ACRN_IOCTL_PM_GET_CPU_STATE: + if (copy_from_user(&cstate_cmd, (void *)ioctl_param, + sizeof(cstate_cmd))) + return -EFAULT; + + ret = pmcmd_ioctl(cstate_cmd, (void __user *)ioctl_param); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index a8813397a3fe..e640632366f0 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -39,6 +39,9 @@ #define HC_ASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x05) #define HC_DEASSIGN_PCIDEV _HC_ID(HC_ID, HC_ID_PCI_BASE + 0x06) +#define HC_ID_PM_BASE 0x80UL +#define HC_PM_GET_CPU_STATE _HC_ID(HC_ID, HC_ID_PM_BASE + 0x00) + /** * hcall_create_vm() - Create a User VM * @vminfo: Service VM GPA of info of User VM creation @@ -225,4 +228,13 @@ static inline long hcall_reset_ptdev_intr(u64 vmid, u64 irq) return acrn_hypercall2(HC_RESET_PTDEV_INTR, vmid, irq); } +/* + * hcall_get_cpu_state() - Get P-states and C-states info from the hypervisor + * @state: Service VM GPA of buffer of P-states and C-states + */ +static inline long hcall_get_cpu_state(u64 cmd, u64 state) +{ + return acrn_hypercall2(HC_PM_GET_CPU_STATE, cmd, state); +} + #endif /* __ACRN_HSM_HYPERCALL_H */ diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index b1c06b28ebdc..e997d80a0cc7 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -427,6 +427,58 @@ struct acrn_msi_entry { __u64 msi_data; }; +struct acrn_acpi_generic_address { + __u8 space_id; + __u8 bit_width; + __u8 bit_offset; + __u8 access_size; + __u64 address; +} __attribute__ ((__packed__)); + +/** + * struct acrn_cstate_data - A C state package defined in ACPI + * @cx_reg: Register of the C state object + * @type: Type of the C state object + * @latency: The worst-case latency to enter and exit this C state + * @power: The average power consumption when in this C state + */ +struct acrn_cstate_data { + struct acrn_acpi_generic_address cx_reg; + __u8 type; + __u32 latency; + __u64 power; +}; + +/** + * struct acrn_pstate_data - A P state package defined in ACPI + * @core_frequency: CPU frequency (in MHz). + * @power: Power dissipation (in milliwatts). + * @transition_latency: The worst-case latency in microseconds that CPU is + * unavailable during a transition from any P state to + * this P state. + * @bus_master_latency: The worst-case latency in microseconds that Bus Masters + * are prevented from accessing memory during a transition + * from any P state to this P state. + * @control: The value to be written to Performance Control Register + * @status: Transition status. + */ +struct acrn_pstate_data { + __u64 core_frequency; + __u64 power; + __u64 transition_latency; + __u64 bus_master_latency; + __u64 control; + __u64 status; +}; + +#define PMCMD_TYPE_MASK 0x000000ff +enum acrn_pm_cmd_type { + ACRN_PMCMD_GET_PX_CNT, + ACRN_PMCMD_GET_PX_DATA, + ACRN_PMCMD_GET_CX_CNT, + ACRN_PMCMD_GET_CX_DATA, +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -478,4 +530,7 @@ struct acrn_msi_entry { #define ACRN_IOCTL_DEASSIGN_PCIDEV \ _IOW(ACRN_IOCTL_TYPE, 0x56, struct acrn_pcidev) +#define ACRN_IOCTL_PM_GET_CPU_STATE \ + _IOWR(ACRN_IOCTL_TYPE, 0x60, __u64) + #endif /* _UAPI_ACRN_H */ From 5a0c9f176f232513d4114a518cbff835d232f500 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:36 +0800 Subject: [PATCH 333/633] virt: acrn: Introduce I/O ranges operation interfaces An I/O request of a User VM, which is constructed by hypervisor, is distributed by the ACRN Hypervisor Service Module to an I/O client corresponding to the address range of the I/O request. I/O client maintains a list of address ranges. Introduce acrn_ioreq_range_{add,del}() to manage these address ranges. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-15-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/acrn_drv.h | 4 +++ drivers/virt/acrn/ioreq.c | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index 068f0be769f6..8a7d7721f505 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -197,6 +197,10 @@ struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm, void *data, bool is_default, const char *name); void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client); +int acrn_ioreq_range_add(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end); +void acrn_ioreq_range_del(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end); int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data); diff --git a/drivers/virt/acrn/ioreq.c b/drivers/virt/acrn/ioreq.c index d19c05582d38..80b2e3f0e276 100644 --- a/drivers/virt/acrn/ioreq.c +++ b/drivers/virt/acrn/ioreq.c @@ -103,6 +103,66 @@ int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu) return ret; } +/** + * acrn_ioreq_range_add() - Add an iorange monitored by an ioreq client + * @client: The ioreq client + * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO) + * @start: Start address of iorange + * @end: End address of iorange + * + * Return: 0 on success, <0 on error + */ +int acrn_ioreq_range_add(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end) +{ + struct acrn_ioreq_range *range; + + if (end < start) { + dev_err(acrn_dev.this_device, + "Invalid IO range [0x%llx,0x%llx]\n", start, end); + return -EINVAL; + } + + range = kzalloc(sizeof(*range), GFP_KERNEL); + if (!range) + return -ENOMEM; + + range->type = type; + range->start = start; + range->end = end; + + write_lock_bh(&client->range_lock); + list_add(&range->list, &client->range_list); + write_unlock_bh(&client->range_lock); + + return 0; +} + +/** + * acrn_ioreq_range_del() - Del an iorange monitored by an ioreq client + * @client: The ioreq client + * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO) + * @start: Start address of iorange + * @end: End address of iorange + */ +void acrn_ioreq_range_del(struct acrn_ioreq_client *client, + u32 type, u64 start, u64 end) +{ + struct acrn_ioreq_range *range; + + write_lock_bh(&client->range_lock); + list_for_each_entry(range, &client->range_list, list) { + if (type == range->type && + start == range->start && + end == range->end) { + list_del(&range->list); + kfree(range); + break; + } + } + write_unlock_bh(&client->range_lock); +} + /* * ioreq_task() is the execution entity of handler thread of an I/O client. * The handler callback of the I/O client is called within the handler thread. From d8ad515156b66e7e79a6e4c814f997ee54eb47c7 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:37 +0800 Subject: [PATCH 334/633] virt: acrn: Introduce ioeventfd ioeventfd is a mechanism to register PIO/MMIO regions to trigger an eventfd signal when written to by a User VM. ACRN userspace can register any arbitrary I/O address with a corresponding eventfd and then pass the eventfd to a specific end-point of interest for handling. Vhost is a kernel-level virtio server which uses eventfd for signalling. To support vhost on ACRN, ioeventfd is introduced in HSM. A new I/O client dedicated to ioeventfd is associated with a User VM during VM creation. HSM provides ioctls to associate an I/O region with a eventfd. The I/O client signals a eventfd once its corresponding I/O region is matched with an I/O request. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-16-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/Kconfig | 1 + drivers/virt/acrn/Makefile | 2 +- drivers/virt/acrn/acrn_drv.h | 10 ++ drivers/virt/acrn/hsm.c | 11 ++ drivers/virt/acrn/ioeventfd.c | 273 ++++++++++++++++++++++++++++++++++ drivers/virt/acrn/vm.c | 2 + include/uapi/linux/acrn.h | 29 ++++ 7 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 drivers/virt/acrn/ioeventfd.c diff --git a/drivers/virt/acrn/Kconfig b/drivers/virt/acrn/Kconfig index 36c80378c30c..3e1a61c9d8d8 100644 --- a/drivers/virt/acrn/Kconfig +++ b/drivers/virt/acrn/Kconfig @@ -2,6 +2,7 @@ config ACRN_HSM tristate "ACRN Hypervisor Service Module" depends on ACRN_GUEST + select EVENTFD help ACRN Hypervisor Service Module (HSM) is a kernel module which communicates with ACRN userspace through ioctls and talks to diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile index 21721cbf6a80..755b583b32ca 100644 --- a/drivers/virt/acrn/Makefile +++ b/drivers/virt/acrn/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACRN_HSM) := acrn.o -acrn-y := hsm.o vm.o mm.o ioreq.o +acrn-y := hsm.o vm.o mm.o ioreq.o ioeventfd.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index 8a7d7721f505..e3f8190bd972 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -156,6 +156,9 @@ extern rwlock_t acrn_vm_list_lock; * @ioreq_page: The page of the I/O request shared buffer * @pci_conf_addr: Address of a PCI configuration access emulation * @monitor_page: Page of interrupt statistics of User VM + * @ioeventfds_lock: Lock to protect ioeventfds list + * @ioeventfds: List to link all hsm_ioeventfd + * @ioeventfd_client: I/O client for ioeventfds of the VM */ struct acrn_vm { struct list_head list; @@ -172,6 +175,9 @@ struct acrn_vm { struct page *ioreq_page; u32 pci_conf_addr; struct page *monitor_page; + struct mutex ioeventfds_lock; + struct list_head ioeventfds; + struct acrn_ioreq_client *ioeventfd_client; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, @@ -204,4 +210,8 @@ void acrn_ioreq_range_del(struct acrn_ioreq_client *client, int acrn_msi_inject(struct acrn_vm *vm, u64 msi_addr, u64 msi_data); +int acrn_ioeventfd_init(struct acrn_vm *vm); +int acrn_ioeventfd_config(struct acrn_vm *vm, struct acrn_ioeventfd *args); +void acrn_ioeventfd_deinit(struct acrn_vm *vm); + #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index b7f2deddc3e7..0b4e88b0b6bc 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -111,6 +111,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_vcpu_regs *cpu_regs; struct acrn_ioreq_notify notify; struct acrn_ptdev_irq *irq_info; + struct acrn_ioeventfd ioeventfd; struct acrn_vm_memmap memmap; struct acrn_msi_entry *msi; struct acrn_pcidev *pcidev; @@ -336,6 +337,16 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, ret = pmcmd_ioctl(cstate_cmd, (void __user *)ioctl_param); break; + case ACRN_IOCTL_IOEVENTFD: + if (copy_from_user(&ioeventfd, (void __user *)ioctl_param, + sizeof(ioeventfd))) + return -EFAULT; + + if (ioeventfd.reserved != 0) + return -EINVAL; + + ret = acrn_ioeventfd_config(vm, &ioeventfd); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; diff --git a/drivers/virt/acrn/ioeventfd.c b/drivers/virt/acrn/ioeventfd.c new file mode 100644 index 000000000000..ac4037e9f947 --- /dev/null +++ b/drivers/virt/acrn/ioeventfd.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN HSM eventfd - use eventfd objects to signal expected I/O requests + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Shuo Liu + * Yakui Zhao + */ + +#include +#include + +#include "acrn_drv.h" + +/** + * struct hsm_ioeventfd - Properties of HSM ioeventfd + * @list: Entry within &acrn_vm.ioeventfds of ioeventfds of a VM + * @eventfd: Eventfd of the HSM ioeventfd + * @addr: Address of I/O range + * @data: Data for matching + * @length: Length of I/O range + * @type: Type of I/O range (ACRN_IOREQ_TYPE_MMIO/ACRN_IOREQ_TYPE_PORTIO) + * @wildcard: Data matching or not + */ +struct hsm_ioeventfd { + struct list_head list; + struct eventfd_ctx *eventfd; + u64 addr; + u64 data; + int length; + int type; + bool wildcard; +}; + +static inline int ioreq_type_from_flags(int flags) +{ + return flags & ACRN_IOEVENTFD_FLAG_PIO ? + ACRN_IOREQ_TYPE_PORTIO : ACRN_IOREQ_TYPE_MMIO; +} + +static void acrn_ioeventfd_shutdown(struct acrn_vm *vm, struct hsm_ioeventfd *p) +{ + lockdep_assert_held(&vm->ioeventfds_lock); + + eventfd_ctx_put(p->eventfd); + list_del(&p->list); + kfree(p); +} + +static bool hsm_ioeventfd_is_conflict(struct acrn_vm *vm, + struct hsm_ioeventfd *ioeventfd) +{ + struct hsm_ioeventfd *p; + + lockdep_assert_held(&vm->ioeventfds_lock); + + /* Either one is wildcard, the data matching will be skipped. */ + list_for_each_entry(p, &vm->ioeventfds, list) + if (p->eventfd == ioeventfd->eventfd && + p->addr == ioeventfd->addr && + p->type == ioeventfd->type && + (p->wildcard || ioeventfd->wildcard || + p->data == ioeventfd->data)) + return true; + + return false; +} + +/* + * Assign an eventfd to a VM and create a HSM ioeventfd associated with the + * eventfd. The properties of the HSM ioeventfd are built from a &struct + * acrn_ioeventfd. + */ +static int acrn_ioeventfd_assign(struct acrn_vm *vm, + struct acrn_ioeventfd *args) +{ + struct eventfd_ctx *eventfd; + struct hsm_ioeventfd *p; + int ret; + + /* Check for range overflow */ + if (args->addr + args->len < args->addr) + return -EINVAL; + + /* + * Currently, acrn_ioeventfd is used to support vhost. 1,2,4,8 width + * accesses can cover vhost's requirements. + */ + if (!(args->len == 1 || args->len == 2 || + args->len == 4 || args->len == 8)) + return -EINVAL; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + ret = -ENOMEM; + goto fail; + } + + INIT_LIST_HEAD(&p->list); + p->addr = args->addr; + p->length = args->len; + p->eventfd = eventfd; + p->type = ioreq_type_from_flags(args->flags); + + /* + * ACRN_IOEVENTFD_FLAG_DATAMATCH flag is set in virtio 1.0 support, the + * writing of notification register of each virtqueue may trigger the + * notification. There is no data matching requirement. + */ + if (args->flags & ACRN_IOEVENTFD_FLAG_DATAMATCH) + p->data = args->data; + else + p->wildcard = true; + + mutex_lock(&vm->ioeventfds_lock); + + if (hsm_ioeventfd_is_conflict(vm, p)) { + ret = -EEXIST; + goto unlock_fail; + } + + /* register the I/O range into ioreq client */ + ret = acrn_ioreq_range_add(vm->ioeventfd_client, p->type, + p->addr, p->addr + p->length - 1); + if (ret < 0) + goto unlock_fail; + + list_add_tail(&p->list, &vm->ioeventfds); + mutex_unlock(&vm->ioeventfds_lock); + + return 0; + +unlock_fail: + mutex_unlock(&vm->ioeventfds_lock); + kfree(p); +fail: + eventfd_ctx_put(eventfd); + return ret; +} + +static int acrn_ioeventfd_deassign(struct acrn_vm *vm, + struct acrn_ioeventfd *args) +{ + struct hsm_ioeventfd *p; + struct eventfd_ctx *eventfd; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + mutex_lock(&vm->ioeventfds_lock); + list_for_each_entry(p, &vm->ioeventfds, list) { + if (p->eventfd != eventfd) + continue; + + acrn_ioreq_range_del(vm->ioeventfd_client, p->type, + p->addr, p->addr + p->length - 1); + acrn_ioeventfd_shutdown(vm, p); + break; + } + mutex_unlock(&vm->ioeventfds_lock); + + eventfd_ctx_put(eventfd); + return 0; +} + +static struct hsm_ioeventfd *hsm_ioeventfd_match(struct acrn_vm *vm, u64 addr, + u64 data, int len, int type) +{ + struct hsm_ioeventfd *p = NULL; + + lockdep_assert_held(&vm->ioeventfds_lock); + + list_for_each_entry(p, &vm->ioeventfds, list) { + if (p->type == type && p->addr == addr && p->length >= len && + (p->wildcard || p->data == data)) + return p; + } + + return NULL; +} + +static int acrn_ioeventfd_handler(struct acrn_ioreq_client *client, + struct acrn_io_request *req) +{ + struct hsm_ioeventfd *p; + u64 addr, val; + int size; + + if (req->type == ACRN_IOREQ_TYPE_MMIO) { + /* + * I/O requests are dispatched by range check only, so a + * acrn_ioreq_client need process both READ and WRITE accesses + * of same range. READ accesses are safe to be ignored here + * because virtio PCI devices write the notify registers for + * notification. + */ + if (req->reqs.mmio_request.direction == ACRN_IOREQ_DIR_READ) { + /* reading does nothing and return 0 */ + req->reqs.mmio_request.value = 0; + return 0; + } + addr = req->reqs.mmio_request.address; + size = req->reqs.mmio_request.size; + val = req->reqs.mmio_request.value; + } else { + if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_READ) { + /* reading does nothing and return 0 */ + req->reqs.pio_request.value = 0; + return 0; + } + addr = req->reqs.pio_request.address; + size = req->reqs.pio_request.size; + val = req->reqs.pio_request.value; + } + + mutex_lock(&client->vm->ioeventfds_lock); + p = hsm_ioeventfd_match(client->vm, addr, val, size, req->type); + if (p) + eventfd_signal(p->eventfd, 1); + mutex_unlock(&client->vm->ioeventfds_lock); + + return 0; +} + +int acrn_ioeventfd_config(struct acrn_vm *vm, struct acrn_ioeventfd *args) +{ + int ret; + + if (args->flags & ACRN_IOEVENTFD_FLAG_DEASSIGN) + ret = acrn_ioeventfd_deassign(vm, args); + else + ret = acrn_ioeventfd_assign(vm, args); + + return ret; +} + +int acrn_ioeventfd_init(struct acrn_vm *vm) +{ + char name[ACRN_NAME_LEN]; + + mutex_init(&vm->ioeventfds_lock); + INIT_LIST_HEAD(&vm->ioeventfds); + snprintf(name, sizeof(name), "ioeventfd-%u", vm->vmid); + vm->ioeventfd_client = acrn_ioreq_client_create(vm, + acrn_ioeventfd_handler, + NULL, false, name); + if (!vm->ioeventfd_client) { + dev_err(acrn_dev.this_device, "Failed to create ioeventfd ioreq client!\n"); + return -EINVAL; + } + + dev_dbg(acrn_dev.this_device, "VM %u ioeventfd init.\n", vm->vmid); + return 0; +} + +void acrn_ioeventfd_deinit(struct acrn_vm *vm) +{ + struct hsm_ioeventfd *p, *next; + + dev_dbg(acrn_dev.this_device, "VM %u ioeventfd deinit.\n", vm->vmid); + acrn_ioreq_client_destroy(vm->ioeventfd_client); + mutex_lock(&vm->ioeventfds_lock); + list_for_each_entry_safe(p, next, &vm->ioeventfds, list) + acrn_ioeventfd_shutdown(vm, p); + mutex_unlock(&vm->ioeventfds_lock); +} diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c index db597f27690a..2b322d956a8c 100644 --- a/drivers/virt/acrn/vm.c +++ b/drivers/virt/acrn/vm.c @@ -50,6 +50,7 @@ struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, list_add(&vm->list, &acrn_vm_list); write_unlock_bh(&acrn_vm_list_lock); + acrn_ioeventfd_init(vm); dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid); return vm; } @@ -67,6 +68,7 @@ int acrn_vm_destroy(struct acrn_vm *vm) list_del_init(&vm->list); write_unlock_bh(&acrn_vm_list_lock); + acrn_ioeventfd_deinit(vm); acrn_ioreq_deinit(vm); if (vm->monitor_page) { put_page(vm->monitor_page); diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index e997d80a0cc7..132c5ff95967 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -479,6 +479,32 @@ enum acrn_pm_cmd_type { ACRN_PMCMD_GET_CX_DATA, }; +#define ACRN_IOEVENTFD_FLAG_PIO 0x01 +#define ACRN_IOEVENTFD_FLAG_DATAMATCH 0x02 +#define ACRN_IOEVENTFD_FLAG_DEASSIGN 0x04 +/** + * struct acrn_ioeventfd - Data to operate a &struct hsm_ioeventfd + * @fd: The fd of eventfd associated with a hsm_ioeventfd + * @flags: Logical-OR of ACRN_IOEVENTFD_FLAG_* + * @addr: The start address of IO range of ioeventfd + * @len: The length of IO range of ioeventfd + * @reserved: Reserved and should be 0 + * @data: Data for data matching + * + * Without flag ACRN_IOEVENTFD_FLAG_DEASSIGN, ioctl ACRN_IOCTL_IOEVENTFD + * creates a &struct hsm_ioeventfd with properties originated from &struct + * acrn_ioeventfd. With flag ACRN_IOEVENTFD_FLAG_DEASSIGN, ioctl + * ACRN_IOCTL_IOEVENTFD destroys the &struct hsm_ioeventfd matching the fd. + */ +struct acrn_ioeventfd { + __u32 fd; + __u32 flags; + __u64 addr; + __u32 len; + __u32 reserved; + __u64 data; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -533,4 +559,7 @@ enum acrn_pm_cmd_type { #define ACRN_IOCTL_PM_GET_CPU_STATE \ _IOWR(ACRN_IOCTL_TYPE, 0x60, __u64) +#define ACRN_IOCTL_IOEVENTFD \ + _IOW(ACRN_IOCTL_TYPE, 0x70, struct acrn_ioeventfd) + #endif /* _UAPI_ACRN_H */ From aa3b483ff1d71c50b33db154048dff9a8f08ac71 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:38 +0800 Subject: [PATCH 335/633] virt: acrn: Introduce irqfd irqfd is a mechanism to inject a specific interrupt to a User VM using a decoupled eventfd mechanism. Vhost is a kernel-level virtio server which uses eventfd for interrupt injection. To support vhost on ACRN, irqfd is introduced in HSM. HSM provides ioctls to associate a virtual Message Signaled Interrupt (MSI) with an eventfd. The corresponding virtual MSI will be injected into a User VM once the eventfd got signal. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-17-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/Makefile | 2 +- drivers/virt/acrn/acrn_drv.h | 10 ++ drivers/virt/acrn/hsm.c | 7 ++ drivers/virt/acrn/irqfd.c | 235 +++++++++++++++++++++++++++++++++++ drivers/virt/acrn/vm.c | 3 + include/uapi/linux/acrn.h | 15 +++ 6 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 drivers/virt/acrn/irqfd.c diff --git a/drivers/virt/acrn/Makefile b/drivers/virt/acrn/Makefile index 755b583b32ca..08ce641dcfa1 100644 --- a/drivers/virt/acrn/Makefile +++ b/drivers/virt/acrn/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACRN_HSM) := acrn.o -acrn-y := hsm.o vm.o mm.o ioreq.o ioeventfd.o +acrn-y := hsm.o vm.o mm.o ioreq.o ioeventfd.o irqfd.o diff --git a/drivers/virt/acrn/acrn_drv.h b/drivers/virt/acrn/acrn_drv.h index e3f8190bd972..1be54efa666c 100644 --- a/drivers/virt/acrn/acrn_drv.h +++ b/drivers/virt/acrn/acrn_drv.h @@ -159,6 +159,9 @@ extern rwlock_t acrn_vm_list_lock; * @ioeventfds_lock: Lock to protect ioeventfds list * @ioeventfds: List to link all hsm_ioeventfd * @ioeventfd_client: I/O client for ioeventfds of the VM + * @irqfds_lock: Lock to protect irqfds list + * @irqfds: List to link all hsm_irqfd + * @irqfd_wq: Workqueue for irqfd async shutdown */ struct acrn_vm { struct list_head list; @@ -178,6 +181,9 @@ struct acrn_vm { struct mutex ioeventfds_lock; struct list_head ioeventfds; struct acrn_ioreq_client *ioeventfd_client; + struct mutex irqfds_lock; + struct list_head irqfds; + struct workqueue_struct *irqfd_wq; }; struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, @@ -214,4 +220,8 @@ int acrn_ioeventfd_init(struct acrn_vm *vm); int acrn_ioeventfd_config(struct acrn_vm *vm, struct acrn_ioeventfd *args); void acrn_ioeventfd_deinit(struct acrn_vm *vm); +int acrn_irqfd_init(struct acrn_vm *vm); +int acrn_irqfd_config(struct acrn_vm *vm, struct acrn_irqfd *args); +void acrn_irqfd_deinit(struct acrn_vm *vm); + #endif /* __ACRN_HSM_DRV_H */ diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 0b4e88b0b6bc..93f6abe10fa6 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -115,6 +115,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, struct acrn_vm_memmap memmap; struct acrn_msi_entry *msi; struct acrn_pcidev *pcidev; + struct acrn_irqfd irqfd; struct page *page; u64 cstate_cmd; int i, ret = 0; @@ -347,6 +348,12 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, ret = acrn_ioeventfd_config(vm, &ioeventfd); break; + case ACRN_IOCTL_IRQFD: + if (copy_from_user(&irqfd, (void __user *)ioctl_param, + sizeof(irqfd))) + return -EFAULT; + ret = acrn_irqfd_config(vm, &irqfd); + break; default: dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd); ret = -ENOTTY; diff --git a/drivers/virt/acrn/irqfd.c b/drivers/virt/acrn/irqfd.c new file mode 100644 index 000000000000..a8766d528e29 --- /dev/null +++ b/drivers/virt/acrn/irqfd.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ACRN HSM irqfd: use eventfd objects to inject virtual interrupts + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + * + * Authors: + * Shuo Liu + * Yakui Zhao + */ + +#include +#include +#include +#include + +#include "acrn_drv.h" + +static LIST_HEAD(acrn_irqfd_clients); +static DEFINE_MUTEX(acrn_irqfds_mutex); + +/** + * struct hsm_irqfd - Properties of HSM irqfd + * @vm: Associated VM pointer + * @wait: Entry of wait-queue + * @shutdown: Async shutdown work + * @eventfd: Associated eventfd + * @list: Entry within &acrn_vm.irqfds of irqfds of a VM + * @pt: Structure for select/poll on the associated eventfd + * @msi: MSI data + */ +struct hsm_irqfd { + struct acrn_vm *vm; + wait_queue_entry_t wait; + struct work_struct shutdown; + struct eventfd_ctx *eventfd; + struct list_head list; + poll_table pt; + struct acrn_msi_entry msi; +}; + +static void acrn_irqfd_inject(struct hsm_irqfd *irqfd) +{ + struct acrn_vm *vm = irqfd->vm; + + acrn_msi_inject(vm, irqfd->msi.msi_addr, + irqfd->msi.msi_data); +} + +static void hsm_irqfd_shutdown(struct hsm_irqfd *irqfd) +{ + u64 cnt; + + lockdep_assert_held(&irqfd->vm->irqfds_lock); + + /* remove from wait queue */ + list_del_init(&irqfd->list); + eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); + eventfd_ctx_put(irqfd->eventfd); + kfree(irqfd); +} + +static void hsm_irqfd_shutdown_work(struct work_struct *work) +{ + struct hsm_irqfd *irqfd; + struct acrn_vm *vm; + + irqfd = container_of(work, struct hsm_irqfd, shutdown); + vm = irqfd->vm; + mutex_lock(&vm->irqfds_lock); + if (!list_empty(&irqfd->list)) + hsm_irqfd_shutdown(irqfd); + mutex_unlock(&vm->irqfds_lock); +} + +/* Called with wqh->lock held and interrupts disabled */ +static int hsm_irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, + int sync, void *key) +{ + unsigned long poll_bits = (unsigned long)key; + struct hsm_irqfd *irqfd; + struct acrn_vm *vm; + + irqfd = container_of(wait, struct hsm_irqfd, wait); + vm = irqfd->vm; + if (poll_bits & POLLIN) + /* An event has been signaled, inject an interrupt */ + acrn_irqfd_inject(irqfd); + + if (poll_bits & POLLHUP) + /* Do shutdown work in thread to hold wqh->lock */ + queue_work(vm->irqfd_wq, &irqfd->shutdown); + + return 0; +} + +static void hsm_irqfd_poll_func(struct file *file, wait_queue_head_t *wqh, + poll_table *pt) +{ + struct hsm_irqfd *irqfd; + + irqfd = container_of(pt, struct hsm_irqfd, pt); + add_wait_queue(wqh, &irqfd->wait); +} + +/* + * Assign an eventfd to a VM and create a HSM irqfd associated with the + * eventfd. The properties of the HSM irqfd are built from a &struct + * acrn_irqfd. + */ +static int acrn_irqfd_assign(struct acrn_vm *vm, struct acrn_irqfd *args) +{ + struct eventfd_ctx *eventfd = NULL; + struct hsm_irqfd *irqfd, *tmp; + unsigned int events; + struct fd f; + int ret = 0; + + irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); + if (!irqfd) + return -ENOMEM; + + irqfd->vm = vm; + memcpy(&irqfd->msi, &args->msi, sizeof(args->msi)); + INIT_LIST_HEAD(&irqfd->list); + INIT_WORK(&irqfd->shutdown, hsm_irqfd_shutdown_work); + + f = fdget(args->fd); + if (!f.file) { + ret = -EBADF; + goto out; + } + + eventfd = eventfd_ctx_fileget(f.file); + if (IS_ERR(eventfd)) { + ret = PTR_ERR(eventfd); + goto fail; + } + + irqfd->eventfd = eventfd; + + /* + * Install custom wake-up handling to be notified whenever underlying + * eventfd is signaled. + */ + init_waitqueue_func_entry(&irqfd->wait, hsm_irqfd_wakeup); + init_poll_funcptr(&irqfd->pt, hsm_irqfd_poll_func); + + mutex_lock(&vm->irqfds_lock); + list_for_each_entry(tmp, &vm->irqfds, list) { + if (irqfd->eventfd != tmp->eventfd) + continue; + ret = -EBUSY; + mutex_unlock(&vm->irqfds_lock); + goto fail; + } + list_add_tail(&irqfd->list, &vm->irqfds); + mutex_unlock(&vm->irqfds_lock); + + /* Check the pending event in this stage */ + events = f.file->f_op->poll(f.file, &irqfd->pt); + + if (events & POLLIN) + acrn_irqfd_inject(irqfd); + + fdput(f); + return 0; +fail: + if (eventfd && !IS_ERR(eventfd)) + eventfd_ctx_put(eventfd); + + fdput(f); +out: + kfree(irqfd); + return ret; +} + +static int acrn_irqfd_deassign(struct acrn_vm *vm, + struct acrn_irqfd *args) +{ + struct hsm_irqfd *irqfd, *tmp; + struct eventfd_ctx *eventfd; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + mutex_lock(&vm->irqfds_lock); + list_for_each_entry_safe(irqfd, tmp, &vm->irqfds, list) { + if (irqfd->eventfd == eventfd) { + hsm_irqfd_shutdown(irqfd); + break; + } + } + mutex_unlock(&vm->irqfds_lock); + eventfd_ctx_put(eventfd); + + return 0; +} + +int acrn_irqfd_config(struct acrn_vm *vm, struct acrn_irqfd *args) +{ + int ret; + + if (args->flags & ACRN_IRQFD_FLAG_DEASSIGN) + ret = acrn_irqfd_deassign(vm, args); + else + ret = acrn_irqfd_assign(vm, args); + + return ret; +} + +int acrn_irqfd_init(struct acrn_vm *vm) +{ + INIT_LIST_HEAD(&vm->irqfds); + mutex_init(&vm->irqfds_lock); + vm->irqfd_wq = alloc_workqueue("acrn_irqfd-%u", 0, 0, vm->vmid); + if (!vm->irqfd_wq) + return -ENOMEM; + + dev_dbg(acrn_dev.this_device, "VM %u irqfd init.\n", vm->vmid); + return 0; +} + +void acrn_irqfd_deinit(struct acrn_vm *vm) +{ + struct hsm_irqfd *irqfd, *next; + + dev_dbg(acrn_dev.this_device, "VM %u irqfd deinit.\n", vm->vmid); + destroy_workqueue(vm->irqfd_wq); + mutex_lock(&vm->irqfds_lock); + list_for_each_entry_safe(irqfd, next, &vm->irqfds, list) + hsm_irqfd_shutdown(irqfd); + mutex_unlock(&vm->irqfds_lock); +} diff --git a/drivers/virt/acrn/vm.c b/drivers/virt/acrn/vm.c index 2b322d956a8c..7804a2492ad7 100644 --- a/drivers/virt/acrn/vm.c +++ b/drivers/virt/acrn/vm.c @@ -51,6 +51,7 @@ struct acrn_vm *acrn_vm_create(struct acrn_vm *vm, write_unlock_bh(&acrn_vm_list_lock); acrn_ioeventfd_init(vm); + acrn_irqfd_init(vm); dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid); return vm; } @@ -69,7 +70,9 @@ int acrn_vm_destroy(struct acrn_vm *vm) write_unlock_bh(&acrn_vm_list_lock); acrn_ioeventfd_deinit(vm); + acrn_irqfd_deinit(vm); acrn_ioreq_deinit(vm); + if (vm->monitor_page) { put_page(vm->monitor_page); vm->monitor_page = NULL; diff --git a/include/uapi/linux/acrn.h b/include/uapi/linux/acrn.h index 132c5ff95967..353b2a2e4536 100644 --- a/include/uapi/linux/acrn.h +++ b/include/uapi/linux/acrn.h @@ -505,6 +505,19 @@ struct acrn_ioeventfd { __u64 data; }; +#define ACRN_IRQFD_FLAG_DEASSIGN 0x01 +/** + * struct acrn_irqfd - Data to operate a &struct hsm_irqfd + * @fd: The fd of eventfd associated with a hsm_irqfd + * @flags: Logical-OR of ACRN_IRQFD_FLAG_* + * @msi: Info of MSI associated with the irqfd + */ +struct acrn_irqfd { + __s32 fd; + __u32 flags; + struct acrn_msi_entry msi; +}; + /* The ioctl type, documented in ioctl-number.rst */ #define ACRN_IOCTL_TYPE 0xA2 @@ -561,5 +574,7 @@ struct acrn_ioeventfd { #define ACRN_IOCTL_IOEVENTFD \ _IOW(ACRN_IOCTL_TYPE, 0x70, struct acrn_ioeventfd) +#define ACRN_IOCTL_IRQFD \ + _IOW(ACRN_IOCTL_TYPE, 0x71, struct acrn_irqfd) #endif /* _UAPI_ACRN_H */ From 279dcf693ac76c9d16b91ffc41280babaff26bb2 Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:39 +0800 Subject: [PATCH 336/633] virt: acrn: Introduce an interface for Service VM to control vCPU ACRN supports partition mode to achieve real-time requirements. In partition mode, a CPU core can be dedicated to a vCPU of User VM. The local APIC of the dedicated CPU core can be passthrough to the User VM. The Service VM controls the assignment of the CPU cores. Introduce an interface for the Service VM to remove the control of CPU core from hypervisor perspective so that the CPU core can be a dedicated CPU core of User VM. Cc: Zhi Wang Cc: Zhenyu Wang Cc: Yu Wang Cc: Reinette Chatre Cc: Greg Kroah-Hartman Reviewed-by: Zhi Wang Reviewed-by: Reinette Chatre Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-18-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/virt/acrn/hsm.c | 48 +++++++++++++++++++++++++++++++++++ drivers/virt/acrn/hypercall.h | 14 ++++++++++ 2 files changed, 62 insertions(+) diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c index 93f6abe10fa6..1f6b7c54a1a4 100644 --- a/drivers/virt/acrn/hsm.c +++ b/drivers/virt/acrn/hsm.c @@ -9,6 +9,7 @@ * Yakui Zhao */ +#include #include #include #include @@ -371,6 +372,52 @@ static int acrn_dev_release(struct inode *inode, struct file *filp) return 0; } +static ssize_t remove_cpu_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + u64 cpu, lapicid; + int ret; + + if (kstrtoull(buf, 0, &cpu) < 0) + return -EINVAL; + + if (cpu >= num_possible_cpus() || cpu == 0 || !cpu_is_hotpluggable(cpu)) + return -EINVAL; + + if (cpu_online(cpu)) + remove_cpu(cpu); + + lapicid = cpu_data(cpu).apicid; + dev_dbg(dev, "Try to remove cpu %lld with lapicid %lld\n", cpu, lapicid); + ret = hcall_sos_remove_cpu(lapicid); + if (ret < 0) { + dev_err(dev, "Failed to remove cpu %lld!\n", cpu); + goto fail_remove; + } + + return count; + +fail_remove: + add_cpu(cpu); + return ret; +} +static DEVICE_ATTR_WO(remove_cpu); + +static struct attribute *acrn_attrs[] = { + &dev_attr_remove_cpu.attr, + NULL +}; + +static struct attribute_group acrn_attr_group = { + .attrs = acrn_attrs, +}; + +static const struct attribute_group *acrn_attr_groups[] = { + &acrn_attr_group, + NULL +}; + static const struct file_operations acrn_fops = { .owner = THIS_MODULE, .open = acrn_dev_open, @@ -382,6 +429,7 @@ struct miscdevice acrn_dev = { .minor = MISC_DYNAMIC_MINOR, .name = "acrn_hsm", .fops = &acrn_fops, + .groups = acrn_attr_groups, }; static int __init hsm_init(void) diff --git a/drivers/virt/acrn/hypercall.h b/drivers/virt/acrn/hypercall.h index e640632366f0..0cfad05bd1a9 100644 --- a/drivers/virt/acrn/hypercall.h +++ b/drivers/virt/acrn/hypercall.h @@ -13,6 +13,9 @@ #define HC_ID 0x80UL +#define HC_ID_GEN_BASE 0x0UL +#define HC_SOS_REMOVE_CPU _HC_ID(HC_ID, HC_ID_GEN_BASE + 0x01) + #define HC_ID_VM_BASE 0x10UL #define HC_CREATE_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x00) #define HC_DESTROY_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x01) @@ -42,6 +45,17 @@ #define HC_ID_PM_BASE 0x80UL #define HC_PM_GET_CPU_STATE _HC_ID(HC_ID, HC_ID_PM_BASE + 0x00) +/** + * hcall_sos_remove_cpu() - Remove a vCPU of Service VM + * @cpu: The vCPU to be removed + * + * Return: 0 on success, <0 on failure + */ +static inline long hcall_sos_remove_cpu(u64 cpu) +{ + return acrn_hypercall1(HC_SOS_REMOVE_CPU, cpu); +} + /** * hcall_create_vm() - Create a User VM * @vminfo: Service VM GPA of info of User VM creation From 5b06931d7f8b8059bec8563b2e8d531218e03e2f Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Sun, 7 Feb 2021 11:10:40 +0800 Subject: [PATCH 337/633] sample/acrn: Introduce a sample of HSM ioctl interface usage Launch a simple guest (with several instructions as payload) on ACRN with demonstration ioctl usage. Signed-off-by: Shuo Liu Link: https://lore.kernel.org/r/20210207031040.49576-19-shuo.a.liu@intel.com Signed-off-by: Greg Kroah-Hartman --- samples/acrn/Makefile | 12 ++++ samples/acrn/guest.ld | 9 +++ samples/acrn/payload.ld | 9 +++ samples/acrn/vm-sample.c | 136 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 samples/acrn/Makefile create mode 100644 samples/acrn/guest.ld create mode 100644 samples/acrn/payload.ld create mode 100644 samples/acrn/vm-sample.c diff --git a/samples/acrn/Makefile b/samples/acrn/Makefile new file mode 100644 index 000000000000..c8e3ed9785e9 --- /dev/null +++ b/samples/acrn/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +.PHONY: vm-sample + +vm-sample: vm-sample.o payload.o + $(CC) $^ -o $@ + +payload.o: payload.ld guest16.o + $(LD) -T $< -o $@ + +clean: + rm *.o vm-sample diff --git a/samples/acrn/guest.ld b/samples/acrn/guest.ld new file mode 100644 index 000000000000..5127c682bd22 --- /dev/null +++ b/samples/acrn/guest.ld @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +OUTPUT_FORMAT(binary) +SECTIONS +{ + .start : { *(.start) } + .text : { *(.text*) } + .rodata : { *(.rodata) } + .data : { *(.data) } +} diff --git a/samples/acrn/payload.ld b/samples/acrn/payload.ld new file mode 100644 index 000000000000..e8d9a498ad62 --- /dev/null +++ b/samples/acrn/payload.ld @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +SECTIONS +{ + .payload16 0 : { + guest16 = .; + guest16.o(.text) + guest16_end = .; + } +} diff --git a/samples/acrn/vm-sample.c b/samples/acrn/vm-sample.c new file mode 100644 index 000000000000..b2dad47a77a0 --- /dev/null +++ b/samples/acrn/vm-sample.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A sample program to run a User VM on the ACRN hypervisor + * + * This sample runs in a Service VM, which is a privileged VM of ACRN. + * CONFIG_ACRN_HSM need to be enabled in the Service VM. + * + * Guest VM code in guest16.s will be executed after the VM launched. + * + * Copyright (C) 2020 Intel Corporation. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define GUEST_MEMORY_SIZE (1024*1024) +void *guest_memory; + +extern const unsigned char guest16[], guest16_end[]; +static char io_request_page[4096] __attribute__((aligned(4096))); +static struct acrn_io_request *io_req_buf = (struct acrn_io_request *)io_request_page; + +__u16 vcpu_num; +__u16 vmid; +/* POST_STANDARD_VM_UUID1, refer to https://github.com/projectacrn/acrn-hypervisor/blob/master/hypervisor/include/common/vm_uuids.h */ +guid_t vm_uuid = GUID_INIT(0x385479d2, 0xd625, 0xe811, 0x86, 0x4e, 0xcb, 0x7a, 0x18, 0xb3, 0x46, 0x43); + +int hsm_fd; +int is_running = 1; + +void vm_exit(int sig) +{ + sig = sig; + + is_running = 0; + ioctl(hsm_fd, ACRN_IOCTL_PAUSE_VM, vmid); + ioctl(hsm_fd, ACRN_IOCTL_DESTROY_IOREQ_CLIENT, 0); +} + +int main(int argc, char **argv) +{ + int vcpu_id, ret; + struct acrn_vm_creation create_vm = {0}; + struct acrn_vm_memmap ram_map = {0}; + struct acrn_vcpu_regs regs; + struct acrn_io_request *io_req; + struct acrn_ioreq_notify __attribute__((aligned(8))) notify; + + argc = argc; + argv = argv; + + guest_memory = memalign(4096, GUEST_MEMORY_SIZE); + if (!guest_memory) { + printf("No enough memory!\n"); + return -1; + } + hsm_fd = open("/dev/acrn_hsm", O_RDWR|O_CLOEXEC); + + memcpy(&create_vm.uuid, &vm_uuid, 16); + create_vm.ioreq_buf = (__u64)io_req_buf; + ret = ioctl(hsm_fd, ACRN_IOCTL_CREATE_VM, &create_vm); + printf("Created VM! [%d]\n", ret); + vcpu_num = create_vm.vcpu_num; + vmid = create_vm.vmid; + + /* setup guest memory */ + ram_map.type = ACRN_MEMMAP_RAM; + ram_map.vma_base = (__u64)guest_memory; + ram_map.len = GUEST_MEMORY_SIZE; + ram_map.user_vm_pa = 0; + ram_map.attr = ACRN_MEM_ACCESS_RWX; + ret = ioctl(hsm_fd, ACRN_IOCTL_SET_MEMSEG, &ram_map); + printf("Set up VM memory! [%d]\n", ret); + + memcpy(guest_memory, guest16, guest16_end-guest16); + + /* setup vcpu registers */ + memset(®s, 0, sizeof(regs)); + regs.vcpu_id = 0; + regs.vcpu_regs.rip = 0; + + /* CR0_ET | CR0_NE */ + regs.vcpu_regs.cr0 = 0x30U; + regs.vcpu_regs.cs_ar = 0x009FU; + regs.vcpu_regs.cs_sel = 0xF000U; + regs.vcpu_regs.cs_limit = 0xFFFFU; + regs.vcpu_regs.cs_base = 0 & 0xFFFF0000UL; + regs.vcpu_regs.rip = 0 & 0xFFFFUL; + + ret = ioctl(hsm_fd, ACRN_IOCTL_SET_VCPU_REGS, ®s); + printf("Set up VM BSP registers! [%d]\n", ret); + + /* create an ioreq client for this VM */ + ret = ioctl(hsm_fd, ACRN_IOCTL_CREATE_IOREQ_CLIENT, 0); + printf("Created IO request client! [%d]\n", ret); + + /* run vm */ + ret = ioctl(hsm_fd, ACRN_IOCTL_START_VM, vmid); + printf("Start VM! [%d]\n", ret); + + signal(SIGINT, vm_exit); + while (is_running) { + ret = ioctl(hsm_fd, ACRN_IOCTL_ATTACH_IOREQ_CLIENT, 0); + + for (vcpu_id = 0; vcpu_id < vcpu_num; vcpu_id++) { + io_req = &io_req_buf[vcpu_id]; + if ((__sync_add_and_fetch(&io_req->processed, 0) == ACRN_IOREQ_STATE_PROCESSING) + && (!io_req->kernel_handled)) + if (io_req->type == ACRN_IOREQ_TYPE_PORTIO) { + int bytes, port, in; + + port = io_req->reqs.pio_request.address; + bytes = io_req->reqs.pio_request.size; + in = (io_req->reqs.pio_request.direction == ACRN_IOREQ_DIR_READ); + printf("Guest VM %s PIO[%x] with size[%x]\n", in ? "read" : "write", port, bytes); + + notify.vmid = vmid; + notify.vcpu = vcpu_id; + ioctl(hsm_fd, ACRN_IOCTL_NOTIFY_REQUEST_FINISH, ¬ify); + } + } + } + + ret = ioctl(hsm_fd, ACRN_IOCTL_DESTROY_VM, NULL); + printf("Destroy VM! [%d]\n", ret); + close(hsm_fd); + free(guest_memory); + return 0; +} From 1077d4367ab3b97f6db2f66c87289af863652215 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Wed, 3 Feb 2021 15:13:50 +0100 Subject: [PATCH 338/633] firmware: xilinx: Use explicit values for all enum values Based on discussion at https://lore.kernel.org/r/20200318125003.GA2727094@kroah.com we got recommendation to use explicit values for all enum values. The patch is following this recommendation. Signed-off-by: Michal Simek Link: https://lore.kernel.org/r/daeb67ded45d8a8f6a96717d1fb9c84439dd2ae8.1612361627.git.michal.simek@xilinx.com Signed-off-by: Greg Kroah-Hartman --- include/linux/firmware/xlnx-zynqmp.h | 338 +++++++++++++-------------- 1 file changed, 169 insertions(+), 169 deletions(-) diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 0a483249ff30..71177b17eee5 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -64,25 +64,25 @@ enum pm_api_id { PM_GET_API_VERSION = 1, PM_SYSTEM_SHUTDOWN = 12, PM_REQUEST_NODE = 13, - PM_RELEASE_NODE, - PM_SET_REQUIREMENT, + PM_RELEASE_NODE = 14, + PM_SET_REQUIREMENT = 15, PM_RESET_ASSERT = 17, - PM_RESET_GET_STATUS, + PM_RESET_GET_STATUS = 18, PM_PM_INIT_FINALIZE = 21, - PM_FPGA_LOAD, - PM_FPGA_GET_STATUS, + PM_FPGA_LOAD = 22, + PM_FPGA_GET_STATUS = 23, PM_GET_CHIPID = 24, PM_IOCTL = 34, - PM_QUERY_DATA, - PM_CLOCK_ENABLE, - PM_CLOCK_DISABLE, - PM_CLOCK_GETSTATE, - PM_CLOCK_SETDIVIDER, - PM_CLOCK_GETDIVIDER, - PM_CLOCK_SETRATE, - PM_CLOCK_GETRATE, - PM_CLOCK_SETPARENT, - PM_CLOCK_GETPARENT, + PM_QUERY_DATA = 35, + PM_CLOCK_ENABLE = 36, + PM_CLOCK_DISABLE = 37, + PM_CLOCK_GETSTATE = 38, + PM_CLOCK_SETDIVIDER = 39, + PM_CLOCK_GETDIVIDER = 40, + PM_CLOCK_SETRATE = 41, + PM_CLOCK_GETRATE = 42, + PM_CLOCK_SETPARENT = 43, + PM_CLOCK_GETPARENT = 44, PM_SECURE_AES = 47, PM_FEATURE_CHECK = 63, }; @@ -92,21 +92,21 @@ enum pm_ret_status { XST_PM_SUCCESS = 0, XST_PM_NO_FEATURE = 19, XST_PM_INTERNAL = 2000, - XST_PM_CONFLICT, - XST_PM_NO_ACCESS, - XST_PM_INVALID_NODE, - XST_PM_DOUBLE_REQ, - XST_PM_ABORT_SUSPEND, + XST_PM_CONFLICT = 2001, + XST_PM_NO_ACCESS = 2002, + XST_PM_INVALID_NODE = 2003, + XST_PM_DOUBLE_REQ = 2004, + XST_PM_ABORT_SUSPEND = 2005, XST_PM_MULT_USER = 2008, }; enum pm_ioctl_id { IOCTL_SD_DLL_RESET = 6, - IOCTL_SET_SD_TAPDELAY, - IOCTL_SET_PLL_FRAC_MODE, - IOCTL_GET_PLL_FRAC_MODE, - IOCTL_SET_PLL_FRAC_DATA, - IOCTL_GET_PLL_FRAC_DATA, + IOCTL_SET_SD_TAPDELAY = 7, + IOCTL_SET_PLL_FRAC_MODE = 8, + IOCTL_GET_PLL_FRAC_MODE = 9, + IOCTL_SET_PLL_FRAC_DATA = 10, + IOCTL_GET_PLL_FRAC_DATA = 11, IOCTL_WRITE_GGS = 12, IOCTL_READ_GGS = 13, IOCTL_WRITE_PGGS = 14, @@ -116,185 +116,185 @@ enum pm_ioctl_id { }; enum pm_query_id { - PM_QID_INVALID, - PM_QID_CLOCK_GET_NAME, - PM_QID_CLOCK_GET_TOPOLOGY, - PM_QID_CLOCK_GET_FIXEDFACTOR_PARAMS, - PM_QID_CLOCK_GET_PARENTS, - PM_QID_CLOCK_GET_ATTRIBUTES, + PM_QID_INVALID = 0, + PM_QID_CLOCK_GET_NAME = 1, + PM_QID_CLOCK_GET_TOPOLOGY = 2, + PM_QID_CLOCK_GET_FIXEDFACTOR_PARAMS = 3, + PM_QID_CLOCK_GET_PARENTS = 4, + PM_QID_CLOCK_GET_ATTRIBUTES = 5, PM_QID_CLOCK_GET_NUM_CLOCKS = 12, - PM_QID_CLOCK_GET_MAX_DIVISOR, + PM_QID_CLOCK_GET_MAX_DIVISOR = 13, }; enum zynqmp_pm_reset_action { - PM_RESET_ACTION_RELEASE, - PM_RESET_ACTION_ASSERT, - PM_RESET_ACTION_PULSE, + PM_RESET_ACTION_RELEASE = 0, + PM_RESET_ACTION_ASSERT = 1, + PM_RESET_ACTION_PULSE = 2, }; enum zynqmp_pm_reset { ZYNQMP_PM_RESET_START = 1000, ZYNQMP_PM_RESET_PCIE_CFG = ZYNQMP_PM_RESET_START, - ZYNQMP_PM_RESET_PCIE_BRIDGE, - ZYNQMP_PM_RESET_PCIE_CTRL, - ZYNQMP_PM_RESET_DP, - ZYNQMP_PM_RESET_SWDT_CRF, - ZYNQMP_PM_RESET_AFI_FM5, - ZYNQMP_PM_RESET_AFI_FM4, - ZYNQMP_PM_RESET_AFI_FM3, - ZYNQMP_PM_RESET_AFI_FM2, - ZYNQMP_PM_RESET_AFI_FM1, - ZYNQMP_PM_RESET_AFI_FM0, - ZYNQMP_PM_RESET_GDMA, - ZYNQMP_PM_RESET_GPU_PP1, - ZYNQMP_PM_RESET_GPU_PP0, - ZYNQMP_PM_RESET_GPU, - ZYNQMP_PM_RESET_GT, - ZYNQMP_PM_RESET_SATA, - ZYNQMP_PM_RESET_ACPU3_PWRON, - ZYNQMP_PM_RESET_ACPU2_PWRON, - ZYNQMP_PM_RESET_ACPU1_PWRON, - ZYNQMP_PM_RESET_ACPU0_PWRON, - ZYNQMP_PM_RESET_APU_L2, - ZYNQMP_PM_RESET_ACPU3, - ZYNQMP_PM_RESET_ACPU2, - ZYNQMP_PM_RESET_ACPU1, - ZYNQMP_PM_RESET_ACPU0, - ZYNQMP_PM_RESET_DDR, - ZYNQMP_PM_RESET_APM_FPD, - ZYNQMP_PM_RESET_SOFT, - ZYNQMP_PM_RESET_GEM0, - ZYNQMP_PM_RESET_GEM1, - ZYNQMP_PM_RESET_GEM2, - ZYNQMP_PM_RESET_GEM3, - ZYNQMP_PM_RESET_QSPI, - ZYNQMP_PM_RESET_UART0, - ZYNQMP_PM_RESET_UART1, - ZYNQMP_PM_RESET_SPI0, - ZYNQMP_PM_RESET_SPI1, - ZYNQMP_PM_RESET_SDIO0, - ZYNQMP_PM_RESET_SDIO1, - ZYNQMP_PM_RESET_CAN0, - ZYNQMP_PM_RESET_CAN1, - ZYNQMP_PM_RESET_I2C0, - ZYNQMP_PM_RESET_I2C1, - ZYNQMP_PM_RESET_TTC0, - ZYNQMP_PM_RESET_TTC1, - ZYNQMP_PM_RESET_TTC2, - ZYNQMP_PM_RESET_TTC3, - ZYNQMP_PM_RESET_SWDT_CRL, - ZYNQMP_PM_RESET_NAND, - ZYNQMP_PM_RESET_ADMA, - ZYNQMP_PM_RESET_GPIO, - ZYNQMP_PM_RESET_IOU_CC, - ZYNQMP_PM_RESET_TIMESTAMP, - ZYNQMP_PM_RESET_RPU_R50, - ZYNQMP_PM_RESET_RPU_R51, - ZYNQMP_PM_RESET_RPU_AMBA, - ZYNQMP_PM_RESET_OCM, - ZYNQMP_PM_RESET_RPU_PGE, - ZYNQMP_PM_RESET_USB0_CORERESET, - ZYNQMP_PM_RESET_USB1_CORERESET, - ZYNQMP_PM_RESET_USB0_HIBERRESET, - ZYNQMP_PM_RESET_USB1_HIBERRESET, - ZYNQMP_PM_RESET_USB0_APB, - ZYNQMP_PM_RESET_USB1_APB, - ZYNQMP_PM_RESET_IPI, - ZYNQMP_PM_RESET_APM_LPD, - ZYNQMP_PM_RESET_RTC, - ZYNQMP_PM_RESET_SYSMON, - ZYNQMP_PM_RESET_AFI_FM6, - ZYNQMP_PM_RESET_LPD_SWDT, - ZYNQMP_PM_RESET_FPD, - ZYNQMP_PM_RESET_RPU_DBG1, - ZYNQMP_PM_RESET_RPU_DBG0, - ZYNQMP_PM_RESET_DBG_LPD, - ZYNQMP_PM_RESET_DBG_FPD, - ZYNQMP_PM_RESET_APLL, - ZYNQMP_PM_RESET_DPLL, - ZYNQMP_PM_RESET_VPLL, - ZYNQMP_PM_RESET_IOPLL, - ZYNQMP_PM_RESET_RPLL, - ZYNQMP_PM_RESET_GPO3_PL_0, - ZYNQMP_PM_RESET_GPO3_PL_1, - ZYNQMP_PM_RESET_GPO3_PL_2, - ZYNQMP_PM_RESET_GPO3_PL_3, - ZYNQMP_PM_RESET_GPO3_PL_4, - ZYNQMP_PM_RESET_GPO3_PL_5, - ZYNQMP_PM_RESET_GPO3_PL_6, - ZYNQMP_PM_RESET_GPO3_PL_7, - ZYNQMP_PM_RESET_GPO3_PL_8, - ZYNQMP_PM_RESET_GPO3_PL_9, - ZYNQMP_PM_RESET_GPO3_PL_10, - ZYNQMP_PM_RESET_GPO3_PL_11, - ZYNQMP_PM_RESET_GPO3_PL_12, - ZYNQMP_PM_RESET_GPO3_PL_13, - ZYNQMP_PM_RESET_GPO3_PL_14, - ZYNQMP_PM_RESET_GPO3_PL_15, - ZYNQMP_PM_RESET_GPO3_PL_16, - ZYNQMP_PM_RESET_GPO3_PL_17, - ZYNQMP_PM_RESET_GPO3_PL_18, - ZYNQMP_PM_RESET_GPO3_PL_19, - ZYNQMP_PM_RESET_GPO3_PL_20, - ZYNQMP_PM_RESET_GPO3_PL_21, - ZYNQMP_PM_RESET_GPO3_PL_22, - ZYNQMP_PM_RESET_GPO3_PL_23, - ZYNQMP_PM_RESET_GPO3_PL_24, - ZYNQMP_PM_RESET_GPO3_PL_25, - ZYNQMP_PM_RESET_GPO3_PL_26, - ZYNQMP_PM_RESET_GPO3_PL_27, - ZYNQMP_PM_RESET_GPO3_PL_28, - ZYNQMP_PM_RESET_GPO3_PL_29, - ZYNQMP_PM_RESET_GPO3_PL_30, - ZYNQMP_PM_RESET_GPO3_PL_31, - ZYNQMP_PM_RESET_RPU_LS, - ZYNQMP_PM_RESET_PS_ONLY, - ZYNQMP_PM_RESET_PL, - ZYNQMP_PM_RESET_PS_PL0, - ZYNQMP_PM_RESET_PS_PL1, - ZYNQMP_PM_RESET_PS_PL2, - ZYNQMP_PM_RESET_PS_PL3, + ZYNQMP_PM_RESET_PCIE_BRIDGE = 1001, + ZYNQMP_PM_RESET_PCIE_CTRL = 1002, + ZYNQMP_PM_RESET_DP = 1003, + ZYNQMP_PM_RESET_SWDT_CRF = 1004, + ZYNQMP_PM_RESET_AFI_FM5 = 1005, + ZYNQMP_PM_RESET_AFI_FM4 = 1006, + ZYNQMP_PM_RESET_AFI_FM3 = 1007, + ZYNQMP_PM_RESET_AFI_FM2 = 1008, + ZYNQMP_PM_RESET_AFI_FM1 = 1009, + ZYNQMP_PM_RESET_AFI_FM0 = 1010, + ZYNQMP_PM_RESET_GDMA = 1011, + ZYNQMP_PM_RESET_GPU_PP1 = 1012, + ZYNQMP_PM_RESET_GPU_PP0 = 1013, + ZYNQMP_PM_RESET_GPU = 1014, + ZYNQMP_PM_RESET_GT = 1015, + ZYNQMP_PM_RESET_SATA = 1016, + ZYNQMP_PM_RESET_ACPU3_PWRON = 1017, + ZYNQMP_PM_RESET_ACPU2_PWRON = 1018, + ZYNQMP_PM_RESET_ACPU1_PWRON = 1019, + ZYNQMP_PM_RESET_ACPU0_PWRON = 1020, + ZYNQMP_PM_RESET_APU_L2 = 1021, + ZYNQMP_PM_RESET_ACPU3 = 1022, + ZYNQMP_PM_RESET_ACPU2 = 1023, + ZYNQMP_PM_RESET_ACPU1 = 1024, + ZYNQMP_PM_RESET_ACPU0 = 1025, + ZYNQMP_PM_RESET_DDR = 1026, + ZYNQMP_PM_RESET_APM_FPD = 1027, + ZYNQMP_PM_RESET_SOFT = 1028, + ZYNQMP_PM_RESET_GEM0 = 1029, + ZYNQMP_PM_RESET_GEM1 = 1030, + ZYNQMP_PM_RESET_GEM2 = 1031, + ZYNQMP_PM_RESET_GEM3 = 1032, + ZYNQMP_PM_RESET_QSPI = 1033, + ZYNQMP_PM_RESET_UART0 = 1034, + ZYNQMP_PM_RESET_UART1 = 1035, + ZYNQMP_PM_RESET_SPI0 = 1036, + ZYNQMP_PM_RESET_SPI1 = 1037, + ZYNQMP_PM_RESET_SDIO0 = 1038, + ZYNQMP_PM_RESET_SDIO1 = 1039, + ZYNQMP_PM_RESET_CAN0 = 1040, + ZYNQMP_PM_RESET_CAN1 = 1041, + ZYNQMP_PM_RESET_I2C0 = 1042, + ZYNQMP_PM_RESET_I2C1 = 1043, + ZYNQMP_PM_RESET_TTC0 = 1044, + ZYNQMP_PM_RESET_TTC1 = 1045, + ZYNQMP_PM_RESET_TTC2 = 1046, + ZYNQMP_PM_RESET_TTC3 = 1047, + ZYNQMP_PM_RESET_SWDT_CRL = 1048, + ZYNQMP_PM_RESET_NAND = 1049, + ZYNQMP_PM_RESET_ADMA = 1050, + ZYNQMP_PM_RESET_GPIO = 1051, + ZYNQMP_PM_RESET_IOU_CC = 1052, + ZYNQMP_PM_RESET_TIMESTAMP = 1053, + ZYNQMP_PM_RESET_RPU_R50 = 1054, + ZYNQMP_PM_RESET_RPU_R51 = 1055, + ZYNQMP_PM_RESET_RPU_AMBA = 1056, + ZYNQMP_PM_RESET_OCM = 1057, + ZYNQMP_PM_RESET_RPU_PGE = 1058, + ZYNQMP_PM_RESET_USB0_CORERESET = 1059, + ZYNQMP_PM_RESET_USB1_CORERESET = 1060, + ZYNQMP_PM_RESET_USB0_HIBERRESET = 1061, + ZYNQMP_PM_RESET_USB1_HIBERRESET = 1062, + ZYNQMP_PM_RESET_USB0_APB = 1063, + ZYNQMP_PM_RESET_USB1_APB = 1064, + ZYNQMP_PM_RESET_IPI = 1065, + ZYNQMP_PM_RESET_APM_LPD = 1066, + ZYNQMP_PM_RESET_RTC = 1067, + ZYNQMP_PM_RESET_SYSMON = 1068, + ZYNQMP_PM_RESET_AFI_FM6 = 1069, + ZYNQMP_PM_RESET_LPD_SWDT = 1070, + ZYNQMP_PM_RESET_FPD = 1071, + ZYNQMP_PM_RESET_RPU_DBG1 = 1072, + ZYNQMP_PM_RESET_RPU_DBG0 = 1073, + ZYNQMP_PM_RESET_DBG_LPD = 1074, + ZYNQMP_PM_RESET_DBG_FPD = 1075, + ZYNQMP_PM_RESET_APLL = 1076, + ZYNQMP_PM_RESET_DPLL = 1077, + ZYNQMP_PM_RESET_VPLL = 1078, + ZYNQMP_PM_RESET_IOPLL = 1079, + ZYNQMP_PM_RESET_RPLL = 1080, + ZYNQMP_PM_RESET_GPO3_PL_0 = 1081, + ZYNQMP_PM_RESET_GPO3_PL_1 = 1082, + ZYNQMP_PM_RESET_GPO3_PL_2 = 1083, + ZYNQMP_PM_RESET_GPO3_PL_3 = 1084, + ZYNQMP_PM_RESET_GPO3_PL_4 = 1085, + ZYNQMP_PM_RESET_GPO3_PL_5 = 1086, + ZYNQMP_PM_RESET_GPO3_PL_6 = 1087, + ZYNQMP_PM_RESET_GPO3_PL_7 = 1088, + ZYNQMP_PM_RESET_GPO3_PL_8 = 1089, + ZYNQMP_PM_RESET_GPO3_PL_9 = 1090, + ZYNQMP_PM_RESET_GPO3_PL_10 = 1091, + ZYNQMP_PM_RESET_GPO3_PL_11 = 1092, + ZYNQMP_PM_RESET_GPO3_PL_12 = 1093, + ZYNQMP_PM_RESET_GPO3_PL_13 = 1094, + ZYNQMP_PM_RESET_GPO3_PL_14 = 1095, + ZYNQMP_PM_RESET_GPO3_PL_15 = 1096, + ZYNQMP_PM_RESET_GPO3_PL_16 = 1097, + ZYNQMP_PM_RESET_GPO3_PL_17 = 1098, + ZYNQMP_PM_RESET_GPO3_PL_18 = 1099, + ZYNQMP_PM_RESET_GPO3_PL_19 = 1100, + ZYNQMP_PM_RESET_GPO3_PL_20 = 1101, + ZYNQMP_PM_RESET_GPO3_PL_21 = 1102, + ZYNQMP_PM_RESET_GPO3_PL_22 = 1103, + ZYNQMP_PM_RESET_GPO3_PL_23 = 1104, + ZYNQMP_PM_RESET_GPO3_PL_24 = 1105, + ZYNQMP_PM_RESET_GPO3_PL_25 = 1106, + ZYNQMP_PM_RESET_GPO3_PL_26 = 1107, + ZYNQMP_PM_RESET_GPO3_PL_27 = 1108, + ZYNQMP_PM_RESET_GPO3_PL_28 = 1109, + ZYNQMP_PM_RESET_GPO3_PL_29 = 1110, + ZYNQMP_PM_RESET_GPO3_PL_30 = 1111, + ZYNQMP_PM_RESET_GPO3_PL_31 = 1112, + ZYNQMP_PM_RESET_RPU_LS = 1113, + ZYNQMP_PM_RESET_PS_ONLY = 1114, + ZYNQMP_PM_RESET_PL = 1115, + ZYNQMP_PM_RESET_PS_PL0 = 1116, + ZYNQMP_PM_RESET_PS_PL1 = 1117, + ZYNQMP_PM_RESET_PS_PL2 = 1118, + ZYNQMP_PM_RESET_PS_PL3 = 1119, ZYNQMP_PM_RESET_END = ZYNQMP_PM_RESET_PS_PL3 }; enum zynqmp_pm_suspend_reason { SUSPEND_POWER_REQUEST = 201, - SUSPEND_ALERT, - SUSPEND_SYSTEM_SHUTDOWN, + SUSPEND_ALERT = 202, + SUSPEND_SYSTEM_SHUTDOWN = 203, }; enum zynqmp_pm_request_ack { ZYNQMP_PM_REQUEST_ACK_NO = 1, - ZYNQMP_PM_REQUEST_ACK_BLOCKING, - ZYNQMP_PM_REQUEST_ACK_NON_BLOCKING, + ZYNQMP_PM_REQUEST_ACK_BLOCKING = 2, + ZYNQMP_PM_REQUEST_ACK_NON_BLOCKING = 3, }; enum pm_node_id { NODE_SD_0 = 39, - NODE_SD_1, + NODE_SD_1 = 40, }; enum tap_delay_type { PM_TAPDELAY_INPUT = 0, - PM_TAPDELAY_OUTPUT, + PM_TAPDELAY_OUTPUT = 1, }; enum dll_reset_type { - PM_DLL_RESET_ASSERT, - PM_DLL_RESET_RELEASE, - PM_DLL_RESET_PULSE, + PM_DLL_RESET_ASSERT = 0, + PM_DLL_RESET_RELEASE = 1, + PM_DLL_RESET_PULSE = 2, }; enum zynqmp_pm_shutdown_type { - ZYNQMP_PM_SHUTDOWN_TYPE_SHUTDOWN, - ZYNQMP_PM_SHUTDOWN_TYPE_RESET, - ZYNQMP_PM_SHUTDOWN_TYPE_SETSCOPE_ONLY, + ZYNQMP_PM_SHUTDOWN_TYPE_SHUTDOWN = 0, + ZYNQMP_PM_SHUTDOWN_TYPE_RESET = 1, + ZYNQMP_PM_SHUTDOWN_TYPE_SETSCOPE_ONLY = 2, }; enum zynqmp_pm_shutdown_subtype { - ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM, - ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY, - ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM, + ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM = 0, + ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY = 1, + ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM = 2, }; /** From 5f6805327982d1fd45355730e9d1adda616b995b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 26 Jan 2021 22:53:39 +0100 Subject: [PATCH 339/633] firmware: google: make coreboot driver's remove callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All coreboot drivers return 0 unconditionally in their remove callback. Also the device core ignores the return value of the struct bus_type::remove(), so make the coreboot remove callback return void instead of giving driver authors the illusion they could return an error code here. All drivers are adapted accordingly. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210126215339.706021-1-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/google/coreboot_table.c | 5 ++--- drivers/firmware/google/coreboot_table.h | 2 +- drivers/firmware/google/framebuffer-coreboot.c | 4 +--- drivers/firmware/google/memconsole-coreboot.c | 4 +--- drivers/firmware/google/vpd.c | 4 +--- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 0205987a4fd4..dc83ea118c67 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -46,14 +46,13 @@ static int coreboot_bus_probe(struct device *dev) static int coreboot_bus_remove(struct device *dev) { - int ret = 0; struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(dev->driver); if (driver->remove) - ret = driver->remove(device); + driver->remove(device); - return ret; + return 0; } static struct bus_type coreboot_bus_type = { diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index 7b7b4a6eedda..beb778674acd 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -72,7 +72,7 @@ struct coreboot_device { /* A driver for handling devices described in coreboot tables. */ struct coreboot_driver { int (*probe)(struct coreboot_device *); - int (*remove)(struct coreboot_device *); + void (*remove)(struct coreboot_device *); struct device_driver drv; u32 tag; }; diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 916f26adc595..c6dcc1ef93ac 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -72,13 +72,11 @@ static int framebuffer_probe(struct coreboot_device *dev) return PTR_ERR_OR_ZERO(pdev); } -static int framebuffer_remove(struct coreboot_device *dev) +static void framebuffer_remove(struct coreboot_device *dev) { struct platform_device *pdev = dev_get_drvdata(&dev->dev); platform_device_unregister(pdev); - - return 0; } static struct coreboot_driver framebuffer_driver = { diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index d17e4d6ac9bc..74b5286518ee 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -91,11 +91,9 @@ static int memconsole_probe(struct coreboot_device *dev) return memconsole_sysfs_init(); } -static int memconsole_remove(struct coreboot_device *dev) +static void memconsole_remove(struct coreboot_device *dev) { memconsole_exit(); - - return 0; } static struct coreboot_driver memconsole_driver = { diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index d23c5c69ab52..ee6e08c0592b 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -298,14 +298,12 @@ static int vpd_probe(struct coreboot_device *dev) return 0; } -static int vpd_remove(struct coreboot_device *dev) +static void vpd_remove(struct coreboot_device *dev) { vpd_section_destroy(&ro_vpd); vpd_section_destroy(&rw_vpd); kobject_put(vpd_kobj); - - return 0; } static struct coreboot_driver vpd_driver = { From 2adc75fba3289455b9c4349dd6b95cfb7167b7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 27 Jan 2021 22:23:29 +0100 Subject: [PATCH 340/633] vme: make remove callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver core ignores the return value of struct bus_type::remove() because there is only little that can be done. To simplify the quest to make this function return void, let struct vme_driver::remove return void, too. There is only a single vme driver and it already returns 0 unconditionally in .remove(). Also fix the bus remove function to always return 0. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210127212329.98517-1-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vme/devices/vme_user.c | 4 +--- drivers/vme/vme.c | 4 ++-- include/linux/vme.h | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c index fd0ea4dbcb91..35d7260e2271 100644 --- a/drivers/staging/vme/devices/vme_user.c +++ b/drivers/staging/vme/devices/vme_user.c @@ -689,7 +689,7 @@ static int vme_user_probe(struct vme_dev *vdev) return err; } -static int vme_user_remove(struct vme_dev *dev) +static void vme_user_remove(struct vme_dev *dev) { int i; @@ -717,8 +717,6 @@ static int vme_user_remove(struct vme_dev *dev) /* Unregister the major and minor device numbers */ unregister_chrdev_region(MKDEV(VME_MAJOR, 0), VME_DEVS); - - return 0; } static struct vme_driver vme_user_driver = { diff --git a/drivers/vme/vme.c b/drivers/vme/vme.c index 54d7963c1078..1b15afea28ee 100644 --- a/drivers/vme/vme.c +++ b/drivers/vme/vme.c @@ -1997,9 +1997,9 @@ static int vme_bus_remove(struct device *dev) driver = dev->platform_data; if (driver->remove) - return driver->remove(vdev); + driver->remove(vdev); - return -ENODEV; + return 0; } struct bus_type vme_bus_type = { diff --git a/include/linux/vme.h b/include/linux/vme.h index 7e82bf500f01..b204a9b4be1b 100644 --- a/include/linux/vme.h +++ b/include/linux/vme.h @@ -122,7 +122,7 @@ struct vme_driver { const char *name; int (*match)(struct vme_dev *); int (*probe)(struct vme_dev *); - int (*remove)(struct vme_dev *); + void (*remove)(struct vme_dev *); struct device_driver driver; struct list_head devices; }; From b2c852f490e086e0683d929d8eb75ccae068bd7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Tue, 26 Jan 2021 23:14:44 +0100 Subject: [PATCH 341/633] firewire: replace tricky statement by two simple ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct fw_driver::remove returns void, still the comma expression's value is zero. This is harder to parse than necessary, so replace the single expression by two easier ones. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210126221444.713837-1-uwe@kleine-koenig.org Signed-off-by: Greg Kroah-Hartman --- drivers/firewire/core-device.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c index 80db43a22069..68216988391f 100644 --- a/drivers/firewire/core-device.c +++ b/drivers/firewire/core-device.c @@ -192,7 +192,9 @@ static int fw_unit_remove(struct device *dev) struct fw_driver *driver = container_of(dev->driver, struct fw_driver, driver); - return driver->remove(fw_unit(dev)), 0; + driver->remove(fw_unit(dev)); + + return 0; } static int get_modalias(struct fw_unit *unit, char *buffer, size_t buffer_size) From 2fd10bcf0310b9525b2af9e1f7aa9ddd87c3772e Mon Sep 17 00:00:00 2001 From: Sabyrzhan Tasbolatov Date: Tue, 9 Feb 2021 16:26:12 +0600 Subject: [PATCH 342/633] drivers/misc/vmw_vmci: restrict too big queue size in qp_host_alloc_queue syzbot found WARNING in qp_broker_alloc[1] in qp_host_alloc_queue() when num_pages is 0x100001, giving queue_size + queue_page_size bigger than KMALLOC_MAX_SIZE for kzalloc(), resulting order >= MAX_ORDER condition. queue_size + queue_page_size=0x8000d8, where KMALLOC_MAX_SIZE=0x400000. [1] Call Trace: alloc_pages include/linux/gfp.h:547 [inline] kmalloc_order+0x40/0x130 mm/slab_common.c:837 kmalloc_order_trace+0x15/0x70 mm/slab_common.c:853 kmalloc_large include/linux/slab.h:481 [inline] __kmalloc+0x257/0x330 mm/slub.c:3959 kmalloc include/linux/slab.h:557 [inline] kzalloc include/linux/slab.h:682 [inline] qp_host_alloc_queue drivers/misc/vmw_vmci/vmci_queue_pair.c:540 [inline] qp_broker_create drivers/misc/vmw_vmci/vmci_queue_pair.c:1351 [inline] qp_broker_alloc+0x936/0x2740 drivers/misc/vmw_vmci/vmci_queue_pair.c:1739 Reported-by: syzbot+15ec7391f3d6a1a7cc7d@syzkaller.appspotmail.com Signed-off-by: Sabyrzhan Tasbolatov Link: https://lore.kernel.org/r/20210209102612.2112247-1-snovitoll@gmail.com Cc: stable Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index d787ddecee77..880c33ab9f47 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -539,6 +539,9 @@ static struct vmci_queue *qp_host_alloc_queue(u64 size) queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page); + if (queue_size + queue_page_size > KMALLOC_MAX_SIZE) + return NULL; + queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL); if (queue) { queue->q_header = NULL; From 61de21a8315147da08a37ff8db2716739977eadc Mon Sep 17 00:00:00 2001 From: Jie Li Date: Mon, 1 Feb 2021 04:25:59 +0000 Subject: [PATCH 343/633] uio: uio_pci_generic: don't fail probe if pdev->irq equals to IRQ_NOTCONNECTED Some devices use 255 as default value of Interrupt Line register, and this maybe causes pdev->irq is set as IRQ_NOTCONNECTED in some scenarios. For example, NVMe controller connects to Intel Volume Management Device (VMD). In this situation, IRQ_NOTCONNECTED means INTx line is not connected, not fault. If bind uio_pci_generic to these devices, uio frame will return -ENOTCONN through request_irq. This patch allows binding uio_pci_generic to device with dev->irq of IRQ_NOTCONNECTED. Acked-by: Kyungsan Kim Signed-off-by: Jie Li Link: https://lore.kernel.org/r/1612153559-17028-1-git-send-email-jie6.li@samsung.com Signed-off-by: Greg Kroah-Hartman --- drivers/uio/uio_pci_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/uio/uio_pci_generic.c b/drivers/uio/uio_pci_generic.c index b8e44d16279f..c7d681fef198 100644 --- a/drivers/uio/uio_pci_generic.c +++ b/drivers/uio/uio_pci_generic.c @@ -92,7 +92,7 @@ static int probe(struct pci_dev *pdev, gdev->info.version = DRIVER_VERSION; gdev->info.release = release; gdev->pdev = pdev; - if (pdev->irq) { + if (pdev->irq && (pdev->irq != IRQ_NOTCONNECTED)) { gdev->info.irq = pdev->irq; gdev->info.irq_flags = IRQF_SHARED; gdev->info.handler = irqhandler; From 0566752c3e8681ec47fee37374cb38081d801e95 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Sun, 7 Feb 2021 14:05:43 +0100 Subject: [PATCH 344/633] uapi: map_to_7segment: Update example in documentation The device_attribute .show() and .store() methods gained an extra parameter in v2.6.13, but the example in the documentation for the 7-segment header file was never updated. Add the missing parameters. While at it, get rid of the (misspelled) deprecated symbolic permissions, and switch to DEVICE_ATTR_RW(), which was introduced in v3.11 Fixes: 54b6f35c99974e99 ("[PATCH] Driver core: change device_attribute callbacks") Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20210207130543.2128980-1-geert@linux-m68k.org Signed-off-by: Greg Kroah-Hartman --- include/uapi/linux/map_to_7segment.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/include/uapi/linux/map_to_7segment.h b/include/uapi/linux/map_to_7segment.h index 13a06e5e966e..8b02088f96e3 100644 --- a/include/uapi/linux/map_to_7segment.h +++ b/include/uapi/linux/map_to_7segment.h @@ -45,17 +45,22 @@ * In device drivers it is recommended, if required, to make the char map * accessible via the sysfs interface using the following scheme: * - * static ssize_t show_map(struct device *dev, char *buf) { + * static ssize_t map_seg7_show(struct device *dev, + * struct device_attribute *attr, char *buf) + * { * memcpy(buf, &map_seg7, sizeof(map_seg7)); * return sizeof(map_seg7); * } - * static ssize_t store_map(struct device *dev, const char *buf, size_t cnt) { + * static ssize_t map_seg7_store(struct device *dev, + * struct device_attribute *attr, const char *buf, + * size_t cnt) + * { * if(cnt != sizeof(map_seg7)) * return -EINVAL; * memcpy(&map_seg7, buf, cnt); * return cnt; * } - * static DEVICE_ATTR(map_seg7, PERMS_RW, show_map, store_map); + * static DEVICE_ATTR_RW(map_seg7); * * History: * 2005-05-31 RFC linux-kernel@vger.kernel.org From befb0e7523afe4557a59454de230435295e074e8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:39 +0200 Subject: [PATCH 345/633] x86/platform/intel-mid: Remove unused leftovers (msic_audio) Commit 05f4434bc130 ("ASoC: Intel: remove mfld_machine") removed the driver, no need to have support files for it. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 1 - .../device_libs/platform_msic_audio.c | 42 ------------------- 2 files changed, 43 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 918edac9ab9a..e926577fe665 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -8,7 +8,6 @@ obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_SND_MFLD_MACHINE)) += platform_msic_audio.o obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_ocd.o obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c deleted file mode 100644 index e765da78ad8c..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_audio.c: MSIC audio platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void *msic_audio_platform_data(void *info) -{ - struct platform_device *pdev; - - pdev = platform_device_register_simple("sst-platform", -1, NULL, 0); - - if (IS_ERR(pdev)) { - pr_err("failed to create audio platform device\n"); - return NULL; - } - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_AUDIO); -} - -static const struct devs_id msic_audio_dev_id __initconst = { - .name = "msic_audio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_audio_platform_data, -}; - -sfi_device(msic_audio_dev_id); From 4bbf7cfede6175f7f1d52a126c2c30a01d6b229b Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:40 +0200 Subject: [PATCH 346/633] x86/platform/intel-mid: Remove unused leftovers (msic_ocd) It seems msic_ocd driver was never upstreamed. Why should we have dead code in the kernel? Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 1 - .../intel-mid/device_libs/platform_msic_ocd.c | 44 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index e926577fe665..1ab7813cd627 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -9,7 +9,6 @@ obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_ocd.o obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c deleted file mode 100644 index 558c0d974430..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_ocd.c: MSIC OCD platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_ocd_platform_data(void *info) -{ - static struct intel_msic_ocd_pdata msic_ocd_pdata; - int gpio; - - gpio = get_gpio_by_name("ocd_gpio"); - - if (gpio < 0) - return NULL; - - msic_ocd_pdata.gpio = gpio; - msic_pdata.ocd = &msic_ocd_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_OCD); -} - -static const struct devs_id msic_ocd_dev_id __initconst = { - .name = "msic_ocd", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_ocd_platform_data, -}; - -sfi_device(msic_ocd_dev_id); From 2ec51fa7445f7db816de61665c4c6d39c893e9ac Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:41 +0200 Subject: [PATCH 347/633] x86/platform/intel-mid: Remove unused leftovers (msic_battery) It seems msic_battery driver was never upstreamed. Why should we have dead code in the kernel? Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 1 - .../device_libs/platform_msic_battery.c | 32 ------------------- 2 files changed, 33 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 1ab7813cd627..cf041a9882c0 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -9,7 +9,6 @@ obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o # SPI Devices diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c deleted file mode 100644 index f461f84903f8..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_battery.c: MSIC battery platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_battery_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_BATTERY); -} - -static const struct devs_id msic_battery_dev_id __initconst = { - .name = "msic_battery", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_battery_platform_data, -}; - -sfi_device(msic_battery_dev_id); From db47204609c62e12b789b80eb0df954912cd856a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:42 +0200 Subject: [PATCH 348/633] x86/platform/intel-mid: Remove unused leftovers (msic_gpio) There is no driver present, remove the device creation and other leftovers. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 1 - .../device_libs/platform_msic_gpio.c | 43 ------------------- 2 files changed, 44 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index cf041a9882c0..3688e62ccf80 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -8,7 +8,6 @@ obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o # SPI Devices diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c deleted file mode 100644 index 71a7d6db3878..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_gpio.c: MSIC GPIO platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_gpio_platform_data(void *info) -{ - static struct intel_msic_gpio_pdata msic_gpio_pdata; - - int gpio = get_gpio_by_name("msic_gpio_base"); - - if (gpio < 0) - return NULL; - - msic_gpio_pdata.gpio_base = gpio; - msic_pdata.gpio = &msic_gpio_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_GPIO); -} - -static const struct devs_id msic_gpio_dev_id __initconst = { - .name = "msic_gpio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_gpio_platform_data, -}; - -sfi_device(msic_gpio_dev_id); From 4450e93fea0c92c0838e9df30e95f2eb5f68226a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:43 +0200 Subject: [PATCH 349/633] x86/platform/intel-mid: Remove unused leftovers (msic_power_btn) There is no driver present, remove the device creation and other leftovers. Note, for Intel Merrifield there is another driver which is instantiated by a certain MFD one and does not need any support from device_libs. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 2 - .../device_libs/platform_mrfld_power_btn.c | 78 ------------------- .../device_libs/platform_msic_power_btn.c | 31 -------- 3 files changed, 111 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 3688e62ccf80..177e91d565ca 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -8,7 +8,6 @@ obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o # SPI Devices obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o @@ -24,5 +23,4 @@ obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_pcal9555a.o obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o # MISC Devices obj-$(subst m,y,$(CONFIG_KEYBOARD_GPIO)) += platform_gpio_keys.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_mrfld_power_btn.o obj-$(subst m,y,$(CONFIG_RTC_DRV_CMOS)) += platform_mrfld_rtc.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c deleted file mode 100644 index ec2afb41b34a..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield power button support - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include - -#include -#include - -static struct resource mrfld_power_btn_resources[] = { - { - .flags = IORESOURCE_IRQ, - }, -}; - -static struct platform_device mrfld_power_btn_dev = { - .name = "msic_power_btn", - .id = PLATFORM_DEVID_NONE, - .num_resources = ARRAY_SIZE(mrfld_power_btn_resources), - .resource = mrfld_power_btn_resources, -}; - -static int mrfld_power_btn_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&mrfld_power_btn_dev); - return 0; - } - - return platform_device_register(&mrfld_power_btn_dev); -} - -static struct notifier_block mrfld_power_btn_scu_notifier = { - .notifier_call = mrfld_power_btn_scu_status_change, -}; - -static int __init register_mrfld_power_btn(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - /* - * We need to be sure that the SCU IPC is ready before - * PMIC power button device can be registered: - */ - intel_scu_notifier_add(&mrfld_power_btn_scu_notifier); - - return 0; -} -arch_initcall(register_mrfld_power_btn); - -static void __init *mrfld_power_btn_platform_data(void *info) -{ - struct resource *res = mrfld_power_btn_resources; - struct sfi_device_table_entry *pentry = info; - - res->start = res->end = pentry->irq; - return NULL; -} - -static const struct devs_id mrfld_power_btn_dev_id __initconst = { - .name = "bcove_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &mrfld_power_btn_platform_data, -}; - -sfi_device(mrfld_power_btn_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c deleted file mode 100644 index 3d3de2d59726..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_power_btn.c: MSIC power btn platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_power_btn_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_POWER_BTN); -} - -static const struct devs_id msic_power_btn_dev_id __initconst = { - .name = "msic_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_power_btn_platform_data, -}; - -sfi_device(msic_power_btn_dev_id); From f7009c53bb83787d25b4b2997577b2655284ef96 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:44 +0200 Subject: [PATCH 350/633] x86/platform/intel-mid: Remove unused leftovers (msic_thermal) There is no driver present, remove the device creation and other leftovers. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../platform/intel-mid/device_libs/Makefile | 1 - .../device_libs/platform_msic_thermal.c | 32 ------------------- 2 files changed, 33 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 177e91d565ca..4e4db0c2b225 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -8,7 +8,6 @@ obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o # IPC Devices obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o # SPI Devices obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o # I2C Devices diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c deleted file mode 100644 index 4858da1d78c6..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_thermal.c: msic_thermal platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_thermal_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_THERMAL); -} - -static const struct devs_id msic_thermal_dev_id __initconst = { - .name = "msic_thermal", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_thermal_platform_data, -}; - -sfi_device(msic_thermal_dev_id); From 59326a6748ce0ed7ea0c3d63576f37e0d61926be Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:45 +0200 Subject: [PATCH 351/633] x86/platform/intel-mid: Remove unused leftovers (msic) There is no driver present, remove the device creation and other leftovers. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/intel-mid.h | 7 -- .../platform/intel-mid/device_libs/Makefile | 2 - .../intel-mid/device_libs/platform_msic.c | 83 ------------------- .../intel-mid/device_libs/platform_msic.h | 15 ---- arch/x86/platform/intel-mid/sfi.c | 7 -- 5 files changed, 114 deletions(-) delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_msic.h diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index cf0e25f45422..e68ae2155db3 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -37,7 +37,6 @@ struct devs_id { char name[SFI_NAME_LEN + 1]; u8 type; u8 delay; - u8 msic; void *(*get_platform_data)(void *info); }; @@ -83,18 +82,12 @@ static inline enum intel_mid_cpu_type intel_mid_identify_cpu(void) return __intel_mid_cpu_chip; } -static inline bool intel_mid_has_msic(void) -{ - return (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_PENWELL); -} - extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); #else /* !CONFIG_X86_INTEL_MID */ #define intel_mid_identify_cpu() 0 -#define intel_mid_has_msic() 0 static inline void intel_scu_devices_create(void) { } static inline void intel_scu_devices_destroy(void) { } diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile index 4e4db0c2b225..4d008b053ac8 100644 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ b/arch/x86/platform/intel-mid/device_libs/Makefile @@ -6,8 +6,6 @@ obj-$(subst m,y,$(CONFIG_MMC_SDHCI_PCI)) += platform_mrfld_sd.o # WiFi + BT obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o -# IPC Devices -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o # SPI Devices obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o # I2C Devices diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.c b/arch/x86/platform/intel-mid/device_libs/platform_msic.c deleted file mode 100644 index b17783d0d4e7..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic.c: MSIC platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "platform_msic.h" - -struct intel_msic_platform_data msic_pdata; - -static struct resource msic_resources[] = { - { - .start = INTEL_MSIC_IRQ_PHYS_BASE, - .end = INTEL_MSIC_IRQ_PHYS_BASE + 64 - 1, - .flags = IORESOURCE_MEM, - }, -}; - -static struct platform_device msic_device = { - .name = "intel_msic", - .id = -1, - .dev = { - .platform_data = &msic_pdata, - }, - .num_resources = ARRAY_SIZE(msic_resources), - .resource = msic_resources, -}; - -static int msic_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&msic_device); - return 0; - } - - return platform_device_register(&msic_device); -} - -static int __init msic_init(void) -{ - static struct notifier_block msic_scu_notifier = { - .notifier_call = msic_scu_status_change, - }; - - /* - * We need to be sure that the SCU IPC is ready before MSIC device - * can be registered. - */ - if (intel_mid_has_msic()) - intel_scu_notifier_add(&msic_scu_notifier); - - return 0; -} -arch_initcall(msic_init); - -/* - * msic_generic_platform_data - sets generic platform data for the block - * @info: pointer to the SFI device table entry for this block - * @block: MSIC block - * - * Function sets IRQ number from the SFI table entry for given device to - * the MSIC platform data. - */ -void *msic_generic_platform_data(void *info, enum intel_msic_block block) -{ - struct sfi_device_table_entry *entry = info; - - BUG_ON(block < 0 || block >= INTEL_MSIC_BLOCK_LAST); - msic_pdata.irq[block] = entry->irq; - - return NULL; -} diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.h b/arch/x86/platform/intel-mid/device_libs/platform_msic.h deleted file mode 100644 index 91deb2e65b0e..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * platform_msic.h: MSIC platform data header file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#ifndef _PLATFORM_MSIC_H_ -#define _PLATFORM_MSIC_H_ - -extern struct intel_msic_platform_data msic_pdata; - -void *msic_generic_platform_data(void *info, enum intel_msic_block block); - -#endif diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c index 30bd5714a3d4..2b22587bfd3d 100644 --- a/arch/x86/platform/intel-mid/sfi.c +++ b/arch/x86/platform/intel-mid/sfi.c @@ -338,13 +338,6 @@ static void __init sfi_handle_ipc_dev(struct sfi_device_table_entry *pentry, if (IS_ERR(pdata)) return; - /* - * On Medfield the platform device creation is handled by the MSIC - * MFD driver so we don't need to do it here. - */ - if (dev->msic && intel_mid_has_msic()) - return; - pdev = platform_device_alloc(pentry->name, 0); if (pdev == NULL) { pr_err("out of memory for SFI platform device '%s'.\n", From 2468f933b14ed49f15562cdb02bd2592a0aa8248 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:46 +0200 Subject: [PATCH 352/633] x86/platform/intel-mid: Remove unused leftovers (vRTC) There is no driver present, remove the device creation and other leftovers. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/fixmap.h | 3 - arch/x86/include/asm/intel-mid.h | 10 -- arch/x86/include/asm/intel_mid_vrtc.h | 10 -- arch/x86/include/asm/intel_scu_ipc_legacy.h | 5 - arch/x86/platform/intel-mid/Makefile | 2 +- arch/x86/platform/intel-mid/intel-mid.c | 2 - arch/x86/platform/intel-mid/intel_mid_vrtc.c | 173 ------------------- arch/x86/platform/intel-mid/sfi.c | 42 ----- 8 files changed, 1 insertion(+), 246 deletions(-) delete mode 100644 arch/x86/include/asm/intel_mid_vrtc.h delete mode 100644 arch/x86/platform/intel-mid/intel_mid_vrtc.c diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 9f1a0a987e5e..d0dcefb5cc59 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h @@ -108,9 +108,6 @@ enum fixed_addresses { #ifdef CONFIG_PARAVIRT_XXL FIX_PARAVIRT_BOOTMAP, #endif -#ifdef CONFIG_X86_INTEL_MID - FIX_LNW_VRTC, -#endif #ifdef CONFIG_ACPI_APEI_GHES /* Used for GHES mapping from assorted contexts */ diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index e68ae2155db3..9a6f8b1bebcd 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -23,10 +23,7 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); extern int get_gpio_by_name(const char *name); -extern int __init sfi_parse_mrtc(struct sfi_table_header *table); extern int __init sfi_parse_mtmr(struct sfi_table_header *table); -extern int sfi_mrtc_num; -extern struct sfi_rtc_table_entry sfi_mrtc_array[]; /* * Here defines the array of devices platform data that IAFW would export @@ -112,13 +109,6 @@ extern enum intel_mid_timer_options intel_mid_timer_options; #define BSEL_SOC_FUSE_111 0x7 #define SFI_MTMR_MAX_NUM 8 -#define SFI_MRTC_MAX 8 - -/* VRTC timer */ -#define MRST_VRTC_MAP_SZ 1024 -/* #define MRST_VRTC_PGOFFSET 0xc00 */ - -extern void intel_mid_rtc_init(void); /* The offset for the mapping of global gpio pin to irq */ #define INTEL_MID_IRQ_OFFSET 0x100 diff --git a/arch/x86/include/asm/intel_mid_vrtc.h b/arch/x86/include/asm/intel_mid_vrtc.h deleted file mode 100644 index 0b44b1abe4d9..000000000000 --- a/arch/x86/include/asm/intel_mid_vrtc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _INTEL_MID_VRTC_H -#define _INTEL_MID_VRTC_H - -extern unsigned char vrtc_cmos_read(unsigned char reg); -extern void vrtc_cmos_write(unsigned char val, unsigned char reg); -extern void vrtc_get_time(struct timespec64 *now); -extern int vrtc_set_mmss(const struct timespec64 *now); - -#endif diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h index 4cf13fecb673..2232197c24f8 100644 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ b/arch/x86/include/asm/intel_scu_ipc_legacy.h @@ -14,11 +14,6 @@ #define IPCMSG_SOFT_RESET 0xF2 #define IPCMSG_COLD_BOOT 0xF3 -#define IPCMSG_VRTC 0xFA /* Set vRTC device */ -/* Command id associated with message IPCMSG_VRTC */ -#define IPC_CMD_VRTC_SETTIME 1 /* Set time */ -#define IPC_CMD_VRTC_SETALARM 2 /* Set alarm */ - /* Don't call these in new code - they will be removed eventually */ /* Read single register */ diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index cc2549f0ccb1..5794e661050c 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o pwr.o +obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o pwr.o # SFI specific code ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 780728161f7d..ada39fb426dd 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -159,7 +158,6 @@ void __init x86_intel_mid_early_setup(void) x86_init.timers.timer_init = intel_mid_time_init; x86_init.timers.setup_percpu_clockev = x86_init_noop; - x86_init.timers.wallclock_init = intel_mid_rtc_init; x86_init.irqs.pre_vector_init = x86_init_noop; diff --git a/arch/x86/platform/intel-mid/intel_mid_vrtc.c b/arch/x86/platform/intel-mid/intel_mid_vrtc.c deleted file mode 100644 index 2226da4f437a..000000000000 --- a/arch/x86/platform/intel-mid/intel_mid_vrtc.c +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_vrtc.c: Driver for virtual RTC device on Intel MID platform - * - * (C) Copyright 2009 Intel Corporation - * - * Note: - * VRTC is emulated by system controller firmware, the real HW - * RTC is located in the PMIC device. SCU FW shadows PMIC RTC - * in a memory mapped IO space that is visible to the host IA - * processor. - * - * This driver is based on RTC CMOS driver. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -static unsigned char __iomem *vrtc_virt_base; - -unsigned char vrtc_cmos_read(unsigned char reg) -{ - unsigned char retval; - - /* vRTC's registers range from 0x0 to 0xD */ - if (reg > 0xd || !vrtc_virt_base) - return 0xff; - - lock_cmos_prefix(reg); - retval = __raw_readb(vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); - return retval; -} -EXPORT_SYMBOL_GPL(vrtc_cmos_read); - -void vrtc_cmos_write(unsigned char val, unsigned char reg) -{ - if (reg > 0xd || !vrtc_virt_base) - return; - - lock_cmos_prefix(reg); - __raw_writeb(val, vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); -} -EXPORT_SYMBOL_GPL(vrtc_cmos_write); - -void vrtc_get_time(struct timespec64 *now) -{ - u8 sec, min, hour, mday, mon; - unsigned long flags; - u32 year; - - spin_lock_irqsave(&rtc_lock, flags); - - while ((vrtc_cmos_read(RTC_FREQ_SELECT) & RTC_UIP)) - cpu_relax(); - - sec = vrtc_cmos_read(RTC_SECONDS); - min = vrtc_cmos_read(RTC_MINUTES); - hour = vrtc_cmos_read(RTC_HOURS); - mday = vrtc_cmos_read(RTC_DAY_OF_MONTH); - mon = vrtc_cmos_read(RTC_MONTH); - year = vrtc_cmos_read(RTC_YEAR); - - spin_unlock_irqrestore(&rtc_lock, flags); - - /* vRTC YEAR reg contains the offset to 1972 */ - year += 1972; - - pr_info("vRTC: sec: %d min: %d hour: %d day: %d " - "mon: %d year: %d\n", sec, min, hour, mday, mon, year); - - now->tv_sec = mktime64(year, mon, mday, hour, min, sec); - now->tv_nsec = 0; -} - -int vrtc_set_mmss(const struct timespec64 *now) -{ - unsigned long flags; - struct rtc_time tm; - int year; - int retval = 0; - - rtc_time64_to_tm(now->tv_sec, &tm); - if (!rtc_valid_tm(&tm) && tm.tm_year >= 72) { - /* - * tm.year is the number of years since 1900, and the - * vrtc need the years since 1972. - */ - year = tm.tm_year - 72; - spin_lock_irqsave(&rtc_lock, flags); - vrtc_cmos_write(year, RTC_YEAR); - vrtc_cmos_write(tm.tm_mon, RTC_MONTH); - vrtc_cmos_write(tm.tm_mday, RTC_DAY_OF_MONTH); - vrtc_cmos_write(tm.tm_hour, RTC_HOURS); - vrtc_cmos_write(tm.tm_min, RTC_MINUTES); - vrtc_cmos_write(tm.tm_sec, RTC_SECONDS); - spin_unlock_irqrestore(&rtc_lock, flags); - } else { - pr_err("%s: Invalid vRTC value: write of %llx to vRTC failed\n", - __func__, (s64)now->tv_sec); - retval = -EINVAL; - } - return retval; -} - -void __init intel_mid_rtc_init(void) -{ - unsigned long vrtc_paddr; - - sfi_table_parse(SFI_SIG_MRTC, NULL, NULL, sfi_parse_mrtc); - - vrtc_paddr = sfi_mrtc_array[0].phys_addr; - if (!sfi_mrtc_num || !vrtc_paddr) - return; - - vrtc_virt_base = (void __iomem *)set_fixmap_offset_nocache(FIX_LNW_VRTC, - vrtc_paddr); - x86_platform.get_wallclock = vrtc_get_time; - x86_platform.set_wallclock = vrtc_set_mmss; -} - -/* - * The Moorestown platform has a memory mapped virtual RTC device that emulates - * the programming interface of the RTC. - */ - -static struct resource vrtc_resources[] = { - [0] = { - .flags = IORESOURCE_MEM, - }, - [1] = { - .flags = IORESOURCE_IRQ, - } -}; - -static struct platform_device vrtc_device = { - .name = "rtc_mrst", - .id = -1, - .resource = vrtc_resources, - .num_resources = ARRAY_SIZE(vrtc_resources), -}; - -/* Register the RTC device if appropriate */ -static int __init intel_mid_device_create(void) -{ - /* No Moorestown, no device */ - if (!intel_mid_identify_cpu()) - return -ENODEV; - /* No timer, no device */ - if (!sfi_mrtc_num) - return -ENODEV; - - /* iomem resource */ - vrtc_resources[0].start = sfi_mrtc_array[0].phys_addr; - vrtc_resources[0].end = sfi_mrtc_array[0].phys_addr + - MRST_VRTC_MAP_SZ; - /* irq resource */ - vrtc_resources[1].start = sfi_mrtc_array[0].irq; - vrtc_resources[1].end = sfi_mrtc_array[0].irq; - - return platform_device_register(&vrtc_device); -} -device_initcall(intel_mid_device_create); diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c index 2b22587bfd3d..a50698e90a9c 100644 --- a/arch/x86/platform/intel-mid/sfi.c +++ b/arch/x86/platform/intel-mid/sfi.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -54,12 +53,8 @@ static int i2c_next_dev; static int i2c_bus[MAX_SCU_I2C]; static int gpio_num_entry; static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; -int sfi_mrtc_num; int sfi_mtimer_num; -struct sfi_rtc_table_entry sfi_mrtc_array[SFI_MRTC_MAX]; -EXPORT_SYMBOL_GPL(sfi_mrtc_array); - struct blocking_notifier_head intel_scu_notifier = BLOCKING_NOTIFIER_INIT(intel_scu_notifier); EXPORT_SYMBOL_GPL(intel_scu_notifier); @@ -138,43 +133,6 @@ void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr) } } -/* parse all the mrtc info to a global mrtc array */ -int __init sfi_parse_mrtc(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_rtc_table_entry *pentry; - struct mpc_intsrc mp_irq; - - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mrtc_num) { - sfi_mrtc_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_rtc_table_entry); - pentry = (struct sfi_rtc_table_entry *)sb->pentry; - totallen = sfi_mrtc_num * sizeof(*pentry); - memcpy(sfi_mrtc_array, pentry, totallen); - } - - pr_debug("SFI RTC info (num = %d):\n", sfi_mrtc_num); - pentry = sfi_mrtc_array; - for (totallen = 0; totallen < sfi_mrtc_num; totallen++, pentry++) { - pr_debug("RTC[%d]: paddr = 0x%08x, irq = %d\n", - totallen, (u32)pentry->phys_addr, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_LEVEL | MP_IRQPOL_ACTIVE_LOW; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - return 0; -} - - /* * Parsing GPIO table first, since the DEVS table will need this table * to map the pin name to the actual pin. From 1b79fc4f2bfd24efa7f0172dffc712e46b30a582 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:47 +0200 Subject: [PATCH 353/633] x86/apb_timer: Remove driver for deprecated platform Intel Moorestown and Medfield are quite old Intel Atom based 32-bit platforms, which were in limited use in some Android phones, tablets and consumer electronics more than eight years ago. There are no bugs or problems ever reported outside from Intel for breaking any of that platforms for years. It seems no real users exists who run more or less fresh kernel on it. Commit 05f4434bc130 ("ASoC: Intel: remove mfld_machine") is also in align with this theory. Due to above and to reduce a burden of supporting outdated drivers, remove the support for outdated platforms completely. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- .../admin-guide/kernel-parameters.txt | 6 - arch/x86/Kconfig | 12 - arch/x86/include/asm/apb_timer.h | 40 -- arch/x86/include/asm/intel-mid.h | 11 - arch/x86/kernel/Makefile | 1 - arch/x86/kernel/apb_timer.c | 347 ------------------ arch/x86/platform/intel-mid/intel-mid.c | 77 +--- arch/x86/platform/intel-mid/sfi.c | 75 ---- 8 files changed, 3 insertions(+), 566 deletions(-) delete mode 100644 arch/x86/include/asm/apb_timer.h delete mode 100644 arch/x86/kernel/apb_timer.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a10b545c2070..b448ea8b6598 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5932,12 +5932,6 @@ default x2apic cluster mode on platforms supporting x2apic. - x86_intel_mid_timer= [X86-32,APBT] - Choose timer option for x86 Intel MID platform. - Two valid options are apbt timer only and lapic timer - plus one apbt timer for broadcast timer. - x86_intel_mid_timer=apbt_only | lapic_and_apbt - xen_512gb_limit [KNL,X86-64,XEN] Restricts the kernel running paravirtualized under Xen to use only up to 512 GB of RAM. The reason to do so is diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 21f851179ff0..cc73f98c3e42 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -892,18 +892,6 @@ config HPET_EMULATE_RTC def_bool y depends on HPET_TIMER && (RTC=y || RTC=m || RTC_DRV_CMOS=m || RTC_DRV_CMOS=y) -config APB_TIMER - def_bool y if X86_INTEL_MID - prompt "Intel MID APB Timer Support" if X86_INTEL_MID - select DW_APB_TIMER - depends on X86_INTEL_MID && SFI - help - APB timer is the replacement for 8254, HPET on X86 MID platforms. - The APBT provides a stable time base on SMP - systems, unlike the TSC, but it is more expensive to access, - as it is off-chip. APB timers are always running regardless of CPU - C states, they are used as per CPU clockevent device when possible. - # Mark as expert because too many people got it wrong. # The code disables itself when not needed. config DMI diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h deleted file mode 100644 index 87ce8e963215..000000000000 --- a/arch/x86/include/asm/apb_timer.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * apb_timer.h: Driver for Langwell APB timer based on Synopsis DesignWare - * - * (C) Copyright 2009 Intel Corporation - * Author: Jacob Pan (jacob.jun.pan@intel.com) - * - * Note: - */ - -#ifndef ASM_X86_APBT_H -#define ASM_X86_APBT_H -#include - -#ifdef CONFIG_APB_TIMER - -/* default memory mapped register base */ -#define LNW_SCU_ADDR 0xFF100000 -#define LNW_EXT_TIMER_OFFSET 0x1B800 -#define APBT_DEFAULT_BASE (LNW_SCU_ADDR+LNW_EXT_TIMER_OFFSET) -#define LNW_EXT_TIMER_PGOFFSET 0x800 - -/* APBT clock speed range from PCLK to fabric base, 25-100MHz */ -#define APBT_MAX_FREQ 50000000 -#define APBT_MIN_FREQ 1000000 -#define APBT_MMAP_SIZE 1024 - -extern void apbt_time_init(void); -extern void apbt_setup_secondary_clock(void); - -extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint); -extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr); -extern int sfi_mtimer_num; - -#else /* CONFIG_APB_TIMER */ - -static inline void apbt_time_init(void) { } - -#endif -#endif /* ASM_X86_APBT_H */ diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index 9a6f8b1bebcd..7bda0587cf70 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -23,7 +23,6 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); extern int get_gpio_by_name(const char *name); -extern int __init sfi_parse_mtmr(struct sfi_table_header *table); /* * Here defines the array of devices platform data that IAFW would export @@ -91,14 +90,6 @@ static inline void intel_scu_devices_destroy(void) { } #endif /* !CONFIG_X86_INTEL_MID */ -enum intel_mid_timer_options { - INTEL_MID_TIMER_DEFAULT, - INTEL_MID_TIMER_APBT_ONLY, - INTEL_MID_TIMER_LAPIC_APBT, -}; - -extern enum intel_mid_timer_options intel_mid_timer_options; - /* Bus Select SoC Fuse value */ #define BSEL_SOC_FUSE_MASK 0x7 /* FSB 133MHz */ @@ -108,8 +99,6 @@ extern enum intel_mid_timer_options intel_mid_timer_options; /* FSB 83MHz */ #define BSEL_SOC_FUSE_111 0x7 -#define SFI_MTMR_MAX_NUM 8 - /* The offset for the mapping of global gpio pin to irq */ #define INTEL_MID_IRQ_OFFSET 0x100 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 5eeb808eb024..2ddf08351f0b 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -116,7 +116,6 @@ obj-$(CONFIG_VM86) += vm86_32.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o -obj-$(CONFIG_APB_TIMER) += apb_timer.o obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c deleted file mode 100644 index 263eeaddb0aa..000000000000 --- a/arch/x86/kernel/apb_timer.c +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * apb_timer.c: Driver for Langwell APB timers - * - * (C) Copyright 2009 Intel Corporation - * Author: Jacob Pan (jacob.jun.pan@intel.com) - * - * Note: - * Langwell is the south complex of Intel Moorestown MID platform. There are - * eight external timers in total that can be used by the operating system. - * The timer information, such as frequency and addresses, is provided to the - * OS via SFI tables. - * Timer interrupts are routed via FW/HW emulated IOAPIC independently via - * individual redirection table entries (RTE). - * Unlike HPET, there is no master counter, therefore one of the timers are - * used as clocksource. The overall allocation looks like: - * - timer 0 - NR_CPUs for per cpu timer - * - one timer for clocksource - * - one timer for watchdog driver. - * It is also worth notice that APB timer does not support true one-shot mode, - * free-running mode will be used here to emulate one-shot mode. - * APB timer can also be used as broadcast timer along with per cpu local APIC - * timer, but by default APB timer has higher rating than local APIC timers. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define APBT_CLOCKEVENT_RATING 110 -#define APBT_CLOCKSOURCE_RATING 250 - -#define APBT_CLOCKEVENT0_NUM (0) -#define APBT_CLOCKSOURCE_NUM (2) - -static phys_addr_t apbt_address; -static int apb_timer_block_enabled; -static void __iomem *apbt_virt_address; - -/* - * Common DW APB timer info - */ -static unsigned long apbt_freq; - -struct apbt_dev { - struct dw_apb_clock_event_device *timer; - unsigned int num; - int cpu; - unsigned int irq; - char name[10]; -}; - -static struct dw_apb_clocksource *clocksource_apbt; - -static inline void __iomem *adev_virt_addr(struct apbt_dev *adev) -{ - return apbt_virt_address + adev->num * APBTMRS_REG_SIZE; -} - -static DEFINE_PER_CPU(struct apbt_dev, cpu_apbt_dev); - -#ifdef CONFIG_SMP -static unsigned int apbt_num_timers_used; -#endif - -static inline void apbt_set_mapping(void) -{ - struct sfi_timer_table_entry *mtmr; - int phy_cs_timer_id = 0; - - if (apbt_virt_address) { - pr_debug("APBT base already mapped\n"); - return; - } - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return; - } - apbt_address = (phys_addr_t)mtmr->phys_addr; - if (!apbt_address) { - printk(KERN_WARNING "No timer base from SFI, use default\n"); - apbt_address = APBT_DEFAULT_BASE; - } - apbt_virt_address = ioremap(apbt_address, APBT_MMAP_SIZE); - if (!apbt_virt_address) { - pr_debug("Failed mapping APBT phy address at %lu\n",\ - (unsigned long)apbt_address); - goto panic_noapbt; - } - apbt_freq = mtmr->freq_hz; - sfi_free_mtmr(mtmr); - - /* Now figure out the physical timer id for clocksource device */ - mtmr = sfi_get_mtmr(APBT_CLOCKSOURCE_NUM); - if (mtmr == NULL) - goto panic_noapbt; - - /* Now figure out the physical timer id */ - pr_debug("Use timer %d for clocksource\n", - (int)(mtmr->phys_addr & 0xff) / APBTMRS_REG_SIZE); - phy_cs_timer_id = (unsigned int)(mtmr->phys_addr & 0xff) / - APBTMRS_REG_SIZE; - - clocksource_apbt = dw_apb_clocksource_init(APBT_CLOCKSOURCE_RATING, - "apbt0", apbt_virt_address + phy_cs_timer_id * - APBTMRS_REG_SIZE, apbt_freq); - return; - -panic_noapbt: - panic("Failed to setup APB system timer\n"); - -} - -static inline void apbt_clear_mapping(void) -{ - iounmap(apbt_virt_address); - apbt_virt_address = NULL; -} - -static int __init apbt_clockevent_register(void) -{ - struct sfi_timer_table_entry *mtmr; - struct apbt_dev *adev = this_cpu_ptr(&cpu_apbt_dev); - - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return -ENODEV; - } - - adev->num = smp_processor_id(); - adev->timer = dw_apb_clockevent_init(smp_processor_id(), "apbt0", - intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT ? - APBT_CLOCKEVENT_RATING - 100 : APBT_CLOCKEVENT_RATING, - adev_virt_addr(adev), 0, apbt_freq); - /* Firmware does EOI handling for us. */ - adev->timer->eoi = NULL; - - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - global_clock_event = &adev->timer->ced; - printk(KERN_DEBUG "%s clockevent registered as global\n", - global_clock_event->name); - } - - dw_apb_clockevent_register(adev->timer); - - sfi_free_mtmr(mtmr); - return 0; -} - -#ifdef CONFIG_SMP - -static void apbt_setup_irq(struct apbt_dev *adev) -{ - irq_modify_status(adev->irq, 0, IRQ_MOVE_PCNTXT); - irq_set_affinity(adev->irq, cpumask_of(adev->cpu)); -} - -/* Should be called with per cpu */ -void apbt_setup_secondary_clock(void) -{ - struct apbt_dev *adev; - int cpu; - - /* Don't register boot CPU clockevent */ - cpu = smp_processor_id(); - if (!cpu) - return; - - adev = this_cpu_ptr(&cpu_apbt_dev); - if (!adev->timer) { - adev->timer = dw_apb_clockevent_init(cpu, adev->name, - APBT_CLOCKEVENT_RATING, adev_virt_addr(adev), - adev->irq, apbt_freq); - adev->timer->eoi = NULL; - } else { - dw_apb_clockevent_resume(adev->timer); - } - - printk(KERN_INFO "Registering CPU %d clockevent device %s, cpu %08x\n", - cpu, adev->name, adev->cpu); - - apbt_setup_irq(adev); - dw_apb_clockevent_register(adev->timer); - - return; -} - -/* - * this notify handler process CPU hotplug events. in case of S0i3, nonboot - * cpus are disabled/enabled frequently, for performance reasons, we keep the - * per cpu timer irq registered so that we do need to do free_irq/request_irq. - * - * TODO: it might be more reliable to directly disable percpu clockevent device - * without the notifier chain. currently, cpu 0 may get interrupts from other - * cpu timers during the offline process due to the ordering of notification. - * the extra interrupt is harmless. - */ -static int apbt_cpu_dead(unsigned int cpu) -{ - struct apbt_dev *adev = &per_cpu(cpu_apbt_dev, cpu); - - dw_apb_clockevent_pause(adev->timer); - if (system_state == SYSTEM_RUNNING) { - pr_debug("skipping APBT CPU %u offline\n", cpu); - } else { - pr_debug("APBT clockevent for cpu %u offline\n", cpu); - dw_apb_clockevent_stop(adev->timer); - } - return 0; -} - -static __init int apbt_late_init(void) -{ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT || - !apb_timer_block_enabled) - return 0; - return cpuhp_setup_state(CPUHP_X86_APB_DEAD, "x86/apb:dead", NULL, - apbt_cpu_dead); -} -fs_initcall(apbt_late_init); -#else - -void apbt_setup_secondary_clock(void) {} - -#endif /* CONFIG_SMP */ - -static int apbt_clocksource_register(void) -{ - u64 start, now; - u64 t1; - - /* Start the counter, use timer 2 as source, timer 0/1 for event */ - dw_apb_clocksource_start(clocksource_apbt); - - /* Verify whether apbt counter works */ - t1 = dw_apb_clocksource_read(clocksource_apbt); - start = rdtsc(); - - /* - * We don't know the TSC frequency yet, but waiting for - * 200000 TSC cycles is safe: - * 4 GHz == 50us - * 1 GHz == 200us - */ - do { - rep_nop(); - now = rdtsc(); - } while ((now - start) < 200000UL); - - /* APBT is the only always on clocksource, it has to work! */ - if (t1 == dw_apb_clocksource_read(clocksource_apbt)) - panic("APBT counter not counting. APBT disabled\n"); - - dw_apb_clocksource_register(clocksource_apbt); - - return 0; -} - -/* - * Early setup the APBT timer, only use timer 0 for booting then switch to - * per CPU timer if possible. - * returns 1 if per cpu apbt is setup - * returns 0 if no per cpu apbt is chosen - * panic if set up failed, this is the only platform timer on Moorestown. - */ -void __init apbt_time_init(void) -{ -#ifdef CONFIG_SMP - int i; - struct sfi_timer_table_entry *p_mtmr; - struct apbt_dev *adev; -#endif - - if (apb_timer_block_enabled) - return; - apbt_set_mapping(); - if (!apbt_virt_address) - goto out_noapbt; - /* - * Read the frequency and check for a sane value, for ESL model - * we extend the possible clock range to allow time scaling. - */ - - if (apbt_freq < APBT_MIN_FREQ || apbt_freq > APBT_MAX_FREQ) { - pr_debug("APBT has invalid freq 0x%lx\n", apbt_freq); - goto out_noapbt; - } - if (apbt_clocksource_register()) { - pr_debug("APBT has failed to register clocksource\n"); - goto out_noapbt; - } - if (!apbt_clockevent_register()) - apb_timer_block_enabled = 1; - else { - pr_debug("APBT has failed to register clockevent\n"); - goto out_noapbt; - } -#ifdef CONFIG_SMP - /* kernel cmdline disable apb timer, so we will use lapic timers */ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - printk(KERN_INFO "apbt: disabled per cpu timer\n"); - return; - } - pr_debug("%s: %d CPUs online\n", __func__, num_online_cpus()); - if (num_possible_cpus() <= sfi_mtimer_num) - apbt_num_timers_used = num_possible_cpus(); - else - apbt_num_timers_used = 1; - pr_debug("%s: %d APB timers used\n", __func__, apbt_num_timers_used); - - /* here we set up per CPU timer data structure */ - for (i = 0; i < apbt_num_timers_used; i++) { - adev = &per_cpu(cpu_apbt_dev, i); - adev->num = i; - adev->cpu = i; - p_mtmr = sfi_get_mtmr(i); - if (p_mtmr) - adev->irq = p_mtmr->irq; - else - printk(KERN_ERR "Failed to get timer for cpu %d\n", i); - snprintf(adev->name, sizeof(adev->name) - 1, "apbt%d", i); - } -#endif - - return; - -out_noapbt: - apbt_clear_mapping(); - apb_timer_block_enabled = 0; - panic("failed to enable APB timer\n"); -} diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index ada39fb426dd..864b0c158b2f 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -28,32 +28,8 @@ #include #include #include -#include #include -/* - * the clockevent devices on Moorestown/Medfield can be APBT or LAPIC clock, - * cmdline option x86_intel_mid_timer can be used to override the configuration - * to prefer one or the other. - * at runtime, there are basically three timer configurations: - * 1. per cpu apbt clock only - * 2. per cpu always-on lapic clocks only, this is Penwell/Medfield only - * 3. per cpu lapic clock (C3STOP) and one apbt clock, with broadcast. - * - * by default (without cmdline option), platform code first detects cpu type - * to see if we are on lincroft or penwell, then set up both lapic or apbt - * clocks accordingly. - * i.e. by default, medfield uses configuration #2, moorestown uses #1. - * config #3 is supported but not recommended on medfield. - * - * rating and feature summary: - * lapic (with C3STOP) --------- 100 - * apbt (always-on) ------------ 110 - * lapic (always-on,ARAT) ------ 150 - */ - -enum intel_mid_timer_options intel_mid_timer_options; - enum intel_mid_cpu_type __intel_mid_cpu_chip; EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); @@ -71,34 +47,11 @@ static void intel_mid_reboot(void) intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0); } -static void __init intel_mid_setup_bp_timer(void) -{ - apbt_time_init(); - setup_boot_APIC_clock(); -} - static void __init intel_mid_time_init(void) { - sfi_table_parse(SFI_SIG_MTMR, NULL, NULL, sfi_parse_mtmr); - - switch (intel_mid_timer_options) { - case INTEL_MID_TIMER_APBT_ONLY: - break; - case INTEL_MID_TIMER_LAPIC_APBT: - /* Use apbt and local apic */ - x86_init.timers.setup_percpu_clockev = intel_mid_setup_bp_timer; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - default: - if (!boot_cpu_has(X86_FEATURE_ARAT)) - break; - /* Lapic only, no apbt */ - x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - } - - x86_init.timers.setup_percpu_clockev = apbt_time_init; + /* Lapic only, no apbt */ + x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; + x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; } static void intel_mid_arch_setup(void) @@ -163,8 +116,6 @@ void __init x86_intel_mid_early_setup(void) x86_init.oem.arch_setup = intel_mid_arch_setup; - x86_cpuinit.setup_percpu_clockev = apbt_setup_secondary_clock; - x86_platform.get_nmi_reason = intel_mid_get_nmi_reason; x86_init.pci.arch_init = intel_mid_pci_init; @@ -186,25 +137,3 @@ void __init x86_intel_mid_early_setup(void) x86_init.mpparse.get_smp_config = x86_init_uint_noop; set_bit(MP_BUS_ISA, mp_bus_not_pci); } - -/* - * if user does not want to use per CPU apb timer, just give it a lower rating - * than local apic timer and skip the late per cpu timer init. - */ -static inline int __init setup_x86_intel_mid_timer(char *arg) -{ - if (!arg) - return -EINVAL; - - if (strcmp("apbt_only", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_APBT_ONLY; - else if (strcmp("lapic_and_apbt", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_LAPIC_APBT; - else { - pr_warn("X86 INTEL_MID timer option %s not recognised use x86_intel_mid_timer=apbt_only or lapic_and_apbt\n", - arg); - return -EINVAL; - } - return 0; -} -__setup("x86_intel_mid_timer=", setup_x86_intel_mid_timer); diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c index a50698e90a9c..63ae342ffb12 100644 --- a/arch/x86/platform/intel-mid/sfi.c +++ b/arch/x86/platform/intel-mid/sfi.c @@ -34,7 +34,6 @@ #include #include #include -#include #include #define SFI_SIG_OEM0 "OEM0" @@ -46,14 +45,11 @@ static struct platform_device *ipc_devs[MAX_IPCDEVS]; static struct spi_board_info *spi_devs[MAX_SCU_SPI]; static struct i2c_board_info *i2c_devs[MAX_SCU_I2C]; static struct sfi_gpio_table_entry *gpio_table; -static struct sfi_timer_table_entry sfi_mtimer_array[SFI_MTMR_MAX_NUM]; static int ipc_next_dev; static int spi_next_dev; static int i2c_next_dev; static int i2c_bus[MAX_SCU_I2C]; static int gpio_num_entry; -static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; -int sfi_mtimer_num; struct blocking_notifier_head intel_scu_notifier = BLOCKING_NOTIFIER_INIT(intel_scu_notifier); @@ -62,77 +58,6 @@ EXPORT_SYMBOL_GPL(intel_scu_notifier); #define intel_mid_sfi_get_pdata(dev, priv) \ ((dev)->get_platform_data ? (dev)->get_platform_data(priv) : NULL) -/* parse all the mtimer info to a static mtimer array */ -int __init sfi_parse_mtmr(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_timer_table_entry *pentry; - struct mpc_intsrc mp_irq; - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mtimer_num) { - sfi_mtimer_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_timer_table_entry); - pentry = (struct sfi_timer_table_entry *) sb->pentry; - totallen = sfi_mtimer_num * sizeof(*pentry); - memcpy(sfi_mtimer_array, pentry, totallen); - } - - pr_debug("SFI MTIMER info (num = %d):\n", sfi_mtimer_num); - pentry = sfi_mtimer_array; - for (totallen = 0; totallen < sfi_mtimer_num; totallen++, pentry++) { - pr_debug("timer[%d]: paddr = 0x%08x, freq = %dHz, irq = %d\n", - totallen, (u32)pentry->phys_addr, - pentry->freq_hz, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_EDGE | MP_IRQPOL_ACTIVE_HIGH; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - - return 0; -} - -struct sfi_timer_table_entry *sfi_get_mtmr(int hint) -{ - int i; - if (hint < sfi_mtimer_num) { - if (!sfi_mtimer_usage[hint]) { - pr_debug("hint taken for timer %d irq %d\n", - hint, sfi_mtimer_array[hint].irq); - sfi_mtimer_usage[hint] = 1; - return &sfi_mtimer_array[hint]; - } - } - /* take the first timer available */ - for (i = 0; i < sfi_mtimer_num;) { - if (!sfi_mtimer_usage[i]) { - sfi_mtimer_usage[i] = 1; - return &sfi_mtimer_array[i]; - } - i++; - } - return NULL; -} - -void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr) -{ - int i; - for (i = 0; i < sfi_mtimer_num;) { - if (mtmr->irq == sfi_mtimer_array[i].irq) { - sfi_mtimer_usage[i] = 0; - return; - } - i++; - } -} - /* * Parsing GPIO table first, since the DEVS table will need this table * to map the pin name to the actual pin. From ef3c67b6454b8f542f50387ad481633ae30874ac Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 21:39:48 +0200 Subject: [PATCH 354/633] mfd: intel_msic: Remove driver for deprecated platform Intel Moorestown and Medfield are quite old Intel Atom based 32-bit platforms, which were in limited use in some Android phones, tablets and consumer electronics more than eight years ago. There are no bugs or problems ever reported outside from Intel for breaking any of that platforms for years. It seems no real users exists who run more or less fresh kernel on it. Commit 05f4434bc130 ("ASoC: Intel: remove mfld_machine") is also in align with this theory. Due to above and to reduce a burden of supporting outdated drivers, remove the support for outdated platforms completely. Signed-off-by: Andy Shevchenko Acked-by: Mika Westerberg Acked-by: Linus Walleij Acked-by: Lee Jones Signed-off-by: Rafael J. Wysocki --- MAINTAINERS | 2 - arch/x86/include/asm/intel_scu_ipc_legacy.h | 12 - drivers/mfd/Kconfig | 9 - drivers/mfd/Makefile | 1 - drivers/mfd/intel_msic.c | 425 ------------------ include/linux/mfd/intel_msic.h | 453 -------------------- 6 files changed, 902 deletions(-) delete mode 100644 drivers/mfd/intel_msic.c delete mode 100644 include/linux/mfd/intel_msic.h diff --git a/MAINTAINERS b/MAINTAINERS index 904eaa52f5f8..8fdb7fe27537 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9098,9 +9098,7 @@ F: drivers/gpio/gpio-*cove.c INTEL PMIC MULTIFUNCTION DEVICE DRIVERS M: Andy Shevchenko S: Maintained -F: drivers/mfd/intel_msic.c F: drivers/mfd/intel_soc_pmic* -F: include/linux/mfd/intel_msic.h F: include/linux/mfd/intel_soc_pmic* INTEL PMT DRIVER diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h index 2232197c24f8..f8b87d8face7 100644 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ b/arch/x86/include/asm/intel_scu_ipc_legacy.h @@ -16,24 +16,12 @@ /* Don't call these in new code - they will be removed eventually */ -/* Read single register */ -static inline int intel_scu_ipc_ioread8(u16 addr, u8 *data) -{ - return intel_scu_ipc_dev_ioread8(NULL, addr, data); -} - /* Read a vector */ static inline int intel_scu_ipc_readv(u16 *addr, u8 *data, int len) { return intel_scu_ipc_dev_readv(NULL, addr, data, len); } -/* Write single register */ -static inline int intel_scu_ipc_iowrite8(u16 addr, u8 data) -{ - return intel_scu_ipc_dev_iowrite8(NULL, addr, data); -} - /* Write a vector */ static inline int intel_scu_ipc_writev(u16 *addr, u8 *data, int len) { diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index bdfce7b15621..2dc58a92251b 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -659,15 +659,6 @@ config MFD_INTEL_LPSS_PCI I2C, SPI and HS-UART starting from Intel Sunrisepoint (Intel Skylake PCH) in PCI mode. -config MFD_INTEL_MSIC - bool "Intel MSIC" - depends on INTEL_SCU - select MFD_CORE - help - Select this option to enable access to Intel MSIC (Avatele - Passage) chip. This chip embeds audio, battery, GPIO, etc. - devices used in Intel Medfield platforms. - config MFD_INTEL_PMC_BXT tristate "Intel PMC Driver for Broxton" depends on X86 diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 14fdb188af02..be19bc799073 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -214,7 +214,6 @@ obj-$(CONFIG_MFD_ATMEL_SMC) += atmel-smc.o obj-$(CONFIG_MFD_INTEL_LPSS) += intel-lpss.o obj-$(CONFIG_MFD_INTEL_LPSS_PCI) += intel-lpss-pci.o obj-$(CONFIG_MFD_INTEL_LPSS_ACPI) += intel-lpss-acpi.o -obj-$(CONFIG_MFD_INTEL_MSIC) += intel_msic.o obj-$(CONFIG_MFD_INTEL_PMC_BXT) += intel_pmc_bxt.o obj-$(CONFIG_MFD_INTEL_PMT) += intel_pmt.o obj-$(CONFIG_MFD_PALMAS) += palmas.o diff --git a/drivers/mfd/intel_msic.c b/drivers/mfd/intel_msic.c deleted file mode 100644 index daa772f8146b..000000000000 --- a/drivers/mfd/intel_msic.c +++ /dev/null @@ -1,425 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Driver for Intel MSIC - * - * Copyright (C) 2011, Intel Corporation - * Author: Mika Westerberg - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define MSIC_VENDOR(id) ((id >> 6) & 3) -#define MSIC_VERSION(id) (id & 0x3f) -#define MSIC_MAJOR(id) ('A' + ((id >> 3) & 7)) -#define MSIC_MINOR(id) (id & 7) - -/* - * MSIC interrupt tree is readable from SRAM at INTEL_MSIC_IRQ_PHYS_BASE. - * Since IRQ block starts from address 0x002 we need to subtract that from - * the actual IRQ status register address. - */ -#define MSIC_IRQ_STATUS(x) (INTEL_MSIC_IRQ_PHYS_BASE + ((x) - 2)) -#define MSIC_IRQ_STATUS_ACCDET MSIC_IRQ_STATUS(INTEL_MSIC_ACCDET) - -/* - * The SCU hardware has limitation of 16 bytes per read/write buffer on - * Medfield. - */ -#define SCU_IPC_RWBUF_LIMIT 16 - -/** - * struct intel_msic - an MSIC MFD instance - * @pdev: pointer to the platform device - * @vendor: vendor ID - * @version: chip version - * @irq_base: base address of the mapped MSIC SRAM interrupt tree - */ -struct intel_msic { - struct platform_device *pdev; - unsigned vendor; - unsigned version; - void __iomem *irq_base; -}; - -static const struct resource msic_touch_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_adc_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_battery_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_gpio_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_audio_resources[] = { - DEFINE_RES_IRQ_NAMED(0, "IRQ"), - /* - * We will pass IRQ_BASE to the driver now but this can be removed - * when/if the driver starts to use intel_msic_irq_read(). - */ - DEFINE_RES_MEM_NAMED(MSIC_IRQ_STATUS_ACCDET, 1, "IRQ_BASE"), -}; - -static const struct resource msic_hdmi_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_thermal_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_power_btn_resources[] = { - DEFINE_RES_IRQ(0), -}; - -static const struct resource msic_ocd_resources[] = { - DEFINE_RES_IRQ(0), -}; - -/* - * Devices that are part of the MSIC and are available via firmware - * populated SFI DEVS table. - */ -static struct mfd_cell msic_devs[] = { - [INTEL_MSIC_BLOCK_TOUCH] = { - .name = "msic_touch", - .num_resources = ARRAY_SIZE(msic_touch_resources), - .resources = msic_touch_resources, - }, - [INTEL_MSIC_BLOCK_ADC] = { - .name = "msic_adc", - .num_resources = ARRAY_SIZE(msic_adc_resources), - .resources = msic_adc_resources, - }, - [INTEL_MSIC_BLOCK_BATTERY] = { - .name = "msic_battery", - .num_resources = ARRAY_SIZE(msic_battery_resources), - .resources = msic_battery_resources, - }, - [INTEL_MSIC_BLOCK_GPIO] = { - .name = "msic_gpio", - .num_resources = ARRAY_SIZE(msic_gpio_resources), - .resources = msic_gpio_resources, - }, - [INTEL_MSIC_BLOCK_AUDIO] = { - .name = "msic_audio", - .num_resources = ARRAY_SIZE(msic_audio_resources), - .resources = msic_audio_resources, - }, - [INTEL_MSIC_BLOCK_HDMI] = { - .name = "msic_hdmi", - .num_resources = ARRAY_SIZE(msic_hdmi_resources), - .resources = msic_hdmi_resources, - }, - [INTEL_MSIC_BLOCK_THERMAL] = { - .name = "msic_thermal", - .num_resources = ARRAY_SIZE(msic_thermal_resources), - .resources = msic_thermal_resources, - }, - [INTEL_MSIC_BLOCK_POWER_BTN] = { - .name = "msic_power_btn", - .num_resources = ARRAY_SIZE(msic_power_btn_resources), - .resources = msic_power_btn_resources, - }, - [INTEL_MSIC_BLOCK_OCD] = { - .name = "msic_ocd", - .num_resources = ARRAY_SIZE(msic_ocd_resources), - .resources = msic_ocd_resources, - }, -}; - -/* - * Other MSIC related devices which are not directly available via SFI DEVS - * table. These can be pseudo devices, regulators etc. which are needed for - * different purposes. - * - * These devices appear only after the MSIC driver itself is initialized so - * we can guarantee that the SCU IPC interface is ready. - */ -static const struct mfd_cell msic_other_devs[] = { - /* Audio codec in the MSIC */ - { - .id = -1, - .name = "sn95031", - }, -}; - -/** - * intel_msic_reg_read - read a single MSIC register - * @reg: register to read - * @val: register value is placed here - * - * Read a single register from MSIC. Returns %0 on success and negative - * errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_read(unsigned short reg, u8 *val) -{ - return intel_scu_ipc_ioread8(reg, val); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_read); - -/** - * intel_msic_reg_write - write a single MSIC register - * @reg: register to write - * @val: value to write to that register - * - * Write a single MSIC register. Returns 0 on success and negative - * errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_write(unsigned short reg, u8 val) -{ - return intel_scu_ipc_iowrite8(reg, val); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_write); - -/** - * intel_msic_reg_update - update a single MSIC register - * @reg: register to update - * @val: value to write to the register - * @mask: specifies which of the bits are updated (%0 = don't update, - * %1 = update) - * - * Perform an update to a register @reg. @mask is used to specify which - * bits are updated. Returns %0 in case of success and negative errno in - * case of failure. - * - * Function may sleep. - */ -int intel_msic_reg_update(unsigned short reg, u8 val, u8 mask) -{ - return intel_scu_ipc_update_register(reg, val, mask); -} -EXPORT_SYMBOL_GPL(intel_msic_reg_update); - -/** - * intel_msic_bulk_read - read an array of registers - * @reg: array of register addresses to read - * @buf: array where the read values are placed - * @count: number of registers to read - * - * Function reads @count registers from the MSIC using addresses passed in - * @reg. Read values are placed in @buf. Reads are performed atomically - * wrt. MSIC. - * - * Returns %0 in case of success and negative errno in case of failure. - * - * Function may sleep. - */ -int intel_msic_bulk_read(unsigned short *reg, u8 *buf, size_t count) -{ - if (WARN_ON(count > SCU_IPC_RWBUF_LIMIT)) - return -EINVAL; - - return intel_scu_ipc_readv(reg, buf, count); -} -EXPORT_SYMBOL_GPL(intel_msic_bulk_read); - -/** - * intel_msic_bulk_write - write an array of values to the MSIC registers - * @reg: array of registers to write - * @buf: values to write to each register - * @count: number of registers to write - * - * Function writes @count registers in @buf to MSIC. Writes are performed - * atomically wrt MSIC. Returns %0 in case of success and negative errno in - * case of failure. - * - * Function may sleep. - */ -int intel_msic_bulk_write(unsigned short *reg, u8 *buf, size_t count) -{ - if (WARN_ON(count > SCU_IPC_RWBUF_LIMIT)) - return -EINVAL; - - return intel_scu_ipc_writev(reg, buf, count); -} -EXPORT_SYMBOL_GPL(intel_msic_bulk_write); - -/** - * intel_msic_irq_read - read a register from an MSIC interrupt tree - * @msic: MSIC instance - * @reg: interrupt register (between %INTEL_MSIC_IRQLVL1 and - * %INTEL_MSIC_RESETIRQ2) - * @val: value of the register is placed here - * - * This function can be used by an MSIC subdevice interrupt handler to read - * a register value from the MSIC interrupt tree. In this way subdevice - * drivers don't have to map in the interrupt tree themselves but can just - * call this function instead. - * - * Function doesn't sleep and is callable from interrupt context. - * - * Returns %-EINVAL if @reg is outside of the allowed register region. - */ -int intel_msic_irq_read(struct intel_msic *msic, unsigned short reg, u8 *val) -{ - if (WARN_ON(reg < INTEL_MSIC_IRQLVL1 || reg > INTEL_MSIC_RESETIRQ2)) - return -EINVAL; - - *val = readb(msic->irq_base + (reg - INTEL_MSIC_IRQLVL1)); - return 0; -} -EXPORT_SYMBOL_GPL(intel_msic_irq_read); - -static int intel_msic_init_devices(struct intel_msic *msic) -{ - struct platform_device *pdev = msic->pdev; - struct intel_msic_platform_data *pdata = dev_get_platdata(&pdev->dev); - int ret, i; - - if (pdata->gpio) { - struct mfd_cell *cell = &msic_devs[INTEL_MSIC_BLOCK_GPIO]; - - cell->platform_data = pdata->gpio; - cell->pdata_size = sizeof(*pdata->gpio); - } - - if (pdata->ocd) { - unsigned gpio = pdata->ocd->gpio; - - ret = devm_gpio_request_one(&pdev->dev, gpio, - GPIOF_IN, "ocd_gpio"); - if (ret) { - dev_err(&pdev->dev, "failed to register OCD GPIO\n"); - return ret; - } - - ret = gpio_to_irq(gpio); - if (ret < 0) { - dev_err(&pdev->dev, "no IRQ number for OCD GPIO\n"); - return ret; - } - - /* Update the IRQ number for the OCD */ - pdata->irq[INTEL_MSIC_BLOCK_OCD] = ret; - } - - for (i = 0; i < ARRAY_SIZE(msic_devs); i++) { - if (!pdata->irq[i]) - continue; - - ret = mfd_add_devices(&pdev->dev, -1, &msic_devs[i], 1, NULL, - pdata->irq[i], NULL); - if (ret) - goto fail; - } - - ret = mfd_add_devices(&pdev->dev, 0, msic_other_devs, - ARRAY_SIZE(msic_other_devs), NULL, 0, NULL); - if (ret) - goto fail; - - return 0; - -fail: - mfd_remove_devices(&pdev->dev); - - return ret; -} - -static void intel_msic_remove_devices(struct intel_msic *msic) -{ - struct platform_device *pdev = msic->pdev; - - mfd_remove_devices(&pdev->dev); -} - -static int intel_msic_probe(struct platform_device *pdev) -{ - struct intel_msic_platform_data *pdata = dev_get_platdata(&pdev->dev); - struct intel_msic *msic; - struct resource *res; - u8 id0, id1; - int ret; - - if (!pdata) { - dev_err(&pdev->dev, "no platform data passed\n"); - return -EINVAL; - } - - /* First validate that we have an MSIC in place */ - ret = intel_scu_ipc_ioread8(INTEL_MSIC_ID0, &id0); - if (ret) { - dev_err(&pdev->dev, "failed to identify the MSIC chip (ID0)\n"); - return -ENXIO; - } - - ret = intel_scu_ipc_ioread8(INTEL_MSIC_ID1, &id1); - if (ret) { - dev_err(&pdev->dev, "failed to identify the MSIC chip (ID1)\n"); - return -ENXIO; - } - - if (MSIC_VENDOR(id0) != MSIC_VENDOR(id1)) { - dev_err(&pdev->dev, "invalid vendor ID: %x, %x\n", id0, id1); - return -ENXIO; - } - - msic = devm_kzalloc(&pdev->dev, sizeof(*msic), GFP_KERNEL); - if (!msic) - return -ENOMEM; - - msic->vendor = MSIC_VENDOR(id0); - msic->version = MSIC_VERSION(id0); - msic->pdev = pdev; - - /* - * Map in the MSIC interrupt tree area in SRAM. This is exposed to - * the clients via intel_msic_irq_read(). - */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - msic->irq_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(msic->irq_base)) - return PTR_ERR(msic->irq_base); - - platform_set_drvdata(pdev, msic); - - ret = intel_msic_init_devices(msic); - if (ret) { - dev_err(&pdev->dev, "failed to initialize MSIC devices\n"); - return ret; - } - - dev_info(&pdev->dev, "Intel MSIC version %c%d (vendor %#x)\n", - MSIC_MAJOR(msic->version), MSIC_MINOR(msic->version), - msic->vendor); - - return 0; -} - -static int intel_msic_remove(struct platform_device *pdev) -{ - struct intel_msic *msic = platform_get_drvdata(pdev); - - intel_msic_remove_devices(msic); - - return 0; -} - -static struct platform_driver intel_msic_driver = { - .probe = intel_msic_probe, - .remove = intel_msic_remove, - .driver = { - .name = "intel_msic", - }, -}; -builtin_platform_driver(intel_msic_driver); diff --git a/include/linux/mfd/intel_msic.h b/include/linux/mfd/intel_msic.h deleted file mode 100644 index 317e8608cf41..000000000000 --- a/include/linux/mfd/intel_msic.h +++ /dev/null @@ -1,453 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Core interface for Intel MSIC - * - * Copyright (C) 2011, Intel Corporation - * Author: Mika Westerberg - */ - -#ifndef __LINUX_MFD_INTEL_MSIC_H__ -#define __LINUX_MFD_INTEL_MSIC_H__ - -/* ID */ -#define INTEL_MSIC_ID0 0x000 /* RO */ -#define INTEL_MSIC_ID1 0x001 /* RO */ - -/* IRQ */ -#define INTEL_MSIC_IRQLVL1 0x002 -#define INTEL_MSIC_ADC1INT 0x003 -#define INTEL_MSIC_CCINT 0x004 -#define INTEL_MSIC_PWRSRCINT 0x005 -#define INTEL_MSIC_PWRSRCINT1 0x006 -#define INTEL_MSIC_CHRINT 0x007 -#define INTEL_MSIC_CHRINT1 0x008 -#define INTEL_MSIC_RTCIRQ 0x009 -#define INTEL_MSIC_GPIO0LVIRQ 0x00a -#define INTEL_MSIC_GPIO1LVIRQ 0x00b -#define INTEL_MSIC_GPIOHVIRQ 0x00c -#define INTEL_MSIC_VRINT 0x00d -#define INTEL_MSIC_OCAUDIO 0x00e -#define INTEL_MSIC_ACCDET 0x00f -#define INTEL_MSIC_RESETIRQ1 0x010 -#define INTEL_MSIC_RESETIRQ2 0x011 -#define INTEL_MSIC_MADC1INT 0x012 -#define INTEL_MSIC_MCCINT 0x013 -#define INTEL_MSIC_MPWRSRCINT 0x014 -#define INTEL_MSIC_MPWRSRCINT1 0x015 -#define INTEL_MSIC_MCHRINT 0x016 -#define INTEL_MSIC_MCHRINT1 0x017 -#define INTEL_MSIC_RTCIRQMASK 0x018 -#define INTEL_MSIC_GPIO0LVIRQMASK 0x019 -#define INTEL_MSIC_GPIO1LVIRQMASK 0x01a -#define INTEL_MSIC_GPIOHVIRQMASK 0x01b -#define INTEL_MSIC_VRINTMASK 0x01c -#define INTEL_MSIC_OCAUDIOMASK 0x01d -#define INTEL_MSIC_ACCDETMASK 0x01e -#define INTEL_MSIC_RESETIRQ1MASK 0x01f -#define INTEL_MSIC_RESETIRQ2MASK 0x020 -#define INTEL_MSIC_IRQLVL1MSK 0x021 -#define INTEL_MSIC_PBCONFIG 0x03e -#define INTEL_MSIC_PBSTATUS 0x03f /* RO */ - -/* GPIO */ -#define INTEL_MSIC_GPIO0LV7CTLO 0x040 -#define INTEL_MSIC_GPIO0LV6CTLO 0x041 -#define INTEL_MSIC_GPIO0LV5CTLO 0x042 -#define INTEL_MSIC_GPIO0LV4CTLO 0x043 -#define INTEL_MSIC_GPIO0LV3CTLO 0x044 -#define INTEL_MSIC_GPIO0LV2CTLO 0x045 -#define INTEL_MSIC_GPIO0LV1CTLO 0x046 -#define INTEL_MSIC_GPIO0LV0CTLO 0x047 -#define INTEL_MSIC_GPIO1LV7CTLOS 0x048 -#define INTEL_MSIC_GPIO1LV6CTLO 0x049 -#define INTEL_MSIC_GPIO1LV5CTLO 0x04a -#define INTEL_MSIC_GPIO1LV4CTLO 0x04b -#define INTEL_MSIC_GPIO1LV3CTLO 0x04c -#define INTEL_MSIC_GPIO1LV2CTLO 0x04d -#define INTEL_MSIC_GPIO1LV1CTLO 0x04e -#define INTEL_MSIC_GPIO1LV0CTLO 0x04f -#define INTEL_MSIC_GPIO0LV7CTLI 0x050 -#define INTEL_MSIC_GPIO0LV6CTLI 0x051 -#define INTEL_MSIC_GPIO0LV5CTLI 0x052 -#define INTEL_MSIC_GPIO0LV4CTLI 0x053 -#define INTEL_MSIC_GPIO0LV3CTLI 0x054 -#define INTEL_MSIC_GPIO0LV2CTLI 0x055 -#define INTEL_MSIC_GPIO0LV1CTLI 0x056 -#define INTEL_MSIC_GPIO0LV0CTLI 0x057 -#define INTEL_MSIC_GPIO1LV7CTLIS 0x058 -#define INTEL_MSIC_GPIO1LV6CTLI 0x059 -#define INTEL_MSIC_GPIO1LV5CTLI 0x05a -#define INTEL_MSIC_GPIO1LV4CTLI 0x05b -#define INTEL_MSIC_GPIO1LV3CTLI 0x05c -#define INTEL_MSIC_GPIO1LV2CTLI 0x05d -#define INTEL_MSIC_GPIO1LV1CTLI 0x05e -#define INTEL_MSIC_GPIO1LV0CTLI 0x05f -#define INTEL_MSIC_PWM0CLKDIV1 0x061 -#define INTEL_MSIC_PWM0CLKDIV0 0x062 -#define INTEL_MSIC_PWM1CLKDIV1 0x063 -#define INTEL_MSIC_PWM1CLKDIV0 0x064 -#define INTEL_MSIC_PWM2CLKDIV1 0x065 -#define INTEL_MSIC_PWM2CLKDIV0 0x066 -#define INTEL_MSIC_PWM0DUTYCYCLE 0x067 -#define INTEL_MSIC_PWM1DUTYCYCLE 0x068 -#define INTEL_MSIC_PWM2DUTYCYCLE 0x069 -#define INTEL_MSIC_GPIO0HV3CTLO 0x06d -#define INTEL_MSIC_GPIO0HV2CTLO 0x06e -#define INTEL_MSIC_GPIO0HV1CTLO 0x06f -#define INTEL_MSIC_GPIO0HV0CTLO 0x070 -#define INTEL_MSIC_GPIO1HV3CTLO 0x071 -#define INTEL_MSIC_GPIO1HV2CTLO 0x072 -#define INTEL_MSIC_GPIO1HV1CTLO 0x073 -#define INTEL_MSIC_GPIO1HV0CTLO 0x074 -#define INTEL_MSIC_GPIO0HV3CTLI 0x075 -#define INTEL_MSIC_GPIO0HV2CTLI 0x076 -#define INTEL_MSIC_GPIO0HV1CTLI 0x077 -#define INTEL_MSIC_GPIO0HV0CTLI 0x078 -#define INTEL_MSIC_GPIO1HV3CTLI 0x079 -#define INTEL_MSIC_GPIO1HV2CTLI 0x07a -#define INTEL_MSIC_GPIO1HV1CTLI 0x07b -#define INTEL_MSIC_GPIO1HV0CTLI 0x07c - -/* SVID */ -#define INTEL_MSIC_SVIDCTRL0 0x080 -#define INTEL_MSIC_SVIDCTRL1 0x081 -#define INTEL_MSIC_SVIDCTRL2 0x082 -#define INTEL_MSIC_SVIDTXLASTPKT3 0x083 /* RO */ -#define INTEL_MSIC_SVIDTXLASTPKT2 0x084 /* RO */ -#define INTEL_MSIC_SVIDTXLASTPKT1 0x085 /* RO */ -#define INTEL_MSIC_SVIDTXLASTPKT0 0x086 /* RO */ -#define INTEL_MSIC_SVIDPKTOUTBYTE3 0x087 -#define INTEL_MSIC_SVIDPKTOUTBYTE2 0x088 -#define INTEL_MSIC_SVIDPKTOUTBYTE1 0x089 -#define INTEL_MSIC_SVIDPKTOUTBYTE0 0x08a -#define INTEL_MSIC_SVIDRXVPDEBUG1 0x08b -#define INTEL_MSIC_SVIDRXVPDEBUG0 0x08c -#define INTEL_MSIC_SVIDRXLASTPKT3 0x08d /* RO */ -#define INTEL_MSIC_SVIDRXLASTPKT2 0x08e /* RO */ -#define INTEL_MSIC_SVIDRXLASTPKT1 0x08f /* RO */ -#define INTEL_MSIC_SVIDRXLASTPKT0 0x090 /* RO */ -#define INTEL_MSIC_SVIDRXCHKSTATUS3 0x091 /* RO */ -#define INTEL_MSIC_SVIDRXCHKSTATUS2 0x092 /* RO */ -#define INTEL_MSIC_SVIDRXCHKSTATUS1 0x093 /* RO */ -#define INTEL_MSIC_SVIDRXCHKSTATUS0 0x094 /* RO */ - -/* VREG */ -#define INTEL_MSIC_VCCLATCH 0x0c0 -#define INTEL_MSIC_VNNLATCH 0x0c1 -#define INTEL_MSIC_VCCCNT 0x0c2 -#define INTEL_MSIC_SMPSRAMP 0x0c3 -#define INTEL_MSIC_VNNCNT 0x0c4 -#define INTEL_MSIC_VNNAONCNT 0x0c5 -#define INTEL_MSIC_VCC122AONCNT 0x0c6 -#define INTEL_MSIC_V180AONCNT 0x0c7 -#define INTEL_MSIC_V500CNT 0x0c8 -#define INTEL_MSIC_VIHFCNT 0x0c9 -#define INTEL_MSIC_LDORAMP1 0x0ca -#define INTEL_MSIC_LDORAMP2 0x0cb -#define INTEL_MSIC_VCC108AONCNT 0x0cc -#define INTEL_MSIC_VCC108ASCNT 0x0cd -#define INTEL_MSIC_VCC108CNT 0x0ce -#define INTEL_MSIC_VCCA100ASCNT 0x0cf -#define INTEL_MSIC_VCCA100CNT 0x0d0 -#define INTEL_MSIC_VCC180AONCNT 0x0d1 -#define INTEL_MSIC_VCC180CNT 0x0d2 -#define INTEL_MSIC_VCC330CNT 0x0d3 -#define INTEL_MSIC_VUSB330CNT 0x0d4 -#define INTEL_MSIC_VCCSDIOCNT 0x0d5 -#define INTEL_MSIC_VPROG1CNT 0x0d6 -#define INTEL_MSIC_VPROG2CNT 0x0d7 -#define INTEL_MSIC_VEMMCSCNT 0x0d8 -#define INTEL_MSIC_VEMMC1CNT 0x0d9 -#define INTEL_MSIC_VEMMC2CNT 0x0da -#define INTEL_MSIC_VAUDACNT 0x0db -#define INTEL_MSIC_VHSPCNT 0x0dc -#define INTEL_MSIC_VHSNCNT 0x0dd -#define INTEL_MSIC_VHDMICNT 0x0de -#define INTEL_MSIC_VOTGCNT 0x0df -#define INTEL_MSIC_V1P35CNT 0x0e0 -#define INTEL_MSIC_V330AONCNT 0x0e1 - -/* RESET */ -#define INTEL_MSIC_CHIPCNTRL 0x100 /* WO */ -#define INTEL_MSIC_ERCONFIG 0x101 - -/* BURST */ -#define INTEL_MSIC_BATCURRENTLIMIT12 0x102 -#define INTEL_MSIC_BATTIMELIMIT12 0x103 -#define INTEL_MSIC_BATTIMELIMIT3 0x104 -#define INTEL_MSIC_BATTIMEDB 0x105 -#define INTEL_MSIC_BRSTCONFIGOUTPUTS 0x106 -#define INTEL_MSIC_BRSTCONFIGACTIONS 0x107 -#define INTEL_MSIC_BURSTCONTROLSTATUS 0x108 - -/* RTC */ -#define INTEL_MSIC_RTCB1 0x140 /* RO */ -#define INTEL_MSIC_RTCB2 0x141 /* RO */ -#define INTEL_MSIC_RTCB3 0x142 /* RO */ -#define INTEL_MSIC_RTCB4 0x143 /* RO */ -#define INTEL_MSIC_RTCOB1 0x144 -#define INTEL_MSIC_RTCOB2 0x145 -#define INTEL_MSIC_RTCOB3 0x146 -#define INTEL_MSIC_RTCOB4 0x147 -#define INTEL_MSIC_RTCAB1 0x148 -#define INTEL_MSIC_RTCAB2 0x149 -#define INTEL_MSIC_RTCAB3 0x14a -#define INTEL_MSIC_RTCAB4 0x14b -#define INTEL_MSIC_RTCWAB1 0x14c -#define INTEL_MSIC_RTCWAB2 0x14d -#define INTEL_MSIC_RTCWAB3 0x14e -#define INTEL_MSIC_RTCWAB4 0x14f -#define INTEL_MSIC_RTCSC1 0x150 -#define INTEL_MSIC_RTCSC2 0x151 -#define INTEL_MSIC_RTCSC3 0x152 -#define INTEL_MSIC_RTCSC4 0x153 -#define INTEL_MSIC_RTCSTATUS 0x154 /* RO */ -#define INTEL_MSIC_RTCCONFIG1 0x155 -#define INTEL_MSIC_RTCCONFIG2 0x156 - -/* CHARGER */ -#define INTEL_MSIC_BDTIMER 0x180 -#define INTEL_MSIC_BATTRMV 0x181 -#define INTEL_MSIC_VBUSDET 0x182 -#define INTEL_MSIC_VBUSDET1 0x183 -#define INTEL_MSIC_ADPHVDET 0x184 -#define INTEL_MSIC_ADPLVDET 0x185 -#define INTEL_MSIC_ADPDETDBDM 0x186 -#define INTEL_MSIC_LOWBATTDET 0x187 -#define INTEL_MSIC_CHRCTRL 0x188 -#define INTEL_MSIC_CHRCVOLTAGE 0x189 -#define INTEL_MSIC_CHRCCURRENT 0x18a -#define INTEL_MSIC_SPCHARGER 0x18b -#define INTEL_MSIC_CHRTTIME 0x18c -#define INTEL_MSIC_CHRCTRL1 0x18d -#define INTEL_MSIC_PWRSRCLMT 0x18e -#define INTEL_MSIC_CHRSTWDT 0x18f -#define INTEL_MSIC_WDTWRITE 0x190 /* WO */ -#define INTEL_MSIC_CHRSAFELMT 0x191 -#define INTEL_MSIC_SPWRSRCINT 0x192 /* RO */ -#define INTEL_MSIC_SPWRSRCINT1 0x193 /* RO */ -#define INTEL_MSIC_CHRLEDPWM 0x194 -#define INTEL_MSIC_CHRLEDCTRL 0x195 - -/* ADC */ -#define INTEL_MSIC_ADC1CNTL1 0x1c0 -#define INTEL_MSIC_ADC1CNTL2 0x1c1 -#define INTEL_MSIC_ADC1CNTL3 0x1c2 -#define INTEL_MSIC_ADC1OFFSETH 0x1c3 /* RO */ -#define INTEL_MSIC_ADC1OFFSETL 0x1c4 /* RO */ -#define INTEL_MSIC_ADC1ADDR0 0x1c5 -#define INTEL_MSIC_ADC1ADDR1 0x1c6 -#define INTEL_MSIC_ADC1ADDR2 0x1c7 -#define INTEL_MSIC_ADC1ADDR3 0x1c8 -#define INTEL_MSIC_ADC1ADDR4 0x1c9 -#define INTEL_MSIC_ADC1ADDR5 0x1ca -#define INTEL_MSIC_ADC1ADDR6 0x1cb -#define INTEL_MSIC_ADC1ADDR7 0x1cc -#define INTEL_MSIC_ADC1ADDR8 0x1cd -#define INTEL_MSIC_ADC1ADDR9 0x1ce -#define INTEL_MSIC_ADC1ADDR10 0x1cf -#define INTEL_MSIC_ADC1ADDR11 0x1d0 -#define INTEL_MSIC_ADC1ADDR12 0x1d1 -#define INTEL_MSIC_ADC1ADDR13 0x1d2 -#define INTEL_MSIC_ADC1ADDR14 0x1d3 -#define INTEL_MSIC_ADC1SNS0H 0x1d4 /* RO */ -#define INTEL_MSIC_ADC1SNS0L 0x1d5 /* RO */ -#define INTEL_MSIC_ADC1SNS1H 0x1d6 /* RO */ -#define INTEL_MSIC_ADC1SNS1L 0x1d7 /* RO */ -#define INTEL_MSIC_ADC1SNS2H 0x1d8 /* RO */ -#define INTEL_MSIC_ADC1SNS2L 0x1d9 /* RO */ -#define INTEL_MSIC_ADC1SNS3H 0x1da /* RO */ -#define INTEL_MSIC_ADC1SNS3L 0x1db /* RO */ -#define INTEL_MSIC_ADC1SNS4H 0x1dc /* RO */ -#define INTEL_MSIC_ADC1SNS4L 0x1dd /* RO */ -#define INTEL_MSIC_ADC1SNS5H 0x1de /* RO */ -#define INTEL_MSIC_ADC1SNS5L 0x1df /* RO */ -#define INTEL_MSIC_ADC1SNS6H 0x1e0 /* RO */ -#define INTEL_MSIC_ADC1SNS6L 0x1e1 /* RO */ -#define INTEL_MSIC_ADC1SNS7H 0x1e2 /* RO */ -#define INTEL_MSIC_ADC1SNS7L 0x1e3 /* RO */ -#define INTEL_MSIC_ADC1SNS8H 0x1e4 /* RO */ -#define INTEL_MSIC_ADC1SNS8L 0x1e5 /* RO */ -#define INTEL_MSIC_ADC1SNS9H 0x1e6 /* RO */ -#define INTEL_MSIC_ADC1SNS9L 0x1e7 /* RO */ -#define INTEL_MSIC_ADC1SNS10H 0x1e8 /* RO */ -#define INTEL_MSIC_ADC1SNS10L 0x1e9 /* RO */ -#define INTEL_MSIC_ADC1SNS11H 0x1ea /* RO */ -#define INTEL_MSIC_ADC1SNS11L 0x1eb /* RO */ -#define INTEL_MSIC_ADC1SNS12H 0x1ec /* RO */ -#define INTEL_MSIC_ADC1SNS12L 0x1ed /* RO */ -#define INTEL_MSIC_ADC1SNS13H 0x1ee /* RO */ -#define INTEL_MSIC_ADC1SNS13L 0x1ef /* RO */ -#define INTEL_MSIC_ADC1SNS14H 0x1f0 /* RO */ -#define INTEL_MSIC_ADC1SNS14L 0x1f1 /* RO */ -#define INTEL_MSIC_ADC1BV0H 0x1f2 /* RO */ -#define INTEL_MSIC_ADC1BV0L 0x1f3 /* RO */ -#define INTEL_MSIC_ADC1BV1H 0x1f4 /* RO */ -#define INTEL_MSIC_ADC1BV1L 0x1f5 /* RO */ -#define INTEL_MSIC_ADC1BV2H 0x1f6 /* RO */ -#define INTEL_MSIC_ADC1BV2L 0x1f7 /* RO */ -#define INTEL_MSIC_ADC1BV3H 0x1f8 /* RO */ -#define INTEL_MSIC_ADC1BV3L 0x1f9 /* RO */ -#define INTEL_MSIC_ADC1BI0H 0x1fa /* RO */ -#define INTEL_MSIC_ADC1BI0L 0x1fb /* RO */ -#define INTEL_MSIC_ADC1BI1H 0x1fc /* RO */ -#define INTEL_MSIC_ADC1BI1L 0x1fd /* RO */ -#define INTEL_MSIC_ADC1BI2H 0x1fe /* RO */ -#define INTEL_MSIC_ADC1BI2L 0x1ff /* RO */ -#define INTEL_MSIC_ADC1BI3H 0x200 /* RO */ -#define INTEL_MSIC_ADC1BI3L 0x201 /* RO */ -#define INTEL_MSIC_CCCNTL 0x202 -#define INTEL_MSIC_CCOFFSETH 0x203 /* RO */ -#define INTEL_MSIC_CCOFFSETL 0x204 /* RO */ -#define INTEL_MSIC_CCADCHA 0x205 /* RO */ -#define INTEL_MSIC_CCADCLA 0x206 /* RO */ - -/* AUDIO */ -#define INTEL_MSIC_AUDPLLCTRL 0x240 -#define INTEL_MSIC_DMICBUF0123 0x241 -#define INTEL_MSIC_DMICBUF45 0x242 -#define INTEL_MSIC_DMICGPO 0x244 -#define INTEL_MSIC_DMICMUX 0x245 -#define INTEL_MSIC_DMICCLK 0x246 -#define INTEL_MSIC_MICBIAS 0x247 -#define INTEL_MSIC_ADCCONFIG 0x248 -#define INTEL_MSIC_MICAMP1 0x249 -#define INTEL_MSIC_MICAMP2 0x24a -#define INTEL_MSIC_NOISEMUX 0x24b -#define INTEL_MSIC_AUDIOMUX12 0x24c -#define INTEL_MSIC_AUDIOMUX34 0x24d -#define INTEL_MSIC_AUDIOSINC 0x24e -#define INTEL_MSIC_AUDIOTXEN 0x24f -#define INTEL_MSIC_HSEPRXCTRL 0x250 -#define INTEL_MSIC_IHFRXCTRL 0x251 -#define INTEL_MSIC_VOICETXVOL 0x252 -#define INTEL_MSIC_SIDETONEVOL 0x253 -#define INTEL_MSIC_MUSICSHARVOL 0x254 -#define INTEL_MSIC_VOICETXCTRL 0x255 -#define INTEL_MSIC_HSMIXER 0x256 -#define INTEL_MSIC_DACCONFIG 0x257 -#define INTEL_MSIC_SOFTMUTE 0x258 -#define INTEL_MSIC_HSLVOLCTRL 0x259 -#define INTEL_MSIC_HSRVOLCTRL 0x25a -#define INTEL_MSIC_IHFLVOLCTRL 0x25b -#define INTEL_MSIC_IHFRVOLCTRL 0x25c -#define INTEL_MSIC_DRIVEREN 0x25d -#define INTEL_MSIC_LINEOUTCTRL 0x25e -#define INTEL_MSIC_VIB1CTRL1 0x25f -#define INTEL_MSIC_VIB1CTRL2 0x260 -#define INTEL_MSIC_VIB1CTRL3 0x261 -#define INTEL_MSIC_VIB1SPIPCM_1 0x262 -#define INTEL_MSIC_VIB1SPIPCM_2 0x263 -#define INTEL_MSIC_VIB1CTRL5 0x264 -#define INTEL_MSIC_VIB2CTRL1 0x265 -#define INTEL_MSIC_VIB2CTRL2 0x266 -#define INTEL_MSIC_VIB2CTRL3 0x267 -#define INTEL_MSIC_VIB2SPIPCM_1 0x268 -#define INTEL_MSIC_VIB2SPIPCM_2 0x269 -#define INTEL_MSIC_VIB2CTRL5 0x26a -#define INTEL_MSIC_BTNCTRL1 0x26b -#define INTEL_MSIC_BTNCTRL2 0x26c -#define INTEL_MSIC_PCM1TXSLOT01 0x26d -#define INTEL_MSIC_PCM1TXSLOT23 0x26e -#define INTEL_MSIC_PCM1TXSLOT45 0x26f -#define INTEL_MSIC_PCM1RXSLOT0123 0x270 -#define INTEL_MSIC_PCM1RXSLOT045 0x271 -#define INTEL_MSIC_PCM2TXSLOT01 0x272 -#define INTEL_MSIC_PCM2TXSLOT23 0x273 -#define INTEL_MSIC_PCM2TXSLOT45 0x274 -#define INTEL_MSIC_PCM2RXSLOT01 0x275 -#define INTEL_MSIC_PCM2RXSLOT23 0x276 -#define INTEL_MSIC_PCM2RXSLOT45 0x277 -#define INTEL_MSIC_PCM1CTRL1 0x278 -#define INTEL_MSIC_PCM1CTRL2 0x279 -#define INTEL_MSIC_PCM1CTRL3 0x27a -#define INTEL_MSIC_PCM2CTRL1 0x27b -#define INTEL_MSIC_PCM2CTRL2 0x27c - -/* HDMI */ -#define INTEL_MSIC_HDMIPUEN 0x280 -#define INTEL_MSIC_HDMISTATUS 0x281 /* RO */ - -/* Physical address of the start of the MSIC interrupt tree in SRAM */ -#define INTEL_MSIC_IRQ_PHYS_BASE 0xffff7fc0 - -/** - * struct intel_msic_gpio_pdata - platform data for the MSIC GPIO driver - * @gpio_base: base number for the GPIOs - */ -struct intel_msic_gpio_pdata { - unsigned gpio_base; -}; - -/** - * struct intel_msic_ocd_pdata - platform data for the MSIC OCD driver - * @gpio: GPIO number used for OCD interrupts - * - * The MSIC MFD driver converts @gpio into an IRQ number and passes it to - * the OCD driver as %IORESOURCE_IRQ. - */ -struct intel_msic_ocd_pdata { - unsigned gpio; -}; - -/* MSIC embedded blocks (subdevices) */ -enum intel_msic_block { - INTEL_MSIC_BLOCK_TOUCH, - INTEL_MSIC_BLOCK_ADC, - INTEL_MSIC_BLOCK_BATTERY, - INTEL_MSIC_BLOCK_GPIO, - INTEL_MSIC_BLOCK_AUDIO, - INTEL_MSIC_BLOCK_HDMI, - INTEL_MSIC_BLOCK_THERMAL, - INTEL_MSIC_BLOCK_POWER_BTN, - INTEL_MSIC_BLOCK_OCD, - - INTEL_MSIC_BLOCK_LAST, -}; - -/** - * struct intel_msic_platform_data - platform data for the MSIC driver - * @irq: array of interrupt numbers, one per device. If @irq is set to %0 - * for a given block, the corresponding platform device is not - * created. For devices which don't have an interrupt, use %0xff - * (this is same as in SFI spec). - * @gpio: platform data for the MSIC GPIO driver - * @ocd: platform data for the MSIC OCD driver - * - * Once the MSIC driver is initialized, the register interface is ready to - * use. All the platform devices for subdevices are created after the - * register interface is ready so that we can guarantee its availability to - * the subdevice drivers. - * - * Interrupt numbers are passed to the subdevices via %IORESOURCE_IRQ - * resources of the created platform device. - */ -struct intel_msic_platform_data { - int irq[INTEL_MSIC_BLOCK_LAST]; - struct intel_msic_gpio_pdata *gpio; - struct intel_msic_ocd_pdata *ocd; -}; - -struct intel_msic; - -extern int intel_msic_reg_read(unsigned short reg, u8 *val); -extern int intel_msic_reg_write(unsigned short reg, u8 val); -extern int intel_msic_reg_update(unsigned short reg, u8 val, u8 mask); -extern int intel_msic_bulk_read(unsigned short *reg, u8 *buf, size_t count); -extern int intel_msic_bulk_write(unsigned short *reg, u8 *buf, size_t count); - -/* - * pdev_to_intel_msic - gets an MSIC instance from the platform device - * @pdev: platform device pointer - * - * The client drivers need to have pointer to the MSIC instance if they - * want to call intel_msic_irq_read(). This macro can be used for - * convenience to get the MSIC pointer from @pdev where needed. This is - * _only_ valid for devices which are managed by the MSIC. - */ -#define pdev_to_intel_msic(pdev) (dev_get_drvdata(pdev->dev.parent)) - -extern int intel_msic_irq_read(struct intel_msic *msic, unsigned short reg, - u8 *val); - -#endif /* __LINUX_MFD_INTEL_MSIC_H__ */ From bfb44502b8fc865c9962ca335ae9877579ff4a9c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 4 Feb 2021 16:40:04 +0100 Subject: [PATCH 355/633] remoteproc: qcom: fix glink dependencies Building the remoteproc drivers into the kernel while the qcom_glink code is in a loadable module results in a link error: ld.lld: error: undefined symbol: qcom_glink_ssr_notify >>> referenced by vmlinux.o:(glink_subdev_unprepare) Add a Kconfig dependency to avoid this. Reviewed-by: Alex Elder Fixes: 8527efc59d45 ("rpmsg: glink: Guard qcom_glink_ssr_notify() with correct config") Fixes: 5d1f2e3c8090 ("soc: qcom: glink_ssr: Internalize ssr_notifiers") Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20210204154010.1585457-1-arnd@kernel.org Signed-off-by: Bjorn Andersson --- drivers/remoteproc/Kconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index bb31b115fdbd..15d1574d129b 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -155,6 +155,7 @@ config QCOM_Q6V5_ADSP depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_PIL_INFO select QCOM_MDT_LOADER @@ -173,6 +174,7 @@ config QCOM_Q6V5_MSS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_MDT_LOADER select QCOM_PIL_INFO @@ -191,6 +193,7 @@ config QCOM_Q6V5_PAS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_PIL_INFO select QCOM_MDT_LOADER @@ -211,6 +214,7 @@ config QCOM_Q6V5_WCSS depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select MFD_SYSCON select QCOM_MDT_LOADER select QCOM_PIL_INFO @@ -244,6 +248,7 @@ config QCOM_WCNSS_PIL depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n depends on QCOM_SMEM depends on QCOM_SYSMON || QCOM_SYSMON=n + depends on RPMSG_QCOM_GLINK || RPMSG_QCOM_GLINK=n select QCOM_MDT_LOADER select QCOM_PIL_INFO select QCOM_RPROC_COMMON From 9a1d27148543da8966aaabb44c5403f3a81cebcb Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 3 Feb 2021 10:46:42 +0800 Subject: [PATCH 356/633] remoteproc: qcom_wcnss: remove unneeded semicolon Eliminate the following coccicheck warning: ./drivers/remoteproc/qcom_wcnss.c:573:2-3: Unneeded semicolon Reported-by: Abaci Robot Signed-off-by: Yang Li Link: https://lore.kernel.org/r/1612320402-3313-1-git-send-email-yang.lee@linux.alibaba.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/qcom_wcnss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index f95854255c70..2a6a23cb14ca 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -570,7 +570,7 @@ static int wcnss_probe(struct platform_device *pdev) if (IS_ERR(mmio)) { ret = PTR_ERR(mmio); goto free_rproc; - }; + } ret = wcnss_alloc_memory_region(wcnss); if (ret) From 8c545f52dce44368fff524e13116e696e005c074 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 27 Jan 2021 16:20:46 +0800 Subject: [PATCH 357/633] remoteproc/mediatek: acknowledge watchdog IRQ after handled Acknowledges watchdog IRQ after handled or kernel keeps receiving the interrupt. Fixes: fd0b6c1ff85a ("remoteproc/mediatek: Add support for mt8192 SCP") Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20210127082046.3735157-1-tzungbi@google.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/mtk_common.h | 1 + drivers/remoteproc/mtk_scp.c | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index 988edb4977c3..bcab38511bf3 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -47,6 +47,7 @@ #define MT8192_CORE0_SW_RSTN_CLR 0x10000 #define MT8192_CORE0_SW_RSTN_SET 0x10004 +#define MT8192_CORE0_WDT_IRQ 0x10030 #define MT8192_CORE0_WDT_CFG 0x10034 #define SCP_FW_VER_LEN 32 diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index e0c235690361..eba825b46696 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -197,17 +197,19 @@ static void mt8192_scp_irq_handler(struct mtk_scp *scp) scp_to_host = readl(scp->reg_base + MT8192_SCP2APMCU_IPC_SET); - if (scp_to_host & MT8192_SCP_IPC_INT_BIT) + if (scp_to_host & MT8192_SCP_IPC_INT_BIT) { scp_ipi_handler(scp); - else - scp_wdt_handler(scp, scp_to_host); - /* - * SCP won't send another interrupt until we clear - * MT8192_SCP2APMCU_IPC. - */ - writel(MT8192_SCP_IPC_INT_BIT, - scp->reg_base + MT8192_SCP2APMCU_IPC_CLR); + /* + * SCP won't send another interrupt until we clear + * MT8192_SCP2APMCU_IPC. + */ + writel(MT8192_SCP_IPC_INT_BIT, + scp->reg_base + MT8192_SCP2APMCU_IPC_CLR); + } else { + scp_wdt_handler(scp, scp_to_host); + writel(1, scp->reg_base + MT8192_CORE0_WDT_IRQ); + } } static irqreturn_t scp_irq_handler(int irq, void *priv) From ec8207ae39dc1f498962722d9fdea7bbc3a8233b Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Sat, 23 Jan 2021 14:29:56 +0000 Subject: [PATCH 358/633] remoteproc: ingenic: Add module parameter 'auto_boot' Add a 'auto_boot' module parameter that instructs the remoteproc driver whether or not it should auto-boot the remote processor, which will default to "false", since the VPU in Ingenic SoCs does not really have any predetermined function. Reviewed-by: Mathieu Poirier Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20210123142956.17865-1-paul@crapouillou.net Signed-off-by: Bjorn Andersson --- drivers/remoteproc/ingenic_rproc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/remoteproc/ingenic_rproc.c b/drivers/remoteproc/ingenic_rproc.c index 26e19e6143b7..e2618c36eaab 100644 --- a/drivers/remoteproc/ingenic_rproc.c +++ b/drivers/remoteproc/ingenic_rproc.c @@ -27,6 +27,11 @@ #define AUX_CTRL_NMI BIT(1) #define AUX_CTRL_SW_RESET BIT(0) +static bool auto_boot; +module_param(auto_boot, bool, 0400); +MODULE_PARM_DESC(auto_boot, + "Auto-boot the remote processor [default=false]"); + struct vpu_mem_map { const char *name; unsigned int da; @@ -172,6 +177,8 @@ static int ingenic_rproc_probe(struct platform_device *pdev) if (!rproc) return -ENOMEM; + rproc->auto_boot = auto_boot; + vpu = rproc->priv; vpu->dev = &pdev->dev; platform_set_drvdata(pdev, vpu); From 2e88e8fcdfcd2e5569180944789ff299114c2bf7 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 27 Jan 2021 16:31:33 +0800 Subject: [PATCH 359/633] remoteproc/mediatek: use devm_platform_ioremap_resource_byname Replaces platform_get_resource_byname() and devm_ioremap_resource() pairs to devm_platform_ioremap_resource_byname(). Note that, not every pairs are applicable to replace. Especially when it needs to access the resource struct from platform_get_resource_byname(). For example: scp->sram_size = resource_size(res); Reviewed-by: Mathieu Poirier Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20210127083136.3745652-2-tzungbi@google.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/mtk_scp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index eba825b46696..d83e1164f02f 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -721,8 +721,7 @@ static int scp_probe(struct platform_device *pdev) for (i = 0; i < SCP_IPI_MAX; i++) mutex_init(&scp->ipi_desc[i].lock); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); - scp->reg_base = devm_ioremap_resource(dev, res); + scp->reg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); if (IS_ERR((__force void *)scp->reg_base)) { dev_err(dev, "Failed to parse and map cfg memory\n"); ret = PTR_ERR((__force void *)scp->reg_base); From ff3ea536023e8a40c499f884bdc3cc5aec5b1e25 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 27 Jan 2021 16:31:34 +0800 Subject: [PATCH 360/633] remoteproc/mediatek: enable MPU for all memory regions in MT8192 SCP The register MT8192_CORE0_MEM_ATT_PREDEF contains attributes for each memory region. It defines whether a memory region can be managed by MPU or not. In the past, due to the default settings in the register, MT8192 SCP works luckily. After enabling L1TCM, SCP starts to access memory region that is not included in the default settings. As a result, SCP hangs. Enables MPU for all memory regions in MT8192 SCP. Note that the register is read only once when SCP resets. Thus, it must be set from kernel side. Reviewed-by: Mathieu Poirier Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20210127083136.3745652-3-tzungbi@google.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/mtk_common.h | 1 + drivers/remoteproc/mtk_scp.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index bcab38511bf3..204691138677 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -47,6 +47,7 @@ #define MT8192_CORE0_SW_RSTN_CLR 0x10000 #define MT8192_CORE0_SW_RSTN_SET 0x10004 +#define MT8192_CORE0_MEM_ATT_PREDEF 0x10008 #define MT8192_CORE0_WDT_IRQ 0x10030 #define MT8192_CORE0_WDT_CFG 0x10034 diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index d83e1164f02f..05b157689121 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -371,6 +371,9 @@ static int mt8192_scp_before_load(struct mtk_scp *scp) mt8192_power_on_sram(scp->reg_base + MT8192_L1TCM_SRAM_PDN); mt8192_power_on_sram(scp->reg_base + MT8192_CPU0_SRAM_PD); + /* enable MPU for all memory regions */ + writel(0xff, scp->reg_base + MT8192_CORE0_MEM_ATT_PREDEF); + return 0; } From 503c64cc42f15799ba0fdc654b6640911aaf7c34 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 27 Jan 2021 16:31:35 +0800 Subject: [PATCH 361/633] dt-bindings: remoteproc: mediatek: add L1TCM memory region Adds L1TCM memory region. The reg-name is "l1tcm". Acked-by: Rob Herring Reviewed-by: Mathieu Poirier Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20210127083136.3745652-4-tzungbi@google.com Signed-off-by: Bjorn Andersson --- Documentation/devicetree/bindings/remoteproc/mtk,scp.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt index 3ba668bab14b..3f5f78764b60 100644 --- a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt +++ b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt @@ -6,10 +6,10 @@ Mediatek SoCs. Required properties: - compatible Should be "mediatek,mt8183-scp" -- reg Should contain the address ranges for the two memory - regions, SRAM and CFG. -- reg-names Contains the corresponding names for the two memory - regions. These should be named "sram" & "cfg". +- reg Should contain the address ranges for memory regions: + SRAM, CFG, and L1TCM. +- reg-names Contains the corresponding names for the memory regions: + "sram", "cfg", and "l1tcm". - clocks Clock for co-processor (See: ../clock/clock-bindings.txt) - clock-names Contains the corresponding name for the clock. This should be named "main". From ca23ecfdbd44bac02cb6964a3793c565a389af96 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 27 Jan 2021 16:31:36 +0800 Subject: [PATCH 362/633] remoteproc/mediatek: support L1TCM L1TCM is a high performance memory region in MT8192 SCP. Reads L1TCM memory region from DTS to determine if the machine supports. Loads L1TCM memory region to SCP sys if the firmware provides. Starts from MT8192 SCP, the firmware contains physical addresses for each memory region, for instance: Program Headers: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align LOAD 0xXXXXXX 0xXXXXXXXX 0x10500000 0xXXXXX 0xXXXXX XXX 0xXXXX LOAD 0xXXXXXX 0xXXXXXXXX 0x10700000 0xXXXXX 0xXXXXX XXX 0xXXXX LOAD 0xXXXXXX 0xXXXXXXXX 0x50000000 0xXXXXX 0xXXXXX XXX 0xXXXX Kernel driver can use the "PhysAddr" (i.e. da in the da_to_va callbacks) to know the ELF segment belongs to which region. To backward compatible to MT8183 SCP, separates the da_to_va callbacks for new and legacy version. Reviewed-by: Mathieu Poirier Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20210127083136.3745652-5-tzungbi@google.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/mtk_common.h | 5 +++ drivers/remoteproc/mtk_scp.c | 56 +++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/drivers/remoteproc/mtk_common.h b/drivers/remoteproc/mtk_common.h index 204691138677..61901f5efa05 100644 --- a/drivers/remoteproc/mtk_common.h +++ b/drivers/remoteproc/mtk_common.h @@ -77,6 +77,7 @@ struct mtk_scp_of_data { void (*scp_reset_assert)(struct mtk_scp *scp); void (*scp_reset_deassert)(struct mtk_scp *scp); void (*scp_stop)(struct mtk_scp *scp); + void *(*scp_da_to_va)(struct mtk_scp *scp, u64 da, size_t len); u32 host_to_scp_reg; u32 host_to_scp_int_bit; @@ -91,6 +92,10 @@ struct mtk_scp { void __iomem *reg_base; void __iomem *sram_base; size_t sram_size; + phys_addr_t sram_phys; + void __iomem *l1tcm_base; + size_t l1tcm_size; + phys_addr_t l1tcm_phys; const struct mtk_scp_of_data *data; diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 05b157689121..ce727598c41c 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -463,9 +463,8 @@ static int scp_start(struct rproc *rproc) return ret; } -static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) +static void *mt8183_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) { - struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; int offset; if (da < scp->sram_size) { @@ -481,6 +480,42 @@ static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) return NULL; } +static void *mt8192_scp_da_to_va(struct mtk_scp *scp, u64 da, size_t len) +{ + int offset; + + if (da >= scp->sram_phys && + (da + len) <= scp->sram_phys + scp->sram_size) { + offset = da - scp->sram_phys; + return (void __force *)scp->sram_base + offset; + } + + /* optional memory region */ + if (scp->l1tcm_size && + da >= scp->l1tcm_phys && + (da + len) <= scp->l1tcm_phys + scp->l1tcm_size) { + offset = da - scp->l1tcm_phys; + return (void __force *)scp->l1tcm_base + offset; + } + + /* optional memory region */ + if (scp->dram_size && + da >= scp->dma_addr && + (da + len) <= scp->dma_addr + scp->dram_size) { + offset = da - scp->dma_addr; + return scp->cpu_addr + offset; + } + + return NULL; +} + +static void *scp_da_to_va(struct rproc *rproc, u64 da, size_t len) +{ + struct mtk_scp *scp = (struct mtk_scp *)rproc->priv; + + return scp->data->scp_da_to_va(scp, da, len); +} + static void mt8183_scp_stop(struct mtk_scp *scp) { /* Disable SCP watchdog */ @@ -719,6 +754,21 @@ static int scp_probe(struct platform_device *pdev) goto free_rproc; } scp->sram_size = resource_size(res); + scp->sram_phys = res->start; + + /* l1tcm is an optional memory region */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "l1tcm"); + scp->l1tcm_base = devm_ioremap_resource(dev, res); + if (IS_ERR((__force void *)scp->l1tcm_base)) { + ret = PTR_ERR((__force void *)scp->l1tcm_base); + if (ret != -EINVAL) { + dev_err(dev, "Failed to map l1tcm memory\n"); + goto free_rproc; + } + } else { + scp->l1tcm_size = resource_size(res); + scp->l1tcm_phys = res->start; + } mutex_init(&scp->send_lock); for (i = 0; i < SCP_IPI_MAX; i++) @@ -807,6 +857,7 @@ static const struct mtk_scp_of_data mt8183_of_data = { .scp_reset_assert = mt8183_scp_reset_assert, .scp_reset_deassert = mt8183_scp_reset_deassert, .scp_stop = mt8183_scp_stop, + .scp_da_to_va = mt8183_scp_da_to_va, .host_to_scp_reg = MT8183_HOST_TO_SCP, .host_to_scp_int_bit = MT8183_HOST_IPC_INT_BIT, .ipi_buf_offset = 0x7bdb0, @@ -818,6 +869,7 @@ static const struct mtk_scp_of_data mt8192_of_data = { .scp_reset_assert = mt8192_scp_reset_assert, .scp_reset_deassert = mt8192_scp_reset_deassert, .scp_stop = mt8192_scp_stop, + .scp_da_to_va = mt8192_scp_da_to_va, .host_to_scp_reg = MT8192_GIPC_IN_SET, .host_to_scp_int_bit = MT8192_HOST_IPC_INT_BIT, }; From 387db89ca00fc6db5dc20571e822af9bddbc9695 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Tue, 9 Feb 2021 11:22:39 -0600 Subject: [PATCH 363/633] dt-bindings: hwlock: Update OMAP HwSpinlock binding for AM64x SoCs Update the existing OMAP HwSpinlock binding to include the info for AM64x SoCs. There are some minor IP integration differences between the AM64x SoCs and the previous AM65x and J721E SoC families. Signed-off-by: Suman Anna Link: https://lore.kernel.org/r/20210209172240.2305-2-s-anna@ti.com Signed-off-by: Bjorn Andersson --- Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml index ac35491a6f65..ae1b37dbee75 100644 --- a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml @@ -13,6 +13,7 @@ properties: compatible: enum: - ti,omap4-hwspinlock # for OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs + - ti,am64-hwspinlock # for K3 AM64x SoCs - ti,am654-hwspinlock # for K3 AM65x, J721E and J7200 SoCs reg: From b9ddb2500e7e544410f38476ab928fc2fe01e381 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Tue, 9 Feb 2021 11:22:40 -0600 Subject: [PATCH 364/633] hwspinlock: omap: Add support for K3 AM64x SoCs The AM64x SoC contains a HwSpinlock IP instance in the MAIN domain, and is a minor variant of the IP on the current TI K3 SoCs such as AM64x, J721E or J7200 SoCs. The IP is not built with the K3 safety feature in hardware, and has slightly different integration into the overall SoC. Add the support for this IP through a new compatible. Signed-off-by: Suman Anna Link: https://lore.kernel.org/r/20210209172240.2305-3-s-anna@ti.com Signed-off-by: Bjorn Andersson --- drivers/hwspinlock/omap_hwspinlock.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 3b05560456ea..33eeff94fc2a 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -2,11 +2,12 @@ /* * OMAP hardware spinlock driver * - * Copyright (C) 2010-2015 Texas Instruments Incorporated - http://www.ti.com + * Copyright (C) 2010-2021 Texas Instruments Incorporated - https://www.ti.com * * Contact: Simon Que * Hari Kanigeri * Ohad Ben-Cohen + * Suman Anna */ #include @@ -164,6 +165,7 @@ static int omap_hwspinlock_remove(struct platform_device *pdev) static const struct of_device_id omap_hwspinlock_of_match[] = { { .compatible = "ti,omap4-hwspinlock", }, + { .compatible = "ti,am64-hwspinlock", }, { .compatible = "ti,am654-hwspinlock", }, { /* end */ }, }; From 8f1fc1c15329a9d53bde5636e85ca98ece2ec7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Hundeb=C3=B8ll?= Date: Mon, 8 Feb 2021 16:01:57 +0100 Subject: [PATCH 365/633] PCI: Add Silicom Denmark vendor ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update pci_ids.h with the vendor ID for Silicom Denmark. The define is going to be referenced in driver(s) for FPGA accelerated smart NICs. Link: https://lore.kernel.org/r/20210208150158.2877414-1-mhu@silicom.dk Signed-off-by: Martin Hundebøll Signed-off-by: Bjorn Helgaas Reviewed-by: Tom Rix Reviewed-by: Krzysztof Wilczyński --- include/linux/pci_ids.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index d8156a5dbee8..aae07d512ca6 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2588,6 +2588,8 @@ #define PCI_VENDOR_ID_REDHAT 0x1b36 +#define PCI_VENDOR_ID_SILICOM_DENMARK 0x1c2c + #define PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS 0x1c36 #define PCI_VENDOR_ID_CIRCUITCO 0x1cc8 From e8e9aababe60a12928172b5f018d15de3c2cdf31 Mon Sep 17 00:00:00 2001 From: Junhao He Date: Thu, 4 Feb 2021 19:30:15 +0800 Subject: [PATCH 366/633] PCI: Apply CONFIG_PCI_DEBUG to entire drivers/pci hierarchy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONFIG_PCI_DEBUG=y adds -DDEBUG to CFLAGS, which enables things like pr_debug() and dev_dbg() (and hence pci_dbg()). Previously we added -DDEBUG for files in drivers/pci/, but not files in subdirectories of drivers/pci/. Add -DDEBUG to CFLAGS for all files below drivers/pci/ so CONFIG_PCI_DEBUG applies to the entire hierarchy. [bhelgaas: commit log] Link: https://lore.kernel.org/r/1612438215-33105-1-git-send-email-yangyicong@hisilicon.com Signed-off-by: Junhao He Signed-off-by: Yicong Yang Signed-off-by: Bjorn Helgaas Reviewed-by: Krzysztof Wilczyński --- drivers/pci/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 11cc79411e2d..d62c4ac4ae1b 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -36,4 +36,4 @@ obj-$(CONFIG_PCI_ENDPOINT) += endpoint/ obj-y += controller/ obj-y += switch/ -ccflags-$(CONFIG_PCI_DEBUG) := -DDEBUG +subdir-ccflags-$(CONFIG_PCI_DEBUG) := -DDEBUG From 43d3f2c715cefcfb89b10675728e9bf0d8bb98e3 Mon Sep 17 00:00:00 2001 From: Daniele Alessandrelli Date: Mon, 18 Jan 2021 16:59:04 +0000 Subject: [PATCH 367/633] remoteproc: core: Fix rproc->firmware free in rproc_set_firmware() rproc_alloc_firmware() (called by rproc_alloc()) can allocate rproc->firmware using kstrdup_const() and therefore should be freed using kfree_const(); however, rproc_set_firmware() frees it using the simple kfree(). This causes a kernel oops if a constant string is passed to rproc_alloc() and rproc_set_firmware() is subsequently called. Fix the above issue by using kfree_const() to free rproc->firmware in rproc_set_firmware(). Reviewed-by: Mathieu Poirier Signed-off-by: Daniele Alessandrelli Link: https://lore.kernel.org/r/20210118165904.719999-1-daniele.alessandrelli@linux.intel.com Signed-off-by: Bjorn Andersson --- drivers/remoteproc/remoteproc_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 2394eef383e3..ab150765d124 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1988,7 +1988,7 @@ int rproc_set_firmware(struct rproc *rproc, const char *fw_name) goto out; } - kfree(rproc->firmware); + kfree_const(rproc->firmware); rproc->firmware = p; out: From db4e8de1935b0202960e9ebb88ab93e8bd1e66b1 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Wed, 10 Feb 2021 13:55:38 +0530 Subject: [PATCH 368/633] mhi: Fix double dma free mhi_deinit_chan_ctxt functionthat takes care of unitializing channel resources, including unmapping coherent MHI areas, can be called from different path in case of controller unregistering/removal: - From a client driver remove callback, via mhi_unprepare_channel - From mhi_driver_remove that unitialize all channels mhi_driver_remove() |-> driver->remove() | |-> mhi_unprepare_channel() | |-> mhi_deinit_chan_ctxt() |... |-> mhi_deinit_chan_ctxt() This leads to double dma freeing... Fix that by preventing deinit for already uninitialized channel. Link: https://lore.kernel.org/r/1612894264-15956-1-git-send-email-loic.poulain@linaro.org Fixes: a7f422f2f89e ("bus: mhi: Fix channel close issue on driver remove") Reported-by: Kalle Valo Tested-by: Kalle Valo Reviewed-by: Manivannan Sadhasivam Signed-off-by: Loic Poulain Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20210210082538.2494-2-manivannan.sadhasivam@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/bus/mhi/core/init.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index aa575d3fb3ae..be4eebb0971b 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -557,6 +557,9 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, tre_ring = &mhi_chan->tre_ring; chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + if (!chan_ctxt->rbase) /* Already uninitialized */ + return; + mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size, tre_ring->pre_aligned, tre_ring->dma_handle); vfree(buf_ring->base); From 647bd7e7a93c494d7981ae66c8e7262a266d21c8 Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Wed, 10 Feb 2021 13:01:16 +0200 Subject: [PATCH 369/633] MAINTAINERS: replace my with email with replacements This email will become inactive in a few weeks. This change removes it from the MAINTAINERS file and adds the people that will be responsible for the parts moving forward. Acked-by: Michael Hennerich Acked-by: Nuno Sa Signed-off-by: Alexandru Ardelean Link: https://lore.kernel.org/r/20210210110116.49955-1-alexandru.ardelean@analog.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 6b5990a28b44..65b2a9ea74da 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1025,7 +1025,7 @@ F: Documentation/devicetree/bindings/mux/adi,adgs1408.txt F: drivers/mux/adgs1408.c ANALOG DEVICES INC ADIN DRIVER -M: Alexandru Ardelean +M: Michael Hennerich L: netdev@vger.kernel.org S: Supported W: http://ez.analog.com/community/linux-device-drivers @@ -1033,7 +1033,7 @@ F: Documentation/devicetree/bindings/net/adi,adin.yaml F: drivers/net/phy/adin.c ANALOG DEVICES INC ADIS DRIVER LIBRARY -M: Alexandru Ardelean +M: Nuno Sa L: linux-iio@vger.kernel.org S: Supported F: drivers/iio/imu/adis.c From 4740b969aaf58adeca6829947a3ad8da423976cf Mon Sep 17 00:00:00 2001 From: Nadeem Athani Date: Tue, 9 Feb 2021 15:46:21 +0100 Subject: [PATCH 370/633] PCI: cadence: Retrain Link to work around Gen2 training defect Cadence controller will not initiate autonomous speed change if strapped as Gen2. The Retrain Link bit is set as quirk to enable this speed change. Link: https://lore.kernel.org/r/20210209144622.26683-3-nadeem@cadence.com Signed-off-by: Nadeem Athani Signed-off-by: Lorenzo Pieralisi --- drivers/pci/controller/cadence/pci-j721e.c | 3 + .../controller/cadence/pcie-cadence-host.c | 81 ++++++++++++++----- drivers/pci/controller/cadence/pcie-cadence.h | 11 ++- 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index dac1ac8a7615..849f1e416ea5 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -64,6 +64,7 @@ enum j721e_pcie_mode { struct j721e_pcie_data { enum j721e_pcie_mode mode; + bool quirk_retrain_flag; }; static inline u32 j721e_pcie_user_readl(struct j721e_pcie *pcie, u32 offset) @@ -280,6 +281,7 @@ static struct pci_ops cdns_ti_pcie_host_ops = { static const struct j721e_pcie_data j721e_pcie_rc_data = { .mode = PCI_MODE_RC, + .quirk_retrain_flag = true, }; static const struct j721e_pcie_data j721e_pcie_ep_data = { @@ -388,6 +390,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) bridge->ops = &cdns_ti_pcie_host_ops; rc = pci_host_bridge_priv(bridge); + rc->quirk_retrain_flag = data->quirk_retrain_flag; cdns_pcie = &rc->pcie; cdns_pcie->dev = dev; diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 811c1cb2e8de..6f591d382578 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -77,6 +77,68 @@ static struct pci_ops cdns_pcie_host_ops = { .write = pci_generic_config_write, }; +static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) +{ + struct device *dev = pcie->dev; + int retries; + + /* Check if the link is up or not */ + for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { + if (cdns_pcie_link_up(pcie)) { + dev_info(dev, "Link up\n"); + return 0; + } + usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); + } + + return -ETIMEDOUT; +} + +static int cdns_pcie_retrain(struct cdns_pcie *pcie) +{ + u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; + u16 lnk_stat, lnk_ctl; + int ret = 0; + + /* + * Set retrain bit if current speed is 2.5 GB/s, + * but the PCIe root port support is > 2.5 GB/s. + */ + + lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + + PCI_EXP_LNKCAP)); + if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) + return ret; + + lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); + if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { + lnk_ctl = cdns_pcie_rp_readw(pcie, + pcie_cap_off + PCI_EXP_LNKCTL); + lnk_ctl |= PCI_EXP_LNKCTL_RL; + cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, + lnk_ctl); + + ret = cdns_pcie_host_wait_for_link(pcie); + } + return ret; +} + +static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + int ret; + + ret = cdns_pcie_host_wait_for_link(pcie); + + /* + * Retrain link for Gen2 training defect + * if quirk flag is set. + */ + if (!ret && rc->quirk_retrain_flag) + ret = cdns_pcie_retrain(pcie); + + return ret; +} static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) { @@ -398,23 +460,6 @@ static int cdns_pcie_host_init(struct device *dev, return cdns_pcie_host_init_address_translation(rc); } -static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie) -{ - struct device *dev = pcie->dev; - int retries; - - /* Check if the link is up or not */ - for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) { - if (cdns_pcie_link_up(pcie)) { - dev_info(dev, "Link up\n"); - return 0; - } - usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX); - } - - return -ETIMEDOUT; -} - int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) { struct device *dev = rc->pcie.dev; @@ -457,7 +502,7 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) return ret; } - ret = cdns_pcie_host_wait_for_link(pcie); + ret = cdns_pcie_host_start_link(rc); if (ret) dev_dbg(dev, "PCIe link never came up\n"); diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 30eba6cafe2c..254d2570f8c9 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -119,7 +119,7 @@ * Root Port Registers (PCI configuration space for the root port function) */ #define CDNS_PCIE_RP_BASE 0x00200000 - +#define CDNS_PCIE_RP_CAP_OFFSET 0xc0 /* * Address Translation Registers @@ -291,6 +291,7 @@ struct cdns_pcie { * @device_id: PCI device ID * @avail_ib_bar: Satus of RP_BAR0, RP_BAR1 and RP_NO_BAR if it's free or * available + * @quirk_retrain_flag: Retrain link as quirk for PCIe Gen2 */ struct cdns_pcie_rc { struct cdns_pcie pcie; @@ -299,6 +300,7 @@ struct cdns_pcie_rc { u32 vendor_id; u32 device_id; bool avail_ib_bar[CDNS_PCIE_RP_MAX_IB]; + bool quirk_retrain_flag; }; /** @@ -414,6 +416,13 @@ static inline void cdns_pcie_rp_writew(struct cdns_pcie *pcie, cdns_pcie_write_sz(addr, 0x2, value); } +static inline u16 cdns_pcie_rp_readw(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x2); +} + /* Endpoint Function register access */ static inline void cdns_pcie_ep_fn_writeb(struct cdns_pcie *pcie, u8 fn, u32 reg, u8 value) From 15f720aabe71a5662c4198b22532d95bbeec80ef Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:42 +0100 Subject: [PATCH 371/633] x86/entry: Fix instrumentation annotation Embracing a callout into instrumentation_begin() / instrumentation_begin() does not really make sense. Make the latter instrumentation_end(). Fixes: 2f6474e4636b ("x86/entry: Switch XEN/PV hypercall entry to IDTENTRY") Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210210002512.106502464@linutronix.de --- arch/x86/entry/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 18d8f17f755c..c4efffbe24da 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -266,7 +266,7 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) instrumentation_begin(); run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); - instrumentation_begin(); + instrumentation_end(); set_irq_regs(old_regs); From e7f89001797148e8dc7060c335df2c56e73a8c7a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:43 +0100 Subject: [PATCH 372/633] x86/irq: Sanitize irq stack tracking The recursion protection for hard interrupt stacks is an unsigned int per CPU variable initialized to -1 named __irq_count. The irq stack switching is only done when the variable is -1, which creates worse code than just checking for 0. When the stack switching happens it uses this_cpu_add/sub(1), but there is no reason to do so. It simply can use straight writes. This is a historical leftover from the low level ASM code which used inc and jz to make a decision. Rename it to hardirq_stack_inuse, make it a bool and use plain stores. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.228830141@linutronix.de --- arch/x86/include/asm/irq_stack.h | 14 +++++++------- arch/x86/include/asm/processor.h | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/process_64.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 775816965c6a..4487bb989cea 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -9,7 +9,7 @@ #ifdef CONFIG_X86_64 static __always_inline bool irqstack_active(void) { - return __this_cpu_read(irq_count) != -1; + return __this_cpu_read(hardirq_stack_inuse); } void asm_call_on_stack(void *sp, void (*func)(void), void *arg); @@ -22,9 +22,9 @@ static __always_inline void __run_on_irqstack(void (*func)(void)) { void *tos = __this_cpu_read(hardirq_stack_ptr); - __this_cpu_add(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, true); asm_call_on_stack(tos - 8, func, NULL); - __this_cpu_sub(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, false); } static __always_inline void @@ -33,9 +33,9 @@ __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), { void *tos = __this_cpu_read(hardirq_stack_ptr); - __this_cpu_add(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, true); asm_call_sysvec_on_stack(tos - 8, func, regs); - __this_cpu_sub(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, false); } static __always_inline void @@ -44,9 +44,9 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), { void *tos = __this_cpu_read(hardirq_stack_ptr); - __this_cpu_add(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, true); asm_call_irq_on_stack(tos - 8, func, desc); - __this_cpu_sub(irq_count, 1); + __this_cpu_write(hardirq_stack_inuse, false); } #else /* CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c20a52b5534b..11d10f4f3059 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -454,7 +454,7 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu); } -DECLARE_PER_CPU(unsigned int, irq_count); +DECLARE_PER_CPU(bool, hardirq_stack_inuse); extern asmlinkage void ignore_sysret(void); /* Save actual FS/GS selectors and bases to current->thread */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 35ad8480c464..845c8a46cf7b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1740,7 +1740,7 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = EXPORT_PER_CPU_SYMBOL(current_task); DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); -DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; +DEFINE_PER_CPU(bool, hardirq_stack_inuse); DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ad582f9ac5a6..d08307df69ad 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -539,7 +539,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) int cpu = smp_processor_id(); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && - this_cpu_read(irq_count) != -1); + this_cpu_read(hardirq_stack_inuse)); if (!test_thread_flag(TIF_NEED_FPU_LOAD)) switch_fpu_prepare(prev_fpu, cpu); From 951c2a51ae75382d519839e2308394ad43ce4b40 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:44 +0100 Subject: [PATCH 373/633] x86/irq/64: Adjust the per CPU irq stack pointer by 8 The per CPU hardirq_stack_ptr contains the pointer to the irq stack in the form that it is ready to be assigned to [ER]SP so that the first push ends up on the top entry of the stack. But the stack switching on 64 bit has the following rules: 1) Store the current stack pointer (RSP) in the top most stack entry to allow the unwinder to link back to the previous stack 2) Set RSP to the top most stack entry 3) Invoke functions on the irq stack 4) Pop RSP from the top most stack entry (stored in #1) so it's back to the original stack. That requires all stack switching code to decrement the stored pointer by 8 in order to be able to store the current RSP and then set RSP to that location. That's a pointless exercise. Do the -8 adjustment right when storing the pointer and make the data type a void pointer to avoid confusion vs. the struct irq_stack data type which is on 64bit only used to declare the backing store. Move the definition next to the inuse flag so they likely end up in the same cache line. Sticking them into a struct to enforce it is a seperate change. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.354260928@linutronix.de --- arch/x86/include/asm/irq_stack.h | 6 +++--- arch/x86/include/asm/processor.h | 7 +++---- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/dumpstack_64.c | 22 ++++++++++++++++------ arch/x86/kernel/irq_64.c | 6 ++++-- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 4487bb989cea..554f84bca1c3 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -23,7 +23,7 @@ static __always_inline void __run_on_irqstack(void (*func)(void)) void *tos = __this_cpu_read(hardirq_stack_ptr); __this_cpu_write(hardirq_stack_inuse, true); - asm_call_on_stack(tos - 8, func, NULL); + asm_call_on_stack(tos, func, NULL); __this_cpu_write(hardirq_stack_inuse, false); } @@ -34,7 +34,7 @@ __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), void *tos = __this_cpu_read(hardirq_stack_ptr); __this_cpu_write(hardirq_stack_inuse, true); - asm_call_sysvec_on_stack(tos - 8, func, regs); + asm_call_sysvec_on_stack(tos, func, regs); __this_cpu_write(hardirq_stack_inuse, false); } @@ -45,7 +45,7 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), void *tos = __this_cpu_read(hardirq_stack_ptr); __this_cpu_write(hardirq_stack_inuse, true); - asm_call_irq_on_stack(tos - 8, func, desc); + asm_call_irq_on_stack(tos, func, desc); __this_cpu_write(hardirq_stack_inuse, false); } diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 11d10f4f3059..dc6d149bf851 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -426,8 +426,6 @@ struct irq_stack { char stack[IRQ_STACK_SIZE]; } __aligned(IRQ_STACK_SIZE); -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); - #ifdef CONFIG_X86_32 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); #else @@ -454,6 +452,7 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu); } +DECLARE_PER_CPU(void *, hardirq_stack_ptr); DECLARE_PER_CPU(bool, hardirq_stack_inuse); extern asmlinkage void ignore_sysret(void); @@ -473,9 +472,9 @@ struct stack_canary { }; DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); #endif -/* Per CPU softirq stack pointer */ +DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr); -#endif /* X86_64 */ +#endif /* !X86_64 */ extern unsigned int fpu_kernel_xstate_size; extern unsigned int fpu_user_xstate_size; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 845c8a46cf7b..c5e23f351f67 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1739,7 +1739,7 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = &init_task; EXPORT_PER_CPU_SYMBOL(current_task); -DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); +DEFINE_PER_CPU(void *, hardirq_stack_ptr); DEFINE_PER_CPU(bool, hardirq_stack_inuse); DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 1dd851397bd9..5601b95944fa 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -128,12 +128,21 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info) { - unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); - unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long)); + unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); + unsigned long *begin; /* - * This is a software stack, so 'end' can be a valid stack pointer. - * It just means the stack is empty. + * @end points directly to the top most stack entry to avoid a -8 + * adjustment in the stack switch hotpath. Adjust it back before + * calculating @begin. + */ + end++; + begin = end - (IRQ_STACK_SIZE / sizeof(long)); + + /* + * Due to the switching logic RSP can never be == @end because the + * final operation is 'popq %rsp' which means after that RSP points + * to the original stack and not to @end. */ if (stack < begin || stack >= end) return false; @@ -143,8 +152,9 @@ static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info info->end = end; /* - * The next stack pointer is the first thing pushed by the entry code - * after switching to the irq stack. + * The next stack pointer is stored at the top of the irq stack + * before switching to the irq stack. Actual stack entries are all + * below that. */ info->next_sp = (unsigned long *)*(end - 1); diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 440eed558558..7103f9889930 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -48,7 +48,8 @@ static int map_irq_stack(unsigned int cpu) if (!va) return -ENOMEM; - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #else @@ -60,7 +61,8 @@ static int map_irq_stack(unsigned int cpu) { void *va = per_cpu_ptr(&irq_stack_backing_store, cpu); - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #endif From 3c5e0267ec3e6ed7d3f1793273cbf0beb4f86a74 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:45 +0100 Subject: [PATCH 374/633] x86/apic: Split out spurious handling code sysvec_spurious_apic_interrupt() calls into the handling body of __spurious_interrupt() which is not obvious as that function is declared inside the DEFINE_IDTENTRY_IRQ(spurious_interrupt) macro. As __spurious_interrupt() is currently always inlined this ends up with two copies of the same code for no reason. Split the handling function out and invoke it from both entry points. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.469379641@linutronix.de --- arch/x86/kernel/apic/apic.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 6bd20c0de8bc..02956c0d18e6 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2133,18 +2133,11 @@ void __init register_lapic_address(unsigned long address) * Local APIC interrupts */ -/** - * spurious_interrupt - Catch all for interrupts raised on unused vectors - * @regs: Pointer to pt_regs on stack - * @vector: The vector number - * - * This is invoked from ASM entry code to catch all interrupts which - * trigger on an entry which is routed to the common_spurious idtentry - * point. - * - * Also called from sysvec_spurious_apic_interrupt(). +/* + * Common handling code for spurious_interrupt and spurious_vector entry + * points below. No point in allowing the compiler to inline it twice. */ -DEFINE_IDTENTRY_IRQ(spurious_interrupt) +static noinline void handle_spurious_interrupt(u8 vector) { u32 v; @@ -2179,9 +2172,23 @@ DEFINE_IDTENTRY_IRQ(spurious_interrupt) trace_spurious_apic_exit(vector); } +/** + * spurious_interrupt - Catch all for interrupts raised on unused vectors + * @regs: Pointer to pt_regs on stack + * @vector: The vector number + * + * This is invoked from ASM entry code to catch all interrupts which + * trigger on an entry which is routed to the common_spurious idtentry + * point. + */ +DEFINE_IDTENTRY_IRQ(spurious_interrupt) +{ + handle_spurious_interrupt(vector); +} + DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) { - __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR); + handle_spurious_interrupt(SPURIOUS_APIC_VECTOR); } /* From a0cfc74d0b00c5201e1c09e28b2dc01c8088f809 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:46 +0100 Subject: [PATCH 375/633] x86/irq: Provide macro for inlining irq stack switching The effort to make the ASM entry code slim and unified moved the irq stack switching out of the low level ASM code so that the whole return from interrupt work and state handling can be done in C and the ASM code just handles the low level details of entry and exit. This ended up being a suboptimal implementation for various reasons (including tooling). The main pain points are: - The indirect call which is expensive thanks to retpoline - The inability to stay on the irq stack for softirq processing on return from interrupt - The fact that the stack switching code ends up being an easy to target exploit gadget. Prepare for inlining the stack switching logic into the C entry points by providing a ASM macro which contains the guts of the switching mechanism: 1) Store RSP at the top of the irq stack 2) Switch RSP to the irq stack 3) Invoke code 4) Pop the original RSP back Document the unholy asm() logic while at it to reduce the amount of head scratching required a half year from now. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.578371068@linutronix.de --- arch/x86/include/asm/irq_stack.h | 98 ++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 554f84bca1c3..d5b7bc6b7a58 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -7,6 +7,104 @@ #include #ifdef CONFIG_X86_64 + +/* + * Macro to inline switching to an interrupt stack and invoking function + * calls from there. The following rules apply: + * + * - Ordering: + * + * 1. Write the stack pointer into the top most place of the irq + * stack. This ensures that the various unwinders can link back to the + * original stack. + * + * 2. Switch the stack pointer to the top of the irq stack. + * + * 3. Invoke whatever needs to be done (@asm_call argument) + * + * 4. Pop the original stack pointer from the top of the irq stack + * which brings it back to the original stack where it left off. + * + * - Function invocation: + * + * To allow flexible usage of the macro, the actual function code including + * the store of the arguments in the call ABI registers is handed in via + * the @asm_call argument. + * + * - Local variables: + * + * @tos: + * The @tos variable holds a pointer to the top of the irq stack and + * _must_ be allocated in a non-callee saved register as this is a + * restriction coming from objtool. + * + * Note, that (tos) is both in input and output constraints to ensure + * that the compiler does not assume that R11 is left untouched in + * case this macro is used in some place where the per cpu interrupt + * stack pointer is used again afterwards + * + * - Function arguments: + * The function argument(s), if any, have to be defined in register + * variables at the place where this is invoked. Storing the + * argument(s) in the proper register(s) is part of the @asm_call + * + * - Constraints: + * + * The constraints have to be done very carefully because the compiler + * does not know about the assembly call. + * + * output: + * As documented already above the @tos variable is required to be in + * the output constraints to make the compiler aware that R11 cannot be + * reused after the asm() statement. + * + * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is + * required as well as this prevents certain creative GCC variants from + * misplacing the ASM code. + * + * input: + * - func: + * Immediate, which tells the compiler that the function is referenced. + * + * - tos: + * Register. The actual register is defined by the variable declaration. + * + * - function arguments: + * The constraints are handed in via the 'argconstr' argument list. They + * describe the register arguments which are used in @asm_call. + * + * clobbers: + * Function calls can clobber anything except the callee-saved + * registers. Tell the compiler. + */ +#define call_on_irqstack(func, asm_call, argconstr...) \ +{ \ + register void *tos asm("r11"); \ + \ + tos = ((void *)__this_cpu_read(hardirq_stack_ptr)); \ + \ + asm_inline volatile( \ + "movq %%rsp, (%[tos]) \n" \ + "movq %[tos], %%rsp \n" \ + \ + asm_call \ + \ + "popq %%rsp \n" \ + \ + : "+r" (tos), ASM_CALL_CONSTRAINT \ + : [__func] "i" (func), [tos] "r" (tos) argconstr \ + : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \ + "memory" \ + ); \ +} + +/* Macros to assert type correctness for run_*_on_irqstack macros */ +#define assert_function_type(func, proto) \ + static_assert(__builtin_types_compatible_p(typeof(&func), proto)) + +#define assert_arg_type(arg, proto) \ + static_assert(__builtin_types_compatible_p(typeof(arg), proto)) + static __always_inline bool irqstack_active(void) { return __this_cpu_read(hardirq_stack_inuse); From 569dd8b4eb7ef666b467c41b8e8e4f2820d07f67 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:47 +0100 Subject: [PATCH 376/633] x86/entry: Convert system vectors to irq stack macro To inline the stack switching and to prepare for enabling CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK provide a macro template for system vectors and device interrupts and convert the system vectors over to it. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.676197354@linutronix.de --- arch/x86/entry/entry_64.S | 1 - arch/x86/include/asm/idtentry.h | 2 - arch/x86/include/asm/irq_stack.h | 93 ++++++++++++++++++++++---------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index cad08703c4ad..68643d324f4a 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -762,7 +762,6 @@ SYM_CODE_END(.Lbad_gs) * rdx: Function argument (can be NULL if none) */ SYM_FUNC_START(asm_call_on_stack) -SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL) SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL) /* * Save the frame pointer unconditionally. This allows the ORC diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 247a60a47331..712b3c8d4554 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -237,10 +237,8 @@ __visible noinstr void func(struct pt_regs *regs) \ irqentry_state_t state = irqentry_enter(regs); \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ run_sysvec_on_irqstack_cond(__##func, regs); \ - irq_exit_rcu(); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index d5b7bc6b7a58..05c37e7b3bcc 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -105,14 +105,69 @@ #define assert_arg_type(arg, proto) \ static_assert(__builtin_types_compatible_p(typeof(arg), proto)) +/* + * Macro to invoke system vector and device interrupt C handlers. + */ +#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \ +{ \ + /* \ + * User mode entry and interrupt on the irq stack do not \ + * switch stacks. If from user mode the task stack is empty. \ + */ \ + if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \ + irq_enter_rcu(); \ + func(c_args); \ + irq_exit_rcu(); \ + } else { \ + /* \ + * Mark the irq stack inuse _before_ and unmark _after_ \ + * switching stacks. Interrupts are disabled in both \ + * places. Invoke the stack switch macro with the call \ + * sequence which matches the above direct invocation. \ + */ \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(func, asm_call, constr); \ + __this_cpu_write(hardirq_stack_inuse, false); \ + } \ +} + +/* + * Function call sequence for __call_on_irqstack() for system vectors. + * + * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input + * mechanism because these functions are global and cannot be optimized out + * when compiling a particular source file which uses one of these macros. + * + * The argument (regs) does not need to be pushed or stashed in a callee + * saved register to be safe vs. the irq_enter_rcu() call because the + * clobbers already prevent the compiler from storing it in a callee + * clobbered register. As the compiler has to preserve @regs for the final + * call to idtentry_exit() anyway, it's likely that it does not cause extra + * effort for this asm magic. + */ +#define ASM_CALL_SYSVEC \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" + +#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs) + +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *)); \ + assert_arg_type(regs, struct pt_regs *); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \ + SYSVEC_CONSTRAINTS, regs); \ +} + static __always_inline bool irqstack_active(void) { return __this_cpu_read(hardirq_stack_inuse); } void asm_call_on_stack(void *sp, void (*func)(void), void *arg); -void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), - struct pt_regs *regs); void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), struct irq_desc *desc); @@ -125,17 +180,6 @@ static __always_inline void __run_on_irqstack(void (*func)(void)) __this_cpu_write(hardirq_stack_inuse, false); } -static __always_inline void -__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_write(hardirq_stack_inuse, true); - asm_call_sysvec_on_stack(tos, func, regs); - __this_cpu_write(hardirq_stack_inuse, false); -} - static __always_inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), struct irq_desc *desc) @@ -148,10 +192,17 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), } #else /* CONFIG_X86_64 */ + +/* System vector handlers always run on the stack they interrupted. */ +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + irq_enter_rcu(); \ + func(regs); \ + irq_exit_rcu(); \ +} + static inline bool irqstack_active(void) { return false; } static inline void __run_on_irqstack(void (*func)(void)) { } -static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) { } static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), struct irq_desc *desc) { } #endif /* !CONFIG_X86_64 */ @@ -177,18 +228,6 @@ static __always_inline void run_on_irqstack_cond(void (*func)(void), func(); } -static __always_inline void -run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_sysvec_on_irqstack(func, regs); - else - func(regs); -} - static __always_inline void run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, struct pt_regs *regs) From 5b51e1db9bdc312d53087a0c97d54ea150111c0d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:48 +0100 Subject: [PATCH 377/633] x86/entry: Convert device interrupts to inline stack switching Convert device interrupts to inline stack switching by replacing the existing macro implementation with the new inline version. Tweak the function signature of the actual handler function to have the vector argument as u32. That allows the inline macro to avoid extra intermediates and lets the compiler be smarter about the whole thing. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.769728139@linutronix.de --- arch/x86/entry/entry_64.S | 1 - arch/x86/include/asm/idtentry.h | 9 +++-- arch/x86/include/asm/irq_stack.h | 58 +++++++++++++++++--------------- arch/x86/kernel/irq.c | 2 +- 4 files changed, 36 insertions(+), 34 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 68643d324f4a..f446e9048d07 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -762,7 +762,6 @@ SYM_CODE_END(.Lbad_gs) * rdx: Function argument (can be NULL if none) */ SYM_FUNC_START(asm_call_on_stack) -SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL) /* * Save the frame pointer unconditionally. This allows the ORC * unwinder to handle the stack switch. diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 712b3c8d4554..f294637ed340 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -187,23 +187,22 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) * has to be done in the function body if necessary. */ #define DEFINE_IDTENTRY_IRQ(func) \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector); \ +static void __##func(struct pt_regs *regs, u32 vector); \ \ __visible noinstr void func(struct pt_regs *regs, \ unsigned long error_code) \ { \ irqentry_state_t state = irqentry_enter(regs); \ + u32 vector = (u32)(u8)error_code; \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ - __##func (regs, (u8)error_code); \ - irq_exit_rcu(); \ + run_irq_on_irqstack_cond(__##func, regs, vector); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector) +static noinline void __##func(struct pt_regs *regs, u32 vector) /** * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 05c37e7b3bcc..dabc0cf60df5 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -162,14 +162,35 @@ SYSVEC_CONSTRAINTS, regs); \ } +/* + * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store + * @regs and @vector in callee saved registers. + */ +#define ASM_CALL_IRQ \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "movl %[arg2], %%esi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" + +#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector) + +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *, u32)); \ + assert_arg_type(regs, struct pt_regs *); \ + assert_arg_type(vector, u32); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \ + IRQ_CONSTRAINTS, regs, vector); \ +} + static __always_inline bool irqstack_active(void) { return __this_cpu_read(hardirq_stack_inuse); } void asm_call_on_stack(void *sp, void (*func)(void), void *arg); -void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), - struct irq_desc *desc); static __always_inline void __run_on_irqstack(void (*func)(void)) { @@ -180,17 +201,6 @@ static __always_inline void __run_on_irqstack(void (*func)(void)) __this_cpu_write(hardirq_stack_inuse, false); } -static __always_inline void -__run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_write(hardirq_stack_inuse, true); - asm_call_irq_on_stack(tos, func, desc); - __this_cpu_write(hardirq_stack_inuse, false); -} - #else /* CONFIG_X86_64 */ /* System vector handlers always run on the stack they interrupted. */ @@ -201,10 +211,16 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), irq_exit_rcu(); \ } +/* Switches to the irq stack within func() */ +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + irq_enter_rcu(); \ + func(regs, vector); \ + irq_exit_rcu(); \ +} + static inline bool irqstack_active(void) { return false; } static inline void __run_on_irqstack(void (*func)(void)) { } -static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) { } #endif /* !CONFIG_X86_64 */ static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) @@ -228,16 +244,4 @@ static __always_inline void run_on_irqstack_cond(void (*func)(void), func(); } -static __always_inline void -run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_irq_on_irqstack(func, desc); - else - func(desc); -} - #endif diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index c5dd50369e2f..1507b983cd8d 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -227,7 +227,7 @@ static __always_inline void handle_irq(struct irq_desc *desc, struct pt_regs *regs) { if (IS_ENABLED(CONFIG_X86_64)) - run_irq_on_irqstack_cond(desc->handle_irq, desc, regs); + generic_handle_irq_desc(desc); else __handle_irq(desc, regs); } From 359f01d1816fc1ea0161e6c30722bef1ed6b8abb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:49 +0100 Subject: [PATCH 378/633] x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall To avoid yet another macro implementation reuse the existing run_sysvec_on_irqstack_cond() and move the set_irq_regs() handling into the called function. Makes the code even simpler. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.869753106@linutronix.de --- arch/x86/entry/common.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index c4efffbe24da..60a5fc4da475 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -245,30 +245,23 @@ static __always_inline bool get_and_clear_inhcall(void) { return false; } static __always_inline void restore_inhcall(bool inhcall) { } #endif -static void __xen_pv_evtchn_do_upcall(void) +static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - irq_enter_rcu(); + struct pt_regs *old_regs = set_irq_regs(regs); + inc_irq_stat(irq_hv_callback_count); xen_hvm_evtchn_do_upcall(); - irq_exit_rcu(); + set_irq_regs(old_regs); } __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - struct pt_regs *old_regs; + irqentry_state_t state = irqentry_enter(regs); bool inhcall; - irqentry_state_t state; - state = irqentry_enter(regs); - old_regs = set_irq_regs(regs); - - instrumentation_begin(); - run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); - instrumentation_end(); - - set_irq_regs(old_regs); + run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); inhcall = get_and_clear_inhcall(); if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { From 52d743f3b71265e14560a38f4c835d07b9c6fc4c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:50 +0100 Subject: [PATCH 379/633] x86/softirq: Remove indirection in do_softirq_own_stack() Use the new inline stack switching and remove the old ASM indirect call implementation. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002512.972714001@linutronix.de --- arch/x86/entry/entry_64.S | 39 ------------------------ arch/x86/include/asm/irq_stack.h | 52 ++++++++++---------------------- arch/x86/kernel/irq_64.c | 2 +- 3 files changed, 17 insertions(+), 76 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index f446e9048d07..bd52f675d11d 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -756,45 +756,6 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) SYM_CODE_END(.Lbad_gs) .previous -/* - * rdi: New stack pointer points to the top word of the stack - * rsi: Function pointer - * rdx: Function argument (can be NULL if none) - */ -SYM_FUNC_START(asm_call_on_stack) - /* - * Save the frame pointer unconditionally. This allows the ORC - * unwinder to handle the stack switch. - */ - pushq %rbp - mov %rsp, %rbp - - /* - * The unwinder relies on the word at the top of the new stack - * page linking back to the previous RSP. - */ - mov %rsp, (%rdi) - mov %rdi, %rsp - /* Move the argument to the right place */ - mov %rdx, %rdi - -1: - .pushsection .discard.instr_begin - .long 1b - . - .popsection - - CALL_NOSPEC rsi - -2: - .pushsection .discard.instr_end - .long 2b - . - .popsection - - /* Restore the previous stack pointer from RBP. */ - leaveq - ret -SYM_FUNC_END(asm_call_on_stack) - #ifdef CONFIG_XEN_PV /* * A note on the "critical region" in our callback handler. diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index dabc0cf60df5..fa444c27772a 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -185,20 +185,23 @@ IRQ_CONSTRAINTS, regs, vector); \ } -static __always_inline bool irqstack_active(void) -{ - return __this_cpu_read(hardirq_stack_inuse); -} +#define ASM_CALL_SOFTIRQ \ + "call %P[__func] \n" -void asm_call_on_stack(void *sp, void (*func)(void), void *arg); - -static __always_inline void __run_on_irqstack(void (*func)(void)) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_write(hardirq_stack_inuse, true); - asm_call_on_stack(tos, func, NULL); - __this_cpu_write(hardirq_stack_inuse, false); +/* + * Macro to invoke __do_softirq on the irq stack. Contrary to the above + * the only check which is necessary is whether the interrupt stack is + * in use already. + */ +#define run_softirq_on_irqstack_cond() \ +{ \ + if (__this_cpu_read(hardirq_stack_inuse)) { \ + __do_softirq(); \ + } else { \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ + __this_cpu_write(hardirq_stack_inuse, false); \ + } \ } #else /* CONFIG_X86_64 */ @@ -219,29 +222,6 @@ static __always_inline void __run_on_irqstack(void (*func)(void)) irq_exit_rcu(); \ } -static inline bool irqstack_active(void) { return false; } -static inline void __run_on_irqstack(void (*func)(void)) { } #endif /* !CONFIG_X86_64 */ -static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return false; - if (!regs) - return !irqstack_active(); - return !user_mode(regs) && !irqstack_active(); -} - - -static __always_inline void run_on_irqstack_cond(void (*func)(void), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_on_irqstack(func); - else - func(); -} - #endif diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 7103f9889930..8d9f9a1b49e5 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned int cpu) void do_softirq_own_stack(void) { - run_on_irqstack_cond(__do_softirq, NULL); + run_softirq_on_irqstack_cond(); } From 624db9eabc74597f682c0651047a25b54f7260a1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:51 +0100 Subject: [PATCH 380/633] x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK Now that all invocations of irq_exit_rcu() happen on the irq stack, turn on CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK which causes the core code to invoke __do_softirq() directly without going through do_softirq_own_stack(). That means do_softirq_own_stack() is only invoked from task context which means it can't be on the irq stack. Remove the conditional from run_softirq_on_irqstack_cond() and rename the function accordingly. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002513.068033456@linutronix.de --- arch/x86/Kconfig | 1 + arch/x86/include/asm/irq_stack.h | 19 ++++++++----------- arch/x86/kernel/irq_64.c | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 21f851179ff0..e17ce871bb19 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -187,6 +187,7 @@ config X86 select HAVE_HW_BREAKPOINT select HAVE_IDE select HAVE_IOREMAP_PROT + select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index fa444c27772a..1b82f9230709 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -189,19 +189,16 @@ "call %P[__func] \n" /* - * Macro to invoke __do_softirq on the irq stack. Contrary to the above - * the only check which is necessary is whether the interrupt stack is - * in use already. + * Macro to invoke __do_softirq on the irq stack. This is only called from + * task context when bottom halfs are about to be reenabled and soft + * interrupts are pending to be processed. The interrupt stack cannot be in + * use here. */ -#define run_softirq_on_irqstack_cond() \ +#define run_softirq_on_irqstack() \ { \ - if (__this_cpu_read(hardirq_stack_inuse)) { \ - __do_softirq(); \ - } else { \ - __this_cpu_write(hardirq_stack_inuse, true); \ - call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ - __this_cpu_write(hardirq_stack_inuse, false); \ - } \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ + __this_cpu_write(hardirq_stack_inuse, false); \ } #else /* CONFIG_X86_64 */ diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 8d9f9a1b49e5..b88fdb9686e6 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned int cpu) void do_softirq_own_stack(void) { - run_softirq_on_irqstack_cond(); + run_softirq_on_irqstack(); } From cd1a41ceba8a4caef4d18a3a14d6d0f8c656efe4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:52 +0100 Subject: [PATCH 381/633] softirq: Move __ARCH_HAS_DO_SOFTIRQ to Kconfig To prepare for inlining do_softirq_own_stack() replace __ARCH_HAS_DO_SOFTIRQ with a Kconfig switch and select it in the affected architectures. This allows in the next step to move the function prototype and the inline stub into a seperate asm-generic header file which is required to avoid include recursion. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002513.181713427@linutronix.de --- arch/Kconfig | 6 ++++++ arch/parisc/Kconfig | 1 + arch/parisc/include/asm/hardirq.h | 4 ---- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/irq.h | 2 -- arch/s390/Kconfig | 1 + arch/s390/include/asm/hardirq.h | 1 - arch/sh/Kconfig | 1 + arch/sh/include/asm/irq.h | 1 - arch/sparc/Kconfig | 1 + arch/sparc/include/asm/irq_64.h | 1 - arch/x86/Kconfig | 1 + arch/x86/include/asm/irq.h | 2 -- include/linux/interrupt.h | 2 +- 14 files changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 24862d15f3a3..7eab9d0a4b43 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -759,6 +759,12 @@ config HAVE_IRQ_EXIT_ON_IRQ_STACK This spares a stack switch and improves cache usage on softirq processing. +config HAVE_SOFTIRQ_ON_OWN_STACK + bool + help + Architecture provides a function to run __do_softirq() on a + seperate stack. + config PGTABLE_LEVELS int default 2 diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 78b17621ee4a..7cd88c0a9a32 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -63,6 +63,7 @@ config PARISC select HAVE_FTRACE_MCOUNT_RECORD if HAVE_DYNAMIC_FTRACE select HAVE_KPROBES_ON_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select SET_FS help diff --git a/arch/parisc/include/asm/hardirq.h b/arch/parisc/include/asm/hardirq.h index fad29aa6f45f..1e4fbd0fd944 100644 --- a/arch/parisc/include/asm/hardirq.h +++ b/arch/parisc/include/asm/hardirq.h @@ -12,10 +12,6 @@ #include #include -#ifdef CONFIG_IRQSTACKS -#define __ARCH_HAS_DO_SOFTIRQ -#endif - typedef struct { unsigned int __softirq_pending; unsigned int kernel_stack_usage; diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 107bb4319e0e..888d1ad436ce 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -237,6 +237,7 @@ config PPC select MMU_GATHER_PAGE_SIZE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_IRQ_TIME_ACCOUNTING diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index 4f983ca4030a..f3f264e441a7 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -37,8 +37,6 @@ extern int distribute_irqs; struct pt_regs; -#define __ARCH_HAS_DO_SOFTIRQ - #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) /* * Per-cpu stacks for handling critical, debug and machine check diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index c72874f09741..e00c02ab5706 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -182,6 +182,7 @@ config S390 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_VIRT_CPU_ACCOUNTING_IDLE diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h index dfbc3c6c0674..58668ffb5488 100644 --- a/arch/s390/include/asm/hardirq.h +++ b/arch/s390/include/asm/hardirq.h @@ -18,7 +18,6 @@ #define or_softirq_pending(x) (S390_lowcore.softirq_pending |= (x)) #define __ARCH_IRQ_STAT -#define __ARCH_HAS_DO_SOFTIRQ #define __ARCH_IRQ_EXIT_IRQS_DISABLED static inline void ack_bad_irq(unsigned int irq) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 5fa580219a86..b49d5e033e29 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -56,6 +56,7 @@ config SUPERH select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_UID16 + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING diff --git a/arch/sh/include/asm/irq.h b/arch/sh/include/asm/irq.h index 6d44c32ef047..839551ce398c 100644 --- a/arch/sh/include/asm/irq.h +++ b/arch/sh/include/asm/irq.h @@ -51,7 +51,6 @@ asmlinkage int do_IRQ(unsigned int irq, struct pt_regs *regs); #ifdef CONFIG_IRQSTACKS extern void irq_ctx_init(int cpu); extern void irq_ctx_exit(int cpu); -# define __ARCH_HAS_DO_SOFTIRQ #else # define irq_ctx_init(cpu) do { } while (0) # define irq_ctx_exit(cpu) do { } while (0) diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index c9c34dc52b7d..f0d22d530645 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -97,6 +97,7 @@ config SPARC64 select ARCH_HAS_PTE_SPECIAL select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE + select HAVE_SOFTIRQ_ON_OWN_STACK config ARCH_PROC_KCORE_TEXT def_bool y diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 4d748e93b974..154df2cf19f4 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -93,7 +93,6 @@ void arch_trigger_cpumask_backtrace(const struct cpumask *mask, extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; -#define __ARCH_HAS_DO_SOFTIRQ #define NO_IRQ 0xffffffff diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e17ce871bb19..019e0e1a5ee0 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -221,6 +221,7 @@ config X86 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 528c8a71fe7f..f70d2ca3887e 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -25,8 +25,6 @@ static inline int irq_canonicalize(int irq) extern int irq_init_percpu_irqstack(unsigned int cpu); -#define __ARCH_HAS_DO_SOFTIRQ - struct irq_desc; extern void fixup_irqs(void); diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index bb8ff9083e7d..f0b918f393a2 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -569,7 +569,7 @@ struct softirq_action asmlinkage void do_softirq(void); asmlinkage void __do_softirq(void); -#ifdef __ARCH_HAS_DO_SOFTIRQ +#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void); #else static inline void do_softirq_own_stack(void) From db1cc7aede37eb9235759131ddfefd9c0ea5136f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:53 +0100 Subject: [PATCH 382/633] softirq: Move do_softirq_own_stack() to generic asm header To avoid include recursion hell move the do_softirq_own_stack() related content into a generic asm header and include it from all places in arch/ which need the prototype. This allows architectures to provide an inline implementation of do_softirq_own_stack() without introducing a lot of #ifdeffery all over the place. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002513.289960691@linutronix.de --- arch/parisc/kernel/irq.c | 1 + arch/powerpc/kernel/irq.c | 1 + arch/s390/kernel/irq.c | 1 + arch/sh/kernel/irq.c | 1 + arch/sparc/kernel/irq_64.c | 1 + arch/x86/kernel/irq_32.c | 1 + arch/x86/kernel/irq_64.c | 1 + include/asm-generic/Kbuild | 1 + include/asm-generic/softirq_stack.h | 14 ++++++++++++++ include/linux/interrupt.h | 9 --------- kernel/softirq.c | 2 ++ 11 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 include/asm-generic/softirq_stack.h diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index 49cd6d2caefb..1632d525ad64 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -17,6 +17,7 @@ #include #include +#include #include #include diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 6b1eca53e36c..96296d3383a7 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -65,6 +65,7 @@ #include #include #include +#include #ifdef CONFIG_PPC64 #include diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index f8a8b9428ae2..a1a2f75f3c09 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index ab5f790b0cd2..ef0f0827cf57 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -20,6 +20,7 @@ #include #include #include +#include atomic_t irq_err_count; diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 3ec9f1402aad..c8848bb681a1 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "entry.h" #include "cpumap.h" diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 0b79efc87be5..044902d5a3c4 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -22,6 +22,7 @@ #include #include +#include #ifdef CONFIG_DEBUG_STACKOVERFLOW diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index b88fdb9686e6..f335c3910834 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 267f6dfb8960..bd684188e2dc 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -51,6 +51,7 @@ mandatory-y += sections.h mandatory-y += serial.h mandatory-y += shmparam.h mandatory-y += simd.h +mandatory-y += softirq_stack.h mandatory-y += switch_to.h mandatory-y += timex.h mandatory-y += tlbflush.h diff --git a/include/asm-generic/softirq_stack.h b/include/asm-generic/softirq_stack.h new file mode 100644 index 000000000000..eceeecf6a5bd --- /dev/null +++ b/include/asm-generic/softirq_stack.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_GENERIC_SOFTIRQ_STACK_H +#define __ASM_GENERIC_SOFTIRQ_STACK_H + +#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK +void do_softirq_own_stack(void); +#else +static inline void do_softirq_own_stack(void) +{ + __do_softirq(); +} +#endif + +#endif diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index f0b918f393a2..967e25767153 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -569,15 +569,6 @@ struct softirq_action asmlinkage void do_softirq(void); asmlinkage void __do_softirq(void); -#ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK -void do_softirq_own_stack(void); -#else -static inline void do_softirq_own_stack(void) -{ - __do_softirq(); -} -#endif - extern void open_softirq(int nr, void (*action)(struct softirq_action *)); extern void softirq_init(void); extern void __raise_softirq_irqoff(unsigned int nr); diff --git a/kernel/softirq.c b/kernel/softirq.c index 9d71046ea247..9908ec4a9bfe 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -26,6 +26,8 @@ #include #include +#include + #define CREATE_TRACE_POINTS #include From 72f40a2823d6e16229ab58b898c6f22044e5222f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 10 Feb 2021 00:40:54 +0100 Subject: [PATCH 383/633] x86/softirq/64: Inline do_softirq_own_stack() There is no reason to have this as a seperate function for a single caller. Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210210002513.382806685@linutronix.de --- arch/x86/include/asm/irq_stack.h | 3 +-- arch/x86/include/asm/softirq_stack.h | 11 +++++++++++ arch/x86/kernel/irq_64.c | 5 ----- 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 arch/x86/include/asm/softirq_stack.h diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 1b82f9230709..9b2a0ff76c73 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -194,7 +194,7 @@ * interrupts are pending to be processed. The interrupt stack cannot be in * use here. */ -#define run_softirq_on_irqstack() \ +#define do_softirq_own_stack() \ { \ __this_cpu_write(hardirq_stack_inuse, true); \ call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ @@ -202,7 +202,6 @@ } #else /* CONFIG_X86_64 */ - /* System vector handlers always run on the stack they interrupted. */ #define run_sysvec_on_irqstack_cond(func, regs) \ { \ diff --git a/arch/x86/include/asm/softirq_stack.h b/arch/x86/include/asm/softirq_stack.h new file mode 100644 index 000000000000..889d53d6a0e1 --- /dev/null +++ b/arch/x86/include/asm/softirq_stack.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_SOFTIRQ_STACK_H +#define _ASM_X86_SOFTIRQ_STACK_H + +#ifdef CONFIG_X86_64 +# include +#else +# include +#endif + +#endif diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index f335c3910834..1c0fb96b9e39 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -74,8 +74,3 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; return map_irq_stack(cpu); } - -void do_softirq_own_stack(void) -{ - run_softirq_on_irqstack(); -} From 9a1471222189bdab467db0a0043c66b95142d5a5 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Mon, 8 Feb 2021 20:44:00 +0100 Subject: [PATCH 384/633] PCI: Remove WARN_ON(in_interrupt()) WARN_ON(in_interrupt()) is used for historic reasons to ensure proper usage of down_read() and predates might_sleep() and lockdep. down_read() has might_sleep() which also catches users from preemption disabled regions while in_interrupt() does not. Remove WARN_ON(in_interrupt()) because there are now better debugging facilities. Link: https://lore.kernel.org/r/20210208194400.384003-1-bigeasy@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Bjorn Helgaas --- drivers/pci/search.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/pci/search.c b/drivers/pci/search.c index 2061672954ee..b4c138a6ec02 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -168,7 +168,6 @@ struct pci_bus *pci_find_next_bus(const struct pci_bus *from) struct list_head *n; struct pci_bus *b = NULL; - WARN_ON(in_interrupt()); down_read(&pci_bus_sem); n = from ? from->node.next : pci_root_buses.next; if (n != &pci_root_buses) @@ -196,7 +195,6 @@ struct pci_dev *pci_get_slot(struct pci_bus *bus, unsigned int devfn) { struct pci_dev *dev; - WARN_ON(in_interrupt()); down_read(&pci_bus_sem); list_for_each_entry(dev, &bus->devices, bus_list) { @@ -274,7 +272,6 @@ static struct pci_dev *pci_get_dev_by_id(const struct pci_device_id *id, struct device *dev_start = NULL; struct pci_dev *pdev = NULL; - WARN_ON(in_interrupt()); if (from) dev_start = &from->dev; dev = bus_find_device(&pci_bus_type, dev_start, (void *)id, @@ -381,7 +378,6 @@ int pci_dev_present(const struct pci_device_id *ids) { struct pci_dev *found = NULL; - WARN_ON(in_interrupt()); while (ids->vendor || ids->subvendor || ids->class_mask) { found = pci_get_dev_by_id(ids, NULL); if (found) { From 957e3f797917b36355766807b1d8a54a1ba0cfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=C5=9Alusarz?= Date: Mon, 8 Feb 2021 13:01:03 +0100 Subject: [PATCH 385/633] soundwire: intel: fix possible crash when no device is detected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit acpi_walk_namespace can return success without executing our callback which initializes info->handle. If the random value in this structure is a valid address (which is on the stack, so it's quite possible), then nothing bad will happen, because: sdw_intel_scan_controller -> acpi_bus_get_device -> acpi_get_device_data -> acpi_get_data_full -> acpi_ns_validate_handle will reject this handle. However, if the value from the stack doesn't point to a valid address, we get this: BUG: kernel NULL pointer dereference, address: 0000000000000050 PGD 0 P4D 0 Oops: 0000 [#1] SMP NOPTI CPU: 6 PID: 472 Comm: systemd-udevd Tainted: G W 5.10.0-1-amd64 #1 Debian 5.10.4-1 Hardware name: HP HP Pavilion Laptop 15-cs3xxx/86E2, BIOS F.05 01/01/2020 RIP: 0010:acpi_ns_validate_handle+0x1a/0x23 Code: 00 48 83 c4 10 5b 5d 41 5c 41 5d 41 5e 41 5f c3 0f 1f 44 00 00 48 8d 57 ff 48 89 f8 48 83 fa fd 76 08 48 8b 05 0c b8 67 01 c3 <80> 7f 08 0f 74 02 31 c0 c3 0f 1f 44 00 00 48 8b 3d f6 b7 67 01 e8 RSP: 0000:ffffc388807c7b20 EFLAGS: 00010213 RAX: 0000000000000048 RBX: ffffc388807c7b70 RCX: 0000000000000000 RDX: 0000000000000047 RSI: 0000000000000246 RDI: 0000000000000048 RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 R10: ffffffffc0f5f4d1 R11: ffffffff8f0cb268 R12: 0000000000001001 R13: ffffffff8e33b160 R14: 0000000000000048 R15: 0000000000000000 FS: 00007f24548288c0(0000) GS:ffff9f781fb80000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000050 CR3: 0000000106158004 CR4: 0000000000770ee0 PKRU: 55555554 Call Trace: acpi_get_data_full+0x4d/0x92 acpi_bus_get_device+0x1f/0x40 sdw_intel_acpi_scan+0x59/0x230 [soundwire_intel] ? strstr+0x22/0x60 ? dmi_matches+0x76/0xe0 snd_intel_dsp_driver_probe.cold+0xaf/0x163 [snd_intel_dspcfg] azx_probe+0x7a/0x970 [snd_hda_intel] local_pci_probe+0x42/0x80 ? _cond_resched+0x16/0x40 pci_device_probe+0xfd/0x1b0 really_probe+0x205/0x460 driver_probe_device+0xe1/0x150 device_driver_attach+0xa1/0xb0 __driver_attach+0x8a/0x150 ? device_driver_attach+0xb0/0xb0 ? device_driver_attach+0xb0/0xb0 bus_for_each_dev+0x78/0xc0 bus_add_driver+0x12b/0x1e0 driver_register+0x8b/0xe0 ? 0xffffffffc0f65000 do_one_initcall+0x44/0x1d0 ? do_init_module+0x23/0x250 ? kmem_cache_alloc_trace+0xf5/0x200 do_init_module+0x5c/0x250 __do_sys_finit_module+0xb1/0x110 do_syscall_64+0x33/0x80 entry_SYSCALL_64_after_hwframe+0x44/0xa9 Signed-off-by: Marcin Ślusarz Reviewed-by: Pierre-Louis Bossart Reviewed-by: Rafael J. Wysocki CC: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210208120104.204761-1-marcin.slusarz@gmail.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel_init.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c index cabdadb09a1b..bc8520eb385e 100644 --- a/drivers/soundwire/intel_init.c +++ b/drivers/soundwire/intel_init.c @@ -405,11 +405,12 @@ int sdw_intel_acpi_scan(acpi_handle *parent_handle, { acpi_status status; + info->handle = NULL; status = acpi_walk_namespace(ACPI_TYPE_DEVICE, parent_handle, 1, sdw_intel_acpi_cb, NULL, info, NULL); - if (ACPI_FAILURE(status)) + if (ACPI_FAILURE(status) || info->handle == NULL) return -ENODEV; return sdw_intel_scan_controller(info); From d288a5712ef961e16d588bbdb2d846e00b5ef154 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 22 Jan 2021 15:06:30 +0800 Subject: [PATCH 386/633] regmap: sdw: use _no_pm functions in regmap_read/write sdw_update_slave_status will be invoked when a codec is attached, and the codec driver will initialize the codec with regmap functions while the codec device is pm_runtime suspended. regmap routines currently rely on regular SoundWire IO functions, which will call pm_runtime_get_sync()/put_autosuspend. This causes a deadlock where the resume routine waits for an initialization complete signal that while the initialization complete can only be reached when the resume completes. The only solution if we allow regmap functions to be used in resume operations as well as during codec initialization is to use _no_pm routines. The duty of making sure the bus is operational needs to be handled above the regmap level. Fixes: 7c22ce6e21840 ('regmap: Add SoundWire bus support') Signed-off-by: Bard Liao Acked-by: Mark Brown Link: https://lore.kernel.org/r/20210122070634.12825-6-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/base/regmap/regmap-sdw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c index c83be26434e7..966de8a136d9 100644 --- a/drivers/base/regmap/regmap-sdw.c +++ b/drivers/base/regmap/regmap-sdw.c @@ -13,7 +13,7 @@ static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val) struct device *dev = context; struct sdw_slave *slave = dev_to_sdw_dev(dev); - return sdw_write(slave, reg, val); + return sdw_write_no_pm(slave, reg, val); } static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) @@ -22,7 +22,7 @@ static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) struct sdw_slave *slave = dev_to_sdw_dev(dev); int read; - read = sdw_read(slave, reg); + read = sdw_read_no_pm(slave, reg); if (read < 0) return read; From 4038e54b8f8f7fc2d0765a1b65061407ea79f1f7 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 22 Jan 2021 15:06:31 +0800 Subject: [PATCH 387/633] regmap: sdw: use no_pm routines for SoundWire 1.2 MBQ Use no_pm versions for write and read. Signed-off-by: Bard Liao Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Acked-by: Mark Brown Link: https://lore.kernel.org/r/20210122070634.12825-7-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/base/regmap/regmap-sdw-mbq.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 8ce30650b97c..6675c3a4b829 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -15,11 +15,11 @@ static int regmap_sdw_mbq_write(void *context, unsigned int reg, unsigned int va struct sdw_slave *slave = dev_to_sdw_dev(dev); int ret; - ret = sdw_write(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); + ret = sdw_write_no_pm(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); if (ret < 0) return ret; - return sdw_write(slave, reg, val & 0xff); + return sdw_write_no_pm(slave, reg, val & 0xff); } static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *val) @@ -29,11 +29,11 @@ static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *va int read0; int read1; - read0 = sdw_read(slave, reg); + read0 = sdw_read_no_pm(slave, reg); if (read0 < 0) return read0; - read1 = sdw_read(slave, SDW_SDCA_MBQ_CTL(reg)); + read1 = sdw_read_no_pm(slave, SDW_SDCA_MBQ_CTL(reg)); if (read1 < 0) return read1; From 8d8d958428674518cef083cd97e535ac5ac9f9d8 Mon Sep 17 00:00:00 2001 From: Bard Liao Date: Fri, 22 Jan 2021 15:06:32 +0800 Subject: [PATCH 388/633] regmap: sdw-mbq: use MODULE_LICENSE("GPL") "GPL v2" is the same as "GPL". It exists for historic reasons. See Documentation/process/license-rules.rst Signed-off-by: Pierre-Louis Bossart Signed-off-by: Bard Liao Acked-by: Mark Brown Link: https://lore.kernel.org/r/20210122070634.12825-8-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/base/regmap/regmap-sdw-mbq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 6675c3a4b829..fe3ac26b66ad 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -98,4 +98,4 @@ struct regmap *__devm_regmap_init_sdw_mbq(struct sdw_slave *sdw, EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw_mbq); MODULE_DESCRIPTION("Regmap SoundWire MBQ Module"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); From 48139bad913d660d1f6a72ee215297e6749bc6f1 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 10 Feb 2021 10:34:33 -0800 Subject: [PATCH 389/633] ACRN: update MAINTAINERS: mailing list is subscribers-only Mark the acrn-dev mailing list as subscribers-only. Evidence from a previous patch: acrn-dev@lists.projectacrn.org SMTP error from remote mail server after end of data: 510 5.1.1 Your email address, rdunlap@infradead.org, is not subscribed to that group. Cc: Shuo Liu Cc: acrn-dev@lists.projectacrn.org Cc: Andrew Morton Cc: Greg Kroah-Hartman Signed-off-by: Randy Dunlap Link: https://lore.kernel.org/r/20210210183433.18746-1-rdunlap@infradead.org Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 65b2a9ea74da..11d37eb4d914 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -438,7 +438,7 @@ F: include/uapi/linux/wmi.h ACRN HYPERVISOR SERVICE MODULE M: Shuo Liu -L: acrn-dev@lists.projectacrn.org +L: acrn-dev@lists.projectacrn.org (subscribers-only) S: Supported W: https://projectacrn.org F: Documentation/virt/acrn/ From 53abf3fe831756261f399dad03ccc07235296acf Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Thu, 11 Feb 2021 10:20:36 -0700 Subject: [PATCH 390/633] coresight: etm-perf: Clarify comment on perf options In theory, the options should be arbitrary values and are neutral for any ETM version; so far perf tool uses ETMv3.5/PTM ETMCR config bits except for register's bit definitions, also uses as options. This can introduce confusion, especially if we want to add a new option but the new option is not supported by ETMv3.5/PTM ETMCR. But on the other hand, we cannot change options since these options are generic CoreSight PMU ABI. For easier maintenance and avoid confusion, this patch refines the comment to clarify perf options, and gives out the background info for these bits are coming from ETMv3.5/PTM. Afterwards, we should take these options as general knobs, and if there have any confliction with ETMv3.5/PTM, should consider to define saperate macros for ETMv3.5/PTM ETMCR config bits. Suggested-by: Suzuki K Poulose Reviewed-by: Suzuki K Poulose Signed-off-by: Leo Yan Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210206150833.42120-2-leo.yan@linaro.org Link: https://lore.kernel.org/r/20210211172038.2483517-2-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../hwtracing/coresight/coresight-etm-perf.c | 5 ++++- include/linux/coresight-pmu.h | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index bdc34ca449f7..465ef1aa8c82 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -27,7 +27,10 @@ static bool etm_perf_up; static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); static DEFINE_PER_CPU(struct coresight_device *, csdev_src); -/* ETMv3.5/PTM's ETMCR is 'config' */ +/* + * The PMU formats were orignally for ETMv3.5/PTM's ETMCR 'config'; + * now take them as general formats and apply on all ETMs. + */ PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); PMU_FORMAT_ATTR(contextid, "config:" __stringify(ETM_OPT_CTXTID)); PMU_FORMAT_ATTR(timestamp, "config:" __stringify(ETM_OPT_TS)); diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h index b0e35eec6499..5dc47cfdcf07 100644 --- a/include/linux/coresight-pmu.h +++ b/include/linux/coresight-pmu.h @@ -10,11 +10,18 @@ #define CORESIGHT_ETM_PMU_NAME "cs_etm" #define CORESIGHT_ETM_PMU_SEED 0x10 -/* ETMv3.5/PTM's ETMCR config bit */ -#define ETM_OPT_CYCACC 12 -#define ETM_OPT_CTXTID 14 -#define ETM_OPT_TS 28 -#define ETM_OPT_RETSTK 29 +/* + * Below are the definition of bit offsets for perf option, and works as + * arbitrary values for all ETM versions. + * + * Most of them are orignally from ETMv3.5/PTM's ETMCR config, therefore, + * ETMv3.5/PTM doesn't define ETMCR config bits with prefix "ETM3_" and + * directly use below macros as config bits. + */ +#define ETM_OPT_CYCACC 12 +#define ETM_OPT_CTXTID 14 +#define ETM_OPT_TS 28 +#define ETM_OPT_RETSTK 29 /* ETMv4 CONFIGR programming bits for the ETM OPTs */ #define ETM4_CFG_BIT_CYCACC 4 From 88f11864cf1d1324f620059ec747d74b72d9d736 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 11 Feb 2021 10:20:37 -0700 Subject: [PATCH 391/633] coresight: etm-perf: Support PID tracing for kernel at EL2 When the kernel is running at EL2, the PID is stored in CONTEXTIDR_EL2. So, tracing CONTEXTIDR_EL1 doesn't give us the pid of the process. Thus we should trace the VMID with VMIDOPT set to trace CONTEXTIDR_EL2 instead of CONTEXTIDR_EL1. Given that we have an existing config option "contextid" and this will be useful for tracing virtual machines (when we get to support virtualization). So instead, this patch extends option CTXTID with an extra bit ETM_OPT_CTXTID2 (bit 15), thus on an EL2 kernel, we will have another bit available for the perf tool: ETM_OPT_CTXTID is for kernel running in EL1, ETM_OPT_CTXTID2 is used when kernel runs in EL2 with VHE enabled. The tool must be backward compatible for users, i.e, "contextid" today traces PID and that should remain the same; for this purpose, the perf tool is updated to automatically set corresponding bit for the "contextid" config, therefore, the user doesn't have to bother which EL the kernel is running. i.e, perf record -e cs_etm/contextid/u -- will always do the "pid" tracing, independent of the kernel EL. The driver parses the format "contextid", which traces CONTEXTIDR_EL1 for ETM_OPT_CTXTID (on EL1 kernel) and traces CONTEXTIDR_EL2 for ETM_OPT_CTXTID2 (on EL2 kernel). Besides the enhancement for format "contexid", extra two formats are introduced: "contextid1" and "contextid2". This considers to support tracing both CONTEXTIDR_EL1 and CONTEXTIDR_EL2 when the kernel is running at EL2. Finally, the PMU formats are defined as follow: "contextid1": Available on both EL1 kernel and EL2 kernel. When the kernel is running at EL1, "contextid1" enables the PID tracing; when the kernel is running at EL2, this enables tracing the PID of guest applications. "contextid2": Only usable when the kernel is running at EL2. When selected, enables PID tracing on EL2 kernel. "contextid": Will be an alias for the option that enables PID tracing. I.e, contextid == contextid1, on EL1 kernel. contextid == contextid2, on EL2 kernel. Cc: Mathieu Poirier Cc: Al Grant Cc: Mike Leach Cc: Leo Yan Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose [ Added two config formats: contextid1, contextid2 ] Signed-off-by: Leo Yan Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210206150833.42120-4-leo.yan@linaro.org Link: https://lore.kernel.org/r/20210211172038.2483517-3-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- .../hwtracing/coresight/coresight-etm-perf.c | 27 ++++++++++++++++++- .../coresight/coresight-etm4x-core.c | 13 +++++++++ include/linux/coresight-pmu.h | 3 +++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 465ef1aa8c82..0f603b4094f2 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -32,15 +32,40 @@ static DEFINE_PER_CPU(struct coresight_device *, csdev_src); * now take them as general formats and apply on all ETMs. */ PMU_FORMAT_ATTR(cycacc, "config:" __stringify(ETM_OPT_CYCACC)); -PMU_FORMAT_ATTR(contextid, "config:" __stringify(ETM_OPT_CTXTID)); +/* contextid1 enables tracing CONTEXTIDR_EL1 for ETMv4 */ +PMU_FORMAT_ATTR(contextid1, "config:" __stringify(ETM_OPT_CTXTID)); +/* contextid2 enables tracing CONTEXTIDR_EL2 for ETMv4 */ +PMU_FORMAT_ATTR(contextid2, "config:" __stringify(ETM_OPT_CTXTID2)); PMU_FORMAT_ATTR(timestamp, "config:" __stringify(ETM_OPT_TS)); PMU_FORMAT_ATTR(retstack, "config:" __stringify(ETM_OPT_RETSTK)); /* Sink ID - same for all ETMs */ PMU_FORMAT_ATTR(sinkid, "config2:0-31"); +/* + * contextid always traces the "PID". The PID is in CONTEXTIDR_EL1 + * when the kernel is running at EL1; when the kernel is at EL2, + * the PID is in CONTEXTIDR_EL2. + */ +static ssize_t format_attr_contextid_show(struct device *dev, + struct device_attribute *attr, + char *page) +{ + int pid_fmt = ETM_OPT_CTXTID; + +#if defined(CONFIG_CORESIGHT_SOURCE_ETM4X) + pid_fmt = is_kernel_in_hyp_mode() ? ETM_OPT_CTXTID2 : ETM_OPT_CTXTID; +#endif + return sprintf(page, "config:%d\n", pid_fmt); +} + +struct device_attribute format_attr_contextid = + __ATTR(contextid, 0444, format_attr_contextid_show, NULL); + static struct attribute *etm_config_formats_attr[] = { &format_attr_cycacc.attr, &format_attr_contextid.attr, + &format_attr_contextid1.attr, + &format_attr_contextid2.attr, &format_attr_timestamp.attr, &format_attr_retstack.attr, &format_attr_sinkid.attr, diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index c8ecd91e289e..15016f757828 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -550,6 +550,19 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata, /* bit[6], Context ID tracing bit */ config->cfg |= BIT(ETM4_CFG_BIT_CTXTID); + /* + * If set bit ETM_OPT_CTXTID2 in perf config, this asks to trace VMID + * for recording CONTEXTIDR_EL2. Do not enable VMID tracing if the + * kernel is not running in EL2. + */ + if (attr->config & BIT(ETM_OPT_CTXTID2)) { + if (!is_kernel_in_hyp_mode()) { + ret = -EINVAL; + goto out; + } + config->cfg |= BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT); + } + /* return stack - enable if selected and supported */ if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) /* bit[12], Return stack enable bit */ diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h index 5dc47cfdcf07..4ac5c081af93 100644 --- a/include/linux/coresight-pmu.h +++ b/include/linux/coresight-pmu.h @@ -20,14 +20,17 @@ */ #define ETM_OPT_CYCACC 12 #define ETM_OPT_CTXTID 14 +#define ETM_OPT_CTXTID2 15 #define ETM_OPT_TS 28 #define ETM_OPT_RETSTK 29 /* ETMv4 CONFIGR programming bits for the ETM OPTs */ #define ETM4_CFG_BIT_CYCACC 4 #define ETM4_CFG_BIT_CTXTID 6 +#define ETM4_CFG_BIT_VMID 7 #define ETM4_CFG_BIT_TS 11 #define ETM4_CFG_BIT_RETSTK 12 +#define ETM4_CFG_BIT_VMID_OPT 15 static inline int coresight_get_trace_id(int cpu) { From 06c18e28c402ecfb842df8e22a19a097c35ffca9 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Thu, 11 Feb 2021 10:20:38 -0700 Subject: [PATCH 392/633] Documentation: coresight: Add PID tracing description After support the PID tracing for the kernel in EL1 or EL2, the usage gets more complicated. This patch gives description for the PMU formats of contextID configs, this can help users to understand how to control the knobs for PID tracing when the kernel is in different ELs. Reviewed-by: Suzuki K Poulose Reviewed-by: Mike Leach Signed-off-by: Leo Yan Signed-off-by: Mathieu Poirier Link: https://lore.kernel.org/r/20210206150833.42120-9-leo.yan@linaro.org Link: https://lore.kernel.org/r/20210211172038.2483517-4-mathieu.poirier@linaro.org Signed-off-by: Greg Kroah-Hartman --- Documentation/trace/coresight/coresight.rst | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst index 0b73acb44efa..169749efd8d1 100644 --- a/Documentation/trace/coresight/coresight.rst +++ b/Documentation/trace/coresight/coresight.rst @@ -512,6 +512,38 @@ The --itrace option controls the type and frequency of synthesized events Note that only 64-bit programs are currently supported - further work is required to support instruction decode of 32-bit Arm programs. +2.2) Tracing PID + +The kernel can be built to write the PID value into the PE ContextID registers. +For a kernel running at EL1, the PID is stored in CONTEXTIDR_EL1. A PE may +implement Arm Virtualization Host Extensions (VHE), which the kernel can +run at EL2 as a virtualisation host; in this case, the PID value is stored in +CONTEXTIDR_EL2. + +perf provides PMU formats that program the ETM to insert these values into the +trace data; the PMU formats are defined as below: + + "contextid1": Available on both EL1 kernel and EL2 kernel. When the + kernel is running at EL1, "contextid1" enables the PID + tracing; when the kernel is running at EL2, this enables + tracing the PID of guest applications. + + "contextid2": Only usable when the kernel is running at EL2. When + selected, enables PID tracing on EL2 kernel. + + "contextid": Will be an alias for the option that enables PID + tracing. I.e, + contextid == contextid1, on EL1 kernel. + contextid == contextid2, on EL2 kernel. + +perf will always enable PID tracing at the relevant EL, this is accomplished by +automatically enable the "contextid" config - but for EL2 it is possible to make +specific adjustments using configs "contextid1" and "contextid2", E.g. if a user +wants to trace PIDs for both host and guest, the two configs "contextid1" and +"contextid2" can be set at the same time: + + perf record -e cs_etm/contextid1,contextid2/u -- vm + Generating coverage files for Feedback Directed Optimization: AutoFDO --------------------------------------------------------------------- From 243a8b3c156b890cead4fd752299315222614232 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Wed, 10 Feb 2021 16:15:38 +0530 Subject: [PATCH 393/633] dt-bindings: remoteproc: qcom: pas: Add SM8350 remoteprocs Add the SM8350 audio, compute, modem and sensor remoteprocs to the PAS DT binding. Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20210210104539.340349-1-vkoul@kernel.org Signed-off-by: Bjorn Andersson --- .../devicetree/bindings/remoteproc/qcom,adsp.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt index 54737024da20..1c330a8941f9 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt @@ -25,6 +25,10 @@ on the Qualcomm ADSP Hexagon core. "qcom,sm8250-adsp-pas" "qcom,sm8250-cdsp-pas" "qcom,sm8250-slpi-pas" + "qcom,sm8350-adsp-pas" + "qcom,sm8350-cdsp-pas" + "qcom,sm8350-slpi-pas" + "qcom,sm8350-mpss-pas" - interrupts-extended: Usage: required @@ -51,10 +55,14 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8250-adsp-pas: qcom,sm8250-cdsp-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-adsp-pas: + qcom,sm8350-cdsp-pas: + qcom,sm8350-slpi-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack" qcom,qcs404-wcss-pas: qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack", "shutdown-ack" @@ -114,13 +122,17 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8150-adsp-pas: qcom,sm8150-cdsp-pas: qcom,sm8250-cdsp-pas: + qcom,sm8350-cdsp-pas: must be "cx", "load_state" qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "cx", "load_state", "mss" qcom,sm8250-adsp-pas: + qcom,sm8350-adsp-pas: qcom,sm8150-slpi-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-slpi-pas: must be "lcx", "lmx", "load_state" - memory-region: From e8b4e9a21af77b65ea68bd698acf4abe04afd051 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Wed, 10 Feb 2021 16:15:39 +0530 Subject: [PATCH 394/633] remoteproc: qcom: pas: Add SM8350 PAS remoteprocs Add audio, compute, modem and sensor DSP resources to the Qualcomm PAS driver. Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20210210104539.340349-2-vkoul@kernel.org Signed-off-by: Bjorn Andersson --- drivers/remoteproc/qcom_q6v5_pas.c | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index ee586226e438..e635454d6170 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -565,6 +565,26 @@ static const struct adsp_data sm8250_adsp_resource = { .ssctl_id = 0x14, }; +static const struct adsp_data sm8350_adsp_resource = { + .crash_reason_smem = 423, + .firmware_name = "adsp.mdt", + .pas_id = 1, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "lcx", + "lmx", + NULL + }, + .ssr_name = "lpass", + .sysmon_name = "adsp", + .ssctl_id = 0x14, +}; + static const struct adsp_data msm8998_adsp_resource = { .crash_reason_smem = 423, .firmware_name = "adsp.mdt", @@ -629,6 +649,25 @@ static const struct adsp_data sm8250_cdsp_resource = { .ssctl_id = 0x17, }; +static const struct adsp_data sm8350_cdsp_resource = { + .crash_reason_smem = 601, + .firmware_name = "cdsp.mdt", + .pas_id = 18, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "cx", + NULL + }, + .ssr_name = "cdsp", + .sysmon_name = "cdsp", + .ssctl_id = 0x17, +}; + static const struct adsp_data mpss_resource_init = { .crash_reason_smem = 421, .firmware_name = "modem.mdt", @@ -701,6 +740,26 @@ static const struct adsp_data sm8250_slpi_resource = { .ssctl_id = 0x16, }; +static const struct adsp_data sm8350_slpi_resource = { + .crash_reason_smem = 424, + .firmware_name = "slpi.mdt", + .pas_id = 12, + .has_aggre2_clk = false, + .auto_boot = true, + .active_pd_names = (char*[]){ + "load_state", + NULL + }, + .proxy_pd_names = (char*[]){ + "lcx", + "lmx", + NULL + }, + .ssr_name = "dsps", + .sysmon_name = "slpi", + .ssctl_id = 0x16, +}; + static const struct adsp_data msm8998_slpi_resource = { .crash_reason_smem = 424, .firmware_name = "slpi.mdt", @@ -745,6 +804,10 @@ static const struct of_device_id adsp_of_match[] = { { .compatible = "qcom,sm8250-adsp-pas", .data = &sm8250_adsp_resource}, { .compatible = "qcom,sm8250-cdsp-pas", .data = &sm8250_cdsp_resource}, { .compatible = "qcom,sm8250-slpi-pas", .data = &sm8250_slpi_resource}, + { .compatible = "qcom,sm8350-adsp-pas", .data = &sm8350_adsp_resource}, + { .compatible = "qcom,sm8350-cdsp-pas", .data = &sm8350_cdsp_resource}, + { .compatible = "qcom,sm8350-slpi-pas", .data = &sm8350_slpi_resource}, + { .compatible = "qcom,sm8350-mpss-pas", .data = &mpss_resource_init}, { }, }; MODULE_DEVICE_TABLE(of, adsp_of_match); From d19db80a366576d3ffadf2508ed876b4c1faf959 Mon Sep 17 00:00:00 2001 From: Subbaraman Narayanamurthy Date: Thu, 11 Feb 2021 19:14:17 -0800 Subject: [PATCH 395/633] spmi: spmi-pmic-arb: Fix hw_irq overflow Currently, when handling the SPMI summary interrupt, the hw_irq number is calculated based on SID, Peripheral ID, IRQ index and APID. This is then passed to irq_find_mapping() to see if a mapping exists for this hw_irq and if available, invoke the interrupt handler. Since the IRQ index uses an "int" type, hw_irq which is of unsigned long data type can take a large value when SID has its MSB set to 1 and the type conversion happens. Because of this, irq_find_mapping() returns 0 as there is no mapping for this hw_irq. This ends up invoking cleanup_irq() as if the interrupt is spurious whereas it is actually a valid interrupt. Fix this by using the proper data type (u32) for id. Cc: stable@vger.kernel.org Signed-off-by: Subbaraman Narayanamurthy Link: https://lore.kernel.org/r/1612812784-26369-1-git-send-email-subbaram@codeaurora.org Signed-off-by: Stephen Boyd Link: https://lore.kernel.org/r/20210212031417.3148936-1-sboyd@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/spmi/spmi-pmic-arb.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c index de844b412110..bbbd311eda03 100644 --- a/drivers/spmi/spmi-pmic-arb.c +++ b/drivers/spmi/spmi-pmic-arb.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2012-2015, 2017, The Linux Foundation. All rights reserved. + * Copyright (c) 2012-2015, 2017, 2021, The Linux Foundation. All rights reserved. */ #include #include @@ -505,8 +505,7 @@ static void cleanup_irq(struct spmi_pmic_arb *pmic_arb, u16 apid, int id) static void periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid) { unsigned int irq; - u32 status; - int id; + u32 status, id; u8 sid = (pmic_arb->apid_data[apid].ppid >> 8) & 0xF; u8 per = pmic_arb->apid_data[apid].ppid & 0xFF; From 3c26db8b289589b1a6dce9f612d6c7a26c2a50c4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:00 +0200 Subject: [PATCH 396/633] media: atomisp: Remove unused header sfi.h is not anyhow used by the driver. Remove it. Signed-off-by: Andy Shevchenko Acked-by: Sakari Ailus Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- drivers/staging/media/atomisp/include/linux/atomisp_platform.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h index 5a5121d958ed..8c65733e0255 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h @@ -22,7 +22,6 @@ #include #include -#include #include #include "atomisp.h" From 73f70d6c200ba85f61818ed3efe7f800c78d2953 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:01 +0200 Subject: [PATCH 397/633] cpufreq: sfi-cpufreq: Remove driver for deprecated firmware SFI-based platforms are gone. So does this driver. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Acked-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/Kconfig.x86 | 10 --- drivers/cpufreq/Makefile | 1 - drivers/cpufreq/sfi-cpufreq.c | 127 ---------------------------------- 3 files changed, 138 deletions(-) delete mode 100644 drivers/cpufreq/sfi-cpufreq.c diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 399526289320..92701a18bdd9 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -62,16 +62,6 @@ config X86_ACPI_CPUFREQ_CPB By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. -config X86_SFI_CPUFREQ - tristate "SFI Performance-States driver" - depends on X86_INTEL_MID && SFI - help - This adds a CPUFreq driver for some Silvermont based Intel Atom - architectures like Z34xx and Z35xx which enumerate processor - performance states through SFI. - - If in doubt, say N. - config ELAN_CPUFREQ tristate "AMD Elan SC400 and SC410" depends on MELAN diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index f1b7e3dd6e5d..18c9b0eafd09 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -43,7 +43,6 @@ obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o obj-$(CONFIG_X86_INTEL_PSTATE) += intel_pstate.o obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o -obj-$(CONFIG_X86_SFI_CPUFREQ) += sfi-cpufreq.o ################################################################################## # ARM SoC drivers diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c deleted file mode 100644 index 45cfdf67cf03..000000000000 --- a/drivers/cpufreq/sfi-cpufreq.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SFI Performance States Driver - * - * Author: Vishwesh M Rudramuni - * Author: Srinidhi Kasagar - */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -static struct cpufreq_frequency_table *freq_table; -static struct sfi_freq_table_entry *sfi_cpufreq_array; -static int num_freq_table_entries; - -static int sfi_parse_freq(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_freq_table_entry *pentry; - int totallen; - - sb = (struct sfi_table_simple *)table; - num_freq_table_entries = SFI_GET_NUM_ENTRIES(sb, - struct sfi_freq_table_entry); - if (num_freq_table_entries <= 1) { - pr_err("No p-states discovered\n"); - return -ENODEV; - } - - pentry = (struct sfi_freq_table_entry *)sb->pentry; - totallen = num_freq_table_entries * sizeof(*pentry); - - sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL); - if (!sfi_cpufreq_array) - return -ENOMEM; - - return 0; -} - -static int sfi_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) -{ - unsigned int next_perf_state = 0; /* Index into perf table */ - u32 lo, hi; - - next_perf_state = policy->freq_table[index].driver_data; - - rdmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, &lo, &hi); - lo = (lo & ~INTEL_PERF_CTL_MASK) | - ((u32) sfi_cpufreq_array[next_perf_state].ctrl_val & - INTEL_PERF_CTL_MASK); - wrmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, lo, hi); - - return 0; -} - -static int sfi_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->shared_type = CPUFREQ_SHARED_TYPE_HW; - policy->cpuinfo.transition_latency = 100000; /* 100us */ - policy->freq_table = freq_table; - - return 0; -} - -static struct cpufreq_driver sfi_cpufreq_driver = { - .flags = CPUFREQ_CONST_LOOPS, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = sfi_cpufreq_target, - .init = sfi_cpufreq_cpu_init, - .name = "sfi-cpufreq", - .attr = cpufreq_generic_attr, -}; - -static int __init sfi_cpufreq_init(void) -{ - int ret, i; - - /* parse the freq table from SFI */ - ret = sfi_table_parse(SFI_SIG_FREQ, NULL, NULL, sfi_parse_freq); - if (ret) - return ret; - - freq_table = kcalloc(num_freq_table_entries + 1, sizeof(*freq_table), - GFP_KERNEL); - if (!freq_table) { - ret = -ENOMEM; - goto err_free_array; - } - - for (i = 0; i < num_freq_table_entries; i++) { - freq_table[i].driver_data = i; - freq_table[i].frequency = sfi_cpufreq_array[i].freq_mhz * 1000; - } - freq_table[i].frequency = CPUFREQ_TABLE_END; - - ret = cpufreq_register_driver(&sfi_cpufreq_driver); - if (ret) - goto err_free_tbl; - - return ret; - -err_free_tbl: - kfree(freq_table); -err_free_array: - kfree(sfi_cpufreq_array); - return ret; -} -late_initcall(sfi_cpufreq_init); - -static void __exit sfi_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&sfi_cpufreq_driver); - kfree(freq_table); - kfree(sfi_cpufreq_array); -} -module_exit(sfi_cpufreq_exit); - -MODULE_AUTHOR("Vishwesh M Rudramuni "); -MODULE_DESCRIPTION("SFI Performance-States Driver"); -MODULE_LICENSE("GPL"); From 4590d98f5a4f466d17e5c81d7c9fc796da9a8cee Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:02 +0200 Subject: [PATCH 398/633] sfi: Remove framework for deprecated firmware SFI-based platforms are gone. So does this framework. This removes mention of SFI through the drivers and other code as well. Signed-off-by: Andy Shevchenko Reviewed-by: Hans de Goede Acked-by: Linus Walleij Signed-off-by: Rafael J. Wysocki --- Documentation/ABI/testing/sysfs-firmware-sfi | 15 - Documentation/ABI/testing/sysfs-platform-kim | 2 +- MAINTAINERS | 7 - arch/x86/Kconfig | 7 +- arch/x86/include/asm/intel-mid.h | 37 -- arch/x86/include/asm/intel_scu_ipc_legacy.h | 58 +- arch/x86/include/asm/platform_sst_audio.h | 2 - arch/x86/kernel/apic/io_apic.c | 4 +- arch/x86/kernel/setup.c | 2 - arch/x86/pci/mmconfig-shared.c | 6 +- arch/x86/platform/Makefile | 1 - arch/x86/platform/intel-mid/Makefile | 5 - .../platform/intel-mid/device_libs/Makefile | 23 - .../intel-mid/device_libs/platform_bcm43xx.c | 101 ---- .../intel-mid/device_libs/platform_bma023.c | 16 - .../intel-mid/device_libs/platform_bt.c | 101 ---- .../intel-mid/device_libs/platform_emc1403.c | 39 -- .../device_libs/platform_gpio_keys.c | 81 --- .../intel-mid/device_libs/platform_lis331.c | 37 -- .../intel-mid/device_libs/platform_max7315.c | 77 --- .../intel-mid/device_libs/platform_mpu3050.c | 32 -- .../device_libs/platform_mrfld_pinctrl.c | 39 -- .../device_libs/platform_mrfld_rtc.c | 44 -- .../intel-mid/device_libs/platform_mrfld_sd.c | 43 -- .../device_libs/platform_mrfld_spidev.c | 50 -- .../device_libs/platform_pcal9555a.c | 95 ---- .../intel-mid/device_libs/platform_tc35876x.c | 42 -- .../intel-mid/device_libs/platform_tca6416.c | 53 -- arch/x86/platform/intel-mid/intel-mid.c | 1 - arch/x86/platform/intel-mid/sfi.c | 419 -------------- arch/x86/platform/sfi/Makefile | 2 - arch/x86/platform/sfi/sfi.c | 100 ---- drivers/Makefile | 2 +- drivers/platform/x86/intel_scu_pcidrv.c | 22 +- drivers/sfi/Kconfig | 18 - drivers/sfi/Makefile | 4 - drivers/sfi/sfi_acpi.c | 214 ------- drivers/sfi/sfi_core.c | 522 ------------------ drivers/sfi/sfi_core.h | 81 --- include/linux/sfi.h | 210 ------- include/linux/sfi_acpi.h | 93 ---- init/main.c | 2 - 42 files changed, 14 insertions(+), 2695 deletions(-) delete mode 100644 Documentation/ABI/testing/sysfs-firmware-sfi delete mode 100644 arch/x86/platform/intel-mid/device_libs/Makefile delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_bma023.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_bt.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_emc1403.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_lis331.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_max7315.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c delete mode 100644 arch/x86/platform/intel-mid/device_libs/platform_tca6416.c delete mode 100644 arch/x86/platform/intel-mid/sfi.c delete mode 100644 arch/x86/platform/sfi/Makefile delete mode 100644 arch/x86/platform/sfi/sfi.c delete mode 100644 drivers/sfi/Kconfig delete mode 100644 drivers/sfi/Makefile delete mode 100644 drivers/sfi/sfi_acpi.c delete mode 100644 drivers/sfi/sfi_core.c delete mode 100644 drivers/sfi/sfi_core.h delete mode 100644 include/linux/sfi.h delete mode 100644 include/linux/sfi_acpi.h diff --git a/Documentation/ABI/testing/sysfs-firmware-sfi b/Documentation/ABI/testing/sysfs-firmware-sfi deleted file mode 100644 index 5210e0f06ddb..000000000000 --- a/Documentation/ABI/testing/sysfs-firmware-sfi +++ /dev/null @@ -1,15 +0,0 @@ -What: /sys/firmware/sfi/tables/ -Date: May 2010 -Contact: Len Brown -Description: - SFI defines a number of small static memory tables - so the kernel can get platform information from firmware. - - The tables are defined in the latest SFI specification: - http://simplefirmware.org/documentation - - While the tables are used by the kernel, user-space - can observe them this way:: - - # cd /sys/firmware/sfi/tables - # cat $TABLENAME > $TABLENAME.bin diff --git a/Documentation/ABI/testing/sysfs-platform-kim b/Documentation/ABI/testing/sysfs-platform-kim index a7f81de68046..6a52d6d2b601 100644 --- a/Documentation/ABI/testing/sysfs-platform-kim +++ b/Documentation/ABI/testing/sysfs-platform-kim @@ -7,7 +7,7 @@ Description: is connected. example: "/dev/ttyS0". The device name flows down to architecture specific board - initialization file from the SFI/ATAGS bootloader + initialization file from the ATAGS bootloader firmware. The name exposed is read from the user-space dameon and opens the device when install is requested. diff --git a/MAINTAINERS b/MAINTAINERS index 8fdb7fe27537..fd41b96d71bb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16234,13 +16234,6 @@ S: Maintained F: Documentation/fb/sm712fb.rst F: drivers/video/fbdev/sm712* -SIMPLE FIRMWARE INTERFACE (SFI) -S: Obsolete -W: http://simplefirmware.org/ -F: arch/x86/platform/sfi/ -F: drivers/sfi/ -F: include/linux/sfi*.h - SIMPLEFB FB DRIVER M: Hans de Goede L: linux-fbdev@vger.kernel.org diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index cc73f98c3e42..7f5b77055266 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -444,7 +444,7 @@ config X86_X2APIC If you don't know what to do here, say N. config X86_MPPARSE - bool "Enable MPS table" if ACPI || SFI + bool "Enable MPS table" if ACPI default y depends on X86_LOCAL_APIC help @@ -603,7 +603,6 @@ config X86_INTEL_MID depends on PCI depends on X86_64 || (PCI_GOANY && X86_32) depends on X86_IO_APIC - select SFI select I2C select DW_APB_TIMER select APB_TIMER @@ -2457,8 +2456,6 @@ source "kernel/power/Kconfig" source "drivers/acpi/Kconfig" -source "drivers/sfi/Kconfig" - config X86_APM_BOOT def_bool y depends on APM @@ -2645,7 +2642,7 @@ config PCI_DIRECT config PCI_MMCONFIG bool "Support mmconfig PCI config space access" if X86_64 default y - depends on PCI && (ACPI || SFI || JAILHOUSE_GUEST) + depends on PCI && (ACPI || JAILHOUSE_GUEST) depends on X86_64 || (PCI_GOANY || PCI_GOMMCONFIG) config PCI_OLPC diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index 7bda0587cf70..6306fe3e5c4a 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -7,7 +7,6 @@ #ifndef _ASM_X86_INTEL_MID_H #define _ASM_X86_INTEL_MID_H -#include #include #include @@ -22,39 +21,6 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); -extern int get_gpio_by_name(const char *name); - -/* - * Here defines the array of devices platform data that IAFW would export - * through SFI "DEVS" table, we use name and type to match the device and - * its platform data. - */ -struct devs_id { - char name[SFI_NAME_LEN + 1]; - u8 type; - u8 delay; - void *(*get_platform_data)(void *info); -}; - -#define sfi_device(i) \ - static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ - __section(".x86_intel_mid_dev.init") = &i - -/** -* struct mid_sd_board_info - template for SD device creation -* @name: identifies the driver -* @bus_num: board-specific identifier for a given SD controller -* @max_clk: the maximum frequency device supports -* @platform_data: the particular data stored there is driver-specific -*/ -struct mid_sd_board_info { - char name[SFI_NAME_LEN]; - int bus_num; - unsigned short addr; - u32 max_clk; - void *platform_data; -}; - /* * Medfield is the follow-up of Moorestown, it combines two chip solution into * one. Other than that it also added always-on and constant tsc and lapic @@ -99,7 +65,4 @@ static inline void intel_scu_devices_destroy(void) { } /* FSB 83MHz */ #define BSEL_SOC_FUSE_111 0x7 -/* The offset for the mapping of global gpio pin to irq */ -#define INTEL_MID_IRQ_OFFSET 0x100 - #endif /* _ASM_X86_INTEL_MID_H */ diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h index f8b87d8face7..fa529e5ec142 100644 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ b/arch/x86/include/asm/intel_scu_ipc_legacy.h @@ -2,73 +2,17 @@ #ifndef _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ #define _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ -#include - -#define IPCMSG_INDIRECT_READ 0x02 -#define IPCMSG_INDIRECT_WRITE 0x05 +#include #define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ - -#define IPCMSG_WARM_RESET 0xF0 #define IPCMSG_COLD_RESET 0xF1 -#define IPCMSG_SOFT_RESET 0xF2 -#define IPCMSG_COLD_BOOT 0xF3 /* Don't call these in new code - they will be removed eventually */ -/* Read a vector */ -static inline int intel_scu_ipc_readv(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_readv(NULL, addr, data, len); -} - -/* Write a vector */ -static inline int intel_scu_ipc_writev(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_writev(NULL, addr, data, len); -} - -/* Update single register based on the mask */ -static inline int intel_scu_ipc_update_register(u16 addr, u8 data, u8 mask) -{ - return intel_scu_ipc_dev_update(NULL, addr, data, mask); -} - /* Issue commands to the SCU with or without data */ static inline int intel_scu_ipc_simple_command(int cmd, int sub) { return intel_scu_ipc_dev_simple_command(NULL, cmd, sub); } -static inline int intel_scu_ipc_command(int cmd, int sub, u32 *in, int inlen, - u32 *out, int outlen) -{ - /* New API takes both inlen and outlen as bytes so convert here */ - size_t inbytes = inlen * sizeof(u32); - size_t outbytes = outlen * sizeof(u32); - - return intel_scu_ipc_dev_command_with_size(NULL, cmd, sub, in, inbytes, - inlen, out, outbytes); -} - -extern struct blocking_notifier_head intel_scu_notifier; - -static inline void intel_scu_notifier_add(struct notifier_block *nb) -{ - blocking_notifier_chain_register(&intel_scu_notifier, nb); -} - -static inline void intel_scu_notifier_remove(struct notifier_block *nb) -{ - blocking_notifier_chain_unregister(&intel_scu_notifier, nb); -} - -static inline int intel_scu_notifier_post(unsigned long v, void *p) -{ - return blocking_notifier_call_chain(&intel_scu_notifier, v, p); -} - -#define SCU_AVAILABLE 1 -#define SCU_DOWN 2 - #endif diff --git a/arch/x86/include/asm/platform_sst_audio.h b/arch/x86/include/asm/platform_sst_audio.h index 16b9f220bdeb..40f92270515b 100644 --- a/arch/x86/include/asm/platform_sst_audio.h +++ b/arch/x86/include/asm/platform_sst_audio.h @@ -10,8 +10,6 @@ #ifndef _PLATFORM_SST_AUDIO_H_ #define _PLATFORM_SST_AUDIO_H_ -#include - #define MAX_NUM_STREAMS_MRFLD 25 #define MAX_NUM_STREAMS MAX_NUM_STREAMS_MRFLD diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index e4ab4804b20d..c3b60c37c728 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -198,7 +198,7 @@ static int __init parse_noapic(char *str) } early_param("noapic", parse_noapic); -/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */ +/* Will be called in mpparse/ACPI codes for saving IRQ info */ void mp_save_irq(struct mpc_intsrc *m) { int i; @@ -2863,7 +2863,7 @@ int mp_register_ioapic(int id, u32 address, u32 gsi_base, /* * If mp_register_ioapic() is called during early boot stage when - * walking ACPI/SFI/DT tables, it's too early to create irqdomain, + * walking ACPI/DT tables, it's too early to create irqdomain, * we are still using bootmem allocator. So delay it to setup_IO_APIC(). */ if (hotplug) { diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 740f3bdb3f61..d883176ef2ce 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -1185,7 +1184,6 @@ void __init setup_arch(char **cmdline_p) * Read APIC and some other early information from ACPI tables. */ acpi_boot_init(); - sfi_init(); x86_dtb_init(); /* diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 234998f196d4..de6bf0e7e8f8 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -11,9 +11,9 @@ * themselves. */ +#include #include #include -#include #include #include #include @@ -665,7 +665,7 @@ void __init pci_mmcfg_early_init(void) if (pci_mmcfg_check_hostbridge()) known_bridge = 1; else - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(1); set_apei_filter(); @@ -683,7 +683,7 @@ void __init pci_mmcfg_late_init(void) /* MMCONFIG hasn't been enabled yet, try again */ if (pci_probe & PCI_PROBE_MASK & ~PCI_PROBE_MMCONF) { - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(0); } } diff --git a/arch/x86/platform/Makefile b/arch/x86/platform/Makefile index d0e835470d01..d1f5228225d9 100644 --- a/arch/x86/platform/Makefile +++ b/arch/x86/platform/Makefile @@ -11,6 +11,5 @@ obj-y += intel-mid/ obj-y += intel-quark/ obj-y += olpc/ obj-y += scx200/ -obj-y += sfi/ obj-y += ts5500/ obj-y += uv/ diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index 5794e661050c..ddfc08783fb8 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,7 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o pwr.o - -# SFI specific code -ifdef CONFIG_X86_INTEL_MID -obj-$(CONFIG_SFI) += sfi.o device_libs/ -endif diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile deleted file mode 100644 index 4d008b053ac8..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Family-Level Interface Shim (FLIS) -obj-$(subst m,y,$(CONFIG_PINCTRL_MERRIFIELD)) += platform_mrfld_pinctrl.o -# SDHCI Devices -obj-$(subst m,y,$(CONFIG_MMC_SDHCI_PCI)) += platform_mrfld_sd.o -# WiFi + BT -obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o -obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o -# SPI Devices -obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o -# I2C Devices -obj-$(subst m,y,$(CONFIG_SENSORS_EMC1403)) += platform_emc1403.o -obj-$(subst m,y,$(CONFIG_SENSORS_LIS3LV02D)) += platform_lis331.o -obj-$(subst m,y,$(CONFIG_MPU3050_I2C)) += platform_mpu3050.o -obj-$(subst m,y,$(CONFIG_INPUT_BMA150)) += platform_bma023.o -obj-$(subst m,y,$(CONFIG_DRM_MEDFIELD)) += platform_tc35876x.o -# I2C GPIO Expanders -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_max7315.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_pcal9555a.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o -# MISC Devices -obj-$(subst m,y,$(CONFIG_KEYBOARD_GPIO)) += platform_gpio_keys.o -obj-$(subst m,y,$(CONFIG_RTC_DRV_CMOS)) += platform_mrfld_rtc.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c b/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c deleted file mode 100644 index 564c47c53f3a..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bcm43xx.c: bcm43xx platform data initialization file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include -#include - -#include - -#define WLAN_SFI_GPIO_IRQ_NAME "WLAN-interrupt" -#define WLAN_SFI_GPIO_ENABLE_NAME "WLAN-enable" - -#define WLAN_DEV_NAME "0000:00:01.3" - -static struct regulator_consumer_supply bcm43xx_vmmc_supply = { - .dev_name = WLAN_DEV_NAME, - .supply = "vmmc", -}; - -static struct regulator_init_data bcm43xx_vmmc_data = { - .constraints = { - .valid_ops_mask = REGULATOR_CHANGE_STATUS, - }, - .num_consumer_supplies = 1, - .consumer_supplies = &bcm43xx_vmmc_supply, -}; - -static struct fixed_voltage_config bcm43xx_vmmc = { - .supply_name = "bcm43xx-vmmc-regulator", - /* - * Announce 2.0V here to be compatible with SDIO specification. The - * real voltage and signaling are still 1.8V. - */ - .microvolts = 2000000, /* 1.8V */ - .startup_delay = 250 * 1000, /* 250ms */ - .enabled_at_boot = 0, /* disabled at boot */ - .init_data = &bcm43xx_vmmc_data, -}; - -static struct platform_device bcm43xx_vmmc_regulator = { - .name = "reg-fixed-voltage", - .id = PLATFORM_DEVID_AUTO, - .dev = { - .platform_data = &bcm43xx_vmmc, - }, -}; - -static struct gpiod_lookup_table bcm43xx_vmmc_gpio_table = { - .dev_id = "reg-fixed-voltage.0", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, NULL, GPIO_ACTIVE_LOW), - {} - }, -}; - -static int __init bcm43xx_regulator_register(void) -{ - struct gpiod_lookup_table *table = &bcm43xx_vmmc_gpio_table; - struct gpiod_lookup *lookup = table->table; - int ret; - - lookup[0].chip_hwnum = get_gpio_by_name(WLAN_SFI_GPIO_ENABLE_NAME); - gpiod_add_lookup_table(table); - - ret = platform_device_register(&bcm43xx_vmmc_regulator); - if (ret) { - pr_err("%s: vmmc regulator register failed\n", __func__); - return ret; - } - - return 0; -} - -static void __init *bcm43xx_platform_data(void *info) -{ - int ret; - - ret = bcm43xx_regulator_register(); - if (ret) - return NULL; - - pr_info("Using generic wifi platform data\n"); - - /* For now it's empty */ - return NULL; -} - -static const struct devs_id bcm43xx_clk_vmmc_dev_id __initconst = { - .name = "bcm43xx_clk_vmmc", - .type = SFI_DEV_TYPE_SD, - .get_platform_data = &bcm43xx_platform_data, -}; - -sfi_device(bcm43xx_clk_vmmc_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c b/arch/x86/platform/intel-mid/device_libs/platform_bma023.c deleted file mode 100644 index 32912a17f68e..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bma023.c: bma023 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - */ - -#include - -static const struct devs_id bma023_dev_id __initconst = { - .name = "bma023", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, -}; - -sfi_device(bma023_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bt.c b/arch/x86/platform/intel-mid/device_libs/platform_bt.c deleted file mode 100644 index 31dda18bb370..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bt.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Bluetooth platform data initialization file - * - * (C) Copyright 2017 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include -#include -#include - -struct bt_sfi_data { - struct device *dev; - const char *name; - int (*setup)(struct bt_sfi_data *ddata); -}; - -static struct gpiod_lookup_table tng_bt_sfi_gpio_table = { - .dev_id = "hci_bcm", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "device-wakeup", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "shutdown", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "host-wakeup", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -#define TNG_BT_SFI_GPIO_DEVICE_WAKEUP "bt_wakeup" -#define TNG_BT_SFI_GPIO_SHUTDOWN "BT-reset" -#define TNG_BT_SFI_GPIO_HOST_WAKEUP "bt_uart_enable" - -static int __init tng_bt_sfi_setup(struct bt_sfi_data *ddata) -{ - struct gpiod_lookup_table *table = &tng_bt_sfi_gpio_table; - struct gpiod_lookup *lookup = table->table; - struct pci_dev *pdev; - - /* Connected to /dev/ttyS0 */ - pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(4, 1)); - if (!pdev) - return -ENODEV; - - ddata->dev = &pdev->dev; - ddata->name = table->dev_id; - - lookup[0].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_DEVICE_WAKEUP); - lookup[1].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_SHUTDOWN); - lookup[2].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_HOST_WAKEUP); - - gpiod_add_lookup_table(table); - return 0; -} - -static struct bt_sfi_data tng_bt_sfi_data __initdata = { - .setup = tng_bt_sfi_setup, -}; - -static const struct x86_cpu_id bt_sfi_cpu_ids[] = { - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &tng_bt_sfi_data), - {} -}; - -static int __init bt_sfi_init(void) -{ - struct platform_device_info info; - struct platform_device *pdev; - const struct x86_cpu_id *id; - struct bt_sfi_data *ddata; - int ret; - - id = x86_match_cpu(bt_sfi_cpu_ids); - if (!id) - return -ENODEV; - - ddata = (struct bt_sfi_data *)id->driver_data; - if (!ddata) - return -ENODEV; - - ret = ddata->setup(ddata); - if (ret) - return ret; - - memset(&info, 0, sizeof(info)); - info.fwnode = ddata->dev->fwnode; - info.parent = ddata->dev; - info.name = ddata->name, - info.id = PLATFORM_DEVID_NONE, - - pdev = platform_device_register_full(&info); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); - - dev_info(ddata->dev, "Registered Bluetooth device: %s\n", ddata->name); - return 0; -} -device_initcall(bt_sfi_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c b/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c deleted file mode 100644 index a2508582a0b1..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_emc1403.c: emc1403 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -static void __init *emc1403_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("thermal_int"); - int intr2nd = get_gpio_by_name("thermal_alert"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id emc1403_dev_id __initconst = { - .name = "emc1403", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &emc1403_platform_data, -}; - -sfi_device(emc1403_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c b/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c deleted file mode 100644 index d9435d2196a4..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_gpio_keys.c: gpio_keys platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#define DEVICE_NAME "gpio-keys" - -/* - * we will search these buttons in SFI GPIO table (by name) - * and register them dynamically. Please add all possible - * buttons here, we will shrink them if no GPIO found. - */ -static struct gpio_keys_button gpio_button[] = { - {KEY_POWER, -1, 1, "power_btn", EV_KEY, 0, 3000}, - {KEY_PROG1, -1, 1, "prog_btn1", EV_KEY, 0, 20}, - {KEY_PROG2, -1, 1, "prog_btn2", EV_KEY, 0, 20}, - {SW_LID, -1, 1, "lid_switch", EV_SW, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "vol_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "vol_down", EV_KEY, 0, 20}, - {KEY_MUTE, -1, 1, "mute_enable", EV_KEY, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "volume_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "volume_down", EV_KEY, 0, 20}, - {KEY_CAMERA, -1, 1, "camera_full", EV_KEY, 0, 20}, - {KEY_CAMERA_FOCUS, -1, 1, "camera_half", EV_KEY, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw1", EV_SW, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw2", EV_SW, 0, 20}, -}; - -static struct gpio_keys_platform_data gpio_keys = { - .buttons = gpio_button, - .rep = 1, - .nbuttons = -1, /* will fill it after search */ -}; - -static struct platform_device pb_device = { - .name = DEVICE_NAME, - .id = -1, - .dev = { - .platform_data = &gpio_keys, - }, -}; - -/* - * Shrink the non-existent buttons, register the gpio button - * device if there is some - */ -static int __init pb_keys_init(void) -{ - struct gpio_keys_button *gb = gpio_button; - int i, good = 0; - - for (i = 0; i < ARRAY_SIZE(gpio_button); i++) { - gb[i].gpio = get_gpio_by_name(gb[i].desc); - pr_debug("info[%2d]: name = %s, gpio = %d\n", i, gb[i].desc, - gb[i].gpio); - if (gb[i].gpio < 0) - continue; - - if (i != good) - gb[good] = gb[i]; - good++; - } - - if (good) { - gpio_keys.nbuttons = good; - return platform_device_register(&pb_device); - } - return 0; -} -late_initcall(pb_keys_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c b/arch/x86/platform/intel-mid/device_libs/platform_lis331.c deleted file mode 100644 index a4485cd638c6..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_lis331.c: lis331 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void __init *lis331dl_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("accel_int"); - int intr2nd = get_gpio_by_name("accel_2"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id lis331dl_dev_id __initconst = { - .name = "i2c_accel", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &lis331dl_platform_data, -}; - -sfi_device(lis331dl_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c b/arch/x86/platform/intel-mid/device_libs/platform_max7315.c deleted file mode 100644 index e9287c3184da..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_max7315.c: max7315 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include - -#define MAX7315_NUM 2 - -static void __init *max7315_platform_data(void *info) -{ - static struct pca953x_platform_data max7315_pdata[MAX7315_NUM]; - static int nr; - struct pca953x_platform_data *max7315 = &max7315_pdata[nr]; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - if (nr == MAX7315_NUM) { - pr_err("too many max7315s, we only support %d\n", - MAX7315_NUM); - return NULL; - } - /* we have several max7315 on the board, we only need load several - * instances of the same pca953x driver to cover them - */ - strcpy(i2c_info->type, "max7315"); - if (nr++) { - snprintf(base_pin_name, sizeof(base_pin_name), - "max7315_%d_base", nr); - snprintf(intr_pin_name, sizeof(intr_pin_name), - "max7315_%d_int", nr); - } else { - strcpy(base_pin_name, "max7315_base"); - strcpy(intr_pin_name, "max7315_int"); - } - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - max7315->gpio_base = gpio_base; - if (intr != -1) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - max7315->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - max7315->irq_base = -1; - } - return max7315; -} - -static const struct devs_id max7315_dev_id __initconst = { - .name = "i2c_max7315", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -static const struct devs_id max7315_2_dev_id __initconst = { - .name = "i2c_max7315_2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -sfi_device(max7315_dev_id); -sfi_device(max7315_2_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c b/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c deleted file mode 100644 index 28a182713934..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_mpu3050.c: mpu3050 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void *mpu3050_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("mpu3050_int"); - - if (intr < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - return NULL; -} - -static const struct devs_id mpu3050_dev_id __initconst = { - .name = "mpu3050", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &mpu3050_platform_data, -}; - -sfi_device(mpu3050_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c deleted file mode 100644 index 605e1f94ad89..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield FLIS platform device initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include - -#define FLIS_BASE_ADDR 0xff0c0000 -#define FLIS_LENGTH 0x8000 - -static struct resource mrfld_pinctrl_mmio_resource = { - .start = FLIS_BASE_ADDR, - .end = FLIS_BASE_ADDR + FLIS_LENGTH - 1, - .flags = IORESOURCE_MEM, -}; - -static struct platform_device mrfld_pinctrl_device = { - .name = "pinctrl-merrifield", - .id = PLATFORM_DEVID_NONE, - .resource = &mrfld_pinctrl_mmio_resource, - .num_resources = 1, -}; - -static int __init mrfld_pinctrl_init(void) -{ - if (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_TANGIER) - return platform_device_register(&mrfld_pinctrl_device); - - return -ENODEV; -} -arch_initcall(mrfld_pinctrl_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c deleted file mode 100644 index 40e9808a9634..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield legacy RTC initialization file - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include - -#include -#include -#include -#include -#include - -static int __init mrfld_legacy_rtc_alloc_irq(void) -{ - struct irq_alloc_info info; - int ret; - - if (!x86_platform.legacy.rtc) - return -ENODEV; - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, 0); - ret = mp_map_gsi_to_irq(RTC_IRQ, IOAPIC_MAP_ALLOC, &info); - if (ret < 0) { - pr_info("Failed to allocate RTC interrupt. Disabling RTC\n"); - x86_platform.legacy.rtc = 0; - return ret; - } - - return 0; -} - -static int __init mrfld_legacy_rtc_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - return mrfld_legacy_rtc_alloc_irq(); -} -arch_initcall(mrfld_legacy_rtc_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c deleted file mode 100644 index fe3b7ff975f3..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SDHCI platform data initilisation file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include - -#include - -#include - -#define INTEL_MRFLD_SD 2 -#define INTEL_MRFLD_SD_CD_GPIO 77 - -static struct sdhci_pci_data mrfld_sdhci_pci_data = { - .rst_n_gpio = -EINVAL, - .cd_gpio = INTEL_MRFLD_SD_CD_GPIO, -}; - -static struct sdhci_pci_data * -mrfld_sdhci_pci_get_data(struct pci_dev *pdev, int slotno) -{ - unsigned int func = PCI_FUNC(pdev->devfn); - - if (func == INTEL_MRFLD_SD) - return &mrfld_sdhci_pci_data; - - return NULL; -} - -static int __init mrfld_sd_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - sdhci_pci_get_data = mrfld_sdhci_pci_get_data; - return 0; -} -arch_initcall(mrfld_sd_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c deleted file mode 100644 index b828f4fd40be..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * spidev platform data initialization file - * - * (C) Copyright 2014, 2016 Intel Corporation - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define MRFLD_SPI_DEFAULT_DMA_BURST 8 -#define MRFLD_SPI_DEFAULT_TIMEOUT 500 - -/* GPIO pin for spidev chipselect */ -#define MRFLD_SPIDEV_GPIO_CS 111 - -static struct pxa2xx_spi_chip spidev_spi_chip = { - .dma_burst_size = MRFLD_SPI_DEFAULT_DMA_BURST, - .timeout = MRFLD_SPI_DEFAULT_TIMEOUT, - .gpio_cs = MRFLD_SPIDEV_GPIO_CS, -}; - -static void __init *spidev_platform_data(void *info) -{ - struct spi_board_info *spi_info = info; - - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return ERR_PTR(-ENODEV); - - spi_info->mode = SPI_MODE_0; - spi_info->controller_data = &spidev_spi_chip; - - return NULL; -} - -static const struct devs_id spidev_dev_id __initconst = { - .name = "spidev", - .type = SFI_DEV_TYPE_SPI, - .delay = 0, - .get_platform_data = &spidev_platform_data, -}; - -sfi_device(spidev_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c b/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c deleted file mode 100644 index 5609d8da3978..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * PCAL9555a platform data initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define PCAL9555A_NUM 4 - -static struct pca953x_platform_data pcal9555a_pdata[PCAL9555A_NUM]; -static int nr; - -static void __init *pcal9555a_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - char *type = i2c_info->type; - struct pca953x_platform_data *pcal9555a; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - int gpio_base, intr; - - snprintf(base_pin_name, sizeof(base_pin_name), "%s_base", type); - snprintf(intr_pin_name, sizeof(intr_pin_name), "%s_int", type); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - /* Check if the SFI record valid */ - if (gpio_base == -1) - return NULL; - - if (nr >= PCAL9555A_NUM) { - pr_err("%s: Too many instances, only %d supported\n", __func__, - PCAL9555A_NUM); - return NULL; - } - - pcal9555a = &pcal9555a_pdata[nr++]; - pcal9555a->gpio_base = gpio_base; - - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - pcal9555a->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - pcal9555a->irq_base = -1; - } - - strcpy(type, "pcal9555a"); - return pcal9555a; -} - -static const struct devs_id pcal9555a_1_dev_id __initconst = { - .name = "pcal9555a-1", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_2_dev_id __initconst = { - .name = "pcal9555a-2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_3_dev_id __initconst = { - .name = "pcal9555a-3", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_4_dev_id __initconst = { - .name = "pcal9555a-4", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -sfi_device(pcal9555a_1_dev_id); -sfi_device(pcal9555a_2_dev_id); -sfi_device(pcal9555a_3_dev_id); -sfi_device(pcal9555a_4_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c b/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c deleted file mode 100644 index 139738bbdd36..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tc35876x.c: tc35876x platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include - -static struct gpiod_lookup_table tc35876x_gpio_table = { - .dev_id = "i2c_disp_brig", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "bridge-reset", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "bl-en", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "vadd", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -/*tc35876x DSI_LVDS bridge chip and panel platform data*/ -static void *tc35876x_platform_data(void *data) -{ - struct gpiod_lookup_table *table = &tc35876x_gpio_table; - struct gpiod_lookup *lookup = table->table; - - lookup[0].chip_hwnum = get_gpio_by_name("LCMB_RXEN"); - lookup[1].chip_hwnum = get_gpio_by_name("6S6P_BL_EN"); - lookup[2].chip_hwnum = get_gpio_by_name("EN_VREG_LCD_V3P3"); - gpiod_add_lookup_table(table); - - return NULL; -} - -static const struct devs_id tc35876x_dev_id __initconst = { - .name = "i2c_disp_brig", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &tc35876x_platform_data, -}; - -sfi_device(tc35876x_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c b/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c deleted file mode 100644 index e689d8f61059..000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tca6416.c: tca6416 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -#define TCA6416_NAME "tca6416" -#define TCA6416_BASE "tca6416_base" -#define TCA6416_INTR "tca6416_int" - -static void *tca6416_platform_data(void *info) -{ - static struct pca953x_platform_data tca6416; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - strcpy(i2c_info->type, TCA6416_NAME); - strcpy(base_pin_name, TCA6416_BASE); - strcpy(intr_pin_name, TCA6416_INTR); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - tca6416.gpio_base = gpio_base; - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - tca6416.irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - tca6416.irq_base = -1; - } - return &tca6416; -} - -static const struct devs_id tca6416_dev_id __initconst = { - .name = "tca6416", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &tca6416_platform_data, -}; - -sfi_device(tca6416_dev_id); diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 864b0c158b2f..2c044e260b4b 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c deleted file mode 100644 index 63ae342ffb12..000000000000 --- a/arch/x86/platform/intel-mid/sfi.c +++ /dev/null @@ -1,419 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_sfi.c: Intel MID SFI initialization code - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SFI_SIG_OEM0 "OEM0" -#define MAX_IPCDEVS 24 -#define MAX_SCU_SPI 24 -#define MAX_SCU_I2C 24 - -static struct platform_device *ipc_devs[MAX_IPCDEVS]; -static struct spi_board_info *spi_devs[MAX_SCU_SPI]; -static struct i2c_board_info *i2c_devs[MAX_SCU_I2C]; -static struct sfi_gpio_table_entry *gpio_table; -static int ipc_next_dev; -static int spi_next_dev; -static int i2c_next_dev; -static int i2c_bus[MAX_SCU_I2C]; -static int gpio_num_entry; - -struct blocking_notifier_head intel_scu_notifier = - BLOCKING_NOTIFIER_INIT(intel_scu_notifier); -EXPORT_SYMBOL_GPL(intel_scu_notifier); - -#define intel_mid_sfi_get_pdata(dev, priv) \ - ((dev)->get_platform_data ? (dev)->get_platform_data(priv) : NULL) - -/* - * Parsing GPIO table first, since the DEVS table will need this table - * to map the pin name to the actual pin. - */ -static int __init sfi_parse_gpio(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_gpio_table_entry *pentry; - int num, i; - - if (gpio_table) - return 0; - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_gpio_table_entry); - pentry = (struct sfi_gpio_table_entry *)sb->pentry; - - gpio_table = kmemdup(pentry, num * sizeof(*pentry), GFP_KERNEL); - if (!gpio_table) - return -1; - gpio_num_entry = num; - - pr_debug("GPIO pin info:\n"); - for (i = 0; i < num; i++, pentry++) - pr_debug("info[%2d]: controller = %16.16s, pin_name = %16.16s," - " pin = %d\n", i, - pentry->controller_name, - pentry->pin_name, - pentry->pin_no); - return 0; -} - -int get_gpio_by_name(const char *name) -{ - struct sfi_gpio_table_entry *pentry = gpio_table; - int i; - - if (!pentry) - return -1; - for (i = 0; i < gpio_num_entry; i++, pentry++) { - if (!strncmp(name, pentry->pin_name, SFI_NAME_LEN)) - return pentry->pin_no; - } - return -EINVAL; -} - -static void __init intel_scu_ipc_device_register(struct platform_device *pdev) -{ - if (ipc_next_dev == MAX_IPCDEVS) - pr_err("too many SCU IPC devices"); - else - ipc_devs[ipc_next_dev++] = pdev; -} - -static void __init intel_scu_spi_device_register(struct spi_board_info *sdev) -{ - struct spi_board_info *new_dev; - - if (spi_next_dev == MAX_SCU_SPI) { - pr_err("too many SCU SPI devices"); - return; - } - - new_dev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed spi dev %s\n", - sdev->modalias); - return; - } - *new_dev = *sdev; - - spi_devs[spi_next_dev++] = new_dev; -} - -static void __init intel_scu_i2c_device_register(int bus, - struct i2c_board_info *idev) -{ - struct i2c_board_info *new_dev; - - if (i2c_next_dev == MAX_SCU_I2C) { - pr_err("too many SCU I2C devices"); - return; - } - - new_dev = kzalloc(sizeof(*idev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed i2c dev %s\n", - idev->type); - return; - } - *new_dev = *idev; - - i2c_bus[i2c_next_dev] = bus; - i2c_devs[i2c_next_dev++] = new_dev; -} - -/* Called by IPC driver */ -void intel_scu_devices_create(void) -{ - int i; - - for (i = 0; i < ipc_next_dev; i++) - platform_device_add(ipc_devs[i]); - - for (i = 0; i < spi_next_dev; i++) - spi_register_board_info(spi_devs[i], 1); - - for (i = 0; i < i2c_next_dev; i++) { - struct i2c_adapter *adapter; - struct i2c_client *client; - - adapter = i2c_get_adapter(i2c_bus[i]); - if (adapter) { - client = i2c_new_client_device(adapter, i2c_devs[i]); - if (IS_ERR(client)) - pr_err("can't create i2c device %s\n", - i2c_devs[i]->type); - } else - i2c_register_board_info(i2c_bus[i], i2c_devs[i], 1); - } - intel_scu_notifier_post(SCU_AVAILABLE, NULL); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_create); - -/* Called by IPC driver */ -void intel_scu_devices_destroy(void) -{ - int i; - - intel_scu_notifier_post(SCU_DOWN, NULL); - - for (i = 0; i < ipc_next_dev; i++) - platform_device_del(ipc_devs[i]); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_destroy); - -static void __init install_irq_resource(struct platform_device *pdev, int irq) -{ - /* Single threaded */ - static struct resource res __initdata = { - .name = "IRQ", - .flags = IORESOURCE_IRQ, - }; - res.start = irq; - platform_device_add_resources(pdev, &res, 1); -} - -static void __init sfi_handle_ipc_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct platform_device *pdev; - void *pdata = NULL; - - pr_debug("IPC bus, name = %16.16s, irq = 0x%2x\n", - pentry->name, pentry->irq); - - /* - * We need to call platform init of IPC devices to fill misc_pdata - * structure. It will be used in msic_init for initialization. - */ - pdata = intel_mid_sfi_get_pdata(dev, pentry); - if (IS_ERR(pdata)) - return; - - pdev = platform_device_alloc(pentry->name, 0); - if (pdev == NULL) { - pr_err("out of memory for SFI platform device '%s'.\n", - pentry->name); - return; - } - install_irq_resource(pdev, pentry->irq); - - pdev->dev.platform_data = pdata; - if (dev->delay) - intel_scu_ipc_device_register(pdev); - else - platform_device_add(pdev); -} - -static void __init sfi_handle_spi_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct spi_board_info spi_info; - void *pdata = NULL; - - memset(&spi_info, 0, sizeof(spi_info)); - strncpy(spi_info.modalias, pentry->name, SFI_NAME_LEN); - spi_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - spi_info.bus_num = pentry->host_num; - spi_info.chip_select = pentry->addr; - spi_info.max_speed_hz = pentry->max_freq; - pr_debug("SPI bus=%d, name=%16.16s, irq=0x%2x, max_freq=%d, cs=%d\n", - spi_info.bus_num, - spi_info.modalias, - spi_info.irq, - spi_info.max_speed_hz, - spi_info.chip_select); - - pdata = intel_mid_sfi_get_pdata(dev, &spi_info); - if (IS_ERR(pdata)) - return; - - spi_info.platform_data = pdata; - if (dev->delay) - intel_scu_spi_device_register(&spi_info); - else - spi_register_board_info(&spi_info, 1); -} - -static void __init sfi_handle_i2c_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct i2c_board_info i2c_info; - void *pdata = NULL; - - memset(&i2c_info, 0, sizeof(i2c_info)); - strncpy(i2c_info.type, pentry->name, SFI_NAME_LEN); - i2c_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - i2c_info.addr = pentry->addr; - pr_debug("I2C bus = %d, name = %16.16s, irq = 0x%2x, addr = 0x%x\n", - pentry->host_num, - i2c_info.type, - i2c_info.irq, - i2c_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &i2c_info); - i2c_info.platform_data = pdata; - if (IS_ERR(pdata)) - return; - - if (dev->delay) - intel_scu_i2c_device_register(pentry->host_num, &i2c_info); - else - i2c_register_board_info(pentry->host_num, &i2c_info, 1); -} - -static void __init sfi_handle_sd_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct mid_sd_board_info sd_info; - void *pdata; - - memset(&sd_info, 0, sizeof(sd_info)); - strncpy(sd_info.name, pentry->name, SFI_NAME_LEN); - sd_info.bus_num = pentry->host_num; - sd_info.max_clk = pentry->max_freq; - sd_info.addr = pentry->addr; - pr_debug("SD bus = %d, name = %16.16s, max_clk = %d, addr = 0x%x\n", - sd_info.bus_num, - sd_info.name, - sd_info.max_clk, - sd_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &sd_info); - if (IS_ERR(pdata)) - return; - - /* Nothing we can do with this for now */ - sd_info.platform_data = pdata; - - pr_debug("Successfully registered %16.16s", sd_info.name); -} - -extern struct devs_id *const __x86_intel_mid_dev_start[], - *const __x86_intel_mid_dev_end[]; - -static struct devs_id __init *get_device_id(u8 type, char *name) -{ - struct devs_id *const *dev_table; - - for (dev_table = __x86_intel_mid_dev_start; - dev_table < __x86_intel_mid_dev_end; dev_table++) { - struct devs_id *dev = *dev_table; - if (dev->type == type && - !strncmp(dev->name, name, SFI_NAME_LEN)) { - return dev; - } - } - - return NULL; -} - -static int __init sfi_parse_devs(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_device_table_entry *pentry; - struct devs_id *dev = NULL; - int num, i, ret; - int polarity; - struct irq_alloc_info info; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_device_table_entry); - pentry = (struct sfi_device_table_entry *)sb->pentry; - - for (i = 0; i < num; i++, pentry++) { - int irq = pentry->irq; - - if (irq != (u8)0xff) { /* native RTE case */ - /* these SPI2 devices are not exposed to system as PCI - * devices, but they have separate RTE entry in IOAPIC - * so we have to enable them one by one here - */ - if (intel_mid_identify_cpu() == - INTEL_MID_CPU_CHIP_TANGIER) { - if (!strncmp(pentry->name, "r69001-ts-i2c", 13)) - /* active low */ - polarity = 1; - else if (!strncmp(pentry->name, - "synaptics_3202", 14)) - /* active low */ - polarity = 1; - else if (irq == 41) - /* fast_int_1 */ - polarity = 1; - else - /* active high */ - polarity = 0; - } else { - /* PNW and CLV go with active low */ - polarity = 1; - } - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, polarity); - ret = mp_map_gsi_to_irq(irq, IOAPIC_MAP_ALLOC, &info); - WARN_ON(ret < 0); - } - - dev = get_device_id(pentry->type, pentry->name); - - if (!dev) - continue; - - switch (pentry->type) { - case SFI_DEV_TYPE_IPC: - sfi_handle_ipc_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SPI: - sfi_handle_spi_dev(pentry, dev); - break; - case SFI_DEV_TYPE_I2C: - sfi_handle_i2c_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SD: - sfi_handle_sd_dev(pentry, dev); - break; - case SFI_DEV_TYPE_UART: - case SFI_DEV_TYPE_HSI: - default: - break; - } - } - return 0; -} - -static int __init intel_mid_platform_init(void) -{ - sfi_table_parse(SFI_SIG_GPIO, NULL, NULL, sfi_parse_gpio); - sfi_table_parse(SFI_SIG_DEVS, NULL, NULL, sfi_parse_devs); - return 0; -} -arch_initcall(intel_mid_platform_init); diff --git a/arch/x86/platform/sfi/Makefile b/arch/x86/platform/sfi/Makefile deleted file mode 100644 index 4eba24c2af67..000000000000 --- a/arch/x86/platform/sfi/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_SFI) += sfi.o diff --git a/arch/x86/platform/sfi/sfi.c b/arch/x86/platform/sfi/sfi.c deleted file mode 100644 index 6259563760f9..000000000000 --- a/arch/x86/platform/sfi/sfi.c +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sfi.c - x86 architecture SFI support. - * - * Copyright (c) 2009, Intel Corporation. - */ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#ifdef CONFIG_X86_LOCAL_APIC -static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; - -/* All CPUs enumerated by SFI must be present and enabled */ -static void __init mp_sfi_register_lapic(u8 id) -{ - if (MAX_LOCAL_APIC - id <= 0) { - pr_warn("Processor #%d invalid (max %d)\n", id, MAX_LOCAL_APIC); - return; - } - - pr_info("registering lapic[%d]\n", id); - - generic_processor_info(id, GET_APIC_VERSION(apic_read(APIC_LVR))); -} - -static int __init sfi_parse_cpus(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_cpu_table_entry *pentry; - int i; - int cpu_num; - - sb = (struct sfi_table_simple *)table; - cpu_num = SFI_GET_NUM_ENTRIES(sb, struct sfi_cpu_table_entry); - pentry = (struct sfi_cpu_table_entry *)sb->pentry; - - for (i = 0; i < cpu_num; i++) { - mp_sfi_register_lapic(pentry->apic_id); - pentry++; - } - - smp_found_config = 1; - return 0; -} -#endif /* CONFIG_X86_LOCAL_APIC */ - -#ifdef CONFIG_X86_IO_APIC - -static int __init sfi_parse_ioapic(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_apic_table_entry *pentry; - int i, num; - struct ioapic_domain_cfg cfg = { - .type = IOAPIC_DOMAIN_STRICT, - .ops = &mp_ioapic_irqdomain_ops, - }; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_apic_table_entry); - pentry = (struct sfi_apic_table_entry *)sb->pentry; - - for (i = 0; i < num; i++) { - mp_register_ioapic(i, pentry->phys_addr, gsi_top, &cfg); - pentry++; - } - - WARN(pic_mode, KERN_WARNING - "SFI: pic_mod shouldn't be 1 when IOAPIC table is present\n"); - pic_mode = 0; - return 0; -} -#endif /* CONFIG_X86_IO_APIC */ - -/* - * sfi_platform_init(): register lapics & io-apics - */ -int __init sfi_platform_init(void) -{ -#ifdef CONFIG_X86_LOCAL_APIC - register_lapic_address(sfi_lapic_addr); - sfi_table_parse(SFI_SIG_CPUS, NULL, NULL, sfi_parse_cpus); -#endif -#ifdef CONFIG_X86_IO_APIC - sfi_table_parse(SFI_SIG_APIC, NULL, NULL, sfi_parse_ioapic); -#endif - return 0; -} diff --git a/drivers/Makefile b/drivers/Makefile index fd11b9ac4cc3..b2af1514a6b3 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -27,7 +27,7 @@ obj-y += idle/ obj-y += char/ipmi/ obj-$(CONFIG_ACPI) += acpi/ -obj-$(CONFIG_SFI) += sfi/ + # PnP must come after ACPI since it will eventually need to check if acpi # was used and do nothing if so obj-$(CONFIG_PNP) += pnp/ diff --git a/drivers/platform/x86/intel_scu_pcidrv.c b/drivers/platform/x86/intel_scu_pcidrv.c index 8c5fd8240da9..80abc708e4f2 100644 --- a/drivers/platform/x86/intel_scu_pcidrv.c +++ b/drivers/platform/x86/intel_scu_pcidrv.c @@ -17,7 +17,6 @@ static int intel_scu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - void (*setup_fn)(void) = (void (*)(void))id->driver_data; struct intel_scu_ipc_data scu_data = {}; struct intel_scu_ipc_dev *scu; int ret; @@ -30,27 +29,14 @@ static int intel_scu_pci_probe(struct pci_dev *pdev, scu_data.irq = pdev->irq; scu = intel_scu_ipc_register(&pdev->dev, &scu_data); - if (IS_ERR(scu)) - return PTR_ERR(scu); - - if (setup_fn) - setup_fn(); - return 0; -} - -static void intel_mid_scu_setup(void) -{ - intel_scu_devices_create(); + return PTR_ERR_OR_ZERO(scu); } static const struct pci_device_id pci_ids[] = { - { PCI_VDEVICE(INTEL, 0x080e), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, - { PCI_VDEVICE(INTEL, 0x08ea), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, + { PCI_VDEVICE(INTEL, 0x080e) }, + { PCI_VDEVICE(INTEL, 0x08ea) }, { PCI_VDEVICE(INTEL, 0x0a94) }, - { PCI_VDEVICE(INTEL, 0x11a0), - .driver_data = (kernel_ulong_t)intel_mid_scu_setup }, + { PCI_VDEVICE(INTEL, 0x11a0) }, { PCI_VDEVICE(INTEL, 0x1a94) }, { PCI_VDEVICE(INTEL, 0x5a94) }, {} diff --git a/drivers/sfi/Kconfig b/drivers/sfi/Kconfig deleted file mode 100644 index 3d0b64db214e..000000000000 --- a/drivers/sfi/Kconfig +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# SFI Configuration -# - -menuconfig SFI - bool "SFI (Simple Firmware Interface) Support" - help - The Simple Firmware Interface (SFI) provides a lightweight method - for platform firmware to pass information to the operating system - via static tables in memory. Kernel SFI support is required to - boot on SFI-only platforms. Currently, all SFI-only platforms are - based on the 2nd generation Intel Atom processor platform, - code-named Moorestown. - - For more information, see http://simplefirmware.org - - Say 'Y' here to enable the kernel to boot on SFI-only platforms. diff --git a/drivers/sfi/Makefile b/drivers/sfi/Makefile deleted file mode 100644 index ca9436b12386..000000000000 --- a/drivers/sfi/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y += sfi_acpi.o -obj-y += sfi_core.o - diff --git a/drivers/sfi/sfi_acpi.c b/drivers/sfi/sfi_acpi.c deleted file mode 100644 index d277b36eb389..000000000000 --- a/drivers/sfi/sfi_acpi.c +++ /dev/null @@ -1,214 +0,0 @@ -/* sfi_acpi.c Simple Firmware Interface - ACPI extensions */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include -#include -#include "sfi_core.h" - -/* - * SFI can access ACPI-defined tables via an optional ACPI XSDT. - * - * This allows re-use, and avoids re-definition, of standard tables. - * For example, the "MCFG" table is defined by PCI, reserved by ACPI, - * and is expected to be present many SFI-only systems. - */ - -static struct acpi_table_xsdt *xsdt_va __read_mostly; - -#define XSDT_GET_NUM_ENTRIES(ptable, entry_type) \ - ((ptable->header.length - sizeof(struct acpi_table_header)) / \ - (sizeof(entry_type))) - -static inline struct sfi_table_header *acpi_to_sfi_th( - struct acpi_table_header *th) -{ - return (struct sfi_table_header *)th; -} - -static inline struct acpi_table_header *sfi_to_acpi_th( - struct sfi_table_header *th) -{ - return (struct acpi_table_header *)th; -} - -/* - * sfi_acpi_parse_xsdt() - * - * Parse the ACPI XSDT for later access by sfi_acpi_table_parse(). - */ -static int __init sfi_acpi_parse_xsdt(struct sfi_table_header *th) -{ - struct sfi_table_key key = SFI_ANY_KEY; - int tbl_cnt, i; - void *ret; - - xsdt_va = (struct acpi_table_xsdt *)th; - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(xsdt_va->table_offset_entry[i], &key); - if (IS_ERR(ret)) { - disable_sfi(); - return -1; - } - } - - return 0; -} - -int __init sfi_acpi_init(void) -{ - struct sfi_table_key xsdt_key = { .sig = SFI_SIG_XSDT }; - - sfi_table_parse(SFI_SIG_XSDT, NULL, NULL, sfi_acpi_parse_xsdt); - - /* Only call the get_table to keep the table mapped */ - xsdt_va = (struct acpi_table_xsdt *)sfi_get_table(&xsdt_key); - return 0; -} - -static struct acpi_table_header *sfi_acpi_get_table(struct sfi_table_key *key) -{ - u32 tbl_cnt, i; - void *ret; - - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(xsdt_va->table_offset_entry[i], key); - if (!IS_ERR(ret) && ret) - return sfi_to_acpi_th(ret); - } - - return NULL; -} - -static void sfi_acpi_put_table(struct acpi_table_header *table) -{ - sfi_put_table(acpi_to_sfi_th(table)); -} - -/* - * sfi_acpi_table_parse() - * - * Find specified table in XSDT, run handler on it and return its return value - */ -int sfi_acpi_table_parse(char *signature, char *oem_id, char *oem_table_id, - int(*handler)(struct acpi_table_header *)) -{ - struct acpi_table_header *table = NULL; - struct sfi_table_key key; - int ret = 0; - - if (sfi_disabled) - return -1; - - key.sig = signature; - key.oem_id = oem_id; - key.oem_table_id = oem_table_id; - - table = sfi_acpi_get_table(&key); - if (!table) - return -EINVAL; - - ret = handler(table); - sfi_acpi_put_table(table); - return ret; -} - -static ssize_t sfi_acpi_table_show(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, char *buf, - loff_t offset, size_t count) -{ - struct sfi_table_attr *tbl_attr = - container_of(bin_attr, struct sfi_table_attr, attr); - struct acpi_table_header *th = NULL; - struct sfi_table_key key; - ssize_t cnt; - - key.sig = tbl_attr->name; - key.oem_id = NULL; - key.oem_table_id = NULL; - - th = sfi_acpi_get_table(&key); - if (!th) - return 0; - - cnt = memory_read_from_buffer(buf, count, &offset, - th, th->length); - sfi_acpi_put_table(th); - - return cnt; -} - - -void __init sfi_acpi_sysfs_init(void) -{ - u32 tbl_cnt, i; - struct sfi_table_attr *tbl_attr; - - tbl_cnt = XSDT_GET_NUM_ENTRIES(xsdt_va, u64); - for (i = 0; i < tbl_cnt; i++) { - tbl_attr = - sfi_sysfs_install_table(xsdt_va->table_offset_entry[i]); - tbl_attr->attr.read = sfi_acpi_table_show; - } - - return; -} diff --git a/drivers/sfi/sfi_core.c b/drivers/sfi/sfi_core.c deleted file mode 100644 index a5136901dd8a..000000000000 --- a/drivers/sfi/sfi_core.c +++ /dev/null @@ -1,522 +0,0 @@ -/* sfi_core.c Simple Firmware Interface - core internals */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sfi_core.h" - -#define ON_SAME_PAGE(addr1, addr2) \ - (((unsigned long)(addr1) & PAGE_MASK) == \ - ((unsigned long)(addr2) & PAGE_MASK)) -#define TABLE_ON_PAGE(page, table, size) (ON_SAME_PAGE(page, table) && \ - ON_SAME_PAGE(page, table + size)) - -int sfi_disabled __read_mostly; -EXPORT_SYMBOL(sfi_disabled); - -static u64 syst_pa __read_mostly; -static struct sfi_table_simple *syst_va __read_mostly; - -/* - * FW creates and saves the SFI tables in memory. When these tables get - * used, they may need to be mapped to virtual address space, and the mapping - * can happen before or after the memremap() is ready, so a flag is needed - * to indicating this - */ -static u32 sfi_use_memremap __read_mostly; - -/* - * sfi_un/map_memory calls early_memremap/memunmap which is a __init function - * and introduces section mismatch. So use __ref to make it calm. - */ -static void __iomem * __ref sfi_map_memory(u64 phys, u32 size) -{ - if (!phys || !size) - return NULL; - - if (sfi_use_memremap) - return memremap(phys, size, MEMREMAP_WB); - else - return early_memremap(phys, size); -} - -static void __ref sfi_unmap_memory(void __iomem *virt, u32 size) -{ - if (!virt || !size) - return; - - if (sfi_use_memremap) - memunmap(virt); - else - early_memunmap(virt, size); -} - -static void sfi_print_table_header(unsigned long long pa, - struct sfi_table_header *header) -{ - pr_info("%4.4s %llX, %04X (v%d %6.6s %8.8s)\n", - header->sig, pa, - header->len, header->rev, header->oem_id, - header->oem_table_id); -} - -/* - * sfi_verify_table() - * Sanity check table lengh, calculate checksum - */ -static int sfi_verify_table(struct sfi_table_header *table) -{ - - u8 checksum = 0; - u8 *puchar = (u8 *)table; - u32 length = table->len; - - /* Sanity check table length against arbitrary 1MB limit */ - if (length > 0x100000) { - pr_err("Invalid table length 0x%x\n", length); - return -1; - } - - while (length--) - checksum += *puchar++; - - if (checksum) { - pr_err("Checksum %2.2X should be %2.2X\n", - table->csum, table->csum - checksum); - return -1; - } - return 0; -} - -/* - * sfi_map_table() - * - * Return address of mapped table - * Check for common case that we can re-use mapping to SYST, - * which requires syst_pa, syst_va to be initialized. - */ -static struct sfi_table_header *sfi_map_table(u64 pa) -{ - struct sfi_table_header *th; - u32 length; - - if (!TABLE_ON_PAGE(syst_pa, pa, sizeof(struct sfi_table_header))) - th = sfi_map_memory(pa, sizeof(struct sfi_table_header)); - else - th = (void *)syst_va + (pa - syst_pa); - - /* If table fits on same page as its header, we are done */ - if (TABLE_ON_PAGE(th, th, th->len)) - return th; - - /* Entire table does not fit on same page as SYST */ - length = th->len; - if (!TABLE_ON_PAGE(syst_pa, pa, sizeof(struct sfi_table_header))) - sfi_unmap_memory(th, sizeof(struct sfi_table_header)); - - return sfi_map_memory(pa, length); -} - -/* - * sfi_unmap_table() - * - * Undoes effect of sfi_map_table() by unmapping table - * if it did not completely fit on same page as SYST. - */ -static void sfi_unmap_table(struct sfi_table_header *th) -{ - if (!TABLE_ON_PAGE(syst_va, th, th->len)) - sfi_unmap_memory(th, TABLE_ON_PAGE(th, th, th->len) ? - sizeof(*th) : th->len); -} - -static int sfi_table_check_key(struct sfi_table_header *th, - struct sfi_table_key *key) -{ - - if (strncmp(th->sig, key->sig, SFI_SIGNATURE_SIZE) - || (key->oem_id && strncmp(th->oem_id, - key->oem_id, SFI_OEM_ID_SIZE)) - || (key->oem_table_id && strncmp(th->oem_table_id, - key->oem_table_id, SFI_OEM_TABLE_ID_SIZE))) - return -1; - - return 0; -} - -/* - * This function will be used in 2 cases: - * 1. used to enumerate and verify the tables addressed by SYST/XSDT, - * thus no signature will be given (in kernel boot phase) - * 2. used to parse one specific table, signature must exist, and - * the mapped virt address will be returned, and the virt space - * will be released by call sfi_put_table() later - * - * This two cases are from two different functions with two different - * sections and causes section mismatch warning. So use __ref to tell - * modpost not to make any noise. - * - * Return value: - * NULL: when can't find a table matching the key - * ERR_PTR(error): error value - * virt table address: when a matched table is found - */ -struct sfi_table_header * - __ref sfi_check_table(u64 pa, struct sfi_table_key *key) -{ - struct sfi_table_header *th; - void *ret = NULL; - - th = sfi_map_table(pa); - if (!th) - return ERR_PTR(-ENOMEM); - - if (!key->sig) { - sfi_print_table_header(pa, th); - if (sfi_verify_table(th)) - ret = ERR_PTR(-EINVAL); - } else { - if (!sfi_table_check_key(th, key)) - return th; /* Success */ - } - - sfi_unmap_table(th); - return ret; -} - -/* - * sfi_get_table() - * - * Search SYST for the specified table with the signature in - * the key, and return the mapped table - */ -struct sfi_table_header *sfi_get_table(struct sfi_table_key *key) -{ - struct sfi_table_header *th; - u32 tbl_cnt, i; - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - for (i = 0; i < tbl_cnt; i++) { - th = sfi_check_table(syst_va->pentry[i], key); - if (!IS_ERR(th) && th) - return th; - } - - return NULL; -} - -void sfi_put_table(struct sfi_table_header *th) -{ - sfi_unmap_table(th); -} - -/* Find table with signature, run handler on it */ -int sfi_table_parse(char *signature, char *oem_id, char *oem_table_id, - sfi_table_handler handler) -{ - struct sfi_table_header *table = NULL; - struct sfi_table_key key; - int ret = -EINVAL; - - if (sfi_disabled || !handler || !signature) - goto exit; - - key.sig = signature; - key.oem_id = oem_id; - key.oem_table_id = oem_table_id; - - table = sfi_get_table(&key); - if (!table) - goto exit; - - ret = handler(table); - sfi_put_table(table); -exit: - return ret; -} -EXPORT_SYMBOL_GPL(sfi_table_parse); - -/* - * sfi_parse_syst() - * Checksum all the tables in SYST and print their headers - * - * success: set syst_va, return 0 - */ -static int __init sfi_parse_syst(void) -{ - struct sfi_table_key key = SFI_ANY_KEY; - int tbl_cnt, i; - void *ret; - - syst_va = sfi_map_memory(syst_pa, sizeof(struct sfi_table_simple)); - if (!syst_va) - return -ENOMEM; - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - for (i = 0; i < tbl_cnt; i++) { - ret = sfi_check_table(syst_va->pentry[i], &key); - if (IS_ERR(ret)) - return PTR_ERR(ret); - } - - return 0; -} - -/* - * The OS finds the System Table by searching 16-byte boundaries between - * physical address 0x000E0000 and 0x000FFFFF. The OS shall search this region - * starting at the low address and shall stop searching when the 1st valid SFI - * System Table is found. - * - * success: set syst_pa, return 0 - * fail: return -1 - */ -static __init int sfi_find_syst(void) -{ - unsigned long offset, len; - void *start; - - len = SFI_SYST_SEARCH_END - SFI_SYST_SEARCH_BEGIN; - start = sfi_map_memory(SFI_SYST_SEARCH_BEGIN, len); - if (!start) - return -1; - - for (offset = 0; offset < len; offset += 16) { - struct sfi_table_header *syst_hdr; - - syst_hdr = start + offset; - if (strncmp(syst_hdr->sig, SFI_SIG_SYST, - SFI_SIGNATURE_SIZE)) - continue; - - if (syst_hdr->len > PAGE_SIZE) - continue; - - sfi_print_table_header(SFI_SYST_SEARCH_BEGIN + offset, - syst_hdr); - - if (sfi_verify_table(syst_hdr)) - continue; - - /* - * Enforce SFI spec mandate that SYST reside within a page. - */ - if (!ON_SAME_PAGE(syst_pa, syst_pa + syst_hdr->len)) { - pr_info("SYST 0x%llx + 0x%x crosses page\n", - syst_pa, syst_hdr->len); - continue; - } - - /* Success */ - syst_pa = SFI_SYST_SEARCH_BEGIN + offset; - sfi_unmap_memory(start, len); - return 0; - } - - sfi_unmap_memory(start, len); - return -1; -} - -static struct kobject *sfi_kobj; -static struct kobject *tables_kobj; - -static ssize_t sfi_table_show(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, char *buf, - loff_t offset, size_t count) -{ - struct sfi_table_attr *tbl_attr = - container_of(bin_attr, struct sfi_table_attr, attr); - struct sfi_table_header *th = NULL; - struct sfi_table_key key; - ssize_t cnt; - - key.sig = tbl_attr->name; - key.oem_id = NULL; - key.oem_table_id = NULL; - - if (strncmp(SFI_SIG_SYST, tbl_attr->name, SFI_SIGNATURE_SIZE)) { - th = sfi_get_table(&key); - if (!th) - return 0; - - cnt = memory_read_from_buffer(buf, count, &offset, - th, th->len); - sfi_put_table(th); - } else - cnt = memory_read_from_buffer(buf, count, &offset, - syst_va, syst_va->header.len); - - return cnt; -} - -struct sfi_table_attr __init *sfi_sysfs_install_table(u64 pa) -{ - struct sfi_table_attr *tbl_attr; - struct sfi_table_header *th; - int ret; - - tbl_attr = kzalloc(sizeof(struct sfi_table_attr), GFP_KERNEL); - if (!tbl_attr) - return NULL; - - th = sfi_map_table(pa); - if (!th || !th->sig[0]) { - kfree(tbl_attr); - return NULL; - } - - sysfs_attr_init(&tbl_attr->attr.attr); - memcpy(tbl_attr->name, th->sig, SFI_SIGNATURE_SIZE); - - tbl_attr->attr.size = 0; - tbl_attr->attr.read = sfi_table_show; - tbl_attr->attr.attr.name = tbl_attr->name; - tbl_attr->attr.attr.mode = 0400; - - ret = sysfs_create_bin_file(tables_kobj, - &tbl_attr->attr); - if (ret) { - kfree(tbl_attr); - tbl_attr = NULL; - } - - sfi_unmap_table(th); - return tbl_attr; -} - -static int __init sfi_sysfs_init(void) -{ - int tbl_cnt, i; - - if (sfi_disabled) - return 0; - - sfi_kobj = kobject_create_and_add("sfi", firmware_kobj); - if (!sfi_kobj) - return 0; - - tables_kobj = kobject_create_and_add("tables", sfi_kobj); - if (!tables_kobj) { - kobject_put(sfi_kobj); - return 0; - } - - sfi_sysfs_install_table(syst_pa); - - tbl_cnt = SFI_GET_NUM_ENTRIES(syst_va, u64); - - for (i = 0; i < tbl_cnt; i++) - sfi_sysfs_install_table(syst_va->pentry[i]); - - sfi_acpi_sysfs_init(); - kobject_uevent(sfi_kobj, KOBJ_ADD); - kobject_uevent(tables_kobj, KOBJ_ADD); - pr_info("SFI sysfs interfaces init success\n"); - return 0; -} - -void __init sfi_init(void) -{ - if (!acpi_disabled) - disable_sfi(); - - if (sfi_disabled) - return; - - pr_info("Simple Firmware Interface v0.81 http://simplefirmware.org\n"); - - if (sfi_find_syst() || sfi_parse_syst() || sfi_platform_init()) - disable_sfi(); - - return; -} - -void __init sfi_init_late(void) -{ - int length; - - if (sfi_disabled) - return; - - length = syst_va->header.len; - sfi_unmap_memory(syst_va, sizeof(struct sfi_table_simple)); - - /* Use memremap now after it is ready */ - sfi_use_memremap = 1; - syst_va = sfi_map_memory(syst_pa, length); - - sfi_acpi_init(); -} - -/* - * The reason we put it here because we need wait till the /sys/firmware - * is setup, then our interface can be registered in /sys/firmware/sfi - */ -core_initcall(sfi_sysfs_init); diff --git a/drivers/sfi/sfi_core.h b/drivers/sfi/sfi_core.h deleted file mode 100644 index 1d5cfe854cf7..000000000000 --- a/drivers/sfi/sfi_core.h +++ /dev/null @@ -1,81 +0,0 @@ -/* sfi_core.h Simple Firmware Interface, internal header */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#include - -struct sfi_table_key{ - char *sig; - char *oem_id; - char *oem_table_id; -}; - -/* sysfs interface */ -struct sfi_table_attr { - struct bin_attribute attr; - char name[8]; -}; - -#define SFI_ANY_KEY { .sig = NULL, .oem_id = NULL, .oem_table_id = NULL } - -extern int __init sfi_acpi_init(void); -extern struct sfi_table_header *sfi_check_table(u64 paddr, - struct sfi_table_key *key); -struct sfi_table_header *sfi_get_table(struct sfi_table_key *key); -extern void sfi_put_table(struct sfi_table_header *table); -extern struct sfi_table_attr __init *sfi_sysfs_install_table(u64 pa); -extern void __init sfi_acpi_sysfs_init(void); diff --git a/include/linux/sfi.h b/include/linux/sfi.h deleted file mode 100644 index e0e1597ef9e6..000000000000 --- a/include/linux/sfi.h +++ /dev/null @@ -1,210 +0,0 @@ -/* sfi.h Simple Firmware Interface */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef _LINUX_SFI_H -#define _LINUX_SFI_H - -#include -#include - -/* Table signatures reserved by the SFI specification */ -#define SFI_SIG_SYST "SYST" -#define SFI_SIG_FREQ "FREQ" -#define SFI_SIG_IDLE "IDLE" -#define SFI_SIG_CPUS "CPUS" -#define SFI_SIG_MTMR "MTMR" -#define SFI_SIG_MRTC "MRTC" -#define SFI_SIG_MMAP "MMAP" -#define SFI_SIG_APIC "APIC" -#define SFI_SIG_XSDT "XSDT" -#define SFI_SIG_WAKE "WAKE" -#define SFI_SIG_DEVS "DEVS" -#define SFI_SIG_GPIO "GPIO" - -#define SFI_SIGNATURE_SIZE 4 -#define SFI_OEM_ID_SIZE 6 -#define SFI_OEM_TABLE_ID_SIZE 8 - -#define SFI_NAME_LEN 16 - -#define SFI_SYST_SEARCH_BEGIN 0x000E0000 -#define SFI_SYST_SEARCH_END 0x000FFFFF - -#define SFI_GET_NUM_ENTRIES(ptable, entry_type) \ - ((ptable->header.len - sizeof(struct sfi_table_header)) / \ - (sizeof(entry_type))) -/* - * Table structures must be byte-packed to match the SFI specification, - * as they are provided by the BIOS. - */ -struct sfi_table_header { - char sig[SFI_SIGNATURE_SIZE]; - u32 len; - u8 rev; - u8 csum; - char oem_id[SFI_OEM_ID_SIZE]; - char oem_table_id[SFI_OEM_TABLE_ID_SIZE]; -} __packed; - -struct sfi_table_simple { - struct sfi_table_header header; - u64 pentry[1]; -} __packed; - -/* Comply with UEFI spec 2.1 */ -struct sfi_mem_entry { - u32 type; - u64 phys_start; - u64 virt_start; - u64 pages; - u64 attrib; -} __packed; - -struct sfi_cpu_table_entry { - u32 apic_id; -} __packed; - -struct sfi_cstate_table_entry { - u32 hint; /* MWAIT hint */ - u32 latency; /* latency in ms */ -} __packed; - -struct sfi_apic_table_entry { - u64 phys_addr; /* phy base addr for APIC reg */ -} __packed; - -struct sfi_freq_table_entry { - u32 freq_mhz; /* in MHZ */ - u32 latency; /* transition latency in ms */ - u32 ctrl_val; /* value to write to PERF_CTL */ -} __packed; - -struct sfi_wake_table_entry { - u64 phys_addr; /* pointer to where the wake vector locates */ -} __packed; - -struct sfi_timer_table_entry { - u64 phys_addr; /* phy base addr for the timer */ - u32 freq_hz; /* in HZ */ - u32 irq; -} __packed; - -struct sfi_rtc_table_entry { - u64 phys_addr; /* phy base addr for the RTC */ - u32 irq; -} __packed; - -struct sfi_device_table_entry { - u8 type; /* bus type, I2C, SPI or ...*/ -#define SFI_DEV_TYPE_SPI 0 -#define SFI_DEV_TYPE_I2C 1 -#define SFI_DEV_TYPE_UART 2 -#define SFI_DEV_TYPE_HSI 3 -#define SFI_DEV_TYPE_IPC 4 -#define SFI_DEV_TYPE_SD 5 - - u8 host_num; /* attached to host 0, 1...*/ - u16 addr; - u8 irq; - u32 max_freq; - char name[SFI_NAME_LEN]; -} __packed; - -struct sfi_gpio_table_entry { - char controller_name[SFI_NAME_LEN]; - u16 pin_no; - char pin_name[SFI_NAME_LEN]; -} __packed; - -typedef int (*sfi_table_handler) (struct sfi_table_header *table); - -#ifdef CONFIG_SFI -extern void __init sfi_init(void); -extern int __init sfi_platform_init(void); -extern void __init sfi_init_late(void); -extern int sfi_table_parse(char *signature, char *oem_id, char *oem_table_id, - sfi_table_handler handler); - -extern int sfi_disabled; -static inline void disable_sfi(void) -{ - sfi_disabled = 1; -} - -#else /* !CONFIG_SFI */ - -static inline void sfi_init(void) -{ -} - -static inline void sfi_init_late(void) -{ -} - -#define sfi_disabled 0 - -static inline int sfi_table_parse(char *signature, char *oem_id, - char *oem_table_id, - sfi_table_handler handler) -{ - return -1; -} - -#endif /* !CONFIG_SFI */ - -#endif /*_LINUX_SFI_H*/ diff --git a/include/linux/sfi_acpi.h b/include/linux/sfi_acpi.h deleted file mode 100644 index a6e555cbe05c..000000000000 --- a/include/linux/sfi_acpi.h +++ /dev/null @@ -1,93 +0,0 @@ -/* sfi.h Simple Firmware Interface */ - -/* - - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - The full GNU General Public License is included in this distribution - in the file called LICENSE.GPL. - - BSD LICENSE - - Copyright(c) 2009 Intel Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef _LINUX_SFI_ACPI_H -#define _LINUX_SFI_ACPI_H - -#include -#include - -#ifdef CONFIG_SFI -extern int sfi_acpi_table_parse(char *signature, char *oem_id, - char *oem_table_id, - int (*handler)(struct acpi_table_header *)); - -static inline int __init acpi_sfi_table_parse(char *signature, - int (*handler)(struct acpi_table_header *)) -{ - if (!acpi_table_parse(signature, handler)) - return 0; - - return sfi_acpi_table_parse(signature, NULL, NULL, handler); -} -#else /* !CONFIG_SFI */ -static inline int sfi_acpi_table_parse(char *signature, char *oem_id, - char *oem_table_id, - int (*handler)(struct acpi_table_header *)) -{ - return -1; -} - -static inline int __init acpi_sfi_table_parse(char *signature, - int (*handler)(struct acpi_table_header *)) -{ - return acpi_table_parse(signature, handler); -} -#endif /* !CONFIG_SFI */ - -#endif /*_LINUX_SFI_ACPI_H*/ diff --git a/init/main.c b/init/main.c index a626e78dbf06..e9933cbf60d4 100644 --- a/init/main.c +++ b/init/main.c @@ -74,7 +74,6 @@ #include #include #include -#include #include #include #include @@ -1054,7 +1053,6 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) acpi_subsystem_init(); arch_post_acpi_subsys_init(); - sfi_init_late(); kcsan_init(); /* Do the rest non-__init'ed, we're now alive */ From 3cc00862a5ddf49e884eb7bb2d198ecb1a9c78f1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:03 +0200 Subject: [PATCH 399/633] x86/PCI: Get rid of custom x86 model comparison Switch the platform code to use x86_id_table and accompanying API instead of custom comparison against x86 CPU model. This is one of the last users of custom API for that and following changes will remove it for the good. Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- arch/x86/pci/intel_mid_pci.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c index 95e2e6bd8d8c..938a8b7bfe7f 100644 --- a/arch/x86/pci/intel_mid_pci.c +++ b/arch/x86/pci/intel_mid_pci.c @@ -28,10 +28,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -212,10 +214,17 @@ static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, where, size, value); } +static const struct x86_cpu_id intel_mid_cpu_ids[] = { + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, NULL), + {} +}; + static int intel_mid_pci_irq_enable(struct pci_dev *dev) { + const struct x86_cpu_id *id; struct irq_alloc_info info; bool polarity_low; + u16 model = 0; int ret; u8 gsi; @@ -228,8 +237,12 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) return ret; } - switch (intel_mid_identify_cpu()) { - case INTEL_MID_CPU_CHIP_TANGIER: + id = x86_match_cpu(intel_mid_cpu_ids); + if (id) + model = id->model; + + switch (model) { + case INTEL_FAM6_ATOM_SILVERMONT_MID: polarity_low = false; /* Special treatment for IRQ0 */ From 24c92537ccacb6aded2ba7a02144ddb2e9cf0d62 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:04 +0200 Subject: [PATCH 400/633] x86/PCI: Describe @reg for type1_access_ok() Describe missed parameter in documentation of type1_access_ok(). Otherwise "make W=1 arch/x86/pci/" produces the following warning: CHECK arch/x86/pci/intel_mid_pci.c CC arch/x86/pci/intel_mid_pci.o arch/x86/pci/intel_mid_pci.c:152: warning: Function parameter or member 'reg' not described in 'type1_access_ok' Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- arch/x86/pci/intel_mid_pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c index 938a8b7bfe7f..8edd62206604 100644 --- a/arch/x86/pci/intel_mid_pci.c +++ b/arch/x86/pci/intel_mid_pci.c @@ -142,6 +142,7 @@ static int pci_device_update_fixed(struct pci_bus *bus, unsigned int devfn, * type1_access_ok - check whether to use type 1 * @bus: bus number * @devfn: device & function in question + * @reg: configuration register offset * * If the bus is on a Lincroft chip and it exists, or is not on a Lincroft at * all, the we can go ahead with any reads & writes. If it's on a Lincroft, From 6517da7aac9df9d5dda4e1e6989db429a8f32de7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:05 +0200 Subject: [PATCH 401/633] x86/platform/intel-mid: Get rid of intel_scu_ipc_legacy.h The header is used by a single user. Move header content to that user. Signed-off-by: Andy Shevchenko Reviewed-by: Mika Westerberg Acked-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/intel_scu_ipc.h | 2 -- arch/x86/include/asm/intel_scu_ipc_legacy.h | 18 ------------------ arch/x86/platform/intel-mid/intel-mid.c | 7 +++++-- 3 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 arch/x86/include/asm/intel_scu_ipc_legacy.h diff --git a/arch/x86/include/asm/intel_scu_ipc.h b/arch/x86/include/asm/intel_scu_ipc.h index 11d457af68c5..8537f597d20a 100644 --- a/arch/x86/include/asm/intel_scu_ipc.h +++ b/arch/x86/include/asm/intel_scu_ipc.h @@ -65,6 +65,4 @@ static inline int intel_scu_ipc_dev_command(struct intel_scu_ipc_dev *scu, int c inlen, out, outlen); } -#include - #endif diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h deleted file mode 100644 index fa529e5ec142..000000000000 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ -#define _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ - -#include - -#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ -#define IPCMSG_COLD_RESET 0xF1 - -/* Don't call these in new code - they will be removed eventually */ - -/* Issue commands to the SCU with or without data */ -static inline int intel_scu_ipc_simple_command(int cmd, int sub) -{ - return intel_scu_ipc_dev_simple_command(NULL, cmd, sub); -} - -#endif diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 2c044e260b4b..846b2ded39d9 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -29,6 +29,9 @@ #include #include +#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ +#define IPCMSG_COLD_RESET 0xF1 + enum intel_mid_cpu_type __intel_mid_cpu_chip; EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); @@ -38,12 +41,12 @@ static void intel_mid_power_off(void) intel_mid_pwr_power_off(); /* Only for Tangier, the rest will ignore this command */ - intel_scu_ipc_simple_command(IPCMSG_COLD_OFF, 1); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_OFF, 1); }; static void intel_mid_reboot(void) { - intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_RESET, 0); } static void __init intel_mid_time_init(void) From 043698c580f441446a1716ea506ecec90c18093a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:06 +0200 Subject: [PATCH 402/633] x86/platform/intel-mid: Drop unused __intel_mid_cpu_chip and Co. Since there is no more user of this global variable and associated custom API, we may safely drop this legacy reinvented a wheel from the kernel sources. Signed-off-by: Andy Shevchenko Acked-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/intel-mid.h | 23 ----------------------- arch/x86/platform/intel-mid/intel-mid.c | 17 ----------------- 2 files changed, 40 deletions(-) diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index 6306fe3e5c4a..c2c7da4c60cf 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -21,36 +21,13 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); -/* - * Medfield is the follow-up of Moorestown, it combines two chip solution into - * one. Other than that it also added always-on and constant tsc and lapic - * timers. Medfield is the platform name, and the chip name is called Penwell - * we treat Medfield/Penwell as a variant of Moorestown. Penwell can be - * identified via MSRs. - */ -enum intel_mid_cpu_type { - /* 1 was Moorestown */ - INTEL_MID_CPU_CHIP_PENWELL = 2, - INTEL_MID_CPU_CHIP_CLOVERVIEW, - INTEL_MID_CPU_CHIP_TANGIER, -}; - -extern enum intel_mid_cpu_type __intel_mid_cpu_chip; - #ifdef CONFIG_X86_INTEL_MID -static inline enum intel_mid_cpu_type intel_mid_identify_cpu(void) -{ - return __intel_mid_cpu_chip; -} - extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); #else /* !CONFIG_X86_INTEL_MID */ -#define intel_mid_identify_cpu() 0 - static inline void intel_scu_devices_create(void) { } static inline void intel_scu_devices_destroy(void) { } diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 846b2ded39d9..2802b5e4637b 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -32,9 +32,6 @@ #define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ #define IPCMSG_COLD_RESET 0xF1 -enum intel_mid_cpu_type __intel_mid_cpu_chip; -EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); - static void intel_mid_power_off(void) { /* Shut down South Complex via PWRMU */ @@ -58,29 +55,15 @@ static void __init intel_mid_time_init(void) static void intel_mid_arch_setup(void) { - if (boot_cpu_data.x86 != 6) { - pr_err("Unknown Intel MID CPU (%d:%d), default to Penwell\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; - goto out; - } - switch (boot_cpu_data.x86_model) { - case 0x35: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_CLOVERVIEW; - break; case 0x3C: case 0x4A: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_TANGIER; x86_platform.legacy.rtc = 1; break; - case 0x27: default: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; break; } -out: /* * Intel MID platforms are using explicitly defined regulators. * From 6b80df1787b35287edc099ef61238ab350711f6f Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:07 +0200 Subject: [PATCH 403/633] x86/platform/intel-mid: Remove unused header inclusion in intel-mid.h After the commit f1be6cdaf57c ("x86/platform/intel-mid: Make intel_scu_device_register() static") the platform_device.h is not being used anymore by intel-mid.h. Remove it. Signed-off-by: Andy Shevchenko Acked-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/intel-mid.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index c2c7da4c60cf..c043e91f45ad 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -8,7 +8,6 @@ #define _ASM_X86_INTEL_MID_H #include -#include extern int intel_mid_pci_init(void); extern int intel_mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state); From c9c26882776a7adddb0173778957e690ac47b195 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 11 Feb 2021 15:40:08 +0200 Subject: [PATCH 404/633] x86/platform/intel-mid: Update Copyright year and drop file names Update Copyright year and drop file names from files themselves. Signed-off-by: Andy Shevchenko Acked-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- arch/x86/include/asm/intel-mid.h | 4 ++-- arch/x86/platform/intel-mid/intel-mid.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index c043e91f45ad..c201083b34f6 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * intel-mid.h: Intel MID specific setup code + * Intel MID specific setup code * - * (C) Copyright 2009 Intel Corporation + * (C) Copyright 2009, 2021 Intel Corporation */ #ifndef _ASM_X86_INTEL_MID_H #define _ASM_X86_INTEL_MID_H diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 2802b5e4637b..f4592dc7a1c1 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * intel-mid.c: Intel MID platform setup code + * Intel MID platform setup code * - * (C) Copyright 2008, 2012 Intel Corporation + * (C) Copyright 2008, 2012, 2021 Intel Corporation * Author: Jacob Pan (jacob.jun.pan@intel.com) * Author: Sathyanarayanan Kuppuswamy */ From 3aac798a917be3b8f2f647b834bb06bf2f8df4f1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 16 Feb 2021 10:23:14 +0100 Subject: [PATCH 405/633] um: Enforce the usage of asm-generic/softirq_stack.h The recent rework of the X86 irq stack switching mechanism broke UM as UM pulls in the X86 specific variant of softirq_stack.h. Enforce the usage of the asm-generic variant. Fixes: 72f40a2823d6 ("x86/softirq/64: Inline do_softirq_own_stack()") Reported-by: Guenter Roeck Signed-off-by: Thomas Gleixner Cc: Richard Weinberger --- arch/um/include/asm/Kbuild | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 1c63b260ecc4..18f86450af72 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -21,6 +21,7 @@ generic-y += param.h generic-y += pci.h generic-y += percpu.h generic-y += preempt.h +generic-y += softirq_stack.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h From f8ee579d53aca887d93f5f411462f25c085a5106 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 2 Feb 2021 17:07:46 +0000 Subject: [PATCH 406/633] PCI: pci-bridge-emul: Fix array overruns, improve safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We allow up to PCI_EXP_SLTSTA2 registers to be accessed, but the pcie_cap_regs_behavior[] array only covers up to PCI_EXP_RTSTA. Expand this array to avoid walking off the end of it. Do the same for pci_regs_behavior for consistency[], and add a BUILD_BUG_ON() to also check the bridge->conf structure size. Fixes: 23a5fba4d941 ("PCI: Introduce PCI bridge emulated config space common logic") Link: https://lore.kernel.org/r/E1l6z9W-0006Re-MQ@rmk-PC.armlinux.org.uk Signed-off-by: Russell King Signed-off-by: Bjorn Helgaas Reviewed-by: Pali Rohár --- drivers/pci/pci-bridge-emul.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index 139869d50eb2..fdaf86a888b7 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -21,8 +21,9 @@ #include "pci-bridge-emul.h" #define PCI_BRIDGE_CONF_END PCI_STD_HEADER_SIZEOF +#define PCI_CAP_PCIE_SIZEOF (PCI_EXP_SLTSTA2 + 2) #define PCI_CAP_PCIE_START PCI_BRIDGE_CONF_END -#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_EXP_SLTSTA2 + 2) +#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_CAP_PCIE_SIZEOF) /** * struct pci_bridge_reg_behavior - register bits behaviors @@ -46,7 +47,8 @@ struct pci_bridge_reg_behavior { u32 w1c; }; -static const struct pci_bridge_reg_behavior pci_regs_behavior[] = { +static const +struct pci_bridge_reg_behavior pci_regs_behavior[PCI_STD_HEADER_SIZEOF / 4] = { [PCI_VENDOR_ID / 4] = { .ro = ~0 }, [PCI_COMMAND / 4] = { .rw = (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | @@ -164,7 +166,8 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = { }, }; -static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = { +static const +struct pci_bridge_reg_behavior pcie_cap_regs_behavior[PCI_CAP_PCIE_SIZEOF / 4] = { [PCI_CAP_LIST_ID / 4] = { /* * Capability ID, Next Capability Pointer and @@ -260,6 +263,8 @@ static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = { int pci_bridge_emul_init(struct pci_bridge_emul *bridge, unsigned int flags) { + BUILD_BUG_ON(sizeof(bridge->conf) != PCI_BRIDGE_CONF_END); + bridge->conf.class_revision |= cpu_to_le32(PCI_CLASS_BRIDGE_PCI << 16); bridge->conf.header_type = PCI_HEADER_TYPE_BRIDGE; bridge->conf.cache_line_size = 0x10; From f6bda644fa3a7070621c3bf12cd657f69a42f170 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 2 Feb 2021 11:03:32 +0100 Subject: [PATCH 407/633] PCI: Fix pci_register_io_range() memory leak Kmemleak reports: unreferenced object 0xc328de40 (size 64): comm "kworker/1:1", pid 21, jiffies 4294938212 (age 1484.670s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 e0 d8 fc eb 00 00 00 00 ................ 00 00 10 fe 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [] pci_register_io_range+0x3c/0x80 [<2c7f139e>] of_pci_range_to_resource+0x48/0xc0 [] devm_of_pci_get_host_bridge_resources.constprop.0+0x2ac/0x3ac [] devm_of_pci_bridge_init+0x60/0x1b8 [] devm_pci_alloc_host_bridge+0x54/0x64 [] rcar_pcie_probe+0x2c/0x644 In case a PCI host driver's probe is deferred, the same I/O range may be allocated again, and be ignored, causing a memory leak. Fix this by (a) letting logic_pio_register_range() return -EEXIST if the passed range already exists, so pci_register_io_range() will free it, and by (b) making pci_register_io_range() not consider -EEXIST an error condition. Link: https://lore.kernel.org/r/20210202100332.829047-1-geert+renesas@glider.be Signed-off-by: Geert Uytterhoeven Signed-off-by: Bjorn Helgaas --- drivers/pci/pci.c | 4 ++++ lib/logic_pio.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index b9fecc25d213..50b55a1e3d76 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4029,6 +4029,10 @@ int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, ret = logic_pio_register_range(range); if (ret) kfree(range); + + /* Ignore duplicates due to deferred probing */ + if (ret == -EEXIST) + ret = 0; #endif return ret; diff --git a/lib/logic_pio.c b/lib/logic_pio.c index f32fe481b492..07b4b9a1f54b 100644 --- a/lib/logic_pio.c +++ b/lib/logic_pio.c @@ -28,6 +28,8 @@ static DEFINE_MUTEX(io_range_mutex); * @new_range: pointer to the IO range to be registered. * * Returns 0 on success, the error code in case of failure. + * If the range already exists, -EEXIST will be returned, which should be + * considered a success. * * Register a new IO range node in the IO range list. */ @@ -51,6 +53,7 @@ int logic_pio_register_range(struct logic_pio_hwaddr *new_range) list_for_each_entry(range, &io_range_list, list) { if (range->fwnode == new_range->fwnode) { /* range already there */ + ret = -EEXIST; goto end_register; } if (range->flags == LOGIC_PIO_CPU_MMIO && From 1002573ee33efef0988a9a546c075a9fa37d2498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Wilczy=C5=84ski?= Date: Tue, 16 Feb 2021 20:59:35 +0000 Subject: [PATCH 408/633] PCI: cadence: Fix DMA range mapping early return error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Function cdns_pcie_host_map_dma_ranges() iterates over a PCIe host bridge DMA ranges using the resource_list_for_each_entry() iterator, returning an error if cdns_pcie_host_bar_config() fails. 49e427e6bdd1 ("Merge branch 'pci/host-probe-refactor'") botched a merge so it *always* returned after the first DMA range, even if no error occurred. Fix the error checking so we return early only when an error occurs. [bhelgaas: commit log] Fixes: 49e427e6bdd1 ("Merge branch 'pci/host-probe-refactor'") Link: https://lore.kernel.org/r/20210216205935.3112661-1-kw@linux.com Signed-off-by: Krzysztof Wilczyński Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/cadence/pcie-cadence-host.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index 811c1cb2e8de..1cb7cfc75d6e 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -321,9 +321,10 @@ static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc) resource_list_for_each_entry(entry, &bridge->dma_ranges) { err = cdns_pcie_host_bar_config(rc, entry); - if (err) + if (err) { dev_err(dev, "Fail to configure IB using dma-ranges\n"); - return err; + return err; + } } return 0; From 35ac5991cdec9d920a683e74b64fda8512bdd3e9 Mon Sep 17 00:00:00 2001 From: Tian Tao Date: Thu, 18 Feb 2021 10:17:29 +0800 Subject: [PATCH 409/633] vfio/iommu_type1: Fix duplicate included kthread.h linux/kthread.h is included more than once, remove the one that isn't necessary. Fixes: 898b9eaeb3fe ("vfio/type1: block on invalid vaddr") Signed-off-by: Tian Tao Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index ec9fd95a138b..b3df383d7028 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include From fc235fcb0f7c1865ccb2d1f50267eef299a4f3fb Mon Sep 17 00:00:00 2001 From: Chen Lin Date: Tue, 16 Feb 2021 10:38:40 +0800 Subject: [PATCH 410/633] PCI: acpiphp: Remove unused acpiphp_callback typedef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the 'acpiphp_callback' typedef as it is not used. Link: https://lore.kernel.org/r/1613443120-4279-1-git-send-email-chen45464546@163.com Signed-off-by: Chen Lin Signed-off-by: Bjorn Helgaas Reviewed-by: Krzysztof Wilczyński Reviewed-by: Rafael J. Wysocki --- drivers/pci/hotplug/acpiphp.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/pci/hotplug/acpiphp.h b/drivers/pci/hotplug/acpiphp.h index a2094c07af6a..a74b274a8c45 100644 --- a/drivers/pci/hotplug/acpiphp.h +++ b/drivers/pci/hotplug/acpiphp.h @@ -176,9 +176,6 @@ int acpiphp_unregister_attention(struct acpiphp_attention_info *info); int acpiphp_register_hotplug_slot(struct acpiphp_slot *slot, unsigned int sun); void acpiphp_unregister_hotplug_slot(struct acpiphp_slot *slot); -/* acpiphp_glue.c */ -typedef int (*acpiphp_callback)(struct acpiphp_slot *slot, void *data); - int acpiphp_enable_slot(struct acpiphp_slot *slot); int acpiphp_disable_slot(struct acpiphp_slot *slot); u8 acpiphp_get_power_status(struct acpiphp_slot *slot); From b9abef43a08ef7faa33477cccb0c08c64eb2b8bf Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 18 Feb 2021 10:44:35 +0000 Subject: [PATCH 411/633] vfio/pci: remove CONFIG_VFIO_PCI_ZDEV from Kconfig In case we're running on s390 system always expose the capabilities for configuration of zPCI devices. In case we're running on different platform, continue as usual. Signed-off-by: Max Gurtovoy Reviewed-by: Matthew Rosato Reviewed-by: Cornelia Huck Signed-off-by: Alex Williamson --- drivers/vfio/pci/Kconfig | 12 ------------ drivers/vfio/pci/Makefile | 2 +- drivers/vfio/pci/vfio_pci.c | 12 +++++------- drivers/vfio/pci/vfio_pci_private.h | 2 +- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 40a223381ab6..ac3c1dd3edef 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -45,15 +45,3 @@ config VFIO_PCI_NVLINK2 depends on VFIO_PCI && PPC_POWERNV help VFIO PCI support for P9 Witherspoon machine with NVIDIA V100 GPUs - -config VFIO_PCI_ZDEV - bool "VFIO PCI ZPCI device CLP support" - depends on VFIO_PCI && S390 - default y - help - Enabling this option exposes VFIO capabilities containing hardware - configuration for zPCI devices. This enables userspace (e.g. QEMU) - to supply proper configuration values instead of hard-coded defaults - for zPCI devices passed through via VFIO on s390. - - Say Y here. diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 781e0809d6ee..eff97a7cd9f1 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -3,6 +3,6 @@ vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o vfio-pci-$(CONFIG_VFIO_PCI_NVLINK2) += vfio_pci_nvlink2.o -vfio-pci-$(CONFIG_VFIO_PCI_ZDEV) += vfio_pci_zdev.o +vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o obj-$(CONFIG_VFIO_PCI) += vfio-pci.o diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 706de3ef94bb..65e7e6b44578 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -807,6 +807,7 @@ static long vfio_pci_ioctl(void *device_data, struct vfio_device_info info; struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; unsigned long capsz; + int ret; minsz = offsetofend(struct vfio_device_info, num_irqs); @@ -832,13 +833,10 @@ static long vfio_pci_ioctl(void *device_data, info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; - if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV)) { - int ret = vfio_pci_info_zdev_add_caps(vdev, &caps); - - if (ret && ret != -ENODEV) { - pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); - return ret; - } + ret = vfio_pci_info_zdev_add_caps(vdev, &caps); + if (ret && ret != -ENODEV) { + pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); + return ret; } if (caps.size) { diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h index 5c90e560c5c7..9cd1882a05af 100644 --- a/drivers/vfio/pci/vfio_pci_private.h +++ b/drivers/vfio/pci/vfio_pci_private.h @@ -214,7 +214,7 @@ static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev) } #endif -#ifdef CONFIG_VFIO_PCI_ZDEV +#ifdef CONFIG_S390 extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev, struct vfio_info_cap *caps); #else From 07956b6269d3ed05d854233d5bb776dca91751dd Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Tue, 16 Feb 2021 15:49:34 -0700 Subject: [PATCH 412/633] vfio/type1: Use follow_pte() follow_pfn() doesn't make sure that we're using the correct page protections, get the pte with follow_pte() so that we can test protections and get the pfn from the pte. Fixes: 5cbf3264bc71 ("vfio/type1: Fix VA->PA translation for PFNMAP VMAs in vaddr_get_pfn()") Reviewed-by: Jason Gunthorpe Reviewed-by: Cornelia Huck Reviewed-by: Peter Xu Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index b3df383d7028..ed03f3fcb07e 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -462,9 +463,11 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, unsigned long vaddr, unsigned long *pfn, bool write_fault) { + pte_t *ptep; + spinlock_t *ptl; int ret; - ret = follow_pfn(vma, vaddr, pfn); + ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); if (ret) { bool unlocked = false; @@ -478,9 +481,17 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, if (ret) return ret; - ret = follow_pfn(vma, vaddr, pfn); + ret = follow_pte(vma->vm_mm, vaddr, &ptep, &ptl); + if (ret) + return ret; } + if (write_fault && !pte_write(*ptep)) + ret = -EFAULT; + else + *pfn = pte_pfn(*ptep); + + pte_unmap_unlock(ptep, ptl); return ret; } From 724c8a23d589d8a002d2e39633c2f9a5a429616f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 18 Feb 2021 17:14:10 +0100 Subject: [PATCH 413/633] objtool: Fix stack-swizzle for FRAME_POINTER=y When objtool encounters the stack-swizzle: mov %rsp, (%[tos]) mov %[tos], %rsp ... pop %rsp Inside a FRAME_POINTER=y build, things go a little screwy because clearly we're not adjusting the cfa->base. This then results in the pop %rsp not being detected as a restore of cfa->base so it will turn into a regular POP and offset the stack, resulting in: kernel/softirq.o: warning: objtool: do_softirq()+0xdb: return with modified stack frame Therefore, have "mov %[tos], %rsp" act like a PUSH (it sorta is anyway) to balance the things out. We're not too concerned with the actual stack_size for frame-pointer builds, since we don't generate ORC data for them anyway. Fixes: aafeb14e9da2 ("objtool: Support stack-swizzle") Reported-by: kernel test robot Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner Acked-by: Josh Poimboeuf Link: https://lkml.kernel.org/r/YC6UC+rc9KKmQrkd@hirez.programming.kicks-ass.net --- tools/objtool/check.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 8e74210f4235..2087974c70d3 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1983,6 +1983,20 @@ static int update_cfi_state(struct instruction *insn, struct cfi_state *cfi, } } + else if (op->dest.reg == CFI_SP && + cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && + cfi->vals[op->src.reg].offset == cfa->offset) { + + /* + * The same stack swizzle case 2) as above. But + * because we can't change cfa->base, case 3) + * will become a regular POP. Pretend we're a + * PUSH so things don't go unbalanced. + */ + cfi->stack_size += 8; + } + + break; case OP_SRC_ADD: From be16c1fd99f41abebc0bf965d5d29cd18c9d271e Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Fri, 19 Feb 2021 11:13:03 -0500 Subject: [PATCH 414/633] vfio/type1: Change success value of vaddr_get_pfn() vaddr_get_pfn() simply returns 0 on success. Have it report the number of pfns successfully gotten instead, whether from page pinning or follow_fault_pfn(), which will be used later when batching pinning. Change the last check in vfio_pin_pages_remote() for consistency with the other two. Signed-off-by: Daniel Jordan Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index ed03f3fcb07e..82ed8bf34c11 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -495,6 +495,10 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, return ret; } +/* + * Returns the positive number of pfns successfully obtained or a negative + * error code. + */ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, int prot, unsigned long *pfn) { @@ -511,7 +515,6 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, page, NULL, NULL); if (ret == 1) { *pfn = page_to_pfn(page[0]); - ret = 0; goto done; } @@ -525,8 +528,12 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, if (ret == -EAGAIN) goto retry; - if (!ret && !is_invalid_reserved_pfn(*pfn)) - ret = -EFAULT; + if (!ret) { + if (is_invalid_reserved_pfn(*pfn)) + ret = 1; + else + ret = -EFAULT; + } } done: mmap_read_unlock(mm); @@ -607,7 +614,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, return -ENODEV; ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); - if (ret) + if (ret < 0) return ret; pinned++; @@ -634,7 +641,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); - if (ret) + if (ret < 0) break; if (pfn != *pfn_base + pinned || @@ -660,7 +667,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, ret = vfio_lock_acct(dma, lock_acct, false); unpin_out: - if (ret) { + if (ret < 0) { if (!rsvd) { for (pfn = *pfn_base ; pinned ; pfn++, pinned--) put_pfn(pfn, dma->prot); @@ -704,7 +711,7 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, return -ENODEV; ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); - if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { + if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { ret = vfio_lock_acct(dma, 1, true); if (ret) { put_pfn(*pfn_base, dma->prot); From 4b6c33b3229678e38a6b0bbd4367d4b91366b523 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Fri, 19 Feb 2021 11:13:04 -0500 Subject: [PATCH 415/633] vfio/type1: Prepare for batched pinning with struct vfio_batch Get ready to pin more pages at once with struct vfio_batch, which represents a batch of pinned pages. The struct has a fallback page pointer to avoid two unlikely scenarios: pointlessly allocating a page if disable_hugepages is enabled or failing the whole pinning operation if the kernel can't allocate memory. vaddr_get_pfn() becomes vaddr_get_pfns() to prepare for handling multiple pages, though for now only one page is stored in the pages array. Signed-off-by: Daniel Jordan Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 71 +++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 82ed8bf34c11..1c86375aca2d 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -103,6 +103,12 @@ struct vfio_dma { unsigned long *bitmap; }; +struct vfio_batch { + struct page **pages; /* for pin_user_pages_remote */ + struct page *fallback_page; /* if pages alloc fails */ + int capacity; /* length of pages array */ +}; + struct vfio_group { struct iommu_group *iommu_group; struct list_head next; @@ -459,6 +465,31 @@ static int put_pfn(unsigned long pfn, int prot) return 0; } +#define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *)) + +static void vfio_batch_init(struct vfio_batch *batch) +{ + if (unlikely(disable_hugepages)) + goto fallback; + + batch->pages = (struct page **) __get_free_page(GFP_KERNEL); + if (!batch->pages) + goto fallback; + + batch->capacity = VFIO_BATCH_MAX_CAPACITY; + return; + +fallback: + batch->pages = &batch->fallback_page; + batch->capacity = 1; +} + +static void vfio_batch_fini(struct vfio_batch *batch) +{ + if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) + free_page((unsigned long)batch->pages); +} + static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, unsigned long vaddr, unsigned long *pfn, bool write_fault) @@ -499,10 +530,10 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, * Returns the positive number of pfns successfully obtained or a negative * error code. */ -static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, - int prot, unsigned long *pfn) +static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, + long npages, int prot, unsigned long *pfn, + struct page **pages) { - struct page *page[1]; struct vm_area_struct *vma; unsigned int flags = 0; int ret; @@ -511,10 +542,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr, flags |= FOLL_WRITE; mmap_read_lock(mm); - ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM, - page, NULL, NULL); - if (ret == 1) { - *pfn = page_to_pfn(page[0]); + ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM, + pages, NULL, NULL); + if (ret > 0) { + *pfn = page_to_pfn(pages[0]); goto done; } @@ -602,7 +633,7 @@ static int vfio_wait_all_valid(struct vfio_iommu *iommu) */ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, long npage, unsigned long *pfn_base, - unsigned long limit) + unsigned long limit, struct vfio_batch *batch) { unsigned long pfn = 0; long ret, pinned = 0, lock_acct = 0; @@ -613,7 +644,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, if (!current->mm) return -ENODEV; - ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, pfn_base); + ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, pfn_base, + batch->pages); if (ret < 0) return ret; @@ -640,7 +672,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, /* Lock all the consecutive pages from pfn_base */ for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { - ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn); + ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, &pfn, + batch->pages); if (ret < 0) break; @@ -703,6 +736,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, unsigned long *pfn_base, bool do_accounting) { + struct page *pages[1]; struct mm_struct *mm; int ret; @@ -710,7 +744,7 @@ static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr, if (!mm) return -ENODEV; - ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base); + ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, pages); if (ret == 1 && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) { ret = vfio_lock_acct(dma, 1, true); if (ret) { @@ -1404,15 +1438,19 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, { dma_addr_t iova = dma->iova; unsigned long vaddr = dma->vaddr; + struct vfio_batch batch; size_t size = map_size; long npage; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + vfio_batch_init(&batch); + while (size) { /* Pin a contiguous chunk of memory */ npage = vfio_pin_pages_remote(dma, vaddr + dma->size, - size >> PAGE_SHIFT, &pfn, limit); + size >> PAGE_SHIFT, &pfn, limit, + &batch); if (npage <= 0) { WARN_ON(!npage); ret = (int)npage; @@ -1432,6 +1470,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, dma->size += npage << PAGE_SHIFT; } + vfio_batch_fini(&batch); dma->iommu_mapped = true; if (ret) @@ -1608,6 +1647,7 @@ static int vfio_bus_type(struct device *dev, void *data) static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_domain *domain) { + struct vfio_batch batch; struct vfio_domain *d = NULL; struct rb_node *n; unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; @@ -1622,6 +1662,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, d = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + vfio_batch_init(&batch); + n = rb_first(&iommu->dma_list); for (; n; n = rb_next(n)) { @@ -1669,7 +1711,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, npage = vfio_pin_pages_remote(dma, vaddr, n >> PAGE_SHIFT, - &pfn, limit); + &pfn, limit, + &batch); if (npage <= 0) { WARN_ON(!npage); ret = (int)npage; @@ -1702,6 +1745,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, dma->iommu_mapped = true; } + vfio_batch_fini(&batch); return 0; unwind: @@ -1742,6 +1786,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, } } + vfio_batch_fini(&batch); return ret; } From 4d83de6da265cd84e74c19d876055fa5f261cde4 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Fri, 19 Feb 2021 11:13:05 -0500 Subject: [PATCH 416/633] vfio/type1: Batch page pinning Pinning one 4K page at a time is inefficient, so do it in batches of 512 instead. This is just an optimization with no functional change intended, and in particular the driver still calls iommu_map() with the largest physically contiguous range possible. Add two fields in vfio_batch to remember where to start between calls to vfio_pin_pages_remote(), and use vfio_batch_unpin() to handle remaining pages in the batch in case of error. qemu pins pages for guests around 8% faster on my test system, a two-node Broadwell server with 128G memory per node. The qemu process was bound to one node with its allocations constrained there as well. base test guest ---------------- ---------------- mem (GB) speedup avg sec (std) avg sec (std) 1 7.4% 0.61 (0.00) 0.56 (0.00) 2 8.3% 0.93 (0.00) 0.85 (0.00) 4 8.4% 1.46 (0.00) 1.34 (0.00) 8 8.6% 2.54 (0.01) 2.32 (0.00) 16 8.3% 4.66 (0.00) 4.27 (0.01) 32 8.3% 8.94 (0.01) 8.20 (0.01) 64 8.2% 17.47 (0.01) 16.04 (0.03) 120 8.5% 32.45 (0.13) 29.69 (0.01) perf diff confirms less time spent in pup. Here are the top ten functions: Baseline Delta Abs Symbol 78.63% +6.64% clear_page_erms 1.50% -1.50% __gup_longterm_locked 1.27% -0.78% __get_user_pages +0.76% kvm_zap_rmapp.constprop.0 0.54% -0.53% vmacache_find 0.55% -0.51% get_pfnblock_flags_mask 0.48% -0.48% __get_user_pages_remote +0.39% slot_rmap_walk_next +0.32% vfio_pin_map_dma +0.26% kvm_handle_hva_range ... Suggested-by: Matthew Wilcox (Oracle) Signed-off-by: Daniel Jordan Signed-off-by: Alex Williamson --- drivers/vfio/vfio_iommu_type1.c | 139 +++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 48 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 1c86375aca2d..4bb162c1d649 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -107,6 +107,8 @@ struct vfio_batch { struct page **pages; /* for pin_user_pages_remote */ struct page *fallback_page; /* if pages alloc fails */ int capacity; /* length of pages array */ + int size; /* of batch currently */ + int offset; /* of next entry in pages */ }; struct vfio_group { @@ -469,6 +471,9 @@ static int put_pfn(unsigned long pfn, int prot) static void vfio_batch_init(struct vfio_batch *batch) { + batch->size = 0; + batch->offset = 0; + if (unlikely(disable_hugepages)) goto fallback; @@ -484,6 +489,17 @@ static void vfio_batch_init(struct vfio_batch *batch) batch->capacity = 1; } +static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma) +{ + while (batch->size) { + unsigned long pfn = page_to_pfn(batch->pages[batch->offset]); + + put_pfn(pfn, dma->prot); + batch->offset++; + batch->size--; + } +} + static void vfio_batch_fini(struct vfio_batch *batch) { if (batch->capacity == VFIO_BATCH_MAX_CAPACITY) @@ -635,65 +651,88 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, long npage, unsigned long *pfn_base, unsigned long limit, struct vfio_batch *batch) { - unsigned long pfn = 0; + unsigned long pfn; + struct mm_struct *mm = current->mm; long ret, pinned = 0, lock_acct = 0; bool rsvd; dma_addr_t iova = vaddr - dma->vaddr + dma->iova; /* This code path is only user initiated */ - if (!current->mm) + if (!mm) return -ENODEV; - ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, pfn_base, - batch->pages); - if (ret < 0) - return ret; - - pinned++; - rsvd = is_invalid_reserved_pfn(*pfn_base); - - /* - * Reserved pages aren't counted against the user, externally pinned - * pages are already counted against the user. - */ - if (!rsvd && !vfio_find_vpfn(dma, iova)) { - if (!dma->lock_cap && current->mm->locked_vm + 1 > limit) { - put_pfn(*pfn_base, dma->prot); - pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, - limit << PAGE_SHIFT); - return -ENOMEM; - } - lock_acct++; + if (batch->size) { + /* Leftover pages in batch from an earlier call. */ + *pfn_base = page_to_pfn(batch->pages[batch->offset]); + pfn = *pfn_base; + rsvd = is_invalid_reserved_pfn(*pfn_base); + } else { + *pfn_base = 0; } - if (unlikely(disable_hugepages)) - goto out; + while (npage) { + if (!batch->size) { + /* Empty batch, so refill it. */ + long req_pages = min_t(long, npage, batch->capacity); - /* Lock all the consecutive pages from pfn_base */ - for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage; - pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) { - ret = vaddr_get_pfns(current->mm, vaddr, 1, dma->prot, &pfn, - batch->pages); - if (ret < 0) - break; - - if (pfn != *pfn_base + pinned || - rsvd != is_invalid_reserved_pfn(pfn)) { - put_pfn(pfn, dma->prot); - break; - } - - if (!rsvd && !vfio_find_vpfn(dma, iova)) { - if (!dma->lock_cap && - current->mm->locked_vm + lock_acct + 1 > limit) { - put_pfn(pfn, dma->prot); - pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", - __func__, limit << PAGE_SHIFT); - ret = -ENOMEM; + ret = vaddr_get_pfns(mm, vaddr, req_pages, dma->prot, + &pfn, batch->pages); + if (ret < 0) goto unpin_out; + + batch->size = ret; + batch->offset = 0; + + if (!*pfn_base) { + *pfn_base = pfn; + rsvd = is_invalid_reserved_pfn(*pfn_base); } - lock_acct++; } + + /* + * pfn is preset for the first iteration of this inner loop and + * updated at the end to handle a VM_PFNMAP pfn. In that case, + * batch->pages isn't valid (there's no struct page), so allow + * batch->pages to be touched only when there's more than one + * pfn to check, which guarantees the pfns are from a + * !VM_PFNMAP vma. + */ + while (true) { + if (pfn != *pfn_base + pinned || + rsvd != is_invalid_reserved_pfn(pfn)) + goto out; + + /* + * Reserved pages aren't counted against the user, + * externally pinned pages are already counted against + * the user. + */ + if (!rsvd && !vfio_find_vpfn(dma, iova)) { + if (!dma->lock_cap && + mm->locked_vm + lock_acct + 1 > limit) { + pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", + __func__, limit << PAGE_SHIFT); + ret = -ENOMEM; + goto unpin_out; + } + lock_acct++; + } + + pinned++; + npage--; + vaddr += PAGE_SIZE; + iova += PAGE_SIZE; + batch->offset++; + batch->size--; + + if (!batch->size) + break; + + pfn = page_to_pfn(batch->pages[batch->offset]); + } + + if (unlikely(disable_hugepages)) + break; } out: @@ -701,10 +740,11 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, unpin_out: if (ret < 0) { - if (!rsvd) { + if (pinned && !rsvd) { for (pfn = *pfn_base ; pinned ; pfn++, pinned--) put_pfn(pfn, dma->prot); } + vfio_batch_unpin(batch, dma); return ret; } @@ -1463,6 +1503,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, if (ret) { vfio_unpin_pages_remote(dma, iova + dma->size, pfn, npage, true); + vfio_batch_unpin(&batch, dma); break; } @@ -1726,11 +1767,13 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, ret = iommu_map(domain->domain, iova, phys, size, dma->prot | domain->prot); if (ret) { - if (!dma->iommu_mapped) + if (!dma->iommu_mapped) { vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT, size >> PAGE_SHIFT, true); + vfio_batch_unpin(&batch, dma); + } goto unwind; } From 791c9f143c77f847232b46ee9c1c990f60825c8e Mon Sep 17 00:00:00 2001 From: Daire McNamara Date: Mon, 25 Jan 2021 16:29:31 +0000 Subject: [PATCH 417/633] PCI: Call platform_set_drvdata earlier in devm_pci_alloc_host_bridge Many drivers can now use pci_host_common_probe() directly. Their hardware window setup can be moved from their 'custom' probe functions to individual driver init functions. Link: https://lore.kernel.org/r/20210125162934.5335-2-daire.mcnamara@microchip.com Signed-off-by: Daire McNamara Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Rob Herring --- drivers/pci/controller/pci-host-common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c index 6ce34a1deecb..6ab694f8d283 100644 --- a/drivers/pci/controller/pci-host-common.c +++ b/drivers/pci/controller/pci-host-common.c @@ -64,6 +64,8 @@ int pci_host_common_probe(struct platform_device *pdev) if (!bridge) return -ENOMEM; + platform_set_drvdata(pdev, bridge); + of_pci_check_probe_only(); /* Parse and map our Configuration Space windows */ @@ -78,8 +80,6 @@ int pci_host_common_probe(struct platform_device *pdev) bridge->sysdata = cfg; bridge->ops = (struct pci_ops *)&ops->pci_ops; - platform_set_drvdata(pdev, bridge); - return pci_host_probe(bridge); } EXPORT_SYMBOL_GPL(pci_host_common_probe); From 6ee6c89aac358cf5f951a4d82868012bec64b30e Mon Sep 17 00:00:00 2001 From: Daire McNamara Date: Mon, 25 Jan 2021 16:29:32 +0000 Subject: [PATCH 418/633] dt-bindings: PCI: microchip: Add Microchip PolarFire host binding Add device tree bindings for the Microchip PolarFire PCIe controller when configured in host (Root Complex) mode. Link: https://lore.kernel.org/r/20210125162934.5335-3-daire.mcnamara@microchip.com Signed-off-by: Daire McNamara Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Rob Herring --- .../bindings/pci/microchip,pcie-host.yaml | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml new file mode 100644 index 000000000000..04251d71f56b --- /dev/null +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/microchip,pcie-host.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip PCIe Root Port Bridge Controller Device Tree Bindings + +maintainers: + - Daire McNamara + +allOf: + - $ref: /schemas/pci/pci-bus.yaml# + +properties: + compatible: + const: microchip,pcie-host-1.0 # PolarFire + + reg: + maxItems: 2 + + reg-names: + items: + - const: cfg + - const: apb + + interrupts: + minItems: 1 + maxItems: 2 + items: + - description: PCIe host controller + - description: builtin MSI controller + + interrupt-names: + minItems: 1 + maxItems: 2 + items: + - const: pcie + - const: msi + + ranges: + maxItems: 1 + + msi-controller: + description: Identifies the node as an MSI controller. + + msi-parent: + description: MSI controller the device is capable of using. + +required: + - reg + - reg-names + - "#interrupt-cells" + - interrupts + - interrupt-map-mask + - interrupt-map + - msi-controller + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + pcie0: pcie@2030000000 { + compatible = "microchip,pcie-host-1.0"; + reg = <0x0 0x70000000 0x0 0x08000000>, + <0x0 0x43000000 0x0 0x00010000>; + reg-names = "cfg", "apb"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + interrupts = <119>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + interrupt-parent = <&plic0>; + msi-parent = <&pcie0>; + msi-controller; + bus-range = <0x00 0x7f>; + ranges = <0x03000000 0x0 0x78000000 0x0 0x78000000 0x0 0x04000000>; + pcie_intc0: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; From 6f15a9c9f94133bee0d861a4bf25e10aaa95219d Mon Sep 17 00:00:00 2001 From: Daire McNamara Date: Mon, 25 Jan 2021 16:29:33 +0000 Subject: [PATCH 419/633] PCI: microchip: Add Microchip PolarFire PCIe controller driver Add support for the Microchip PolarFire PCIe controller when configured in host (Root Complex) mode. [bhelgaas: wrap lines to fit in 80 columns, fix trivial style issues] Link: https://lore.kernel.org/r/20210125162934.5335-4-daire.mcnamara@microchip.com Signed-off-by: Daire McNamara [lorenzo.pieralisi@arm.com: minor comments tweak] Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Rob Herring --- drivers/pci/controller/Kconfig | 10 + drivers/pci/controller/Makefile | 1 + drivers/pci/controller/pcie-microchip-host.c | 1138 ++++++++++++++++++ 3 files changed, 1149 insertions(+) create mode 100644 drivers/pci/controller/pcie-microchip-host.c diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 64e2f5e379aa..bca2f8949510 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -298,6 +298,16 @@ config PCI_LOONGSON Say Y here if you want to enable PCI controller support on Loongson systems. +config PCIE_MICROCHIP_HOST + bool "Microchip AXI PCIe host bridge support" + depends on PCI_MSI && OF + select PCI_MSI_IRQ_DOMAIN + select GENERIC_MSI_IRQ_DOMAIN + select PCI_HOST_COMMON + help + Say Y here if you want kernel to support the Microchip AXI PCIe + Host Bridge driver. + config PCIE_HISI_ERR depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST) bool "HiSilicon HIP PCIe controller error handling driver" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 04c6edc285c5..b85fcd574ff6 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -28,6 +28,7 @@ obj-$(CONFIG_PCIE_ROCKCHIP_EP) += pcie-rockchip-ep.o obj-$(CONFIG_PCIE_ROCKCHIP_HOST) += pcie-rockchip-host.o obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o +obj-$(CONFIG_PCIE_MICROCHIP_HOST) += pcie-microchip-host.o obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o diff --git a/drivers/pci/controller/pcie-microchip-host.c b/drivers/pci/controller/pcie-microchip-host.c new file mode 100644 index 000000000000..04c19ff81aff --- /dev/null +++ b/drivers/pci/controller/pcie-microchip-host.c @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip AXI PCIe Bridge host controller driver + * + * Copyright (c) 2018 - 2020 Microchip Corporation. All rights reserved. + * + * Author: Daire McNamara + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../pci.h" + +/* Number of MSI IRQs */ +#define MC_NUM_MSI_IRQS 32 +#define MC_NUM_MSI_IRQS_CODED 5 + +/* PCIe Bridge Phy and Controller Phy offsets */ +#define MC_PCIE1_BRIDGE_ADDR 0x00008000u +#define MC_PCIE1_CTRL_ADDR 0x0000a000u + +#define MC_PCIE_BRIDGE_ADDR (MC_PCIE1_BRIDGE_ADDR) +#define MC_PCIE_CTRL_ADDR (MC_PCIE1_CTRL_ADDR) + +/* PCIe Controller Phy Regs */ +#define SEC_ERROR_CNT 0x20 +#define DED_ERROR_CNT 0x24 +#define SEC_ERROR_INT 0x28 +#define SEC_ERROR_INT_TX_RAM_SEC_ERR_INT GENMASK(3, 0) +#define SEC_ERROR_INT_RX_RAM_SEC_ERR_INT GENMASK(7, 4) +#define SEC_ERROR_INT_PCIE2AXI_RAM_SEC_ERR_INT GENMASK(11, 8) +#define SEC_ERROR_INT_AXI2PCIE_RAM_SEC_ERR_INT GENMASK(15, 12) +#define NUM_SEC_ERROR_INTS (4) +#define SEC_ERROR_INT_MASK 0x2c +#define DED_ERROR_INT 0x30 +#define DED_ERROR_INT_TX_RAM_DED_ERR_INT GENMASK(3, 0) +#define DED_ERROR_INT_RX_RAM_DED_ERR_INT GENMASK(7, 4) +#define DED_ERROR_INT_PCIE2AXI_RAM_DED_ERR_INT GENMASK(11, 8) +#define DED_ERROR_INT_AXI2PCIE_RAM_DED_ERR_INT GENMASK(15, 12) +#define NUM_DED_ERROR_INTS (4) +#define DED_ERROR_INT_MASK 0x34 +#define ECC_CONTROL 0x38 +#define ECC_CONTROL_TX_RAM_INJ_ERROR_0 BIT(0) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_1 BIT(1) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_2 BIT(2) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_3 BIT(3) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_0 BIT(4) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_1 BIT(5) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_2 BIT(6) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_3 BIT(7) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_0 BIT(8) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_1 BIT(9) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_2 BIT(10) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_3 BIT(11) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_0 BIT(12) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_1 BIT(13) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_2 BIT(14) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_3 BIT(15) +#define ECC_CONTROL_TX_RAM_ECC_BYPASS BIT(24) +#define ECC_CONTROL_RX_RAM_ECC_BYPASS BIT(25) +#define ECC_CONTROL_PCIE2AXI_RAM_ECC_BYPASS BIT(26) +#define ECC_CONTROL_AXI2PCIE_RAM_ECC_BYPASS BIT(27) +#define LTSSM_STATE 0x5c +#define LTSSM_L0_STATE 0x10 +#define PCIE_EVENT_INT 0x14c +#define PCIE_EVENT_INT_L2_EXIT_INT BIT(0) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT BIT(1) +#define PCIE_EVENT_INT_DLUP_EXIT_INT BIT(2) +#define PCIE_EVENT_INT_MASK GENMASK(2, 0) +#define PCIE_EVENT_INT_L2_EXIT_INT_MASK BIT(16) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT_MASK BIT(17) +#define PCIE_EVENT_INT_DLUP_EXIT_INT_MASK BIT(18) +#define PCIE_EVENT_INT_ENB_MASK GENMASK(18, 16) +#define PCIE_EVENT_INT_ENB_SHIFT 16 +#define NUM_PCIE_EVENTS (3) + +/* PCIe Bridge Phy Regs */ +#define PCIE_PCI_IDS_DW1 0x9c + +/* PCIe Config space MSI capability structure */ +#define MC_MSI_CAP_CTRL_OFFSET 0xe0u +#define MC_MSI_MAX_Q_AVAIL (MC_NUM_MSI_IRQS_CODED << 1) +#define MC_MSI_Q_SIZE (MC_NUM_MSI_IRQS_CODED << 4) + +#define IMASK_LOCAL 0x180 +#define DMA_END_ENGINE_0_MASK 0x00000000u +#define DMA_END_ENGINE_0_SHIFT 0 +#define DMA_END_ENGINE_1_MASK 0x00000000u +#define DMA_END_ENGINE_1_SHIFT 1 +#define DMA_ERROR_ENGINE_0_MASK 0x00000100u +#define DMA_ERROR_ENGINE_0_SHIFT 8 +#define DMA_ERROR_ENGINE_1_MASK 0x00000200u +#define DMA_ERROR_ENGINE_1_SHIFT 9 +#define A_ATR_EVT_POST_ERR_MASK 0x00010000u +#define A_ATR_EVT_POST_ERR_SHIFT 16 +#define A_ATR_EVT_FETCH_ERR_MASK 0x00020000u +#define A_ATR_EVT_FETCH_ERR_SHIFT 17 +#define A_ATR_EVT_DISCARD_ERR_MASK 0x00040000u +#define A_ATR_EVT_DISCARD_ERR_SHIFT 18 +#define A_ATR_EVT_DOORBELL_MASK 0x00000000u +#define A_ATR_EVT_DOORBELL_SHIFT 19 +#define P_ATR_EVT_POST_ERR_MASK 0x00100000u +#define P_ATR_EVT_POST_ERR_SHIFT 20 +#define P_ATR_EVT_FETCH_ERR_MASK 0x00200000u +#define P_ATR_EVT_FETCH_ERR_SHIFT 21 +#define P_ATR_EVT_DISCARD_ERR_MASK 0x00400000u +#define P_ATR_EVT_DISCARD_ERR_SHIFT 22 +#define P_ATR_EVT_DOORBELL_MASK 0x00000000u +#define P_ATR_EVT_DOORBELL_SHIFT 23 +#define PM_MSI_INT_INTA_MASK 0x01000000u +#define PM_MSI_INT_INTA_SHIFT 24 +#define PM_MSI_INT_INTB_MASK 0x02000000u +#define PM_MSI_INT_INTB_SHIFT 25 +#define PM_MSI_INT_INTC_MASK 0x04000000u +#define PM_MSI_INT_INTC_SHIFT 26 +#define PM_MSI_INT_INTD_MASK 0x08000000u +#define PM_MSI_INT_INTD_SHIFT 27 +#define PM_MSI_INT_INTX_MASK 0x0f000000u +#define PM_MSI_INT_INTX_SHIFT 24 +#define PM_MSI_INT_MSI_MASK 0x10000000u +#define PM_MSI_INT_MSI_SHIFT 28 +#define PM_MSI_INT_AER_EVT_MASK 0x20000000u +#define PM_MSI_INT_AER_EVT_SHIFT 29 +#define PM_MSI_INT_EVENTS_MASK 0x40000000u +#define PM_MSI_INT_EVENTS_SHIFT 30 +#define PM_MSI_INT_SYS_ERR_MASK 0x80000000u +#define PM_MSI_INT_SYS_ERR_SHIFT 31 +#define NUM_LOCAL_EVENTS 15 +#define ISTATUS_LOCAL 0x184 +#define IMASK_HOST 0x188 +#define ISTATUS_HOST 0x18c +#define MSI_ADDR 0x190 +#define ISTATUS_MSI 0x194 + +/* PCIe Master table init defines */ +#define ATR0_PCIE_WIN0_SRCADDR_PARAM 0x600u +#define ATR0_PCIE_ATR_SIZE 0x25 +#define ATR0_PCIE_ATR_SIZE_SHIFT 1 +#define ATR0_PCIE_WIN0_SRC_ADDR 0x604u +#define ATR0_PCIE_WIN0_TRSL_ADDR_LSB 0x608u +#define ATR0_PCIE_WIN0_TRSL_ADDR_UDW 0x60cu +#define ATR0_PCIE_WIN0_TRSL_PARAM 0x610u + +/* PCIe AXI slave table init defines */ +#define ATR0_AXI4_SLV0_SRCADDR_PARAM 0x800u +#define ATR_SIZE_SHIFT 1 +#define ATR_IMPL_ENABLE 1 +#define ATR0_AXI4_SLV0_SRC_ADDR 0x804u +#define ATR0_AXI4_SLV0_TRSL_ADDR_LSB 0x808u +#define ATR0_AXI4_SLV0_TRSL_ADDR_UDW 0x80cu +#define ATR0_AXI4_SLV0_TRSL_PARAM 0x810u +#define PCIE_TX_RX_INTERFACE 0x00000000u +#define PCIE_CONFIG_INTERFACE 0x00000001u + +#define ATR_ENTRY_SIZE 32 + +#define EVENT_PCIE_L2_EXIT 0 +#define EVENT_PCIE_HOTRST_EXIT 1 +#define EVENT_PCIE_DLUP_EXIT 2 +#define EVENT_SEC_TX_RAM_SEC_ERR 3 +#define EVENT_SEC_RX_RAM_SEC_ERR 4 +#define EVENT_SEC_AXI2PCIE_RAM_SEC_ERR 5 +#define EVENT_SEC_PCIE2AXI_RAM_SEC_ERR 6 +#define EVENT_DED_TX_RAM_DED_ERR 7 +#define EVENT_DED_RX_RAM_DED_ERR 8 +#define EVENT_DED_AXI2PCIE_RAM_DED_ERR 9 +#define EVENT_DED_PCIE2AXI_RAM_DED_ERR 10 +#define EVENT_LOCAL_DMA_END_ENGINE_0 11 +#define EVENT_LOCAL_DMA_END_ENGINE_1 12 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_0 13 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_1 14 +#define EVENT_LOCAL_A_ATR_EVT_POST_ERR 15 +#define EVENT_LOCAL_A_ATR_EVT_FETCH_ERR 16 +#define EVENT_LOCAL_A_ATR_EVT_DISCARD_ERR 17 +#define EVENT_LOCAL_A_ATR_EVT_DOORBELL 18 +#define EVENT_LOCAL_P_ATR_EVT_POST_ERR 19 +#define EVENT_LOCAL_P_ATR_EVT_FETCH_ERR 20 +#define EVENT_LOCAL_P_ATR_EVT_DISCARD_ERR 21 +#define EVENT_LOCAL_P_ATR_EVT_DOORBELL 22 +#define EVENT_LOCAL_PM_MSI_INT_INTX 23 +#define EVENT_LOCAL_PM_MSI_INT_MSI 24 +#define EVENT_LOCAL_PM_MSI_INT_AER_EVT 25 +#define EVENT_LOCAL_PM_MSI_INT_EVENTS 26 +#define EVENT_LOCAL_PM_MSI_INT_SYS_ERR 27 +#define NUM_EVENTS 28 + +#define PCIE_EVENT_CAUSE(x, s) \ + [EVENT_PCIE_ ## x] = { __stringify(x), s } + +#define SEC_ERROR_CAUSE(x, s) \ + [EVENT_SEC_ ## x] = { __stringify(x), s } + +#define DED_ERROR_CAUSE(x, s) \ + [EVENT_DED_ ## x] = { __stringify(x), s } + +#define LOCAL_EVENT_CAUSE(x, s) \ + [EVENT_LOCAL_ ## x] = { __stringify(x), s } + +#define PCIE_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = PCIE_EVENT_INT, \ + .mask_offset = PCIE_EVENT_INT, \ + .mask_high = 1, \ + .mask = PCIE_EVENT_INT_ ## x ## _INT, \ + .enb_mask = PCIE_EVENT_INT_ENB_MASK + +#define SEC_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = SEC_ERROR_INT, \ + .mask_offset = SEC_ERROR_INT_MASK, \ + .mask = SEC_ERROR_INT_ ## x ## _INT, \ + .mask_high = 1, \ + .enb_mask = 0 + +#define DED_EVENT(x) \ + .base = MC_PCIE_CTRL_ADDR, \ + .offset = DED_ERROR_INT, \ + .mask_offset = DED_ERROR_INT_MASK, \ + .mask_high = 1, \ + .mask = DED_ERROR_INT_ ## x ## _INT, \ + .enb_mask = 0 + +#define LOCAL_EVENT(x) \ + .base = MC_PCIE_BRIDGE_ADDR, \ + .offset = ISTATUS_LOCAL, \ + .mask_offset = IMASK_LOCAL, \ + .mask_high = 0, \ + .mask = x ## _MASK, \ + .enb_mask = 0 + +#define PCIE_EVENT_TO_EVENT_MAP(x) \ + { PCIE_EVENT_INT_ ## x ## _INT, EVENT_PCIE_ ## x } + +#define SEC_ERROR_TO_EVENT_MAP(x) \ + { SEC_ERROR_INT_ ## x ## _INT, EVENT_SEC_ ## x } + +#define DED_ERROR_TO_EVENT_MAP(x) \ + { DED_ERROR_INT_ ## x ## _INT, EVENT_DED_ ## x } + +#define LOCAL_STATUS_TO_EVENT_MAP(x) \ + { x ## _MASK, EVENT_LOCAL_ ## x } + +struct event_map { + u32 reg_mask; + u32 event_bit; +}; + +struct mc_msi { + struct mutex lock; /* Protect used bitmap */ + struct irq_domain *msi_domain; + struct irq_domain *dev_domain; + u32 num_vectors; + u64 vector_phy; + DECLARE_BITMAP(used, MC_NUM_MSI_IRQS); +}; + +struct mc_port { + void __iomem *axi_base_addr; + struct device *dev; + struct irq_domain *intx_domain; + struct irq_domain *event_domain; + raw_spinlock_t lock; + struct mc_msi msi; +}; + +struct cause { + const char *sym; + const char *str; +}; + +static const struct cause event_cause[NUM_EVENTS] = { + PCIE_EVENT_CAUSE(L2_EXIT, "L2 exit event"), + PCIE_EVENT_CAUSE(HOTRST_EXIT, "Hot reset exit event"), + PCIE_EVENT_CAUSE(DLUP_EXIT, "DLUP exit event"), + SEC_ERROR_CAUSE(TX_RAM_SEC_ERR, "sec error in tx buffer"), + SEC_ERROR_CAUSE(RX_RAM_SEC_ERR, "sec error in rx buffer"), + SEC_ERROR_CAUSE(PCIE2AXI_RAM_SEC_ERR, "sec error in pcie2axi buffer"), + SEC_ERROR_CAUSE(AXI2PCIE_RAM_SEC_ERR, "sec error in axi2pcie buffer"), + DED_ERROR_CAUSE(TX_RAM_DED_ERR, "ded error in tx buffer"), + DED_ERROR_CAUSE(RX_RAM_DED_ERR, "ded error in rx buffer"), + DED_ERROR_CAUSE(PCIE2AXI_RAM_DED_ERR, "ded error in pcie2axi buffer"), + DED_ERROR_CAUSE(AXI2PCIE_RAM_DED_ERR, "ded error in axi2pcie buffer"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_0, "dma engine 0 error"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_1, "dma engine 1 error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_POST_ERR, "axi write request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_FETCH_ERR, "axi read request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_DISCARD_ERR, "axi read timeout"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_POST_ERR, "pcie write request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_FETCH_ERR, "pcie read request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_DISCARD_ERR, "pcie read timeout"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_AER_EVT, "aer event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_EVENTS, "pm/ltr/hotplug event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_SYS_ERR, "system error"), +}; + +struct event_map pcie_event_to_event[] = { + PCIE_EVENT_TO_EVENT_MAP(L2_EXIT), + PCIE_EVENT_TO_EVENT_MAP(HOTRST_EXIT), + PCIE_EVENT_TO_EVENT_MAP(DLUP_EXIT), +}; + +struct event_map sec_error_to_event[] = { + SEC_ERROR_TO_EVENT_MAP(TX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(RX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_SEC_ERR), +}; + +struct event_map ded_error_to_event[] = { + DED_ERROR_TO_EVENT_MAP(TX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(RX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_DED_ERR), +}; + +struct event_map local_status_to_event[] = { + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_INTX), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_MSI), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_AER_EVT), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_EVENTS), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_SYS_ERR), +}; + +struct { + u32 base; + u32 offset; + u32 mask; + u32 shift; + u32 enb_mask; + u32 mask_high; + u32 mask_offset; +} event_descs[] = { + { PCIE_EVENT(L2_EXIT) }, + { PCIE_EVENT(HOTRST_EXIT) }, + { PCIE_EVENT(DLUP_EXIT) }, + { SEC_EVENT(TX_RAM_SEC_ERR) }, + { SEC_EVENT(RX_RAM_SEC_ERR) }, + { SEC_EVENT(PCIE2AXI_RAM_SEC_ERR) }, + { SEC_EVENT(AXI2PCIE_RAM_SEC_ERR) }, + { DED_EVENT(TX_RAM_DED_ERR) }, + { DED_EVENT(RX_RAM_DED_ERR) }, + { DED_EVENT(PCIE2AXI_RAM_DED_ERR) }, + { DED_EVENT(AXI2PCIE_RAM_DED_ERR) }, + { LOCAL_EVENT(DMA_END_ENGINE_0) }, + { LOCAL_EVENT(DMA_END_ENGINE_1) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_0) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_1) }, + { LOCAL_EVENT(A_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(P_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(PM_MSI_INT_INTX) }, + { LOCAL_EVENT(PM_MSI_INT_MSI) }, + { LOCAL_EVENT(PM_MSI_INT_AER_EVT) }, + { LOCAL_EVENT(PM_MSI_INT_EVENTS) }, + { LOCAL_EVENT(PM_MSI_INT_SYS_ERR) }, +}; + +static char poss_clks[][5] = { "fic0", "fic1", "fic2", "fic3" }; + +static void mc_pcie_enable_msi(struct mc_port *port, void __iomem *base) +{ + struct mc_msi *msi = &port->msi; + u32 cap_offset = MC_MSI_CAP_CTRL_OFFSET; + u16 msg_ctrl = readw_relaxed(base + cap_offset + PCI_MSI_FLAGS); + + msg_ctrl |= PCI_MSI_FLAGS_ENABLE; + msg_ctrl &= ~PCI_MSI_FLAGS_QMASK; + msg_ctrl |= MC_MSI_MAX_Q_AVAIL; + msg_ctrl &= ~PCI_MSI_FLAGS_QSIZE; + msg_ctrl |= MC_MSI_Q_SIZE; + msg_ctrl |= PCI_MSI_FLAGS_64BIT; + + writew_relaxed(msg_ctrl, base + cap_offset + PCI_MSI_FLAGS); + + writel_relaxed(lower_32_bits(msi->vector_phy), + base + cap_offset + PCI_MSI_ADDRESS_LO); + writel_relaxed(upper_32_bits(msi->vector_phy), + base + cap_offset + PCI_MSI_ADDRESS_HI); +} + +static void mc_handle_msi(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + struct device *dev = port->dev; + struct mc_msi *msi = &port->msi; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long status; + u32 bit; + u32 virq; + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_MSI_MASK) { + status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); + for_each_set_bit(bit, &status, msi->num_vectors) { + virq = irq_find_mapping(msi->dev_domain, bit); + if (virq) + generic_handle_irq(virq); + else + dev_err_ratelimited(dev, "bad MSI IRQ %d\n", + bit); + } + } +} + +static void mc_msi_bottom_irq_ack(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + u32 bitpos = data->hwirq; + unsigned long status; + + writel_relaxed(BIT(bitpos), bridge_base_addr + ISTATUS_MSI); + status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); + if (!status) + writel_relaxed(BIT(PM_MSI_INT_MSI_SHIFT), + bridge_base_addr + ISTATUS_LOCAL); +} + +static void mc_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + phys_addr_t addr = port->msi.vector_phy; + + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq; + + dev_dbg(port->dev, "msi#%x address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static int mc_msi_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + return -EINVAL; +} + +static struct irq_chip mc_msi_bottom_irq_chip = { + .name = "Microchip MSI", + .irq_ack = mc_msi_bottom_irq_ack, + .irq_compose_msi_msg = mc_compose_msi_msg, + .irq_set_affinity = mc_msi_set_affinity, +}; + +static int mc_irq_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct mc_port *port = domain->host_data; + struct mc_msi *msi = &port->msi; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long bit; + u32 val; + + mutex_lock(&msi->lock); + bit = find_first_zero_bit(msi->used, msi->num_vectors); + if (bit >= msi->num_vectors) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + set_bit(bit, msi->used); + + irq_domain_set_info(domain, virq, bit, &mc_msi_bottom_irq_chip, + domain->host_data, handle_edge_irq, NULL, NULL); + + /* Enable MSI interrupts */ + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= PM_MSI_INT_MSI_MASK; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + + mutex_unlock(&msi->lock); + + return 0; +} + +static void mc_irq_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mc_port *port = irq_data_get_irq_chip_data(d); + struct mc_msi *msi = &port->msi; + + mutex_lock(&msi->lock); + + if (test_bit(d->hwirq, msi->used)) + __clear_bit(d->hwirq, msi->used); + else + dev_err(port->dev, "trying to free unused MSI%lu\n", d->hwirq); + + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = mc_irq_msi_domain_alloc, + .free = mc_irq_msi_domain_free, +}; + +static struct irq_chip mc_msi_irq_chip = { + .name = "Microchip PCIe MSI", + .irq_ack = irq_chip_ack_parent, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, +}; + +static struct msi_domain_info mc_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX), + .chip = &mc_msi_irq_chip, +}; + +static int mc_allocate_msi_domains(struct mc_port *port) +{ + struct device *dev = port->dev; + struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node); + struct mc_msi *msi = &port->msi; + + mutex_init(&port->msi.lock); + + msi->dev_domain = irq_domain_add_linear(NULL, msi->num_vectors, + &msi_domain_ops, port); + if (!msi->dev_domain) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + msi->msi_domain = pci_msi_create_irq_domain(fwnode, &mc_msi_domain_info, + msi->dev_domain); + if (!msi->msi_domain) { + dev_err(dev, "failed to create MSI domain\n"); + irq_domain_remove(msi->dev_domain); + return -ENOMEM; + } + + return 0; +} + +static void mc_handle_intx(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + struct device *dev = port->dev; + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long status; + u32 bit; + u32 virq; + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_INTX_MASK) { + status &= PM_MSI_INT_INTX_MASK; + status >>= PM_MSI_INT_INTX_SHIFT; + for_each_set_bit(bit, &status, PCI_NUM_INTX) { + virq = irq_find_mapping(port->intx_domain, bit); + if (virq) + generic_handle_irq(virq); + else + dev_err_ratelimited(dev, "bad INTx IRQ %d\n", + bit); + } + } +} + +static void mc_ack_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + + writel_relaxed(mask, bridge_base_addr + ISTATUS_LOCAL); +} + +static void mc_mask_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val &= ~mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void mc_unmask_intx_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip mc_intx_irq_chip = { + .name = "Microchip PCIe INTx", + .irq_ack = mc_ack_intx_irq, + .irq_mask = mc_mask_intx_irq, + .irq_unmask = mc_unmask_intx_irq, +}; + +static int mc_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &mc_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = mc_pcie_intx_map, +}; + +static inline u32 reg_to_event(u32 reg, struct event_map field) +{ + return (reg & field.reg_mask) ? BIT(field.event_bit) : 0; +} + +static u32 pcie_events(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(pcie_event_to_event); i++) + val |= reg_to_event(reg, pcie_event_to_event[i]); + + return val; +} + +static u32 sec_errors(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(sec_error_to_event); i++) + val |= reg_to_event(reg, sec_error_to_event[i]); + + return val; +} + +static u32 ded_errors(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ded_error_to_event); i++) + val |= reg_to_event(reg, ded_error_to_event[i]); + + return val; +} + +static u32 local_events(void __iomem *addr) +{ + u32 reg = readl_relaxed(addr); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(local_status_to_event); i++) + val |= reg_to_event(reg, local_status_to_event[i]); + + return val; +} + +static u32 get_events(struct mc_port *port) +{ + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + void __iomem *ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; + u32 events = 0; + + events |= pcie_events(ctrl_base_addr + PCIE_EVENT_INT); + events |= sec_errors(ctrl_base_addr + SEC_ERROR_INT); + events |= ded_errors(ctrl_base_addr + DED_ERROR_INT); + events |= local_events(bridge_base_addr + ISTATUS_LOCAL); + + return events; +} + +static irqreturn_t mc_event_handler(int irq, void *dev_id) +{ + struct mc_port *port = dev_id; + struct device *dev = port->dev; + struct irq_data *data; + + data = irq_domain_get_irq_data(port->event_domain, irq); + + if (event_cause[data->hwirq].str) + dev_err_ratelimited(dev, "%s\n", event_cause[data->hwirq].str); + else + dev_err_ratelimited(dev, "bad event IRQ %ld\n", data->hwirq); + + return IRQ_HANDLED; +} + +static void mc_handle_event(struct irq_desc *desc) +{ + struct mc_port *port = irq_desc_get_handler_data(desc); + unsigned long events; + u32 bit; + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + events = get_events(port); + + for_each_set_bit(bit, &events, NUM_EVENTS) + generic_handle_irq(irq_find_mapping(port->event_domain, bit)); + + chained_irq_exit(chip, desc); +} + +static void mc_ack_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].offset; + mask = event_descs[event].mask; + mask |= event_descs[event].enb_mask; + + writel_relaxed(mask, addr); +} + +static void mc_mask_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].mask_offset; + mask = event_descs[event].mask; + if (event_descs[event].enb_mask) { + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + mask &= PCIE_EVENT_INT_ENB_MASK; + } + + if (!event_descs[event].mask_high) + mask = ~mask; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val |= mask; + else + val &= mask; + + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static void mc_unmask_event_irq(struct irq_data *data) +{ + struct mc_port *port = irq_data_get_irq_chip_data(data); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + addr = port->axi_base_addr + event_descs[event].base + + event_descs[event].mask_offset; + mask = event_descs[event].mask; + + if (event_descs[event].enb_mask) + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + + if (event_descs[event].mask_high) + mask = ~mask; + + if (event_descs[event].enb_mask) + mask &= PCIE_EVENT_INT_ENB_MASK; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val &= mask; + else + val |= mask; + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip mc_event_irq_chip = { + .name = "Microchip PCIe EVENT", + .irq_ack = mc_ack_event_irq, + .irq_mask = mc_mask_event_irq, + .irq_unmask = mc_unmask_event_irq, +}; + +static int mc_pcie_event_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &mc_event_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops event_domain_ops = { + .map = mc_pcie_event_map, +}; + +static inline struct clk *mc_pcie_init_clk(struct device *dev, const char *id) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get_optional(dev, id); + if (IS_ERR(clk)) + return clk; + if (!clk) + return clk; + + ret = clk_prepare_enable(clk); + if (ret) + return ERR_PTR(ret); + + devm_add_action_or_reset(dev, (void (*) (void *))clk_disable_unprepare, + clk); + + return clk; +} + +static int mc_pcie_init_clks(struct device *dev) +{ + int i; + struct clk *fic; + + /* + * PCIe may be clocked via Fabric Interface using between 1 and 4 + * clocks. Scan DT for clocks and enable them if present + */ + for (i = 0; i < ARRAY_SIZE(poss_clks); i++) { + fic = mc_pcie_init_clk(dev, poss_clks[i]); + if (IS_ERR(fic)) + return PTR_ERR(fic); + } + + return 0; +} + +static int mc_pcie_init_irq_domains(struct mc_port *port) +{ + struct device *dev = port->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "failed to find PCIe Intc node\n"); + return -EINVAL; + } + + port->event_domain = irq_domain_add_linear(pcie_intc_node, NUM_EVENTS, + &event_domain_ops, port); + if (!port->event_domain) { + dev_err(dev, "failed to get event domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->event_domain, DOMAIN_BUS_NEXUS); + + port->intx_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX, + &intx_domain_ops, port); + if (!port->intx_domain) { + dev_err(dev, "failed to get an INTx IRQ domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->intx_domain, DOMAIN_BUS_WIRED); + + of_node_put(pcie_intc_node); + raw_spin_lock_init(&port->lock); + + return mc_allocate_msi_domains(port); +} + +static void mc_pcie_setup_window(void __iomem *bridge_base_addr, u32 index, + phys_addr_t axi_addr, phys_addr_t pci_addr, + size_t size) +{ + u32 atr_sz = ilog2(size) - 1; + u32 val; + + if (index == 0) + val = PCIE_CONFIG_INTERFACE; + else + val = PCIE_TX_RX_INTERFACE; + + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_PARAM); + + val = lower_32_bits(axi_addr) | (atr_sz << ATR_SIZE_SHIFT) | + ATR_IMPL_ENABLE; + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRCADDR_PARAM); + + val = upper_32_bits(axi_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRC_ADDR); + + val = lower_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_LSB); + + val = upper_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_UDW); + + val = readl(bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + val |= (ATR0_PCIE_ATR_SIZE << ATR0_PCIE_ATR_SIZE_SHIFT); + writel(val, bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + writel(0, bridge_base_addr + ATR0_PCIE_WIN0_SRC_ADDR); +} + +static int mc_pcie_setup_windows(struct platform_device *pdev, + struct mc_port *port) +{ + void __iomem *bridge_base_addr = + port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + struct resource_entry *entry; + u64 pci_addr; + u32 index = 1; + + resource_list_for_each_entry(entry, &bridge->windows) { + if (resource_type(entry->res) == IORESOURCE_MEM) { + pci_addr = entry->res->start - entry->offset; + mc_pcie_setup_window(bridge_base_addr, index, + entry->res->start, pci_addr, + resource_size(entry->res)); + index++; + } + } + + return 0; +} + +static int mc_platform_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct platform_device *pdev = to_platform_device(dev); + struct mc_port *port; + void __iomem *bridge_base_addr; + void __iomem *ctrl_base_addr; + int ret; + int irq; + int i, intx_irq, msi_irq, event_irq; + u32 val; + int err; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + port->dev = dev; + + ret = mc_pcie_init_clks(dev); + if (ret) { + dev_err(dev, "failed to get clock resources, error %d\n", ret); + return -ENODEV; + } + + port->axi_base_addr = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(port->axi_base_addr)) + return PTR_ERR(port->axi_base_addr); + + bridge_base_addr = port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; + ctrl_base_addr = port->axi_base_addr + MC_PCIE_CTRL_ADDR; + + port->msi.vector_phy = MSI_ADDR; + port->msi.num_vectors = MC_NUM_MSI_IRQS; + ret = mc_pcie_init_irq_domains(port); + if (ret) { + dev_err(dev, "failed creating IRQ domains\n"); + return ret; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "unable to request IRQ%d\n", irq); + return -ENODEV; + } + + for (i = 0; i < NUM_EVENTS; i++) { + event_irq = irq_create_mapping(port->event_domain, i); + if (!event_irq) { + dev_err(dev, "failed to map hwirq %d\n", i); + return -ENXIO; + } + + err = devm_request_irq(dev, event_irq, mc_event_handler, + 0, event_cause[i].sym, port); + if (err) { + dev_err(dev, "failed to request IRQ %d\n", event_irq); + return err; + } + } + + intx_irq = irq_create_mapping(port->event_domain, + EVENT_LOCAL_PM_MSI_INT_INTX); + if (!intx_irq) { + dev_err(dev, "failed to map INTx interrupt\n"); + return -ENXIO; + } + + /* Plug the INTx chained handler */ + irq_set_chained_handler_and_data(intx_irq, mc_handle_intx, port); + + msi_irq = irq_create_mapping(port->event_domain, + EVENT_LOCAL_PM_MSI_INT_MSI); + if (!msi_irq) + return -ENXIO; + + /* Plug the MSI chained handler */ + irq_set_chained_handler_and_data(msi_irq, mc_handle_msi, port); + + /* Plug the main event chained handler */ + irq_set_chained_handler_and_data(irq, mc_handle_event, port); + + /* Hardware doesn't setup MSI by default */ + mc_pcie_enable_msi(port, cfg->win); + + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= PM_MSI_INT_INTX_MASK; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + + writel_relaxed(val, ctrl_base_addr + ECC_CONTROL); + + val = PCIE_EVENT_INT_L2_EXIT_INT | + PCIE_EVENT_INT_HOTRST_EXIT_INT | + PCIE_EVENT_INT_DLUP_EXIT_INT; + writel_relaxed(val, ctrl_base_addr + PCIE_EVENT_INT); + + val = SEC_ERROR_INT_TX_RAM_SEC_ERR_INT | + SEC_ERROR_INT_RX_RAM_SEC_ERR_INT | + SEC_ERROR_INT_PCIE2AXI_RAM_SEC_ERR_INT | + SEC_ERROR_INT_AXI2PCIE_RAM_SEC_ERR_INT; + writel_relaxed(val, ctrl_base_addr + SEC_ERROR_INT); + writel_relaxed(0, ctrl_base_addr + SEC_ERROR_INT_MASK); + writel_relaxed(0, ctrl_base_addr + SEC_ERROR_CNT); + + val = DED_ERROR_INT_TX_RAM_DED_ERR_INT | + DED_ERROR_INT_RX_RAM_DED_ERR_INT | + DED_ERROR_INT_PCIE2AXI_RAM_DED_ERR_INT | + DED_ERROR_INT_AXI2PCIE_RAM_DED_ERR_INT; + writel_relaxed(val, ctrl_base_addr + DED_ERROR_INT); + writel_relaxed(0, ctrl_base_addr + DED_ERROR_INT_MASK); + writel_relaxed(0, ctrl_base_addr + DED_ERROR_CNT); + + writel_relaxed(0, bridge_base_addr + IMASK_HOST); + writel_relaxed(GENMASK(31, 0), bridge_base_addr + ISTATUS_HOST); + + /* Configure Address Translation Table 0 for PCIe config space */ + mc_pcie_setup_window(bridge_base_addr, 0, cfg->res.start & 0xffffffff, + cfg->res.start, resource_size(&cfg->res)); + + return mc_pcie_setup_windows(pdev, port); +} + +static const struct pci_ecam_ops mc_ecam_ops = { + .init = mc_platform_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static const struct of_device_id mc_pcie_of_match[] = { + { + .compatible = "microchip,pcie-host-1.0", + .data = &mc_ecam_ops, + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, mc_pcie_of_match) + +static struct platform_driver mc_pcie_driver = { + .probe = pci_host_common_probe, + .driver = { + .name = "microchip-pcie", + .of_match_table = mc_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver(mc_pcie_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microchip PCIe host controller driver"); +MODULE_AUTHOR("Daire McNamara "); From daaaf866587ced121e3d33b4e978ec1fa66c18e9 Mon Sep 17 00:00:00 2001 From: Daire McNamara Date: Mon, 25 Jan 2021 16:29:34 +0000 Subject: [PATCH 420/633] MAINTAINERS: Add Daire McNamara as Microchip PCIe driver maintainer Link: https://lore.kernel.org/r/20210125162934.5335-5-daire.mcnamara@microchip.com Signed-off-by: Daire McNamara Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index cc1e6a5ee6e6..b6377e128d26 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13830,6 +13830,13 @@ S: Supported F: Documentation/devicetree/bindings/pci/mediatek* F: drivers/pci/controller/*mediatek* +PCIE DRIVER FOR MICROCHIP +M: Daire McNamara +L: linux-pci@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/pci/microchip* +F: drivers/pci/controller/*microchip* + PCIE DRIVER FOR QUALCOMM MSM M: Stanimir Varbanov L: linux-pci@vger.kernel.org From 13bccf873808ac9516089760efce7ea18b7484a9 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:53 +0530 Subject: [PATCH 421/633] Documentation: PCI: Add specification for the PCI NTB function device Add specification for the PCI NTB function device. The endpoint function driver and the host PCI driver should be created based on this specification. [bhelgaas: fix a few typos] Link: https://lore.kernel.org/r/20210201195809.7342-2-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- Documentation/PCI/endpoint/index.rst | 1 + .../PCI/endpoint/pci-ntb-function.rst | 348 ++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 Documentation/PCI/endpoint/pci-ntb-function.rst diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index 4ca7439fbfc9..ef6861128506 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -11,5 +11,6 @@ PCI Endpoint Framework pci-endpoint-cfs pci-test-function pci-test-howto + pci-ntb-function function/binding/pci-test diff --git a/Documentation/PCI/endpoint/pci-ntb-function.rst b/Documentation/PCI/endpoint/pci-ntb-function.rst new file mode 100644 index 000000000000..3b9d836a4924 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-function.rst @@ -0,0 +1,348 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================= +PCI NTB Function +================= + +:Author: Kishon Vijay Abraham I + +PCI Non-Transparent Bridges (NTB) allow two host systems to communicate +with each other by exposing each host as a device to the other host. +NTBs typically support the ability to generate interrupts on the remote +machine, expose memory ranges as BARs, and perform DMA. They also support +scratchpads, which are areas of memory within the NTB that are accessible +from both machines. + +PCI NTB Function allows two different systems (or hosts) to communicate +with each other by configuring the endpoint instances in such a way that +transactions from one system are routed to the other system. + +In the below diagram, PCI NTB function configures the SoC with multiple +PCI Endpoint (EP) instances in such a way that transactions from one EP +controller are routed to the other EP controller. Once PCI NTB function +configures the SoC with multiple EP instances, HOST1 and HOST2 can +communicate with each other using SoC as a bridge. + +.. code-block:: text + + +-------------+ +-------------+ + | | | | + | HOST1 | | HOST2 | + | | | | + +------^------+ +------^------+ + | | + | | + +---------|-------------------------------------------------|---------+ + | +------v------+ +------v------+ | + | | | | | | + | | EP | | EP | | + | | CONTROLLER1 | | CONTROLLER2 | | + | | <-----------------------------------> | | + | | | | | | + | | | | | | + | | | SoC With Multiple EP Instances | | | + | | | (Configured using NTB Function) | | | + | +-------------+ +-------------+ | + +---------------------------------------------------------------------+ + +Constructs used for Implementing NTB +==================================== + + 1) Config Region + 2) Self Scratchpad Registers + 3) Peer Scratchpad Registers + 4) Doorbell (DB) Registers + 5) Memory Window (MW) + + +Config Region: +-------------- + +Config Region is a construct that is specific to NTB implemented using NTB +Endpoint Function Driver. The host and endpoint side NTB function driver will +exchange information with each other using this region. Config Region has +Control/Status Registers for configuring the Endpoint Controller. Host can +write into this region for configuring the outbound Address Translation Unit +(ATU) and to indicate the link status. Endpoint can indicate the status of +commands issued by host in this region. Endpoint can also indicate the +scratchpad offset and number of memory windows to the host using this region. + +The format of Config Region is given below. All the fields here are 32 bits. + +.. code-block:: text + + +------------------------+ + | COMMAND | + +------------------------+ + | ARGUMENT | + +------------------------+ + | STATUS | + +------------------------+ + | TOPOLOGY | + +------------------------+ + | ADDRESS (LOWER 32) | + +------------------------+ + | ADDRESS (UPPER 32) | + +------------------------+ + | SIZE | + +------------------------+ + | NO OF MEMORY WINDOW | + +------------------------+ + | MEMORY WINDOW1 OFFSET | + +------------------------+ + | SPAD OFFSET | + +------------------------+ + | SPAD COUNT | + +------------------------+ + | DB ENTRY SIZE | + +------------------------+ + | DB DATA | + +------------------------+ + | : | + +------------------------+ + | : | + +------------------------+ + | DB DATA | + +------------------------+ + + + COMMAND: + + NTB function supports three commands: + + CMD_CONFIGURE_DOORBELL (0x1): Command to configure doorbell. Before + invoking this command, the host should allocate and initialize + MSI/MSI-X vectors (i.e., initialize the MSI/MSI-X Capability in the + Endpoint). The endpoint on receiving this command will configure + the outbound ATU such that transactions to Doorbell BAR will be routed + to the MSI/MSI-X address programmed by the host. The ARGUMENT + register should be populated with number of DBs to configure (in the + lower 16 bits) and if MSI or MSI-X should be configured (BIT 16). + + CMD_CONFIGURE_MW (0x2): Command to configure memory window (MW). The + host invokes this command after allocating a buffer that can be + accessed by remote host. The allocated address should be programmed + in the ADDRESS register (64 bit), the size should be programmed in + the SIZE register and the memory window index should be programmed + in the ARGUMENT register. The endpoint on receiving this command + will configure the outbound ATU such that transactions to MW BAR + are routed to the address provided by the host. + + CMD_LINK_UP (0x3): Command to indicate an NTB application is + bound to the EP device on the host side. Once the endpoint + receives this command from both the hosts, the endpoint will + raise a LINK_UP event to both the hosts to indicate the host + NTB applications can start communicating with each other. + + ARGUMENT: + + The value of this register is based on the commands issued in + command register. See COMMAND section for more information. + + TOPOLOGY: + + Set to NTB_TOPO_B2B_USD for Primary interface + Set to NTB_TOPO_B2B_DSD for Secondary interface + + ADDRESS/SIZE: + + Address and Size to be used while configuring the memory window. + See "CMD_CONFIGURE_MW" for more info. + + MEMORY WINDOW1 OFFSET: + + Memory Window 1 and Doorbell registers are packed together in the + same BAR. The initial portion of the region will have doorbell + registers and the latter portion of the region is for memory window 1. + This register will specify the offset of the memory window 1. + + NO OF MEMORY WINDOW: + + Specifies the number of memory windows supported by the NTB device. + + SPAD OFFSET: + + Self scratchpad region and config region are packed together in the + same BAR. The initial portion of the region will have config region + and the latter portion of the region is for self scratchpad. This + register will specify the offset of the self scratchpad registers. + + SPAD COUNT: + + Specifies the number of scratchpad registers supported by the NTB + device. + + DB ENTRY SIZE: + + Used to determine the offset within the DB BAR that should be written + in order to raise doorbell. EPF NTB can use either MSI or MSI-X to + ring doorbell (MSI-X support will be added later). MSI uses same + address for all the interrupts and MSI-X can provide different + addresses for different interrupts. The MSI/MSI-X address is provided + by the host and the address it gives is based on the MSI/MSI-X + implementation supported by the host. For instance, ARM platform + using GIC ITS will have the same MSI-X address for all the interrupts. + In order to support all the combinations and use the same mechanism + for both MSI and MSI-X, EPF NTB allocates a separate region in the + Outbound Address Space for each of the interrupts. This region will + be mapped to the MSI/MSI-X address provided by the host. If a host + provides the same address for all the interrupts, all the regions + will be translated to the same address. If a host provides different + addresses, the regions will be translated to different addresses. This + will ensure there is no difference while raising the doorbell. + + DB DATA: + + EPF NTB supports 32 interrupts, so there are 32 DB DATA registers. + This holds the MSI/MSI-X data that has to be written to MSI address + for raising doorbell interrupt. This will be populated by EPF NTB + while invoking CMD_CONFIGURE_DOORBELL. + +Scratchpad Registers: +--------------------- + + Each host has its own register space allocated in the memory of NTB endpoint + controller. They are both readable and writable from both sides of the bridge. + They are used by applications built over NTB and can be used to pass control + and status information between both sides of a device. + + Scratchpad registers has 2 parts + 1) Self Scratchpad: Host's own register space + 2) Peer Scratchpad: Remote host's register space. + +Doorbell Registers: +------------------- + + Doorbell Registers are used by the hosts to interrupt each other. + +Memory Window: +-------------- + + Actual transfer of data between the two hosts will happen using the + memory window. + +Modeling Constructs: +==================== + +There are 5 or more distinct regions (config, self scratchpad, peer +scratchpad, doorbell, one or more memory windows) to be modeled to achieve +NTB functionality. At least one memory window is required while more than +one is permitted. All these regions should be mapped to BARs for hosts to +access these regions. + +If one 32-bit BAR is allocated for each of these regions, the scheme would +look like this: + +====== =============== +BAR NO CONSTRUCTS USED +====== =============== +BAR0 Config Region +BAR1 Self Scratchpad +BAR2 Peer Scratchpad +BAR3 Doorbell +BAR4 Memory Window 1 +BAR5 Memory Window 2 +====== =============== + +However if we allocate a separate BAR for each of the regions, there would not +be enough BARs for all the regions in a platform that supports only 64-bit +BARs. + +In order to be supported by most of the platforms, the regions should be +packed and mapped to BARs in a way that provides NTB functionality and +also makes sure the host doesn't access any region that it is not supposed +to. + +The following scheme is used in EPF NTB Function: + +====== =============================== +BAR NO CONSTRUCTS USED +====== =============================== +BAR0 Config Region + Self Scratchpad +BAR1 Peer Scratchpad +BAR2 Doorbell + Memory Window 1 +BAR3 Memory Window 2 +BAR4 Memory Window 3 +BAR5 Memory Window 4 +====== =============================== + +With this scheme, for the basic NTB functionality 3 BARs should be sufficient. + +Modeling Config/Scratchpad Region: +---------------------------------- + +.. code-block:: text + + +-----------------+------->+------------------+ +-----------------+ + | BAR0 | | CONFIG REGION | | BAR0 | + +-----------------+----+ +------------------+<-------+-----------------+ + | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + +-----------------+ +-->+------------------+<-------+-----------------+ + | BAR2 | Local Memory | BAR2 | + +-----------------+ +-----------------+ + | BAR3 | | BAR3 | + +-----------------+ +-----------------+ + | BAR4 | | BAR4 | + +-----------------+ +-----------------+ + | BAR5 | | BAR5 | + +-----------------+ +-----------------+ + EP CONTROLLER 1 EP CONTROLLER 2 + +Above diagram shows Config region + Scratchpad region for HOST1 (connected to +EP controller 1) allocated in local memory. The HOST1 can access the config +region and scratchpad region (self scratchpad) using BAR0 of EP controller 1. +The peer host (HOST2 connected to EP controller 2) can also access this +scratchpad region (peer scratchpad) using BAR1 of EP controller 2. This +diagram shows the case where Config region and Scratchpad regions are allocated +for HOST1, however the same is applicable for HOST2. + +Modeling Doorbell/Memory Window 1: +---------------------------------- + +.. code-block:: text + + +-----------------+ +----->+----------------+-----------+-----------------+ + | BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | + +-----------------+ | +----------------+ +-----------------+ + | BAR1 | | | Doorbell 2 +---------+ | | + +-----------------+----+ +----------------+ | | | + | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + +-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | + | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + +-----------------+ | |----------------+ | | | | + | BAR4 | | | | | | +-----------------+ + +-----------------+ | | MW1 +---+ | +-->+ MSI-X ADDRESS 3|| + | BAR5 | | | | | | +-----------------+ + +-----------------+ +----->-----------------+ | | | | + EP CONTROLLER 1 | | | | +-----------------+ + | | | +---->+ MSI-X ADDRESS 4 | + +----------------+ | +-----------------+ + EP CONTROLLER 2 | | | + (OB SPACE) | | | + +-------> MW1 | + | | + | | + +-----------------+ + | | + | | + | | + | | + | | + +-----------------+ + PCI Address Space + (Managed by HOST2) + +Above diagram shows how the doorbell and memory window 1 is mapped so that +HOST1 can raise doorbell interrupt on HOST2 and also how HOST1 can access +buffers exposed by HOST2 using memory window1 (MW1). Here doorbell and +memory window 1 regions are allocated in EP controller 2 outbound (OB) address +space. Allocating and configuring BARs for doorbell and memory window1 +is done during the initialization phase of NTB endpoint function driver. +Mapping from EP controller 2 OB space to PCI address space is done when HOST2 +sends CMD_CONFIGURE_MW/CMD_CONFIGURE_DOORBELL. + +Modeling Optional Memory Windows: +--------------------------------- + +This is modeled the same was as MW1 but each of the additional memory windows +is mapped to separate BARs. From 959a48d0eac0321948c9f3d1707ba22c100e92d5 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:54 +0530 Subject: [PATCH 422/633] PCI: endpoint: Make *_get_first_free_bar() take into account 64 bit BAR pci_epc_get_first_free_bar() uses only "reserved_bar" member in epc_features to get the first unreserved BAR. However if the reserved BAR is also a 64-bit BAR, then the next BAR shouldn't be returned (since 64-bit BAR uses two BARs). Make pci_epc_get_first_free_bar() take into account 64 bit BAR while returning the first free unreserved BAR. Link: https://lore.kernel.org/r/20210201195809.7342-3-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-epc-core.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index cadd3db0cbb0..25e57672e1a1 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -93,12 +93,20 @@ EXPORT_SYMBOL_GPL(pci_epc_get); unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features) { - int free_bar; + unsigned long free_bar; if (!epc_features) return 0; - free_bar = ffz(epc_features->reserved_bar); + /* Find if the reserved BAR is also a 64-bit BAR */ + free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit; + + /* Set the adjacent bit if the reserved BAR is also a 64-bit BAR */ + free_bar <<= 1; + free_bar |= epc_features->reserved_bar; + + /* Now find the free BAR */ + free_bar = ffz(free_bar); if (free_bar > 5) return 0; From fa8fef0e104a23efe568b835d9e7e188d1d97610 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:55 +0530 Subject: [PATCH 423/633] PCI: endpoint: Add helper API to get the 'next' unreserved BAR Add an API to get the next unreserved BAR starting from a given BAR number that can be used by the endpoint function. Link: https://lore.kernel.org/r/20210201195809.7342-4-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-epc-core.c | 26 ++++++++++++++++++++++---- include/linux/pci-epc.h | 2 ++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 25e57672e1a1..1afe5d9afb0d 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -87,17 +87,36 @@ EXPORT_SYMBOL_GPL(pci_epc_get); * pci_epc_get_first_free_bar() - helper to get first unreserved BAR * @epc_features: pci_epc_features structure that holds the reserved bar bitmap * - * Invoke to get the first unreserved BAR that can be used for endpoint + * Invoke to get the first unreserved BAR that can be used by the endpoint * function. For any incorrect value in reserved_bar return '0'. */ unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features) +{ + return pci_epc_get_next_free_bar(epc_features, BAR_0); +} +EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); + +/** + * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar + * @epc_features: pci_epc_features structure that holds the reserved bar bitmap + * @bar: the starting BAR number from where unreserved BAR should be searched + * + * Invoke to get the next unreserved BAR starting from @bar that can be used + * for endpoint function. For any incorrect value in reserved_bar return '0'. + */ +unsigned int pci_epc_get_next_free_bar(const struct pci_epc_features + *epc_features, enum pci_barno bar) { unsigned long free_bar; if (!epc_features) return 0; + /* If 'bar - 1' is a 64-bit BAR, move to the next BAR */ + if ((epc_features->bar_fixed_64bit << 1) & 1 << bar) + bar++; + /* Find if the reserved BAR is also a 64-bit BAR */ free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit; @@ -105,14 +124,13 @@ unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features free_bar <<= 1; free_bar |= epc_features->reserved_bar; - /* Now find the free BAR */ - free_bar = ffz(free_bar); + free_bar = find_next_zero_bit(&free_bar, 6, bar); if (free_bar > 5) return 0; return free_bar; } -EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); +EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar); /** * pci_epc_get_features() - get the features supported by EPC diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index cc66bec8be90..cfe9b427e6b7 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -203,6 +203,8 @@ const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, u8 func_no); unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features); +unsigned int pci_epc_get_next_free_bar(const struct pci_epc_features + *epc_features, enum pci_barno bar); struct pci_epc *pci_epc_get(const char *epc_name); void pci_epc_put(struct pci_epc *epc); From 0e27aeccfa3d1bab7c6a29fb8e6fcedbad7b09a8 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:56 +0530 Subject: [PATCH 424/633] PCI: endpoint: Make *_free_bar() to return error codes on failure Modify pci_epc_get_next_free_bar() and pci_epc_get_first_free_bar() to return error values if there are no free BARs available. Link: https://lore.kernel.org/r/20210201195809.7342-5-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/functions/pci-epf-test.c | 2 ++ drivers/pci/endpoint/pci-epc-core.c | 12 ++++++------ include/linux/pci-epc.h | 8 ++++---- include/linux/pci-epf.h | 1 + 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index e4e51d884553..7a1f3abfde48 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -834,6 +834,8 @@ static int pci_epf_test_bind(struct pci_epf *epf) linkup_notifier = epc_features->linkup_notifier; core_init_notifier = epc_features->core_init_notifier; test_reg_bar = pci_epc_get_first_free_bar(epc_features); + if (test_reg_bar < 0) + return -EINVAL; pci_epf_configure_bar(epf, epc_features); } diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 1afe5d9afb0d..ea7e7465ce7a 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -90,8 +90,8 @@ EXPORT_SYMBOL_GPL(pci_epc_get); * Invoke to get the first unreserved BAR that can be used by the endpoint * function. For any incorrect value in reserved_bar return '0'. */ -unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features - *epc_features) +enum pci_barno +pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features) { return pci_epc_get_next_free_bar(epc_features, BAR_0); } @@ -105,13 +105,13 @@ EXPORT_SYMBOL_GPL(pci_epc_get_first_free_bar); * Invoke to get the next unreserved BAR starting from @bar that can be used * for endpoint function. For any incorrect value in reserved_bar return '0'. */ -unsigned int pci_epc_get_next_free_bar(const struct pci_epc_features - *epc_features, enum pci_barno bar) +enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features + *epc_features, enum pci_barno bar) { unsigned long free_bar; if (!epc_features) - return 0; + return BAR_0; /* If 'bar - 1' is a 64-bit BAR, move to the next BAR */ if ((epc_features->bar_fixed_64bit << 1) & 1 << bar) @@ -126,7 +126,7 @@ unsigned int pci_epc_get_next_free_bar(const struct pci_epc_features free_bar = find_next_zero_bit(&free_bar, 6, bar); if (free_bar > 5) - return 0; + return NO_BAR; return free_bar; } diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index cfe9b427e6b7..88d311bad984 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -201,10 +201,10 @@ int pci_epc_start(struct pci_epc *epc); void pci_epc_stop(struct pci_epc *epc); const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, u8 func_no); -unsigned int pci_epc_get_first_free_bar(const struct pci_epc_features - *epc_features); -unsigned int pci_epc_get_next_free_bar(const struct pci_epc_features - *epc_features, enum pci_barno bar); +enum pci_barno +pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features); +enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features + *epc_features, enum pci_barno bar); struct pci_epc *pci_epc_get(const char *epc_name); void pci_epc_put(struct pci_epc *epc); diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 6644ff3b0702..fa3aca43eb19 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -21,6 +21,7 @@ enum pci_notify_event { }; enum pci_barno { + NO_BAR = -1, BAR_0, BAR_1, BAR_2, From 7e5a51ebb321537c4209cdd0c54c4c19b3ef960d Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:57 +0530 Subject: [PATCH 425/633] PCI: endpoint: Remove unused pci_epf_match_device() Remove unused pci_epf_match_device() function added in pci-epf-core.c Link: https://lore.kernel.org/r/20210201195809.7342-6-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-epf-core.c | 16 ---------------- include/linux/pci-epf.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index c977cf9dce56..e44a317a2a2a 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -282,22 +282,6 @@ struct pci_epf *pci_epf_create(const char *name) } EXPORT_SYMBOL_GPL(pci_epf_create); -const struct pci_epf_device_id * -pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf) -{ - if (!id || !epf) - return NULL; - - while (*id->name) { - if (strcmp(epf->name, id->name) == 0) - return id; - id++; - } - - return NULL; -} -EXPORT_SYMBOL_GPL(pci_epf_match_device); - static void pci_epf_dev_release(struct device *dev) { struct pci_epf *epf = to_pci_epf(dev); diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index fa3aca43eb19..f373a134ac04 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -165,8 +165,6 @@ static inline void *epf_get_drvdata(struct pci_epf *epf) return dev_get_drvdata(&epf->dev); } -const struct pci_epf_device_id * -pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf); struct pci_epf *pci_epf_create(const char *name); void pci_epf_destroy(struct pci_epf *epf); int __pci_epf_register_driver(struct pci_epf_driver *driver, From 63840ff5322373d665b2b9c59cd64233d5f0691e Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:58 +0530 Subject: [PATCH 426/633] PCI: endpoint: Add support to associate secondary EPC with EPF In the case of standard endpoint functions, only one endpoint controller (EPC) will be associated with an endpoint function (EPF). However for providing NTB (non transparent bridge) functionality, two EPCs should be associated with a single EPF. Add support to associate secondary EPC with EPF. This is in preparation for adding NTB endpoint function driver. Link: https://lore.kernel.org/r/20210201195809.7342-7-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/functions/pci-epf-test.c | 11 ++-- drivers/pci/endpoint/pci-ep-cfs.c | 6 +- drivers/pci/endpoint/pci-epc-core.c | 47 +++++++++++---- drivers/pci/endpoint/pci-epf-core.c | 57 +++++++++++++------ include/linux/pci-epc.h | 25 +++++++- include/linux/pci-epf.h | 17 +++++- 6 files changed, 125 insertions(+), 38 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 7a1f3abfde48..c0ac4e9cbe72 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -619,7 +619,8 @@ static void pci_epf_test_unbind(struct pci_epf *epf) if (epf_test->reg[bar]) { pci_epc_clear_bar(epc, epf->func_no, epf_bar); - pci_epf_free_space(epf, epf_test->reg[bar], bar); + pci_epf_free_space(epf, epf_test->reg[bar], bar, + PRIMARY_INTERFACE); } } } @@ -651,7 +652,8 @@ static int pci_epf_test_set_bar(struct pci_epf *epf) ret = pci_epc_set_bar(epc, epf->func_no, epf_bar); if (ret) { - pci_epf_free_space(epf, epf_test->reg[bar], bar); + pci_epf_free_space(epf, epf_test->reg[bar], bar, + PRIMARY_INTERFACE); dev_err(dev, "Failed to set BAR%d\n", bar); if (bar == test_reg_bar) return ret; @@ -771,7 +773,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) } base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar, - epc_features->align); + epc_features->align, PRIMARY_INTERFACE); if (!base) { dev_err(dev, "Failed to allocated register space\n"); return -ENOMEM; @@ -789,7 +791,8 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) continue; base = pci_epf_alloc_space(epf, bar_size[bar], bar, - epc_features->align); + epc_features->align, + PRIMARY_INTERFACE); if (!base) dev_err(dev, "Failed to allocate space for BAR%d\n", bar); diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 3710adf51912..6ca9e2f92460 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -94,13 +94,13 @@ static int pci_epc_epf_link(struct config_item *epc_item, struct pci_epc *epc = epc_group->epc; struct pci_epf *epf = epf_group->epf; - ret = pci_epc_add_epf(epc, epf); + ret = pci_epc_add_epf(epc, epf, PRIMARY_INTERFACE); if (ret) return ret; ret = pci_epf_bind(epf); if (ret) { - pci_epc_remove_epf(epc, epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); return ret; } @@ -120,7 +120,7 @@ static void pci_epc_epf_unlink(struct config_item *epc_item, epc = epc_group->epc; epf = epf_group->epf; pci_epf_unbind(epf); - pci_epc_remove_epf(epc, epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); } static struct configfs_item_operations pci_epc_item_ops = { diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index ea7e7465ce7a..3693eca5b030 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -493,21 +493,28 @@ EXPORT_SYMBOL_GPL(pci_epc_write_header); * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller * @epc: the EPC device to which the endpoint function should be added * @epf: the endpoint function to be added + * @type: Identifies if the EPC is connected to the primary or secondary + * interface of EPF * * A PCI endpoint device can have one or more functions. In the case of PCIe, * the specification allows up to 8 PCIe endpoint functions. Invoke * pci_epc_add_epf() to add a PCI endpoint function to an endpoint controller. */ -int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf) +int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type) { + struct list_head *list; u32 func_no; int ret = 0; - if (epf->epc) + if (IS_ERR_OR_NULL(epc)) + return -EINVAL; + + if (type == PRIMARY_INTERFACE && epf->epc) return -EBUSY; - if (IS_ERR(epc)) - return -EINVAL; + if (type == SECONDARY_INTERFACE && epf->sec_epc) + return -EBUSY; mutex_lock(&epc->lock); func_no = find_first_zero_bit(&epc->function_num_map, @@ -524,11 +531,17 @@ int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf) } set_bit(func_no, &epc->function_num_map); - epf->func_no = func_no; - epf->epc = epc; - - list_add_tail(&epf->list, &epc->pci_epf); + if (type == PRIMARY_INTERFACE) { + epf->func_no = func_no; + epf->epc = epc; + list = &epf->list; + } else { + epf->sec_epc_func_no = func_no; + epf->sec_epc = epc; + list = &epf->sec_epc_list; + } + list_add_tail(list, &epc->pci_epf); ret: mutex_unlock(&epc->lock); @@ -543,14 +556,26 @@ EXPORT_SYMBOL_GPL(pci_epc_add_epf); * * Invoke to remove PCI endpoint function from the endpoint controller. */ -void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf) +void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type) { + struct list_head *list; + u32 func_no = 0; + if (!epc || IS_ERR(epc) || !epf) return; + if (type == PRIMARY_INTERFACE) { + func_no = epf->func_no; + list = &epf->list; + } else { + func_no = epf->sec_epc_func_no; + list = &epf->sec_epc_list; + } + mutex_lock(&epc->lock); - clear_bit(epf->func_no, &epc->function_num_map); - list_del(&epf->list); + clear_bit(func_no, &epc->function_num_map); + list_del(list); epf->epc = NULL; mutex_unlock(&epc->lock); } diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index e44a317a2a2a..79329ec6373c 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -74,24 +74,37 @@ EXPORT_SYMBOL_GPL(pci_epf_bind); * @epf: the EPF device from whom to free the memory * @addr: the virtual address of the PCI EPF register space * @bar: the BAR number corresponding to the register space + * @type: Identifies if the allocated space is for primary EPC or secondary EPC * * Invoke to free the allocated PCI EPF register space. */ -void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar) +void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, + enum pci_epc_interface_type type) { struct device *dev = epf->epc->dev.parent; + struct pci_epf_bar *epf_bar; + struct pci_epc *epc; if (!addr) return; - dma_free_coherent(dev, epf->bar[bar].size, addr, - epf->bar[bar].phys_addr); + if (type == PRIMARY_INTERFACE) { + epc = epf->epc; + epf_bar = epf->bar; + } else { + epc = epf->sec_epc; + epf_bar = epf->sec_epc_bar; + } - epf->bar[bar].phys_addr = 0; - epf->bar[bar].addr = NULL; - epf->bar[bar].size = 0; - epf->bar[bar].barno = 0; - epf->bar[bar].flags = 0; + dev = epc->dev.parent; + dma_free_coherent(dev, epf_bar[bar].size, addr, + epf_bar[bar].phys_addr); + + epf_bar[bar].phys_addr = 0; + epf_bar[bar].addr = NULL; + epf_bar[bar].size = 0; + epf_bar[bar].barno = 0; + epf_bar[bar].flags = 0; } EXPORT_SYMBOL_GPL(pci_epf_free_space); @@ -101,15 +114,18 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space); * @size: the size of the memory that has to be allocated * @bar: the BAR number corresponding to the allocated register space * @align: alignment size for the allocation region + * @type: Identifies if the allocation is for primary EPC or secondary EPC * * Invoke to allocate memory for the PCI EPF register space. */ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, - size_t align) + size_t align, enum pci_epc_interface_type type) { - void *space; - struct device *dev = epf->epc->dev.parent; + struct pci_epf_bar *epf_bar; dma_addr_t phys_addr; + struct pci_epc *epc; + struct device *dev; + void *space; if (size < 128) size = 128; @@ -119,17 +135,26 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, else size = roundup_pow_of_two(size); + if (type == PRIMARY_INTERFACE) { + epc = epf->epc; + epf_bar = epf->bar; + } else { + epc = epf->sec_epc; + epf_bar = epf->sec_epc_bar; + } + + dev = epc->dev.parent; space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL); if (!space) { dev_err(dev, "failed to allocate mem space\n"); return NULL; } - epf->bar[bar].phys_addr = phys_addr; - epf->bar[bar].addr = space; - epf->bar[bar].size = size; - epf->bar[bar].barno = bar; - epf->bar[bar].flags |= upper_32_bits(size) ? + epf_bar[bar].phys_addr = phys_addr; + epf_bar[bar].addr = space; + epf_bar[bar].size = size; + epf_bar[bar].barno = bar; + epf_bar[bar].flags |= upper_32_bits(size) ? PCI_BASE_ADDRESS_MEM_TYPE_64 : PCI_BASE_ADDRESS_MEM_TYPE_32; diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index 88d311bad984..d9cb3944fb87 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -13,6 +13,12 @@ struct pci_epc; +enum pci_epc_interface_type { + UNKNOWN_INTERFACE = -1, + PRIMARY_INTERFACE, + SECONDARY_INTERFACE, +}; + enum pci_epc_irq_type { PCI_EPC_IRQ_UNKNOWN, PCI_EPC_IRQ_LEGACY, @@ -20,6 +26,19 @@ enum pci_epc_irq_type { PCI_EPC_IRQ_MSIX, }; +static inline const char * +pci_epc_interface_string(enum pci_epc_interface_type type) +{ + switch (type) { + case PRIMARY_INTERFACE: + return "primary"; + case SECONDARY_INTERFACE: + return "secondary"; + default: + return "UNKNOWN interface"; + } +} + /** * struct pci_epc_ops - set of function pointers for performing EPC operations * @write_header: ops to populate configuration space header @@ -175,10 +194,12 @@ __pci_epc_create(struct device *dev, const struct pci_epc_ops *ops, struct module *owner); void devm_pci_epc_destroy(struct device *dev, struct pci_epc *epc); void pci_epc_destroy(struct pci_epc *epc); -int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf); +int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type); void pci_epc_linkup(struct pci_epc *epc); void pci_epc_init_notify(struct pci_epc *epc); -void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf); +void pci_epc_remove_epf(struct pci_epc *epc, struct pci_epf *epf, + enum pci_epc_interface_type type); int pci_epc_write_header(struct pci_epc *epc, u8 func_no, struct pci_epf_header *hdr); int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index f373a134ac04..1dc66824f5a8 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -14,6 +14,7 @@ #include struct pci_epf; +enum pci_epc_interface_type; enum pci_notify_event { CORE_INIT, @@ -119,6 +120,11 @@ struct pci_epf_bar { * @list: to add pci_epf as a list of PCI endpoint functions to pci_epc * @nb: notifier block to notify EPF of any EPC events (like linkup) * @lock: mutex to protect pci_epf_ops + * @sec_epc: the secondary EPC device to which this EPF device is bound + * @sec_epc_list: to add pci_epf as list of PCI endpoint functions to secondary + * EPC device + * @sec_epc_bar: represents the BAR of EPF device associated with secondary EPC + * @sec_epc_func_no: unique (physical) function number within the secondary EPC */ struct pci_epf { struct device dev; @@ -135,6 +141,12 @@ struct pci_epf { struct notifier_block nb; /* mutex to protect against concurrent access of pci_epf_ops */ struct mutex lock; + + /* Below members are to attach secondary EPC to an endpoint function */ + struct pci_epc *sec_epc; + struct list_head sec_epc_list; + struct pci_epf_bar sec_epc_bar[6]; + u8 sec_epc_func_no; }; /** @@ -171,8 +183,9 @@ int __pci_epf_register_driver(struct pci_epf_driver *driver, struct module *owner); void pci_epf_unregister_driver(struct pci_epf_driver *driver); void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, - size_t align); -void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar); + size_t align, enum pci_epc_interface_type type); +void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, + enum pci_epc_interface_type type); int pci_epf_bind(struct pci_epf *epf); void pci_epf_unbind(struct pci_epf *epf); #endif /* __LINUX_PCI_EPF_H */ From e85a2d7837622bd99c96f5bbc7f972da90c285a2 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:27:59 +0530 Subject: [PATCH 427/633] PCI: endpoint: Add support in configfs to associate two EPCs with EPF Now that PCI endpoint core supports to add secondary endpoint controller (EPC) with endpoint function (EPF), Add support in configfs to associate two EPCs with EPF. This creates "primary" and "secondary" directory inside the directory created by users for EPF device. Users have to add a symlink of endpoint controller (pci_ep/controllers/) to "primary" or "secondary" directory to bind EPF to primary and secondary EPF interfaces respectively. Existing method of linking directory representing EPF device to directory representing EPC device to associate a single EPC device with a EPF device will continue to work. Link: https://lore.kernel.org/r/20210201195809.7342-8-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- .../PCI/endpoint/pci-endpoint-cfs.rst | 10 ++ drivers/pci/endpoint/pci-ep-cfs.c | 147 ++++++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst index 1bbd81ed06c8..696f8eeb4738 100644 --- a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst +++ b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst @@ -68,6 +68,16 @@ created) ... subsys_vendor_id ... subsys_id ... interrupt_pin + ... primary/ + ... / + ... secondary/ + ... / + +If an EPF device has to be associated with 2 EPCs (like in the case of +Non-transparent bridge), symlink of endpoint controller connected to primary +interface should be added in 'primary' directory and symlink of endpoint +controller connected to secondary interface should be added in 'secondary' +directory. EPC Device ========== diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 6ca9e2f92460..8f750961d6ab 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -21,6 +21,9 @@ static struct config_group *controllers_group; struct pci_epf_group { struct config_group group; + struct config_group primary_epc_group; + struct config_group secondary_epc_group; + struct delayed_work cfs_work; struct pci_epf *epf; int index; }; @@ -41,6 +44,127 @@ static inline struct pci_epc_group *to_pci_epc_group(struct config_item *item) return container_of(to_config_group(item), struct pci_epc_group, group); } +static int pci_secondary_epc_epf_link(struct config_item *epf_item, + struct config_item *epc_item) +{ + int ret; + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc = epc_group->epc; + struct pci_epf *epf = epf_group->epf; + + ret = pci_epc_add_epf(epc, epf, SECONDARY_INTERFACE); + if (ret) + return ret; + + ret = pci_epf_bind(epf); + if (ret) { + pci_epc_remove_epf(epc, epf, SECONDARY_INTERFACE); + return ret; + } + + return 0; +} + +static void pci_secondary_epc_epf_unlink(struct config_item *epc_item, + struct config_item *epf_item) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc; + struct pci_epf *epf; + + WARN_ON_ONCE(epc_group->start); + + epc = epc_group->epc; + epf = epf_group->epf; + pci_epf_unbind(epf); + pci_epc_remove_epf(epc, epf, SECONDARY_INTERFACE); +} + +static struct configfs_item_operations pci_secondary_epc_item_ops = { + .allow_link = pci_secondary_epc_epf_link, + .drop_link = pci_secondary_epc_epf_unlink, +}; + +static const struct config_item_type pci_secondary_epc_type = { + .ct_item_ops = &pci_secondary_epc_item_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_group +*pci_ep_cfs_add_secondary_group(struct pci_epf_group *epf_group) +{ + struct config_group *secondary_epc_group; + + secondary_epc_group = &epf_group->secondary_epc_group; + config_group_init_type_name(secondary_epc_group, "secondary", + &pci_secondary_epc_type); + configfs_register_group(&epf_group->group, secondary_epc_group); + + return secondary_epc_group; +} + +static int pci_primary_epc_epf_link(struct config_item *epf_item, + struct config_item *epc_item) +{ + int ret; + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc = epc_group->epc; + struct pci_epf *epf = epf_group->epf; + + ret = pci_epc_add_epf(epc, epf, PRIMARY_INTERFACE); + if (ret) + return ret; + + ret = pci_epf_bind(epf); + if (ret) { + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); + return ret; + } + + return 0; +} + +static void pci_primary_epc_epf_unlink(struct config_item *epc_item, + struct config_item *epf_item) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); + struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); + struct pci_epc *epc; + struct pci_epf *epf; + + WARN_ON_ONCE(epc_group->start); + + epc = epc_group->epc; + epf = epf_group->epf; + pci_epf_unbind(epf); + pci_epc_remove_epf(epc, epf, PRIMARY_INTERFACE); +} + +static struct configfs_item_operations pci_primary_epc_item_ops = { + .allow_link = pci_primary_epc_epf_link, + .drop_link = pci_primary_epc_epf_unlink, +}; + +static const struct config_item_type pci_primary_epc_type = { + .ct_item_ops = &pci_primary_epc_item_ops, + .ct_owner = THIS_MODULE, +}; + +static struct config_group +*pci_ep_cfs_add_primary_group(struct pci_epf_group *epf_group) +{ + struct config_group *primary_epc_group = &epf_group->primary_epc_group; + + config_group_init_type_name(primary_epc_group, "primary", + &pci_primary_epc_type); + configfs_register_group(&epf_group->group, primary_epc_group); + + return primary_epc_group; +} + static ssize_t pci_epc_start_store(struct config_item *item, const char *page, size_t len) { @@ -372,6 +496,25 @@ static const struct config_item_type pci_epf_type = { .ct_owner = THIS_MODULE, }; +static void pci_epf_cfs_work(struct work_struct *work) +{ + struct pci_epf_group *epf_group; + struct config_group *group; + + epf_group = container_of(work, struct pci_epf_group, cfs_work.work); + group = pci_ep_cfs_add_primary_group(epf_group); + if (IS_ERR(group)) { + pr_err("failed to create 'primary' EPC interface\n"); + return; + } + + group = pci_ep_cfs_add_secondary_group(epf_group); + if (IS_ERR(group)) { + pr_err("failed to create 'secondary' EPC interface\n"); + return; + } +} + static struct config_group *pci_epf_make(struct config_group *group, const char *name) { @@ -414,6 +557,10 @@ static struct config_group *pci_epf_make(struct config_group *group, kfree(epf_name); + INIT_DELAYED_WORK(&epf_group->cfs_work, pci_epf_cfs_work); + queue_delayed_work(system_wq, &epf_group->cfs_work, + msecs_to_jiffies(1)); + return &epf_group->group; free_name: From 87d5972e476f6c4e98a0abce713c54c6f40661b0 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:00 +0530 Subject: [PATCH 428/633] PCI: endpoint: Add pci_epc_ops to map MSI IRQ Add pci_epc_ops to map physical address to MSI address and return MSI data. The physical address is an address in the outbound region. This is required to implement doorbell functionality of NTB (non-transparent bridge) wherein EPC on either side of the interface (primary and secondary) can directly write to the physical address (in outbound region) of the other interface to ring doorbell using MSI. Link: https://lore.kernel.org/r/20210201195809.7342-9-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-epc-core.c | 41 +++++++++++++++++++++++++++++ include/linux/pci-epc.h | 8 ++++++ 2 files changed, 49 insertions(+) diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 3693eca5b030..cc8f9eb2b177 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -230,6 +230,47 @@ int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, } EXPORT_SYMBOL_GPL(pci_epc_raise_irq); +/** + * pci_epc_map_msi_irq() - Map physical address to MSI address and return + * MSI data + * @epc: the EPC device which has the MSI capability + * @func_no: the physical endpoint function number in the EPC device + * @phys_addr: the physical address of the outbound region + * @interrupt_num: the MSI interrupt number + * @entry_size: Size of Outbound address region for each interrupt + * @msi_data: the data that should be written in order to raise MSI interrupt + * with interrupt number as 'interrupt num' + * @msi_addr_offset: Offset of MSI address from the aligned outbound address + * to which the MSI address is mapped + * + * Invoke to map physical address to MSI address and return MSI data. The + * physical address should be an address in the outbound region. This is + * required to implement doorbell functionality of NTB wherein EPC on either + * side of the interface (primary and secondary) can directly write to the + * physical address (in outbound region) of the other interface to ring + * doorbell. + */ +int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, phys_addr_t phys_addr, + u8 interrupt_num, u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset) +{ + int ret; + + if (IS_ERR_OR_NULL(epc)) + return -EINVAL; + + if (!epc->ops->map_msi_irq) + return -EINVAL; + + mutex_lock(&epc->lock); + ret = epc->ops->map_msi_irq(epc, func_no, phys_addr, interrupt_num, + entry_size, msi_data, msi_addr_offset); + mutex_unlock(&epc->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(pci_epc_map_msi_irq); + /** * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated * @epc: the EPC device to which MSI interrupts was requested diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index d9cb3944fb87..b82c9b100e97 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -55,6 +55,7 @@ pci_epc_interface_string(enum pci_epc_interface_type type) * @get_msix: ops to get the number of MSI-X interrupts allocated by the RC * from the MSI-X capability register * @raise_irq: ops to raise a legacy, MSI or MSI-X interrupt + * @map_msi_irq: ops to map physical address to MSI address and return MSI data * @start: ops to start the PCI link * @stop: ops to stop the PCI link * @owner: the module owner containing the ops @@ -77,6 +78,10 @@ struct pci_epc_ops { int (*get_msix)(struct pci_epc *epc, u8 func_no); int (*raise_irq)(struct pci_epc *epc, u8 func_no, enum pci_epc_irq_type type, u16 interrupt_num); + int (*map_msi_irq)(struct pci_epc *epc, u8 func_no, + phys_addr_t phys_addr, u8 interrupt_num, + u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset); int (*start)(struct pci_epc *epc); void (*stop)(struct pci_epc *epc); const struct pci_epc_features* (*get_features)(struct pci_epc *epc, @@ -216,6 +221,9 @@ int pci_epc_get_msi(struct pci_epc *epc, u8 func_no); int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, enum pci_barno, u32 offset); int pci_epc_get_msix(struct pci_epc *epc, u8 func_no); +int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, + phys_addr_t phys_addr, u8 interrupt_num, + u32 entry_size, u32 *msi_data, u32 *msi_addr_offset); int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, enum pci_epc_irq_type type, u16 interrupt_num); int pci_epc_start(struct pci_epc *epc); From 256ae475201b16fd69e00dd6c2d14035e4ea5745 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:01 +0530 Subject: [PATCH 429/633] PCI: endpoint: Add pci_epf_ops to expose function-specific attrs In addition to the attributes that are generic across function drivers documented in Documentation/PCI/endpoint/pci-endpoint-cfs.rst, there could be function-specific attributes that has to be exposed by the function driver to be configured by the user. Add ->add_cfs() in pci_epf_ops to be populated by the function driver if it has to expose any function-specific attributes and pci_epf_type_add_cfs() to be invoked by pci-ep-cfs.c when sub-directory to main function directory is created. Link: https://lore.kernel.org/r/20210201195809.7342-10-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-epf-core.c | 32 +++++++++++++++++++++++++++++ include/linux/pci-epf.h | 5 +++++ 2 files changed, 37 insertions(+) diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index 79329ec6373c..7646c8660d42 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -20,6 +20,38 @@ static DEFINE_MUTEX(pci_epf_mutex); static struct bus_type pci_epf_bus_type; static const struct device_type pci_epf_type; +/** + * pci_epf_type_add_cfs() - Help function drivers to expose function specific + * attributes in configfs + * @epf: the EPF device that has to be configured using configfs + * @group: the parent configfs group (corresponding to entries in + * pci_epf_device_id) + * + * Invoke to expose function specific attributes in configfs. If the function + * driver does not have anything to expose (attributes configured by user), + * return NULL. + */ +struct config_group *pci_epf_type_add_cfs(struct pci_epf *epf, + struct config_group *group) +{ + struct config_group *epf_type_group; + + if (!epf->driver) { + dev_err(&epf->dev, "epf device not bound to driver\n"); + return NULL; + } + + if (!epf->driver->ops->add_cfs) + return NULL; + + mutex_lock(&epf->lock); + epf_type_group = epf->driver->ops->add_cfs(epf, group); + mutex_unlock(&epf->lock); + + return epf_type_group; +} +EXPORT_SYMBOL_GPL(pci_epf_type_add_cfs); + /** * pci_epf_unbind() - Notify the function driver that the binding between the * EPF device and EPC device has been lost diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 1dc66824f5a8..b241e7dd171f 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -62,10 +62,13 @@ struct pci_epf_header { * @bind: ops to perform when a EPC device has been bound to EPF device * @unbind: ops to perform when a binding has been lost between a EPC device * and EPF device + * @add_cfs: ops to initialize function specific configfs attributes */ struct pci_epf_ops { int (*bind)(struct pci_epf *epf); void (*unbind)(struct pci_epf *epf); + struct config_group *(*add_cfs)(struct pci_epf *epf, + struct config_group *group); }; /** @@ -188,4 +191,6 @@ void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, enum pci_epc_interface_type type); int pci_epf_bind(struct pci_epf *epf); void pci_epf_unbind(struct pci_epf *epf); +struct config_group *pci_epf_type_add_cfs(struct pci_epf *epf, + struct config_group *group); #endif /* __LINUX_PCI_EPF_H */ From 38ad827e3bc0f0e94628ee1d8dc31e778d9be40f Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:02 +0530 Subject: [PATCH 430/633] PCI: endpoint: Allow user to create sub-directory of 'EPF Device' directory Documentation/PCI/endpoint/pci-endpoint-cfs.rst explains how a user has to create a directory in-order to create a 'EPF Device' that can be configured/probed by 'EPF Driver'. Allow user to create a sub-directory of 'EPF Device' directory for any function specific attributes that has to be exposed to the user. Link: https://lore.kernel.org/r/20210201195809.7342-11-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/pci-ep-cfs.c | 23 +++++++++++++++++++++++ include/linux/pci-epf.h | 3 +++ 2 files changed, 26 insertions(+) diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 8f750961d6ab..f3a8b833b479 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -490,7 +490,29 @@ static struct configfs_item_operations pci_epf_ops = { .release = pci_epf_release, }; +static struct config_group *pci_epf_type_make(struct config_group *group, + const char *name) +{ + struct pci_epf_group *epf_group = to_pci_epf_group(&group->cg_item); + struct config_group *epf_type_group; + + epf_type_group = pci_epf_type_add_cfs(epf_group->epf, group); + return epf_type_group; +} + +static void pci_epf_type_drop(struct config_group *group, + struct config_item *item) +{ + config_item_put(item); +} + +static struct configfs_group_operations pci_epf_type_group_ops = { + .make_group = &pci_epf_type_make, + .drop_item = &pci_epf_type_drop, +}; + static const struct config_item_type pci_epf_type = { + .ct_group_ops = &pci_epf_type_group_ops, .ct_item_ops = &pci_epf_ops, .ct_attrs = pci_epf_attrs, .ct_owner = THIS_MODULE, @@ -553,6 +575,7 @@ static struct config_group *pci_epf_make(struct config_group *group, goto free_name; } + epf->group = &epf_group->group; epf_group->epf = epf; kfree(epf_name); diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index b241e7dd171f..6833e2160ef1 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -9,6 +9,7 @@ #ifndef __LINUX_PCI_EPF_H #define __LINUX_PCI_EPF_H +#include #include #include #include @@ -128,6 +129,7 @@ struct pci_epf_bar { * EPC device * @sec_epc_bar: represents the BAR of EPF device associated with secondary EPC * @sec_epc_func_no: unique (physical) function number within the secondary EPC + * @group: configfs group associated with the EPF device */ struct pci_epf { struct device dev; @@ -150,6 +152,7 @@ struct pci_epf { struct list_head sec_epc_list; struct pci_epf_bar sec_epc_bar[6]; u8 sec_epc_func_no; + struct config_group *group; }; /** From dbcc542f36086abcaec28a858b17f2c358d57973 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:03 +0530 Subject: [PATCH 431/633] PCI: cadence: Implement ->msi_map_irq() ops Implement ->msi_map_irq() ops in order to map physical address to MSI address and return MSI data. Link: https://lore.kernel.org/r/20210201195809.7342-12-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Tom Joseph --- .../pci/controller/cadence/pcie-cadence-ep.c | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index 9e2b024d32f2..dc88078194cb 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -382,6 +382,57 @@ static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, return 0; } +static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, + phys_addr_t addr, u8 interrupt_num, + u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET; + struct cdns_pcie *pcie = &ep->pcie; + u64 pci_addr, pci_addr_mask = 0xff; + u16 flags, mme, data, data_mask; + u8 msi_count; + int ret; + int i; + + /* Check whether the MSI feature has been enabled by the PCI host. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + if (!(flags & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + /* Get the number of enabled MSIs */ + mme = (flags & PCI_MSI_FLAGS_QSIZE) >> 4; + msi_count = 1 << mme; + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + /* Compute the data value to be written. */ + data_mask = msi_count - 1; + data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64); + data = data & ~data_mask; + + /* Get the PCI address where to write the data into. */ + pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI); + pci_addr <<= 32; + pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO); + pci_addr &= GENMASK_ULL(63, 2); + + for (i = 0; i < interrupt_num; i++) { + ret = cdns_pcie_ep_map_addr(epc, fn, addr, + pci_addr & ~pci_addr_mask, + entry_size); + if (ret) + return ret; + addr = addr + entry_size; + } + + *msi_data = data; + *msi_addr_offset = pci_addr & pci_addr_mask; + + return 0; +} + static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u16 interrupt_num) { @@ -481,6 +532,7 @@ static const struct pci_epc_features cdns_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, .msix_capable = true, + .align = 256, }; static const struct pci_epc_features* @@ -500,6 +552,7 @@ static const struct pci_epc_ops cdns_pcie_epc_ops = { .set_msix = cdns_pcie_ep_set_msix, .get_msix = cdns_pcie_ep_get_msix, .raise_irq = cdns_pcie_ep_raise_irq, + .map_msi_irq = cdns_pcie_ep_map_msi_irq, .start = cdns_pcie_ep_start, .get_features = cdns_pcie_ep_get_features, }; From a62074a9ba856082a60ff60693abd79f4b55177d Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:04 +0530 Subject: [PATCH 432/633] PCI: cadence: Configure LM_EP_FUNC_CFG based on epc->function_num_map The number of functions supported by the endpoint controller is configured in LM_EP_FUNC_CFG based on func_no member of struct pci_epf. Now that an endpoint function can be associated with two endpoint controllers (primary and secondary), just using func_no will not suffice as that will take into account only if the endpoint controller is associated with the primary interface of endpoint function. Instead use epc->function_num_map which will already have the configured functions information (irrespective of whether the endpoint controller is associated with primary or secondary interface). Link: https://lore.kernel.org/r/20210201195809.7342-13-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Tom Joseph --- drivers/pci/controller/cadence/pcie-cadence-ep.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index dc88078194cb..897cdde02bd8 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -506,18 +506,13 @@ static int cdns_pcie_ep_start(struct pci_epc *epc) struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; struct device *dev = pcie->dev; - struct pci_epf *epf; - u32 cfg; int ret; /* * BIT(0) is hardwired to 1, hence function 0 is always enabled * and can't be disabled anyway. */ - cfg = BIT(0); - list_for_each_entry(epf, &epc->pci_epf, list) - cfg |= BIT(epf->func_no); - cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, cfg); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, epc->function_num_map); ret = cdns_pcie_start_link(pcie); if (ret) { From 8b821cf761503b80d0bd052f932adfe1bc1a0088 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:05 +0530 Subject: [PATCH 433/633] PCI: endpoint: Add EP function driver to provide NTB functionality Add a new endpoint function driver to provide NTB functionality using multiple PCIe endpoint instances. [arnd@arndb.de: Select configfs dependency] [yebin10@huawei.com: Fix unused but set variables] [geert+renesas@glider.be: Explain NTB in PCI_EPF_NTB help text] Link: https://lore.kernel.org/r/20210201195809.7342-14-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Arnd Bergmann Signed-off-by: Ye Bin Signed-off-by: Geert Uytterhoeven Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/endpoint/functions/Kconfig | 13 + drivers/pci/endpoint/functions/Makefile | 1 + drivers/pci/endpoint/functions/pci-epf-ntb.c | 2128 ++++++++++++++++++ 3 files changed, 2142 insertions(+) create mode 100644 drivers/pci/endpoint/functions/pci-epf-ntb.c diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig index 8820d0f7ec77..5f1242ca2f4e 100644 --- a/drivers/pci/endpoint/functions/Kconfig +++ b/drivers/pci/endpoint/functions/Kconfig @@ -12,3 +12,16 @@ config PCI_EPF_TEST for PCI Endpoint. If in doubt, say "N" to disable Endpoint test driver. + +config PCI_EPF_NTB + tristate "PCI Endpoint NTB driver" + depends on PCI_ENDPOINT + select CONFIGFS_FS + help + Select this configuration option to enable the Non-Transparent + Bridge (NTB) driver for PCI Endpoint. NTB driver implements NTB + controller functionality using multiple PCIe endpoint instances. + It can support NTB endpoint function devices created using + device tree. + + If in doubt, say "N" to disable Endpoint NTB driver. diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile index d6fafff080e2..96ab932a537a 100644 --- a/drivers/pci/endpoint/functions/Makefile +++ b/drivers/pci/endpoint/functions/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_PCI_EPF_TEST) += pci-epf-test.o +obj-$(CONFIG_PCI_EPF_NTB) += pci-epf-ntb.o diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c new file mode 100644 index 000000000000..338148cf56f5 --- /dev/null +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -0,0 +1,2128 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Endpoint Function Driver to implement Non-Transparent Bridge functionality + * + * Copyright (C) 2020 Texas Instruments + * Author: Kishon Vijay Abraham I + */ + +/* + * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint + * (EP) controller instances (see diagram below) in such a way that + * transactions from one EP controller are routed to the other EP controller. + * Once PCI NTB function driver configures the SoC with multiple EP instances, + * HOST1 and HOST2 can communicate with each other using SoC as a bridge. + * + * +-------------+ +-------------+ + * | | | | + * | HOST1 | | HOST2 | + * | | | | + * +------^------+ +------^------+ + * | | + * | | + * +---------|-------------------------------------------------|---------+ + * | +------v------+ +------v------+ | + * | | | | | | + * | | EP | | EP | | + * | | CONTROLLER1 | | CONTROLLER2 | | + * | | <-----------------------------------> | | + * | | | | | | + * | | | | | | + * | | | SoC With Multiple EP Instances | | | + * | | | (Configured using NTB Function) | | | + * | +-------------+ +-------------+ | + * +---------------------------------------------------------------------+ + */ + +#include +#include +#include +#include + +#include +#include + +static struct workqueue_struct *kpcintb_workqueue; + +#define COMMAND_CONFIGURE_DOORBELL 1 +#define COMMAND_TEARDOWN_DOORBELL 2 +#define COMMAND_CONFIGURE_MW 3 +#define COMMAND_TEARDOWN_MW 4 +#define COMMAND_LINK_UP 5 +#define COMMAND_LINK_DOWN 6 + +#define COMMAND_STATUS_OK 1 +#define COMMAND_STATUS_ERROR 2 + +#define LINK_STATUS_UP BIT(0) + +#define SPAD_COUNT 64 +#define DB_COUNT 4 +#define NTB_MW_OFFSET 2 +#define DB_COUNT_MASK GENMASK(15, 0) +#define MSIX_ENABLE BIT(16) +#define MAX_DB_COUNT 32 +#define MAX_MW 4 + +enum epf_ntb_bar { + BAR_CONFIG, + BAR_PEER_SPAD, + BAR_DB_MW1, + BAR_MW2, + BAR_MW3, + BAR_MW4, +}; + +struct epf_ntb { + u32 num_mws; + u32 db_count; + u32 spad_count; + struct pci_epf *epf; + u64 mws_size[MAX_MW]; + struct config_group group; + struct epf_ntb_epc *epc[2]; +}; + +#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) + +struct epf_ntb_epc { + u8 func_no; + bool linkup; + bool is_msix; + int msix_bar; + u32 spad_size; + struct pci_epc *epc; + struct epf_ntb *epf_ntb; + void __iomem *mw_addr[6]; + size_t msix_table_offset; + struct epf_ntb_ctrl *reg; + struct pci_epf_bar *epf_bar; + enum pci_barno epf_ntb_bar[6]; + struct delayed_work cmd_handler; + enum pci_epc_interface_type type; + const struct pci_epc_features *epc_features; +}; + +struct epf_ntb_ctrl { + u32 command; + u32 argument; + u16 command_status; + u16 link_status; + u32 topology; + u64 addr; + u64 size; + u32 num_mws; + u32 mw1_offset; + u32 spad_offset; + u32 spad_count; + u32 db_entry_size; + u32 db_data[MAX_DB_COUNT]; + u32 db_offset[MAX_DB_COUNT]; +} __packed; + +static struct pci_epf_header epf_ntb_header = { + .vendorid = PCI_ANY_ID, + .deviceid = PCI_ANY_ID, + .baseclass_code = PCI_BASE_CLASS_MEMORY, + .interrupt_pin = PCI_INTERRUPT_INTA, +}; + +/** + * epf_ntb_link_up() - Raise link_up interrupt to both the hosts + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @link_up: true or false indicating Link is UP or Down + * + * Once NTB function in HOST1 and the NTB function in HOST2 invoke + * ntb_link_enable(), this NTB function driver will trigger a link event to + * the NTB client in both the hosts. + */ +static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) +{ + enum pci_epc_interface_type type; + enum pci_epc_irq_type irq_type; + struct epf_ntb_epc *ntb_epc; + struct epf_ntb_ctrl *ctrl; + struct pci_epc *epc; + bool is_msix; + u8 func_no; + int ret; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + is_msix = ntb_epc->is_msix; + ctrl = ntb_epc->reg; + if (link_up) + ctrl->link_status |= LINK_STATUS_UP; + else + ctrl->link_status &= ~LINK_STATUS_UP; + irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; + ret = pci_epc_raise_irq(epc, func_no, irq_type, 1); + if (ret) { + dev_err(&epc->dev, + "%s intf: Failed to raise Link Up IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host + * to access the memory window of other host + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * This function performs stage (B) in the above diagram (see MW1) i.e., map OB + * address space of memory window to PCI address space. + * + * This operation requires 3 parameters + * 1) Address in the outbound address space + * 2) Address in the PCI Address space + * 3) Size of the address region to be mapped + * + * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is + * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3 + * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This + * is populated in epf_ntb_alloc_peer_mem() in this driver. + * + * The address and size of the PCI address region that has to be mapped would + * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is + * connected to HOST2. + * + * Please note Memory window1 (MW1) and Doorbell registers together will be + * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's + * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1], + * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2]. + */ +static int epf_ntb_configure_mw(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u32 mw) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + struct epf_ntb_ctrl *ctrl; + phys_addr_t phys_addr; + struct pci_epc *epc; + u64 addr, size; + int ret = 0; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + phys_addr = peer_epf_bar->phys_addr; + ctrl = ntb_epc->reg; + addr = ctrl->addr; + size = ctrl->size; + if (mw + NTB_MW_OFFSET == BAR_DB_MW1) + phys_addr += ctrl->mw1_offset; + + if (size > ntb->mws_size[mw]) { + dev_err(&epc->dev, + "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n", + pci_epc_interface_string(type), mw, size, + ntb->mws_size[mw]); + ret = -EINVAL; + goto err_invalid_size; + } + + func_no = ntb_epc->func_no; + + ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size); + if (ret) + dev_err(&epc->dev, + "%s intf: Failed to map memory window %d address\n", + pci_epc_interface_string(type), mw); + +err_invalid_size: + + return ret; +} + +/** + * epf_ntb_teardown_mw() - Teardown the configured OB ATU + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @mw: Index of the memory window (either 0, 1, 2 or 3) + * + * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using + * pci_epc_unmap_addr() + */ +static void epf_ntb_teardown_mw(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u32 mw) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + struct epf_ntb_ctrl *ctrl; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + phys_addr = peer_epf_bar->phys_addr; + ctrl = ntb_epc->reg; + if (mw + NTB_MW_OFFSET == BAR_DB_MW1) + phys_addr += ctrl->mw1_offset; + func_no = ntb_epc->func_no; + + pci_epc_unmap_addr(epc, func_no, phys_addr); +} + +/** + * epf_ntb_configure_msi() - Map OB address space to MSI address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Number of doorbell interrupts to map + * + *+-----------------+ +----->+----------------+-----------+-----------------+ + *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS | + *+-----------------+ | +----------------+ | +-----------------+ + *| BAR1 | | | Doorbell 2 +---+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR2 | | Doorbell 3 +---+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR3 | | | Doorbell 4 +---+ | | + *+-----------------+ | |----------------+ | | + *| BAR4 | | | | | | + *+-----------------+ | | MW1 | | | + *| BAR5 | | | | | | + *+-----------------+ +----->-----------------+ | | + * EP CONTROLLER 1 | | | | + * | | | | + * +----------------+ +-----------------+ + * (A) EP CONTROLLER 2 | | + * (OB SPACE) | | + * | MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * + * This function performs stage (B) in the above diagram (see Doorbell 1, + * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to + * doorbell to MSI address in PCI address space. + * + * This operation requires 3 parameters + * 1) Address reserved for doorbell in the outbound address space + * 2) MSI-X address in the PCIe Address space + * 3) Number of MSI-X interrupts that has to be configured + * + * The address in the outbound address space (for the Doorbell) is stored in + * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to + * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along + * with address for MW1. + * + * pci_epc_map_msi_irq() takes the MSI address from MSI capability register + * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI + * address. + * + * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt + * in db_data of the peer's control region. This helps the peer to raise + * doorbell of the other host by writing db_data to the BAR corresponding to + * BAR_DB_MW1. + */ +static int epf_ntb_configure_msi(struct epf_ntb *ntb, + enum pci_epc_interface_type type, u16 db_count) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + u32 db_entry_size, db_data, db_offset; + struct pci_epf_bar *peer_epf_bar; + struct epf_ntb_ctrl *peer_ctrl; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + int ret, i; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + peer_ctrl = peer_ntb_epc->reg; + db_entry_size = peer_ctrl->db_entry_size; + + phys_addr = peer_epf_bar->phys_addr; + func_no = ntb_epc->func_no; + + ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count, + db_entry_size, &db_data, &db_offset); + if (ret) { + dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + + for (i = 0; i < db_count; i++) { + peer_ctrl->db_data[i] = db_data | i; + peer_ctrl->db_offset[i] = db_offset; + } + + return 0; +} + +/** + * epf_ntb_configure_msix() - Map OB address space to MSI-X address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Number of doorbell interrupts to map + * + *+-----------------+ +----->+----------------+-----------+-----------------+ + *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | + *+-----------------+ | +----------------+ +-----------------+ + *| BAR1 | | | Doorbell 2 +---------+ | | + *+-----------------+----+ +----------------+ | | | + *| BAR2 | | Doorbell 3 +-------+ | +-----------------+ + *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | + *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + *+-----------------+ | |----------------+ | | | | + *| BAR4 | | | | | | +-----------------+ + *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3|| + *| BAR5 | | | | | +-----------------+ + *+-----------------+ +----->-----------------+ | | | + * EP CONTROLLER 1 | | | +-----------------+ + * | | +---->+ MSI-X ADDRESS 4 | + * +----------------+ +-----------------+ + * (A) EP CONTROLLER 2 | | + * (OB SPACE) | | + * | MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * This function performs stage (B) in the above diagram (see Doorbell 1, + * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to + * doorbell to MSI-X address in PCI address space. + * + * This operation requires 3 parameters + * 1) Address reserved for doorbell in the outbound address space + * 2) MSI-X address in the PCIe Address space + * 3) Number of MSI-X interrupts that has to be configured + * + * The address in the outbound address space (for the Doorbell) is stored in + * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to + * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along + * with address for MW1. + * + * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and + * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected + * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the + * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix() + * gets the MSI-X address and data. + * + * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt + * in db_data of the peer's control region. This helps the peer to raise + * doorbell of the other host by writing db_data to the BAR corresponding to + * BAR_DB_MW1. + */ +static int epf_ntb_configure_msix(struct epf_ntb *ntb, + enum pci_epc_interface_type type, + u16 db_count) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar, *epf_bar; + struct pci_epf_msix_tbl *msix_tbl; + struct epf_ntb_ctrl *peer_ctrl; + u32 db_entry_size, msg_data; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + size_t align; + u64 msg_addr; + u8 func_no; + int ret, i; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; + msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + phys_addr = peer_epf_bar->phys_addr; + peer_ctrl = peer_ntb_epc->reg; + epc_features = ntb_epc->epc_features; + align = epc_features->align; + + func_no = ntb_epc->func_no; + db_entry_size = peer_ctrl->db_entry_size; + + for (i = 0; i < db_count; i++) { + msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); + msg_data = msix_tbl[i].msg_data; + ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr, + db_entry_size); + if (ret) { + dev_err(&epc->dev, + "%s intf: Failed to configure MSI-X IRQ\n", + pci_epc_interface_string(type)); + return ret; + } + phys_addr = phys_addr + db_entry_size; + peer_ctrl->db_data[i] = msg_data; + peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); + } + ntb_epc->is_msix = true; + + return 0; +} + +/** + * epf_ntb_configure_db() - Configure the Outbound Address Space for one host + * to ring the doorbell of other host + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * @db_count: Count of the number of doorbells that has to be configured + * @msix: Indicates whether MSI-X or MSI should be used + * + * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for + * one HOST to ring the doorbell of other HOST. + */ +static int epf_ntb_configure_db(struct epf_ntb *ntb, + enum pci_epc_interface_type type, + u16 db_count, bool msix) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + int ret; + + if (db_count > MAX_DB_COUNT) + return -EINVAL; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + if (msix) + ret = epf_ntb_configure_msix(ntb, type, db_count); + else + ret = epf_ntb_configure_msi(ntb, type, db_count); + + if (ret) + dev_err(&epc->dev, "%s intf: Failed to configure DB\n", + pci_epc_interface_string(type)); + + return ret; +} + +/** + * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X + * address + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address. + */ +static void +epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar; + enum pci_barno peer_barno; + phys_addr_t phys_addr; + struct pci_epc *epc; + u8 func_no; + + ntb_epc = ntb->epc[type]; + epc = ntb_epc->epc; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + phys_addr = peer_epf_bar->phys_addr; + func_no = ntb_epc->func_no; + + pci_epc_unmap_addr(epc, func_no, phys_addr); +} + +/** + * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host + * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY) + * + * Workqueue function that gets invoked for the two epf_ntb_epc + * periodically (once every 5ms) to see if it has received any commands + * from NTB host. The host can send commands to configure doorbell or + * configure memory window or to update link status. + */ +static void epf_ntb_cmd_handler(struct work_struct *work) +{ + enum pci_epc_interface_type type; + struct epf_ntb_epc *ntb_epc; + struct epf_ntb_ctrl *ctrl; + u32 command, argument; + struct epf_ntb *ntb; + struct device *dev; + u16 db_count; + bool is_msix; + int ret; + + ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work); + ctrl = ntb_epc->reg; + command = ctrl->command; + if (!command) + goto reset_handler; + argument = ctrl->argument; + + ctrl->command = 0; + ctrl->argument = 0; + + ctrl = ntb_epc->reg; + type = ntb_epc->type; + ntb = ntb_epc->epf_ntb; + dev = &ntb->epf->dev; + + switch (command) { + case COMMAND_CONFIGURE_DOORBELL: + db_count = argument & DB_COUNT_MASK; + is_msix = argument & MSIX_ENABLE; + ret = epf_ntb_configure_db(ntb, type, db_count, is_msix); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_DOORBELL: + epf_ntb_teardown_db(ntb, type); + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_CONFIGURE_MW: + ret = epf_ntb_configure_mw(ntb, type, argument); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_TEARDOWN_MW: + epf_ntb_teardown_mw(ntb, type, argument); + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_LINK_UP: + ntb_epc->linkup = true; + if (ntb->epc[PRIMARY_INTERFACE]->linkup && + ntb->epc[SECONDARY_INTERFACE]->linkup) { + ret = epf_ntb_link_up(ntb, true); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + goto reset_handler; + } + ctrl->command_status = COMMAND_STATUS_OK; + break; + case COMMAND_LINK_DOWN: + ntb_epc->linkup = false; + ret = epf_ntb_link_up(ntb, false); + if (ret < 0) + ctrl->command_status = COMMAND_STATUS_ERROR; + else + ctrl->command_status = COMMAND_STATUS_OK; + break; + default: + dev_err(dev, "%s intf UNKNOWN command: %d\n", + pci_epc_interface_string(type), command); + break; + } + +reset_handler: + queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, + msecs_to_jiffies(5)); +} + +/** + * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + *+-----------------+------->+------------------+ +-----------------+ + *| BAR0 | | CONFIG REGION | | BAR0 | + *+-----------------+----+ +------------------+<-------+-----------------+ + *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | + *+-----------------+ +-->+------------------+<-------+-----------------+ + *| BAR2 | Local Memory | BAR2 | + *+-----------------+ +-----------------+ + *| BAR3 | | BAR3 | + *+-----------------+ +-----------------+ + *| BAR4 | | BAR4 | + *+-----------------+ +-----------------+ + *| BAR5 | | BAR5 | + *+-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad + * region. While BAR1 is the default peer scratchpad BAR, an NTB could have + * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). + * This function can get the exact BAR used for peer scratchpad from + * epf_ntb_bar[BAR_PEER_SPAD]. + * + * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function + * gets the address of peer scratchpad from + * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. + */ +static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); +} + +/** + * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + *+-----------------+------->+------------------+ +-----------------+ + *| BAR0 | | CONFIG REGION | | BAR0 | + *+-----------------+----+ +------------------+<-------+-----------------+ + *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | + *+-----------------+ +-->+------------------+<-------+-----------------+ + *| BAR2 | Local Memory | BAR2 | + *+-----------------+ +-----------------+ + *| BAR3 | | BAR3 | + *+-----------------+ +-----------------+ + *| BAR4 | | BAR4 | + *+-----------------+ +-----------------+ + *| BAR5 | | BAR5 | + *+-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad + * region. While BAR1 is the default peer scratchpad BAR, an NTB could have + * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). + * This function can get the exact BAR used for peer scratchpad from + * epf_ntb_bar[BAR_PEER_SPAD]. + * + * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function + * gets the address of peer scratchpad from + * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. + */ +static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *peer_epf_bar, *epf_bar; + enum pci_barno peer_barno, barno; + u32 peer_spad_offset; + struct pci_epc *epc; + struct device *dev; + u8 func_no; + int ret; + + dev = &ntb->epf->dev; + + peer_ntb_epc = ntb->epc[!type]; + peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; + peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; + + ntb_epc = ntb->epc[type]; + barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + epf_bar = &ntb_epc->epf_bar[barno]; + func_no = ntb_epc->func_no; + epc = ntb_epc->epc; + + peer_spad_offset = peer_ntb_epc->reg->spad_offset; + epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; + epf_bar->size = peer_ntb_epc->spad_size; + epf_bar->barno = barno; + epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s intf: peer SPAD BAR set failed\n", + pci_epc_interface_string(type)); + return ret; + } + + return 0; +} + +/** + * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region (removes inbound ATU configuration). While BAR0 is + * the default self scratchpad BAR, an NTB could have other BARs for self + * scratchpad (because of reserved BARs). This function can get the exact BAR + * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. Also note HOST2's peer + * scratchpad is HOST1's self scratchpad. + */ +static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); +} + +/** + * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and + * self scratchpad region. While BAR0 is the default self scratchpad BAR, an + * NTB could have other BARs for self scratchpad (because of reserved BARs). + * This function can get the exact BAR used for self scratchpad from + * epf_ntb_bar[BAR_CONFIG]. + * + * Please note the self scratchpad region and config region is combined to + * a single region and mapped using the same BAR. Also note HOST2's peer + * scratchpad is HOST1's self scratchpad. + */ +static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum pci_barno barno; + struct epf_ntb *ntb; + struct pci_epc *epc; + struct device *dev; + u8 func_no; + int ret; + + ntb = ntb_epc->epf_ntb; + dev = &ntb->epf->dev; + + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + epf_bar = &ntb_epc->epf_bar[barno]; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", + pci_epc_interface_string(ntb_epc->type)); + return ret; + } + + return 0; +} + +/** + * epf_ntb_config_spad_bar_free() - Free the physical memory associated with + * config + scratchpad region + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Free the Local Memory mentioned in the above diagram. After invoking this + * function, any of config + self scratchpad region of HOST1 or peer scratchpad + * region of HOST2 should not be accessed. + */ +static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct epf_ntb_epc *ntb_epc; + enum pci_barno barno; + struct pci_epf *epf; + + epf = ntb->epf; + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ntb_epc = ntb->epc[type]; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + if (ntb_epc->reg) + pci_epf_free_space(epf, ntb_epc->reg, barno, type); + } +} + +/** + * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad + * region + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * +-----------------+------->+------------------+ +-----------------+ + * | BAR0 | | CONFIG REGION | | BAR0 | + * +-----------------+----+ +------------------+<-------+-----------------+ + * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + * +-----------------+ +-->+------------------+<-------+-----------------+ + * | BAR2 | Local Memory | BAR2 | + * +-----------------+ +-----------------+ + * | BAR3 | | BAR3 | + * +-----------------+ +-----------------+ + * | BAR4 | | BAR4 | + * +-----------------+ +-----------------+ + * | BAR5 | | BAR5 | + * +-----------------+ +-----------------+ + * EP CONTROLLER 1 EP CONTROLLER 2 + * + * Allocate the Local Memory mentioned in the above diagram. The size of + * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION + * is obtained from "spad-count" configfs entry. + * + * The size of both config region and scratchpad region has to be aligned, + * since the scratchpad region will also be mapped as PEER SCRATCHPAD of + * other host using a separate BAR. + */ +static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *peer_epc_features, *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + size_t msix_table_size, pba_size, align; + enum pci_barno peer_barno, barno; + struct epf_ntb_ctrl *ctrl; + u32 spad_size, ctrl_size; + u64 size, peer_size; + struct pci_epf *epf; + struct device *dev; + bool msix_capable; + u32 spad_count; + void *base; + + epf = ntb->epf; + dev = &epf->dev; + ntb_epc = ntb->epc[type]; + + epc_features = ntb_epc->epc_features; + barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; + size = epc_features->bar_fixed_size[barno]; + align = epc_features->align; + + peer_ntb_epc = ntb->epc[!type]; + peer_epc_features = peer_ntb_epc->epc_features; + peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; + peer_size = peer_epc_features->bar_fixed_size[peer_barno]; + + /* Check if epc_features is populated incorrectly */ + if ((!IS_ALIGNED(size, align))) + return -EINVAL; + + spad_count = ntb->spad_count; + + ctrl_size = sizeof(struct epf_ntb_ctrl); + spad_size = spad_count * 4; + + msix_capable = epc_features->msix_capable; + if (msix_capable) { + msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; + ctrl_size = ALIGN(ctrl_size, 8); + ntb_epc->msix_table_offset = ctrl_size; + ntb_epc->msix_bar = barno; + /* Align to QWORD or 8 Bytes */ + pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); + ctrl_size = ctrl_size + msix_table_size + pba_size; + } + + if (!align) { + ctrl_size = roundup_pow_of_two(ctrl_size); + spad_size = roundup_pow_of_two(spad_size); + } else { + ctrl_size = ALIGN(ctrl_size, align); + spad_size = ALIGN(spad_size, align); + } + + if (peer_size) { + if (peer_size < spad_size) + spad_count = peer_size / 4; + spad_size = peer_size; + } + + /* + * In order to make sure SPAD offset is aligned to its size, + * expand control region size to the size of SPAD if SPAD size + * is greater than control region size. + */ + if (spad_size > ctrl_size) + ctrl_size = spad_size; + + if (!size) + size = ctrl_size + spad_size; + else if (size < ctrl_size + spad_size) + return -EINVAL; + + base = pci_epf_alloc_space(epf, size, barno, align, type); + if (!base) { + dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", + pci_epc_interface_string(type)); + return -ENOMEM; + } + + ntb_epc->reg = base; + + ctrl = ntb_epc->reg; + ctrl->spad_offset = ctrl_size; + ctrl->spad_count = spad_count; + ctrl->num_mws = ntb->num_mws; + ctrl->db_entry_size = align ? align : 4; + ntb_epc->spad_size = spad_size; + + return 0; +} + +/** + * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config + + * scratchpad region for each of PRIMARY and SECONDARY interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for + * config + scratchpad region for a specific interface + */ +static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_config_spad_bar_alloc(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address + * space + * @ntb_epc: EPC associated with one of the HOST which holds peers outbound + * address regions + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram. + * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, + * MW4). + */ +static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + void __iomem *mw_addr; + phys_addr_t phys_addr; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + size_t size; + + epc = ntb_epc->epc; + + for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { + barno = ntb_epc->epf_ntb_bar[bar]; + mw_addr = ntb_epc->mw_addr[barno]; + epf_bar = &ntb_epc->epf_bar[barno]; + phys_addr = epf_bar->phys_addr; + size = epf_bar->size; + if (mw_addr) { + pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size); + ntb_epc->mw_addr[barno] = NULL; + } + } +} + +/** + * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR + * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound + * address + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above + * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2 + * BAR, MW3 BAR and MW4 BAR). + */ +static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) +{ + struct pci_epf_bar *epf_bar; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + u8 func_no; + + epc = ntb_epc->epc; + + func_no = ntb_epc->func_no; + + for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + pci_epc_clear_bar(epc, func_no, epf_bar); + } +} + +/** + * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory + * allocated in peers outbound address space + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and + * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory. + */ +static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + + ntb_epc = ntb->epc[type]; + peer_ntb_epc = ntb->epc[!type]; + + epf_ntb_db_mw_bar_clear(ntb_epc); + epf_ntb_free_peer_mem(peer_ntb_epc); +} + +/** + * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Configure MSI/MSI-X capability for each interface with number of + * interrupts equal to "db_count" configfs entry. + */ +static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + bool msix_capable, msi_capable; + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct device *dev; + u32 db_count; + u8 func_no; + int ret; + + ntb_epc = ntb->epc[type]; + dev = &ntb->epf->dev; + + epc_features = ntb_epc->epc_features; + msix_capable = epc_features->msix_capable; + msi_capable = epc_features->msi_capable; + + if (!(msix_capable || msi_capable)) { + dev_err(dev, "MSI or MSI-X is required for doorbell\n"); + return -EINVAL; + } + + func_no = ntb_epc->func_no; + + db_count = ntb->db_count; + if (db_count > MAX_DB_COUNT) { + dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); + return -EINVAL; + } + + ntb->db_count = db_count; + epc = ntb_epc->epc; + + if (msi_capable) { + ret = pci_epc_set_msi(epc, func_no, db_count); + if (ret) { + dev_err(dev, "%s intf: MSI configuration failed\n", + pci_epc_interface_string(type)); + return ret; + } + } + + if (msix_capable) { + ret = pci_epc_set_msix(epc, func_no, db_count, + ntb_epc->msix_bar, + ntb_epc->msix_table_offset); + if (ret) { + dev_err(dev, "MSI configuration failed\n"); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space + * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound + * address + * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be + * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4) + * @peer_ntb_epc: EPC associated with HOST whose outbound address space is + * used by @ntb_epc + * @size: Size of the address region that has to be allocated in peers OB SPACE + * + * + * +-----------------+ +---->+----------------+-----------+-----------------+ + * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | + * +-----------------+ | +----------------+ +-----------------+ + * | BAR1 | | | Doorbell 2 +---------+ | | + * +-----------------+----+ +----------------+ | | | + * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | + * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + * +-----------------+ | |----------------+ | | | | + * | BAR4 | | | | | | +-----------------+ + * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| + * | BAR5 | | | | | | +-----------------+ + * +-----------------+ +---->-----------------+ | | | | + * EP CONTROLLER 1 | | | | +-----------------+ + * | | | +---->+ MSI|X ADDRESS 4 | + * +----------------+ | +-----------------+ + * (A) EP CONTROLLER 2 | | | + * (OB SPACE) | | | + * +-------> MW1 | + * | | + * | | + * (B) +-----------------+ + * | | + * | | + * | | + * | | + * | | + * +-----------------+ + * PCI Address Space + * (Managed by HOST2) + * + * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate + * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4). + */ +static int epf_ntb_alloc_peer_mem(struct device *dev, + struct epf_ntb_epc *ntb_epc, + enum epf_ntb_bar bar, + struct epf_ntb_epc *peer_ntb_epc, + size_t size) +{ + const struct pci_epc_features *epc_features; + struct pci_epf_bar *epf_bar; + struct pci_epc *peer_epc; + phys_addr_t phys_addr; + void __iomem *mw_addr; + enum pci_barno barno; + size_t align; + + epc_features = ntb_epc->epc_features; + align = epc_features->align; + + if (size < 128) + size = 128; + + if (align) + size = ALIGN(size, align); + else + size = roundup_pow_of_two(size); + + peer_epc = peer_ntb_epc->epc; + mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size); + if (!mw_addr) { + dev_err(dev, "%s intf: Failed to allocate OB address\n", + pci_epc_interface_string(peer_ntb_epc->type)); + return -ENOMEM; + } + + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + ntb_epc->mw_addr[barno] = mw_addr; + + epf_bar->phys_addr = phys_addr; + epf_bar->size = size; + epf_bar->barno = barno; + epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; + + return 0; +} + +/** + * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates + * memory in OB address space of HOST2 and configures BAR of HOST1 + */ +static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; + struct pci_epf_bar *epf_bar; + struct epf_ntb_ctrl *ctrl; + u32 num_mws, db_count; + enum epf_ntb_bar bar; + enum pci_barno barno; + struct pci_epc *epc; + struct device *dev; + size_t align; + int ret, i; + u8 func_no; + u64 size; + + ntb_epc = ntb->epc[type]; + peer_ntb_epc = ntb->epc[!type]; + + dev = &ntb->epf->dev; + epc_features = ntb_epc->epc_features; + align = epc_features->align; + func_no = ntb_epc->func_no; + epc = ntb_epc->epc; + num_mws = ntb->num_mws; + db_count = ntb->db_count; + + for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) { + if (bar == BAR_DB_MW1) { + align = align ? align : 4; + size = db_count * align; + size = ALIGN(size, ntb->mws_size[i]); + ctrl = ntb_epc->reg; + ctrl->mw1_offset = size; + size += ntb->mws_size[i]; + } else { + size = ntb->mws_size[i]; + } + + ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar, + peer_ntb_epc, size); + if (ret) { + dev_err(dev, "%s intf: DoorBell mem alloc failed\n", + pci_epc_interface_string(type)); + goto err_alloc_peer_mem; + } + + barno = ntb_epc->epf_ntb_bar[bar]; + epf_bar = &ntb_epc->epf_bar[barno]; + + ret = pci_epc_set_bar(epc, func_no, epf_bar); + if (ret) { + dev_err(dev, "%s intf: DoorBell BAR set failed\n", + pci_epc_interface_string(type)); + goto err_alloc_peer_mem; + } + } + + return 0; + +err_alloc_peer_mem: + epf_ntb_db_mw_bar_cleanup(ntb, type); + + return ret; +} + +/** + * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Unbind NTB function device from EPC and relinquish reference to pci_epc + * for each of the interface. + */ +static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct pci_epf *epf; + + if (type < 0) + return; + + epf = ntb->epf; + ntb_epc = ntb->epc[type]; + if (!ntb_epc) + return; + epc = ntb_epc->epc; + pci_epc_remove_epf(epc, epf, type); + pci_epc_put(epc); +} + +/** + * epf_ntb_epc_destroy() - Cleanup NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces + */ +static void epf_ntb_epc_destroy(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) + epf_ntb_epc_destroy_interface(ntb, type); +} + +/** + * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @epc: struct pci_epc to which a particular NTB interface should be associated + * @type: PRIMARY interface or SECONDARY interface + * + * Allocate memory for NTB EPC interface and initialize it. + */ +static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, + struct pci_epc *epc, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct pci_epf_bar *epf_bar; + struct epf_ntb_epc *ntb_epc; + struct pci_epf *epf; + struct device *dev; + u8 func_no; + + dev = &ntb->epf->dev; + + ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL); + if (!ntb_epc) + return -ENOMEM; + + epf = ntb->epf; + if (type == PRIMARY_INTERFACE) { + func_no = epf->func_no; + epf_bar = epf->bar; + } else { + func_no = epf->sec_epc_func_no; + epf_bar = epf->sec_epc_bar; + } + + ntb_epc->linkup = false; + ntb_epc->epc = epc; + ntb_epc->func_no = func_no; + ntb_epc->type = type; + ntb_epc->epf_bar = epf_bar; + ntb_epc->epf_ntb = ntb; + + epc_features = pci_epc_get_features(epc, func_no); + if (!epc_features) + return -EINVAL; + ntb_epc->epc_features = epc_features; + + ntb->epc[type] = ntb_epc; + + return 0; +} + +/** + * epf_ntb_epc_create() - Create and initialize NTB EPC interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Get a reference to EPC device and bind NTB function device to that EPC + * for each of the interface. It is also a wrapper to + * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface + * and initialize it + */ +static int epf_ntb_epc_create(struct epf_ntb *ntb) +{ + struct pci_epf *epf; + struct device *dev; + int ret; + + epf = ntb->epf; + dev = &epf->dev; + + ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); + if (ret) { + dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n"); + return ret; + } + + ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, + SECONDARY_INTERFACE); + if (ret) { + dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); + goto err_epc_create; + } + + return 0; + +err_epc_create: + epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); + + return ret; +} + +/** + * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of + * the NTB constructs (scratchpad region, doorbell, memorywindow) + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD, + * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4. + */ +static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + const struct pci_epc_features *epc_features; + struct epf_ntb_epc *ntb_epc; + enum pci_barno barno; + enum epf_ntb_bar bar; + struct device *dev; + u32 num_mws; + int i; + + barno = BAR_0; + ntb_epc = ntb->epc[type]; + num_mws = ntb->num_mws; + dev = &ntb->epf->dev; + epc_features = ntb_epc->epc_features; + + /* These are required BARs which are mandatory for NTB functionality */ + for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + dev_err(dev, "%s intf: Fail to get NTB function BAR\n", + pci_epc_interface_string(type)); + return barno; + } + ntb_epc->epf_ntb_bar[bar] = barno; + } + + /* These are optional BARs which don't impact NTB functionality */ + for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) { + barno = pci_epc_get_next_free_bar(epc_features, barno); + if (barno < 0) { + ntb->num_mws = i; + dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); + } + ntb_epc->epf_ntb_bar[bar] = barno; + } + + return 0; +} + +/** + * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB + * constructs (scratchpad region, doorbell, memorywindow) + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs + * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2, + * BAR_MW3 and BAR_MW4 for all the interfaces. + */ +static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_init_epc_bar_interface(ntb, type); + if (ret) { + dev_err(dev, "Fail to init EPC bar for %s interface\n", + pci_epc_interface_string(type)); + return ret; + } + } + + return 0; +} + +/** + * epf_ntb_epc_init_interface() - Initialize NTB interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to initialize a particular EPC interface and start the workqueue + * to check for commands from host. This function will write to the + * EP controller HW for configuring it. + */ +static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + struct pci_epc *epc; + struct pci_epf *epf; + struct device *dev; + u8 func_no; + int ret; + + ntb_epc = ntb->epc[type]; + epf = ntb->epf; + dev = &epf->dev; + epc = ntb_epc->epc; + func_no = ntb_epc->func_no; + + ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); + if (ret) { + dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n", + pci_epc_interface_string(type)); + return ret; + } + + ret = epf_ntb_peer_spad_bar_set(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Peer SPAD BAR init failed\n", + pci_epc_interface_string(type)); + goto err_peer_spad_bar_init; + } + + ret = epf_ntb_configure_interrupt(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Interrupt configuration failed\n", + pci_epc_interface_string(type)); + goto err_peer_spad_bar_init; + } + + ret = epf_ntb_db_mw_bar_init(ntb, type); + if (ret) { + dev_err(dev, "%s intf: DB/MW BAR init failed\n", + pci_epc_interface_string(type)); + goto err_db_mw_bar_init; + } + + ret = pci_epc_write_header(epc, func_no, epf->header); + if (ret) { + dev_err(dev, "%s intf: Configuration header write failed\n", + pci_epc_interface_string(type)); + goto err_write_header; + } + + INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); + queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); + + return 0; + +err_write_header: + epf_ntb_db_mw_bar_cleanup(ntb, type); + +err_db_mw_bar_init: + epf_ntb_peer_spad_bar_clear(ntb->epc[type]); + +err_peer_spad_bar_init: + epf_ntb_config_sspad_bar_clear(ntb->epc[type]); + + return ret; +} + +/** + * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * @type: PRIMARY interface or SECONDARY interface + * + * Wrapper to cleanup a particular NTB interface. + */ +static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb, + enum pci_epc_interface_type type) +{ + struct epf_ntb_epc *ntb_epc; + + if (type < 0) + return; + + ntb_epc = ntb->epc[type]; + cancel_delayed_work(&ntb_epc->cmd_handler); + epf_ntb_db_mw_bar_cleanup(ntb, type); + epf_ntb_peer_spad_bar_clear(ntb_epc); + epf_ntb_config_sspad_bar_clear(ntb_epc); +} + +/** + * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper to cleanup all NTB interfaces. + */ +static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) + epf_ntb_epc_cleanup_interface(ntb, type); +} + +/** + * epf_ntb_epc_init() - Initialize all NTB interfaces + * @ntb: NTB device that facilitates communication between HOST1 and HOST2 + * + * Wrapper to initialize all NTB interface and start the workqueue + * to check for commands from host. + */ +static int epf_ntb_epc_init(struct epf_ntb *ntb) +{ + enum pci_epc_interface_type type; + struct device *dev; + int ret; + + dev = &ntb->epf->dev; + + for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { + ret = epf_ntb_epc_init_interface(ntb, type); + if (ret) { + dev_err(dev, "%s intf: Failed to initialize\n", + pci_epc_interface_string(type)); + goto err_init_type; + } + } + + return 0; + +err_init_type: + epf_ntb_epc_cleanup_interface(ntb, type - 1); + + return ret; +} + +/** + * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality + * @epf: NTB endpoint function device + * + * Initialize both the endpoint controllers associated with NTB function device. + * Invoked when a primary interface or secondary interface is bound to EPC + * device. This function will succeed only when EPC is bound to both the + * interfaces. + */ +static int epf_ntb_bind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct device *dev = &epf->dev; + int ret; + + if (!epf->epc) { + dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); + return 0; + } + + if (!epf->sec_epc) { + dev_dbg(dev, "SECONDARY EPC interface not yet bound\n"); + return 0; + } + + ret = epf_ntb_epc_create(ntb); + if (ret) { + dev_err(dev, "Failed to create NTB EPC\n"); + return ret; + } + + ret = epf_ntb_init_epc_bar(ntb); + if (ret) { + dev_err(dev, "Failed to create NTB EPC\n"); + goto err_bar_init; + } + + ret = epf_ntb_config_spad_bar_alloc_interface(ntb); + if (ret) { + dev_err(dev, "Failed to allocate BAR memory\n"); + goto err_bar_alloc; + } + + ret = epf_ntb_epc_init(ntb); + if (ret) { + dev_err(dev, "Failed to initialize EPC\n"); + goto err_bar_alloc; + } + + epf_set_drvdata(epf, ntb); + + return 0; + +err_bar_alloc: + epf_ntb_config_spad_bar_free(ntb); + +err_bar_init: + epf_ntb_epc_destroy(ntb); + + return ret; +} + +/** + * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() + * @epf: NTB endpoint function device + * + * Cleanup the initialization from epf_ntb_bind() + */ +static void epf_ntb_unbind(struct pci_epf *epf) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + + epf_ntb_epc_cleanup(ntb); + epf_ntb_config_spad_bar_free(ntb); + epf_ntb_epc_destroy(ntb); +} + +#define EPF_NTB_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + \ + return sprintf(page, "%d\n", ntb->_name); \ +} + +#define EPF_NTB_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + u32 val; \ + int ret; \ + \ + ret = kstrtou32(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + ntb->_name = val; \ + \ + return len; \ +} + +#define EPF_NTB_MW_R(_name) \ +static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ + char *page) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + int win_no; \ + \ + sscanf(#_name, "mw%d", &win_no); \ + \ + return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ +} + +#define EPF_NTB_MW_W(_name) \ +static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ + const char *page, size_t len) \ +{ \ + struct config_group *group = to_config_group(item); \ + struct epf_ntb *ntb = to_epf_ntb(group); \ + struct device *dev = &ntb->epf->dev; \ + int win_no; \ + u64 val; \ + int ret; \ + \ + ret = kstrtou64(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + if (sscanf(#_name, "mw%d", &win_no) != 1) \ + return -EINVAL; \ + \ + if (ntb->num_mws < win_no) { \ + dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ + return -EINVAL; \ + } \ + \ + ntb->mws_size[win_no - 1] = val; \ + \ + return len; \ +} + +static ssize_t epf_ntb_num_mws_store(struct config_item *item, + const char *page, size_t len) +{ + struct config_group *group = to_config_group(item); + struct epf_ntb *ntb = to_epf_ntb(group); + u32 val; + int ret; + + ret = kstrtou32(page, 0, &val); + if (ret) + return ret; + + if (val > MAX_MW) + return -EINVAL; + + ntb->num_mws = val; + + return len; +} + +EPF_NTB_R(spad_count) +EPF_NTB_W(spad_count) +EPF_NTB_R(db_count) +EPF_NTB_W(db_count) +EPF_NTB_R(num_mws) +EPF_NTB_MW_R(mw1) +EPF_NTB_MW_W(mw1) +EPF_NTB_MW_R(mw2) +EPF_NTB_MW_W(mw2) +EPF_NTB_MW_R(mw3) +EPF_NTB_MW_W(mw3) +EPF_NTB_MW_R(mw4) +EPF_NTB_MW_W(mw4) + +CONFIGFS_ATTR(epf_ntb_, spad_count); +CONFIGFS_ATTR(epf_ntb_, db_count); +CONFIGFS_ATTR(epf_ntb_, num_mws); +CONFIGFS_ATTR(epf_ntb_, mw1); +CONFIGFS_ATTR(epf_ntb_, mw2); +CONFIGFS_ATTR(epf_ntb_, mw3); +CONFIGFS_ATTR(epf_ntb_, mw4); + +static struct configfs_attribute *epf_ntb_attrs[] = { + &epf_ntb_attr_spad_count, + &epf_ntb_attr_db_count, + &epf_ntb_attr_num_mws, + &epf_ntb_attr_mw1, + &epf_ntb_attr_mw2, + &epf_ntb_attr_mw3, + &epf_ntb_attr_mw4, + NULL, +}; + +static const struct config_item_type ntb_group_type = { + .ct_attrs = epf_ntb_attrs, + .ct_owner = THIS_MODULE, +}; + +/** + * epf_ntb_add_cfs() - Add configfs directory specific to NTB + * @epf: NTB endpoint function device + * + * Add configfs directory specific to NTB. This directory will hold + * NTB specific properties like db_count, spad_count, num_mws etc., + */ +static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, + struct config_group *group) +{ + struct epf_ntb *ntb = epf_get_drvdata(epf); + struct config_group *ntb_group = &ntb->group; + struct device *dev = &epf->dev; + + config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); + + return ntb_group; +} + +/** + * epf_ntb_probe() - Probe NTB function driver + * @epf: NTB endpoint function device + * + * Probe NTB function driver when endpoint function bus detects a NTB + * endpoint function. + */ +static int epf_ntb_probe(struct pci_epf *epf) +{ + struct epf_ntb *ntb; + struct device *dev; + + dev = &epf->dev; + + ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); + if (!ntb) + return -ENOMEM; + + epf->header = &epf_ntb_header; + ntb->epf = epf; + epf_set_drvdata(epf, ntb); + + return 0; +} + +static struct pci_epf_ops epf_ntb_ops = { + .bind = epf_ntb_bind, + .unbind = epf_ntb_unbind, + .add_cfs = epf_ntb_add_cfs, +}; + +static const struct pci_epf_device_id epf_ntb_ids[] = { + { + .name = "pci_epf_ntb", + }, + {}, +}; + +static struct pci_epf_driver epf_ntb_driver = { + .driver.name = "pci_epf_ntb", + .probe = epf_ntb_probe, + .id_table = epf_ntb_ids, + .ops = &epf_ntb_ops, + .owner = THIS_MODULE, +}; + +static int __init epf_ntb_init(void) +{ + int ret; + + kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | + WQ_HIGHPRI, 0); + ret = pci_epf_register_driver(&epf_ntb_driver); + if (ret) { + destroy_workqueue(kpcintb_workqueue); + pr_err("Failed to register pci epf ntb driver --> %d\n", ret); + return ret; + } + + return 0; +} +module_init(epf_ntb_init); + +static void __exit epf_ntb_exit(void) +{ + pci_epf_unregister_driver(&epf_ntb_driver); + destroy_workqueue(kpcintb_workqueue); +} +module_exit(epf_ntb_exit); + +MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); +MODULE_AUTHOR("Kishon Vijay Abraham I "); +MODULE_LICENSE("GPL v2"); From 599f86872f9ce8a0a0bd111a23442b18e8ee7059 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:06 +0530 Subject: [PATCH 434/633] PCI: Add TI J721E device to PCI IDs Add TI J721E device to the PCI ID database. Since this device has a configurable PCIe endpoint, it could be used with different drivers. Link: https://lore.kernel.org/r/20210201195809.7342-15-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/misc/pci_endpoint_test.c | 1 - include/linux/pci_ids.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index eff481ce08ee..1b2868ca4f2a 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -68,7 +68,6 @@ #define PCI_ENDPOINT_TEST_FLAGS 0x2c #define FLAG_USE_DMA BIT(0) -#define PCI_DEVICE_ID_TI_J721E 0xb00d #define PCI_DEVICE_ID_TI_AM654 0xb00c #define PCI_DEVICE_ID_LS1088A 0x80c0 diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index d8156a5dbee8..f968fcda338e 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -881,6 +881,7 @@ #define PCI_DEVICE_ID_TI_X620 0xac8d #define PCI_DEVICE_ID_TI_X420 0xac8e #define PCI_DEVICE_ID_TI_XX20_FM 0xac8f +#define PCI_DEVICE_ID_TI_J721E 0xb00d #define PCI_DEVICE_ID_TI_DRA74x 0xb500 #define PCI_DEVICE_ID_TI_DRA72x 0xb501 From 812ce2f8d14ea791edd88c36ebcc9017bf4c88cb Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:07 +0530 Subject: [PATCH 435/633] NTB: Add support for EPF PCI Non-Transparent Bridge Add support for EPF PCI Non-Transparent Bridge (NTB) devices. This driver is platform independent and may be used by any platform that has multiple PCI endpoint instances configured using the pci-epf-ntb driver. The driver connnects to the standard NTB subsystem interface. The EPF NTB device has a configurable number of memory windows (max 4), a configurable number of doorbells (max 32), and a configurable number of scratch-pad registers. Link: https://lore.kernel.org/r/20210201195809.7342-16-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Dave Jiang --- drivers/ntb/hw/Kconfig | 1 + drivers/ntb/hw/Makefile | 1 + drivers/ntb/hw/epf/Kconfig | 6 + drivers/ntb/hw/epf/Makefile | 1 + drivers/ntb/hw/epf/ntb_hw_epf.c | 753 ++++++++++++++++++++++++++++++++ 5 files changed, 762 insertions(+) create mode 100644 drivers/ntb/hw/epf/Kconfig create mode 100644 drivers/ntb/hw/epf/Makefile create mode 100644 drivers/ntb/hw/epf/ntb_hw_epf.c diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig index e77c587060ff..c325be526b80 100644 --- a/drivers/ntb/hw/Kconfig +++ b/drivers/ntb/hw/Kconfig @@ -2,4 +2,5 @@ source "drivers/ntb/hw/amd/Kconfig" source "drivers/ntb/hw/idt/Kconfig" source "drivers/ntb/hw/intel/Kconfig" +source "drivers/ntb/hw/epf/Kconfig" source "drivers/ntb/hw/mscc/Kconfig" diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile index 4714d6238845..223ca592b5f9 100644 --- a/drivers/ntb/hw/Makefile +++ b/drivers/ntb/hw/Makefile @@ -2,4 +2,5 @@ obj-$(CONFIG_NTB_AMD) += amd/ obj-$(CONFIG_NTB_IDT) += idt/ obj-$(CONFIG_NTB_INTEL) += intel/ +obj-$(CONFIG_NTB_EPF) += epf/ obj-$(CONFIG_NTB_SWITCHTEC) += mscc/ diff --git a/drivers/ntb/hw/epf/Kconfig b/drivers/ntb/hw/epf/Kconfig new file mode 100644 index 000000000000..6197d1aab344 --- /dev/null +++ b/drivers/ntb/hw/epf/Kconfig @@ -0,0 +1,6 @@ +config NTB_EPF + tristate "Generic EPF Non-Transparent Bridge support" + depends on m + help + This driver supports EPF NTB on configurable endpoint. + If unsure, say N. diff --git a/drivers/ntb/hw/epf/Makefile b/drivers/ntb/hw/epf/Makefile new file mode 100644 index 000000000000..2f560a422bc6 --- /dev/null +++ b/drivers/ntb/hw/epf/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_NTB_EPF) += ntb_hw_epf.o diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c new file mode 100644 index 000000000000..b019755e4e21 --- /dev/null +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c @@ -0,0 +1,753 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Host side endpoint driver to implement Non-Transparent Bridge functionality + * + * Copyright (C) 2020 Texas Instruments + * Author: Kishon Vijay Abraham I + */ + +#include +#include +#include +#include +#include + +#define NTB_EPF_COMMAND 0x0 +#define CMD_CONFIGURE_DOORBELL 1 +#define CMD_TEARDOWN_DOORBELL 2 +#define CMD_CONFIGURE_MW 3 +#define CMD_TEARDOWN_MW 4 +#define CMD_LINK_UP 5 +#define CMD_LINK_DOWN 6 + +#define NTB_EPF_ARGUMENT 0x4 +#define MSIX_ENABLE BIT(16) + +#define NTB_EPF_CMD_STATUS 0x8 +#define COMMAND_STATUS_OK 1 +#define COMMAND_STATUS_ERROR 2 + +#define NTB_EPF_LINK_STATUS 0x0A +#define LINK_STATUS_UP BIT(0) + +#define NTB_EPF_TOPOLOGY 0x0C +#define NTB_EPF_LOWER_ADDR 0x10 +#define NTB_EPF_UPPER_ADDR 0x14 +#define NTB_EPF_LOWER_SIZE 0x18 +#define NTB_EPF_UPPER_SIZE 0x1C +#define NTB_EPF_MW_COUNT 0x20 +#define NTB_EPF_MW1_OFFSET 0x24 +#define NTB_EPF_SPAD_OFFSET 0x28 +#define NTB_EPF_SPAD_COUNT 0x2C +#define NTB_EPF_DB_ENTRY_SIZE 0x30 +#define NTB_EPF_DB_DATA(n) (0x34 + (n) * 4) +#define NTB_EPF_DB_OFFSET(n) (0xB4 + (n) * 4) + +#define NTB_EPF_MIN_DB_COUNT 3 +#define NTB_EPF_MAX_DB_COUNT 31 +#define NTB_EPF_MW_OFFSET 2 + +#define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ + +enum pci_barno { + BAR_0, + BAR_1, + BAR_2, + BAR_3, + BAR_4, + BAR_5, +}; + +struct ntb_epf_dev { + struct ntb_dev ntb; + struct device *dev; + /* Mutex to protect providing commands to NTB EPF */ + struct mutex cmd_lock; + + enum pci_barno ctrl_reg_bar; + enum pci_barno peer_spad_reg_bar; + enum pci_barno db_reg_bar; + + unsigned int mw_count; + unsigned int spad_count; + unsigned int db_count; + + void __iomem *ctrl_reg; + void __iomem *db_reg; + void __iomem *peer_spad_reg; + + unsigned int self_spad; + unsigned int peer_spad; + + int db_val; + u64 db_valid_mask; +}; + +#define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) + +struct ntb_epf_data { + /* BAR that contains both control region and self spad region */ + enum pci_barno ctrl_reg_bar; + /* BAR that contains peer spad region */ + enum pci_barno peer_spad_reg_bar; + /* BAR that contains Doorbell region and Memory window '1' */ + enum pci_barno db_reg_bar; +}; + +static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, + u32 argument) +{ + ktime_t timeout; + bool timedout; + int ret = 0; + u32 status; + + mutex_lock(&ndev->cmd_lock); + writel(argument, ndev->ctrl_reg + NTB_EPF_ARGUMENT); + writel(command, ndev->ctrl_reg + NTB_EPF_COMMAND); + + timeout = ktime_add_ms(ktime_get(), NTB_EPF_COMMAND_TIMEOUT); + while (1) { + timedout = ktime_after(ktime_get(), timeout); + status = readw(ndev->ctrl_reg + NTB_EPF_CMD_STATUS); + + if (status == COMMAND_STATUS_ERROR) { + ret = -EINVAL; + break; + } + + if (status == COMMAND_STATUS_OK) + break; + + if (WARN_ON(timedout)) { + ret = -ETIMEDOUT; + break; + } + + usleep_range(5, 10); + } + + writew(0, ndev->ctrl_reg + NTB_EPF_CMD_STATUS); + mutex_unlock(&ndev->cmd_lock); + + return ret; +} + +static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx) +{ + struct device *dev = ndev->dev; + + if (idx < 0 || idx > ndev->mw_count) { + dev_err(dev, "Unsupported Memory Window index %d\n", idx); + return -EINVAL; + } + + return idx + 2; +} + +static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + return ndev->mw_count; +} + +static int ntb_epf_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int bar; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + bar = ntb_epf_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; + + if (addr_align) + *addr_align = SZ_4K; + + if (size_align) + *size_align = 1; + + if (size_max) + *size_max = pci_resource_len(ndev->ntb.pdev, bar); + + return 0; +} + +static u64 ntb_epf_link_is_up(struct ntb_dev *ntb, + enum ntb_speed *speed, + enum ntb_width *width) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 status; + + status = readw(ndev->ctrl_reg + NTB_EPF_LINK_STATUS); + + return status & LINK_STATUS_UP; +} + +static u32 ntb_epf_spad_read(struct ntb_dev *ntb, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "READ: Invalid ScratchPad Index %d\n", idx); + return 0; + } + + offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); + offset += (idx << 2); + + return readl(ndev->ctrl_reg + offset); +} + +static int ntb_epf_spad_write(struct ntb_dev *ntb, + int idx, u32 val) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = readl(ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); + offset += (idx << 2); + writel(val, ndev->ctrl_reg + offset); + + return 0; +} + +static u32 ntb_epf_peer_spad_read(struct ntb_dev *ntb, int pidx, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = (idx << 2); + return readl(ndev->peer_spad_reg + offset); +} + +static int ntb_epf_peer_spad_write(struct ntb_dev *ntb, int pidx, + int idx, u32 val) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + u32 offset; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + if (idx < 0 || idx >= ndev->spad_count) { + dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n", idx); + return -EINVAL; + } + + offset = (idx << 2); + writel(val, ndev->peer_spad_reg + offset); + + return 0; +} + +static int ntb_epf_link_enable(struct ntb_dev *ntb, + enum ntb_speed max_speed, + enum ntb_width max_width) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret; + + ret = ntb_epf_send_command(ndev, CMD_LINK_UP, 0); + if (ret) { + dev_err(dev, "Fail to enable link\n"); + return ret; + } + + return 0; +} + +static int ntb_epf_link_disable(struct ntb_dev *ntb) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret; + + ret = ntb_epf_send_command(ndev, CMD_LINK_DOWN, 0); + if (ret) { + dev_err(dev, "Fail to disable link\n"); + return ret; + } + + return 0; +} + +static irqreturn_t ntb_epf_vec_isr(int irq, void *dev) +{ + struct ntb_epf_dev *ndev = dev; + int irq_no; + + irq_no = irq - pci_irq_vector(ndev->ntb.pdev, 0); + ndev->db_val = irq_no + 1; + + if (irq_no == 0) + ntb_link_event(&ndev->ntb); + else + ntb_db_event(&ndev->ntb, irq_no); + + return IRQ_HANDLED; +} + +static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + struct device *dev = ndev->dev; + u32 argument = MSIX_ENABLE; + int irq; + int ret; + int i; + + irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, PCI_IRQ_MSIX); + if (irq < 0) { + dev_dbg(dev, "Failed to get MSIX interrupts\n"); + irq = pci_alloc_irq_vectors(pdev, msi_min, msi_max, + PCI_IRQ_MSI); + if (irq < 0) { + dev_err(dev, "Failed to get MSI interrupts\n"); + return irq; + } + argument &= ~MSIX_ENABLE; + } + + for (i = 0; i < irq; i++) { + ret = request_irq(pci_irq_vector(pdev, i), ntb_epf_vec_isr, + 0, "ntb_epf", ndev); + if (ret) { + dev_err(dev, "Failed to request irq\n"); + goto err_request_irq; + } + } + + ndev->db_count = irq - 1; + + ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, + argument | irq); + if (ret) { + dev_err(dev, "Failed to configure doorbell\n"); + goto err_configure_db; + } + + return 0; + +err_configure_db: + for (i = 0; i < ndev->db_count + 1; i++) + free_irq(pci_irq_vector(pdev, i), ndev); + +err_request_irq: + pci_free_irq_vectors(pdev); + + return ret; +} + +static int ntb_epf_peer_mw_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->mw_count; +} + +static int ntb_epf_spad_count(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->spad_count; +} + +static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb) +{ + return ntb_ndev(ntb)->db_valid_mask; +} + +static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) +{ + return 0; +} + +static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, + dma_addr_t addr, resource_size_t size) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + resource_size_t mw_size; + int bar; + + if (pidx != NTB_DEF_PEER_IDX) { + dev_err(dev, "Unsupported Peer ID %d\n", pidx); + return -EINVAL; + } + + bar = idx + NTB_EPF_MW_OFFSET; + + mw_size = pci_resource_len(ntb->pdev, bar); + + if (size > mw_size) { + dev_err(dev, "Size:%pa is greater than the MW size %pa\n", + &size, &mw_size); + return -EINVAL; + } + + writel(lower_32_bits(addr), ndev->ctrl_reg + NTB_EPF_LOWER_ADDR); + writel(upper_32_bits(addr), ndev->ctrl_reg + NTB_EPF_UPPER_ADDR); + writel(lower_32_bits(size), ndev->ctrl_reg + NTB_EPF_LOWER_SIZE); + writel(upper_32_bits(size), ndev->ctrl_reg + NTB_EPF_UPPER_SIZE); + ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, idx); + + return 0; +} + +static int ntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + struct device *dev = ndev->dev; + int ret = 0; + + ntb_epf_send_command(ndev, CMD_TEARDOWN_MW, idx); + if (ret) + dev_err(dev, "Failed to teardown memory window\n"); + + return ret; +} + +static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, + phys_addr_t *base, resource_size_t *size) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 offset = 0; + int bar; + + if (idx == 0) + offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); + + bar = idx + NTB_EPF_MW_OFFSET; + + if (base) + *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; + + if (size) + *size = pci_resource_len(ndev->ntb.pdev, bar) - offset; + + return 0; +} + +static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + u32 interrupt_num = ffs(db_bits) + 1; + struct device *dev = ndev->dev; + u32 db_entry_size; + u32 db_offset; + u32 db_data; + + if (interrupt_num > ndev->db_count) { + dev_err(dev, "DB interrupt %d greater than Max Supported %d\n", + interrupt_num, ndev->db_count); + return -EINVAL; + } + + db_entry_size = readl(ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE); + + db_data = readl(ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); + db_offset = readl(ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); + writel(db_data, ndev->db_reg + (db_entry_size * interrupt_num) + + db_offset); + + return 0; +} + +static u64 ntb_epf_db_read(struct ntb_dev *ntb) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + + return ndev->db_val; +} + +static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) +{ + return 0; +} + +static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits) +{ + struct ntb_epf_dev *ndev = ntb_ndev(ntb); + + ndev->db_val = 0; + + return 0; +} + +static const struct ntb_dev_ops ntb_epf_ops = { + .mw_count = ntb_epf_mw_count, + .spad_count = ntb_epf_spad_count, + .peer_mw_count = ntb_epf_peer_mw_count, + .db_valid_mask = ntb_epf_db_valid_mask, + .db_set_mask = ntb_epf_db_set_mask, + .mw_set_trans = ntb_epf_mw_set_trans, + .mw_clear_trans = ntb_epf_mw_clear_trans, + .peer_mw_get_addr = ntb_epf_peer_mw_get_addr, + .link_enable = ntb_epf_link_enable, + .spad_read = ntb_epf_spad_read, + .spad_write = ntb_epf_spad_write, + .peer_spad_read = ntb_epf_peer_spad_read, + .peer_spad_write = ntb_epf_peer_spad_write, + .peer_db_set = ntb_epf_peer_db_set, + .db_read = ntb_epf_db_read, + .mw_get_align = ntb_epf_mw_get_align, + .link_is_up = ntb_epf_link_is_up, + .db_clear_mask = ntb_epf_db_clear_mask, + .db_clear = ntb_epf_db_clear, + .link_disable = ntb_epf_link_disable, +}; + +static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev, + struct pci_dev *pdev) +{ + ndev->ntb.pdev = pdev; + ndev->ntb.topo = NTB_TOPO_NONE; + ndev->ntb.ops = &ntb_epf_ops; +} + +static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) +{ + struct device *dev = ndev->dev; + int ret; + + /* One Link interrupt and rest doorbell interrupt */ + ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1, + NTB_EPF_MAX_DB_COUNT + 1); + if (ret) { + dev_err(dev, "Failed to init ISR\n"); + return ret; + } + + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT); + ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); + + return 0; +} + +static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, + struct pci_dev *pdev) +{ + struct device *dev = ndev->dev; + int ret; + + pci_set_drvdata(pdev, ndev); + + ret = pci_enable_device(pdev); + if (ret) { + dev_err(dev, "Cannot enable PCI device\n"); + goto err_pci_enable; + } + + ret = pci_request_regions(pdev, "ntb"); + if (ret) { + dev_err(dev, "Cannot obtain PCI resources\n"); + goto err_pci_regions; + } + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "Cannot set DMA mask\n"); + goto err_dma_mask; + } + dev_warn(&pdev->dev, "Cannot DMA highmem\n"); + } + + ndev->ctrl_reg = pci_iomap(pdev, ndev->ctrl_reg_bar, 0); + if (!ndev->ctrl_reg) { + ret = -EIO; + goto err_dma_mask; + } + + ndev->peer_spad_reg = pci_iomap(pdev, ndev->peer_spad_reg_bar, 0); + if (!ndev->peer_spad_reg) { + ret = -EIO; + goto err_dma_mask; + } + + ndev->db_reg = pci_iomap(pdev, ndev->db_reg_bar, 0); + if (!ndev->db_reg) { + ret = -EIO; + goto err_dma_mask; + } + + return 0; + +err_dma_mask: + pci_clear_master(pdev); + +err_pci_regions: + pci_disable_device(pdev); + +err_pci_enable: + pci_set_drvdata(pdev, NULL); + + return ret; +} + +static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + + pci_iounmap(pdev, ndev->ctrl_reg); + pci_iounmap(pdev, ndev->peer_spad_reg); + pci_iounmap(pdev, ndev->db_reg); + + pci_clear_master(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) +{ + struct pci_dev *pdev = ndev->ntb.pdev; + int i; + + ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1); + + for (i = 0; i < ndev->db_count + 1; i++) + free_irq(pci_irq_vector(pdev, i), ndev); + pci_free_irq_vectors(pdev); +} + +static int ntb_epf_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + enum pci_barno peer_spad_reg_bar = BAR_1; + enum pci_barno ctrl_reg_bar = BAR_0; + enum pci_barno db_reg_bar = BAR_2; + struct device *dev = &pdev->dev; + struct ntb_epf_data *data; + struct ntb_epf_dev *ndev; + int ret; + + if (pci_is_bridge(pdev)) + return -ENODEV; + + ndev = devm_kzalloc(dev, sizeof(*ndev), GFP_KERNEL); + if (!ndev) + return -ENOMEM; + + data = (struct ntb_epf_data *)id->driver_data; + if (data) { + if (data->peer_spad_reg_bar) + peer_spad_reg_bar = data->peer_spad_reg_bar; + if (data->ctrl_reg_bar) + ctrl_reg_bar = data->ctrl_reg_bar; + if (data->db_reg_bar) + db_reg_bar = data->db_reg_bar; + } + + ndev->peer_spad_reg_bar = peer_spad_reg_bar; + ndev->ctrl_reg_bar = ctrl_reg_bar; + ndev->db_reg_bar = db_reg_bar; + ndev->dev = dev; + + ntb_epf_init_struct(ndev, pdev); + mutex_init(&ndev->cmd_lock); + + ret = ntb_epf_init_pci(ndev, pdev); + if (ret) { + dev_err(dev, "Failed to init PCI\n"); + return ret; + } + + ret = ntb_epf_init_dev(ndev); + if (ret) { + dev_err(dev, "Failed to init device\n"); + goto err_init_dev; + } + + ret = ntb_register_device(&ndev->ntb); + if (ret) { + dev_err(dev, "Failed to register NTB device\n"); + goto err_register_dev; + } + + return 0; + +err_register_dev: + ntb_epf_cleanup_isr(ndev); + +err_init_dev: + ntb_epf_deinit_pci(ndev); + + return ret; +} + +static void ntb_epf_pci_remove(struct pci_dev *pdev) +{ + struct ntb_epf_dev *ndev = pci_get_drvdata(pdev); + + ntb_unregister_device(&ndev->ntb); + ntb_epf_cleanup_isr(ndev); + ntb_epf_deinit_pci(ndev); +} + +static const struct ntb_epf_data j721e_data = { + .ctrl_reg_bar = BAR_0, + .peer_spad_reg_bar = BAR_1, + .db_reg_bar = BAR_2, +}; + +static const struct pci_device_id ntb_epf_pci_tbl[] = { + { + PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), + .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, + .driver_data = (kernel_ulong_t)&j721e_data, + }, + { }, +}; + +static struct pci_driver ntb_epf_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = ntb_epf_pci_tbl, + .probe = ntb_epf_pci_probe, + .remove = ntb_epf_pci_remove, +}; +module_pci_driver(ntb_epf_pci_driver); + +MODULE_DESCRIPTION("PCI ENDPOINT NTB HOST DRIVER"); +MODULE_AUTHOR("Kishon Vijay Abraham I "); +MODULE_LICENSE("GPL v2"); From 250c475be70a6ef0201ec2455d3e109e0c71f4b9 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:08 +0530 Subject: [PATCH 436/633] Documentation: PCI: Add configfs binding documentation for pci-ntb endpoint function Add binding documentation for pci-ntb endpoint function that helps in adding and configuring pci-ntb endpoint function. Link: https://lore.kernel.org/r/20210201195809.7342-17-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- .../PCI/endpoint/function/binding/pci-ntb.rst | 38 +++++++++++++++++++ Documentation/PCI/endpoint/index.rst | 1 + 2 files changed, 39 insertions(+) create mode 100644 Documentation/PCI/endpoint/function/binding/pci-ntb.rst diff --git a/Documentation/PCI/endpoint/function/binding/pci-ntb.rst b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst new file mode 100644 index 000000000000..40253d3d5163 --- /dev/null +++ b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +PCI NTB Endpoint Function +========================== + +1) Create a subdirectory to pci_epf_ntb directory in configfs. + +Standard EPF Configurable Fields: + +================ =========================================================== +vendorid should be 0x104c +deviceid should be 0xb00d for TI's J721E SoC +revid don't care +progif_code don't care +subclass_code should be 0x00 +baseclass_code should be 0x5 +cache_line_size don't care +subsys_vendor_id don't care +subsys_id don't care +interrupt_pin don't care +msi_interrupts don't care +msix_interrupts don't care +================ =========================================================== + +2) Create a subdirectory to directory created in 1 + +NTB EPF specific configurable fields: + +================ =========================================================== +db_count Number of doorbells; default = 4 +mw1 size of memory window1 +mw2 size of memory window2 +mw3 size of memory window3 +mw4 size of memory window4 +num_mws Number of memory windows; max = 4 +spad_count Number of scratchpad registers; default = 64 +================ =========================================================== diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index ef6861128506..9cb6e5f3c4d5 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -14,3 +14,4 @@ PCI Endpoint Framework pci-ntb-function function/binding/pci-test + function/binding/pci-ntb From b28a23676e0945934df983fcc8df0bfd9b24f866 Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Tue, 2 Feb 2021 01:28:09 +0530 Subject: [PATCH 437/633] Documentation: PCI: Add PCI endpoint NTB function user guide Add documentation to help users use pci-epf-ntb function driver and existing host side NTB infrastructure for NTB functionality. [bhelgaas: fix a few typos] Link: https://lore.kernel.org/r/20210201195809.7342-18-kishon@ti.com Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Randy Dunlap --- Documentation/PCI/endpoint/index.rst | 1 + Documentation/PCI/endpoint/pci-ntb-howto.rst | 161 +++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 Documentation/PCI/endpoint/pci-ntb-howto.rst diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index 9cb6e5f3c4d5..38ea1f604b6d 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -12,6 +12,7 @@ PCI Endpoint Framework pci-test-function pci-test-howto pci-ntb-function + pci-ntb-howto function/binding/pci-test function/binding/pci-ntb diff --git a/Documentation/PCI/endpoint/pci-ntb-howto.rst b/Documentation/PCI/endpoint/pci-ntb-howto.rst new file mode 100644 index 000000000000..1884bf29caba --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-howto.rst @@ -0,0 +1,161 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================================== +PCI Non-Transparent Bridge (NTB) Endpoint Function (EPF) User Guide +=================================================================== + +:Author: Kishon Vijay Abraham I + +This document is a guide to help users use pci-epf-ntb function driver +and ntb_hw_epf host driver for NTB functionality. The list of steps to +be followed in the host side and EP side is given below. For the hardware +configuration and internals of NTB using configurable endpoints see +Documentation/PCI/endpoint/pci-ntb-function.rst + +Endpoint Device +=============== + +Endpoint Controller Devices +--------------------------- + +For implementing NTB functionality at least two endpoint controller devices +are required. + +To find the list of endpoint controller devices in the system:: + + # ls /sys/class/pci_epc/ + 2900000.pcie-ep 2910000.pcie-ep + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/controllers + 2900000.pcie-ep 2910000.pcie-ep + + +Endpoint Function Drivers +------------------------- + +To find the list of endpoint function drivers in the system:: + + # ls /sys/bus/pci-epf/drivers + pci_epf_ntb pci_epf_ntb + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/functions + pci_epf_ntb pci_epf_ntb + + +Creating pci-epf-ntb Device +---------------------------- + +PCI endpoint function device can be created using the configfs. To create +pci-epf-ntb device, the following commands can be used:: + + # mount -t configfs none /sys/kernel/config + # cd /sys/kernel/config/pci_ep/ + # mkdir functions/pci_epf_ntb/func1 + +The "mkdir func1" above creates the pci-epf-ntb function device that will +be probed by pci_epf_ntb driver. + +The PCI endpoint framework populates the directory with the following +configurable fields:: + + # ls functions/pci_epf_ntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + progif_code secondary subsys_id vendorid + cache_line_size interrupt_pin msix_interrupts primary + revid subclass_code subsys_vendor_id + +The PCI endpoint function driver populates these entries with default values +when the device is bound to the driver. The pci-epf-ntb driver populates +vendorid with 0xffff and interrupt_pin with 0x0001:: + + # cat functions/pci_epf_ntb/func1/vendorid + 0xffff + # cat functions/pci_epf_ntb/func1/interrupt_pin + 0x0001 + + +Configuring pci-epf-ntb Device +------------------------------- + +The user can configure the pci-epf-ntb device using its configfs entry. In order +to change the vendorid and the deviceid, the following +commands can be used:: + + # echo 0x104c > functions/pci_epf_ntb/func1/vendorid + # echo 0xb00d > functions/pci_epf_ntb/func1/deviceid + +In order to configure NTB specific attributes, a new sub-directory to func1 +should be created:: + + # mkdir functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + +The NTB function driver will populate this directory with various attributes +that can be configured by the user:: + + # ls functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + db_count mw1 mw2 mw3 mw4 num_mws + spad_count + +A sample configuration for NTB function is given below:: + + # echo 4 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/db_count + # echo 128 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/spad_count + # echo 2 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/num_mws + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw1 + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw2 + +Binding pci-epf-ntb Device to EP Controller +-------------------------------------------- + +NTB function device should be attached to two PCI endpoint controllers +connected to the two hosts. Use the 'primary' and 'secondary' entries +inside NTB function device to attach one PCI endpoint controller to +primary interface and the other PCI endpoint controller to the secondary +interface:: + + # ln -s controllers/2900000.pcie-ep/ functions/pci-epf-ntb/func1/primary + # ln -s controllers/2910000.pcie-ep/ functions/pci-epf-ntb/func1/secondary + +Once the above step is completed, both the PCI endpoint controllers are ready to +establish a link with the host. + + +Start the Link +-------------- + +In order for the endpoint device to establish a link with the host, the _start_ +field should be populated with '1'. For NTB, both the PCI endpoint controllers +should establish link with the host:: + + # echo 1 > controllers/2900000.pcie-ep/start + # echo 1 > controllers/2910000.pcie-ep/start + + +RootComplex Device +================== + +lspci Output +------------ + +Note that the devices listed here correspond to the values populated in +"Creating pci-epf-ntb Device" section above:: + + # lspci + 0000:00:00.0 PCI bridge: Texas Instruments Device b00d + 0000:01:00.0 RAM memory: Texas Instruments Device b00d + + +Using ntb_hw_epf Device +----------------------- + +The host side software follows the standard NTB software architecture in Linux. +All the existing client side NTB utilities like NTB Transport Client and NTB +Netdev, NTB Ping Pong Test Client and NTB Tool Test Client can be used with NTB +function device. + +For more information on NTB see +:doc:`Non-Transparent Bridge <../../driver-api/ntb>` From 58adbfb3ebec460e8b58875c682bafd866808e80 Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Fri, 22 Jan 2021 00:23:18 +0800 Subject: [PATCH 438/633] PCI: rockchip: Make 'ep-gpios' DT property optional The Rockchip PCIe controller DT binding clearly states that 'ep-gpios' is an optional property. And indeed there are boards that don't require it. Make the driver follow the binding by using devm_gpiod_get_optional() instead of devm_gpiod_get(). [bhelgaas: tidy whitespace] Link: https://lore.kernel.org/r/20210121162321.4538-2-wens@kernel.org Fixes: e77f847df54c ("PCI: rockchip: Add Rockchip PCIe controller support") Fixes: 956cd99b35a8 ("PCI: rockchip: Separate common code from RC driver") Fixes: 964bac9455be ("PCI: rockchip: Split out rockchip_pcie_parse_dt() to parse DT") Signed-off-by: Chen-Yu Tsai Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/pcie-rockchip.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-rockchip.c b/drivers/pci/controller/pcie-rockchip.c index 904dec0d3a88..990a00e08bc5 100644 --- a/drivers/pci/controller/pcie-rockchip.c +++ b/drivers/pci/controller/pcie-rockchip.c @@ -82,7 +82,7 @@ int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) } rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev, - "mgmt-sticky"); + "mgmt-sticky"); if (IS_ERR(rockchip->mgmt_sticky_rst)) { if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER) dev_err(dev, "missing mgmt-sticky reset property in node\n"); @@ -118,11 +118,11 @@ int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) } if (rockchip->is_rc) { - rockchip->ep_gpio = devm_gpiod_get(dev, "ep", GPIOD_OUT_HIGH); - if (IS_ERR(rockchip->ep_gpio)) { - dev_err(dev, "missing ep-gpios property in node\n"); - return PTR_ERR(rockchip->ep_gpio); - } + rockchip->ep_gpio = devm_gpiod_get_optional(dev, "ep", + GPIOD_OUT_HIGH); + if (IS_ERR(rockchip->ep_gpio)) + return dev_err_probe(dev, PTR_ERR(rockchip->ep_gpio), + "failed to get ep GPIO\n"); } rockchip->aclk_pcie = devm_clk_get(dev, "aclk"); From 8e61744de4d34162467b0b3e49aa9de8bf817180 Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Fri, 22 Jan 2021 00:23:19 +0800 Subject: [PATCH 439/633] dt-bindings: arm: rockchip: Add FriendlyARM NanoPi M4B The NanoPi M4B is a minor revision of the original M4. The differences against the original Nanopi M4 that are common with the other M4V2 revision include: - microphone header removed - power button added - recovery button added Additional changes specific to the M4B: - USB 3.0 hub removed; board now has 2x USB 3.0 type-A ports and 2x USB 2.0 ports - ADB toggle switch added; this changes the top USB 3.0 host port to a peripheral port - Type-C port no longer supports data or PD - WiFi/Bluetooth combo chip switched to AP6256, which supports BT 5.0 but only 1T1R (down from 2T2R) for WiFi Add a compatible string for the new board revision. Link: https://lore.kernel.org/r/20210121162321.4538-3-wens@kernel.org Signed-off-by: Chen-Yu Tsai Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Rob Herring --- Documentation/devicetree/bindings/arm/rockchip.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml index ef4544ad6f82..773fd4c531ed 100644 --- a/Documentation/devicetree/bindings/arm/rockchip.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip.yaml @@ -132,6 +132,7 @@ properties: - enum: - friendlyarm,nanopc-t4 - friendlyarm,nanopi-m4 + - friendlyarm,nanopi-m4b - friendlyarm,nanopi-neo4 - const: rockchip,rk3399 From 7d7cbeaba5b7aea8e1e4eb988d6b5e7cb3c34490 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 4 Jan 2021 15:02:56 -0800 Subject: [PATCH 440/633] PCI/ERR: Clear status of the reporting device Error handling operates on the first Downstream Port above the detected error, but the error may have been reported by a downstream device. Clear the AER status of the device that reported the error rather than the first Downstream Port. Link: https://lore.kernel.org/r/20210104230300.1277180-2-kbusch@kernel.org Tested-by: Hedi Berriche Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas Acked-by: Sean V Kelley Acked-by: Hedi Berriche --- drivers/pci/pcie/err.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index 510f31f0ef6d..a84f0bf4c1e2 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -231,15 +231,14 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, pci_walk_bridge(bridge, report_resume, &status); /* - * If we have native control of AER, clear error status in the Root - * Port or Downstream Port that signaled the error. If the - * platform retained control of AER, it is responsible for clearing - * this status. In that case, the signaling device may not even be - * visible to the OS. + * If we have native control of AER, clear error status in the device + * that detected the error. If the platform retained control of AER, + * it is responsible for clearing this status. In that case, the + * signaling device may not even be visible to the OS. */ if (host->native_aer || pcie_ports_native) { - pcie_clear_device_status(bridge); - pci_aer_clear_nonfatal_status(bridge); + pcie_clear_device_status(dev); + pci_aer_clear_nonfatal_status(dev); } pci_info(bridge, "device recovery successful\n"); return status; From 7a8a22be35a5058366429e311017e05206c43137 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 4 Jan 2021 15:02:57 -0800 Subject: [PATCH 441/633] PCI/AER: Clear AER status from Root Port when resetting Downstream Port The pci_dev parameter given to aer_root_reset() may be a Downstream Port rather than the Root Port. Get the Root Port from the provided device in order to clear the root's AER status. Link: https://lore.kernel.org/r/20210104230300.1277180-3-kbusch@kernel.org Tested-by: Hedi Berriche Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas Acked-by: Sean V Kelley Acked-by: Hedi Berriche --- drivers/pci/pcie/aer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 77b0f2c45bc0..3fd4aaaa627e 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1388,7 +1388,7 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) if (type == PCI_EXP_TYPE_RC_END) root = dev->rcec; else - root = dev; + root = pcie_find_root_port(dev); /* * If the platform retained control of AER, an RCiEP may not have From 387c72cdd7fb6bef650fb078d0f6ae9682abf631 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 4 Jan 2021 15:02:58 -0800 Subject: [PATCH 442/633] PCI/ERR: Retain status from error notification Overwriting the frozen detected status with the result of the link reset loses the NEED_RESET result that drivers are depending on for error handling to report the .slot_reset() callback. Retain this status so that subsequent error handling has the correct flow. Link: https://lore.kernel.org/r/20210104230300.1277180-4-kbusch@kernel.org Reported-by: Hinko Kocevar Tested-by: Hedi Berriche Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas Acked-by: Sean V Kelley Acked-by: Hedi Berriche --- drivers/pci/pcie/err.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index a84f0bf4c1e2..b576aa890c76 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -198,8 +198,7 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, pci_dbg(bridge, "broadcast error_detected message\n"); if (state == pci_channel_io_frozen) { pci_walk_bridge(bridge, report_frozen_detected, &status); - status = reset_subordinates(bridge); - if (status != PCI_ERS_RESULT_RECOVERED) { + if (reset_subordinates(bridge) != PCI_ERS_RESULT_RECOVERED) { pci_warn(bridge, "subordinate device reset failed\n"); goto failed; } From 33ac78bd3b509d36e7f109a447e28af42e637cb2 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 4 Jan 2021 15:02:59 -0800 Subject: [PATCH 443/633] PCI/AER: Specify the type of Port that was reset The AER driver may be called upon to reset either a Downstream or a Root Port. Check which type it is to properly identify it when logging that the reset occurred. Link: https://lore.kernel.org/r/20210104230300.1277180-5-kbusch@kernel.org Tested-by: Hedi Berriche Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas Acked-by: Hedi Berriche --- drivers/pci/pcie/aer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 3fd4aaaa627e..ba22388342d1 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1414,7 +1414,8 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) } } else { rc = pci_bus_error_reset(dev); - pci_info(dev, "Root Port link has been reset (%d)\n", rc); + pci_info(dev, "%s Port link has been reset (%d)\n", + pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); } if ((host->native_aer || pcie_ports_native) && aer) { From ba952824e6c106f979c07814c8e3ef7405dd7b29 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 4 Jan 2021 15:03:00 -0800 Subject: [PATCH 444/633] PCI/portdrv: Report reset for frozen channel The PCI error recovery always resets the link for a frozen state, so the port driver should return that a reset is required for its result. This will get the .slot_reset() callback invoked, which is necessary to restore the port's config space. Without this, the driver had been relying on downstream drivers to return this status. Link: https://lore.kernel.org/r/20210104230300.1277180-6-kbusch@kernel.org Tested-by: Hedi Berriche Signed-off-by: Keith Busch Signed-off-by: Bjorn Helgaas Acked-by: Hedi Berriche --- drivers/pci/pcie/portdrv_pci.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 0b250bc5f405..de141bfb0bc2 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -153,7 +153,8 @@ static void pcie_portdrv_remove(struct pci_dev *dev) static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, pci_channel_state_t error) { - /* Root Port has no impact. Always recovers. */ + if (error == pci_channel_io_frozen) + return PCI_ERS_RESULT_NEED_RESET; return PCI_ERS_RESULT_CAN_RECOVER; } From 6104033bd25ef48d2013220f66632d8b0fc8cddb Mon Sep 17 00:00:00 2001 From: Vidya Sagar Date: Wed, 30 Dec 2020 22:27:23 +0530 Subject: [PATCH 445/633] PCI: dwc: Work around ECRC configuration issue DesignWare core has a TLP digest (TD) override bit in one of the control registers of ATU. This bit also needs to be programmed for proper ECRC functionality. This is currently identified as an issue with DesignWare IP version 4.90a. [bhelgaas: fix typos/grammar errors] Link: https://lore.kernel.org/r/20201230165723.673-1-vidyas@nvidia.com Signed-off-by: Vidya Sagar Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Bjorn Helgaas --- drivers/pci/controller/dwc/pcie-designware.c | 49 +++++++++++++++++++- drivers/pci/controller/dwc/pcie-designware.h | 1 + 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 645fa1892375..c49c8b5b4800 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -225,6 +225,47 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } +static inline u32 dw_pcie_enable_ecrc(u32 val) +{ + /* + * DesignWare core version 4.90A has a design issue where the 'TD' + * bit in the Control register-1 of the ATU outbound region acts + * like an override for the ECRC setting, i.e., the presence of TLP + * Digest (ECRC) in the outgoing TLPs is solely determined by this + * bit. This is contrary to the PCIe spec which says that the + * enablement of the ECRC is solely determined by the AER + * registers. + * + * Because of this, even when the ECRC is enabled through AER + * registers, the transactions going through ATU won't have TLP + * Digest as there is no way the PCI core AER code could program + * the TD bit which is specific to the DesignWare core. + * + * The best way to handle this scenario is to program the TD bit + * always. It affects only the traffic from root port to downstream + * devices. + * + * At this point, + * When ECRC is enabled in AER registers, everything works normally + * When ECRC is NOT enabled in AER registers, then, + * on Root Port:- TLP Digest (DWord size) gets appended to each packet + * even through it is not required. Since downstream + * TLPs are mostly for configuration accesses and BAR + * accesses, they are not in critical path and won't + * have much negative effect on the performance. + * on End Point:- TLP Digest is received for some/all the packets coming + * from the root port. TLP Digest is ignored because, + * as per the PCIe Spec r5.0 v1.0 section 2.2.3 + * "TLP Digest Rules", when an endpoint receives TLP + * Digest when its ECRC check functionality is disabled + * in AER registers, received TLP Digest is just ignored. + * Since there is no issue or error reported either side, best way to + * handle the scenario is to program TD bit by default. + */ + + return val | PCIE_ATU_TD; +} + static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, @@ -248,6 +289,8 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, val = type | PCIE_ATU_FUNC_NUM(func_no); val = upper_32_bits(size - 1) ? val | PCIE_ATU_INCREASE_REGION_SIZE : val; + if (pci->version == 0x490A) + val = dw_pcie_enable_ecrc(val); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, val); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, PCIE_ATU_ENABLE); @@ -294,8 +337,10 @@ static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, lower_32_bits(pci_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(pci_addr)); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | - PCIE_ATU_FUNC_NUM(func_no)); + val = type | PCIE_ATU_FUNC_NUM(func_no); + if (pci->version == 0x490A) + val = dw_pcie_enable_ecrc(val); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val); dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE); /* diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 0207840756c4..5d979953800d 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -86,6 +86,7 @@ #define PCIE_ATU_TYPE_IO 0x2 #define PCIE_ATU_TYPE_CFG0 0x4 #define PCIE_ATU_TYPE_CFG1 0x5 +#define PCIE_ATU_TD BIT(8) #define PCIE_ATU_FUNC_NUM(pf) ((pf) << 20) #define PCIE_ATU_CR2 0x908 #define PCIE_ATU_ENABLE BIT(31) From 792b6aa97e56d34aca23d314ca330843fdcaed33 Mon Sep 17 00:00:00 2001 From: Hou Zhiqiang Date: Mon, 26 Oct 2020 13:14:47 +0800 Subject: [PATCH 446/633] dt-bindings: PCI: layerscape: Add LX2160A rev2 compatible strings Add PCIe Endpoint mode compatible string "fsl,lx2160ar2-pcie-ep" Link: https://lore.kernel.org/r/20201026051448.1913-1-Zhiqiang.Hou@nxp.com Signed-off-by: Hou Zhiqiang Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Rob Herring --- Documentation/devicetree/bindings/pci/layerscape-pci.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt index daa99f7d4c3f..6d898dd4a8e2 100644 --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt @@ -26,6 +26,7 @@ Required properties: "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls1088a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls2088a-pcie-ep", "fsl,ls-pcie-ep" + "fsl,lx2160ar2-pcie-ep", "fsl,ls-pcie-ep" - reg: base addresses and lengths of the PCIe controller register blocks. - interrupts: A list of interrupt outputs of the controller. Must contain an entry for each entry in the interrupt-names property. From 5bfb792f210ce6644bc2d72e047e0715ac4a1010 Mon Sep 17 00:00:00 2001 From: Hou Zhiqiang Date: Mon, 26 Oct 2020 13:14:48 +0800 Subject: [PATCH 447/633] PCI: layerscape: Add LX2160A rev2 EP mode support The LX2160A rev2 uses the same PCIe IP as LS2088A, but LX2160A rev2 PCIe controller is integrated with different stride between PFs' register address. Link: https://lore.kernel.org/r/20201026051448.1913-2-Zhiqiang.Hou@nxp.com Signed-off-by: Hou Zhiqiang Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/dwc/pci-layerscape-ep.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 4d12efdacd2f..39fe2ed5a6a2 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -115,10 +115,17 @@ static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; +static const struct ls_pcie_ep_drvdata lx2_ep_drvdata = { + .func_offset = 0x8000, + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + static const struct of_device_id ls_pcie_ep_of_match[] = { { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,lx2160ar2-pcie-ep", .data = &lx2_ep_drvdata }, { }, }; From 7007b745a508735dc168637294404d6ac0a2d475 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Wed, 20 Jan 2021 11:52:46 +0100 Subject: [PATCH 448/633] PCI: layerscape: Convert to builtin_platform_driver() fw_devlink will defer the probe until all suppliers are ready. We can't use builtin_platform_driver_probe() because it doesn't retry after probe deferral. Convert it to builtin_platform_driver(). Link: https://lore.kernel.org/r/20210120105246.23218-1-michael@walle.cc Signed-off-by: Michael Walle Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/dwc/pci-layerscape.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c index 44ad34cdc3bc..5b9c625df7b8 100644 --- a/drivers/pci/controller/dwc/pci-layerscape.c +++ b/drivers/pci/controller/dwc/pci-layerscape.c @@ -232,7 +232,7 @@ static const struct of_device_id ls_pcie_of_match[] = { { }, }; -static int __init ls_pcie_probe(struct platform_device *pdev) +static int ls_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct dw_pcie *pci; @@ -271,10 +271,11 @@ static int __init ls_pcie_probe(struct platform_device *pdev) } static struct platform_driver ls_pcie_driver = { + .probe = ls_pcie_probe, .driver = { .name = "layerscape-pcie", .of_match_table = ls_pcie_of_match, .suppress_bind_attrs = true, }, }; -builtin_platform_driver_probe(ls_pcie_driver, ls_pcie_probe); +builtin_platform_driver(ls_pcie_driver); From 2f5ab5afe018a8c208bcefe37fbd26ff1afc25a2 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 15 Dec 2020 13:41:49 -0600 Subject: [PATCH 449/633] PCI: dwc: Drop support for config space in 'ranges' Since commit a0fd361db8e5 ("PCI: dwc: Move "dbi", "dbi2", and "addr_space" resource setup into common code"), the code setting dbi_base when the config space is defined in 'ranges' property instead of 'reg' is dead code as dbi_base is never NULL. Rather than fix this, let's just drop the code. Using ranges has been deprecated since 2014. The only platforms using this were exynos5440, i.MX6 and Spear13xx. Exynos5440 is dead and has been removed. i.MX6 and Spear13xx had PCIe support added just before this was deprecated and were fixed within a kernel release or 2. Link: https://lore.kernel.org/r/20201215194149.86831-1-robh@kernel.org Reported-by: Dan Carpenter Signed-off-by: Rob Herring Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- .../pci/controller/dwc/pcie-designware-host.c | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 8a84c005f32b..bcb5bd7ec5ef 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -305,8 +305,13 @@ int dw_pcie_host_init(struct pcie_port *pp) if (cfg_res) { pp->cfg0_size = resource_size(cfg_res); pp->cfg0_base = cfg_res->start; - } else if (!pp->va_cfg0_base) { + + pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, cfg_res); + if (IS_ERR(pp->va_cfg0_base)) + return PTR_ERR(pp->va_cfg0_base); + } else { dev_err(dev, "Missing *config* reg space\n"); + return -ENODEV; } if (!pci->dbi_base) { @@ -322,38 +327,12 @@ int dw_pcie_host_init(struct pcie_port *pp) pp->bridge = bridge; - /* Get the I/O and memory ranges from DT */ - resource_list_for_each_entry(win, &bridge->windows) { - switch (resource_type(win->res)) { - case IORESOURCE_IO: - pp->io_size = resource_size(win->res); - pp->io_bus_addr = win->res->start - win->offset; - pp->io_base = pci_pio_to_address(win->res->start); - break; - case 0: - dev_err(dev, "Missing *config* reg space\n"); - pp->cfg0_size = resource_size(win->res); - pp->cfg0_base = win->res->start; - if (!pci->dbi_base) { - pci->dbi_base = devm_pci_remap_cfgspace(dev, - pp->cfg0_base, - pp->cfg0_size); - if (!pci->dbi_base) { - dev_err(dev, "Error with ioremap\n"); - return -ENOMEM; - } - } - break; - } - } - - if (!pp->va_cfg0_base) { - pp->va_cfg0_base = devm_pci_remap_cfgspace(dev, - pp->cfg0_base, pp->cfg0_size); - if (!pp->va_cfg0_base) { - dev_err(dev, "Error with ioremap in function\n"); - return -ENOMEM; - } + /* Get the I/O range from DT */ + win = resource_list_first_type(&bridge->windows, IORESOURCE_IO); + if (win) { + pp->io_size = resource_size(win->res); + pp->io_bus_addr = win->res->start - win->offset; + pp->io_base = pci_pio_to_address(win->res->start); } if (pci->link_gen < 1) From 3856e1c5b88e5d363c251a2bc0d9fd0efdc6184a Mon Sep 17 00:00:00 2001 From: Shradha Todi Date: Wed, 6 Jan 2021 16:15:00 +0530 Subject: [PATCH 450/633] PCI: dwc: Change size to u64 for EP outbound iATU Since outbound iATU permits size to be greater than 4GB for which the support is also available, allow EP function to send u64 size instead of truncating to u32. Link: https://lore.kernel.org/r/1609929900-19082-1-git-send-email-shradha.t@samsung.com Signed-off-by: Shradha Todi Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Pankaj Dubey --- drivers/pci/controller/dwc/pcie-designware.c | 2 +- drivers/pci/controller/dwc/pcie-designware.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index c49c8b5b4800..e7b9a7d7c9a2 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -366,7 +366,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, - u32 size) + u64 size) { __dw_pcie_prog_outbound_atu(pci, func_no, index, type, cpu_addr, pci_addr, size); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 5d979953800d..d8d2e0a7ac09 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -298,7 +298,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, u64 size); void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 cpu_addr, u64 pci_addr, - u32 size); + u64 size); int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, int bar, u64 cpu_addr, enum dw_pcie_as_type as_type); From 5b4cf0f6532434537818e4a3c656b9f11c81729b Mon Sep 17 00:00:00 2001 From: Shradha Todi Date: Tue, 2 Feb 2021 12:58:38 +0530 Subject: [PATCH 451/633] PCI: dwc: Add upper limit address for outbound iATU The size parameter is unsigned long type which can accept size > 4GB. In that case, the upper limit address must be programmed. Add support to program the upper limit address and set INCREASE_REGION_SIZE in case size > 4GB. Link: https://lore.kernel.org/r/1612250918-19610-1-git-send-email-shradha.t@samsung.com Signed-off-by: Shradha Todi Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Pankaj Dubey Reviewed-by: Rob Herring --- drivers/pci/controller/dwc/pcie-designware.c | 5 +++++ drivers/pci/controller/dwc/pcie-designware.h | 1 + 2 files changed, 6 insertions(+) diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index e7b9a7d7c9a2..fb637830bc71 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -333,11 +333,16 @@ static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, upper_32_bits(cpu_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT, lower_32_bits(cpu_addr + size - 1)); + if (pci->version >= 0x460A) + dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_LIMIT, + upper_32_bits(cpu_addr + size - 1)); dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET, lower_32_bits(pci_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(pci_addr)); val = type | PCIE_ATU_FUNC_NUM(func_no); + val = ((upper_32_bits(size - 1)) && (pci->version >= 0x460A)) ? + val | PCIE_ATU_INCREASE_REGION_SIZE : val; if (pci->version == 0x490A) val = dw_pcie_enable_ecrc(val); dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index d8d2e0a7ac09..7247c8b01f04 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -100,6 +100,7 @@ #define PCIE_ATU_DEV(x) FIELD_PREP(GENMASK(23, 19), x) #define PCIE_ATU_FUNC(x) FIELD_PREP(GENMASK(18, 16), x) #define PCIE_ATU_UPPER_TARGET 0x91C +#define PCIE_ATU_UPPER_LIMIT 0x924 #define PCIE_MISC_CONTROL_1_OFF 0x8BC #define PCIE_DBI_RO_WR_EN BIT(0) From a2f882d84406ac3a31af09ebd2ec2410fda3e80d Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 28 Jan 2021 14:42:58 +0800 Subject: [PATCH 452/633] PCI: dwc: Don't assume the ops in dw_pcie always exist Some dwc-based device drivers, especially host-only drivers, may work well with the default read_dbi/write_dbi/link_up implementations in pcie-designware.c, so remove the assumption that every driver implements them to simplify those drivers. Link: https://lore.kernel.org/r/20210128144258.10329aa4@xhacker.debian Signed-off-by: Jisheng Zhang Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- drivers/pci/controller/dwc/pcie-designware-ep.c | 8 +++----- drivers/pci/controller/dwc/pcie-designware-host.c | 2 +- drivers/pci/controller/dwc/pcie-designware.c | 14 +++++++------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index bcd1cd9ba8c8..1c25d8337151 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -434,10 +434,8 @@ static void dw_pcie_ep_stop(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - if (!pci->ops->stop_link) - return; - - pci->ops->stop_link(pci); + if (pci->ops && pci->ops->stop_link) + pci->ops->stop_link(pci); } static int dw_pcie_ep_start(struct pci_epc *epc) @@ -445,7 +443,7 @@ static int dw_pcie_ep_start(struct pci_epc *epc) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - if (!pci->ops->start_link) + if (!pci->ops || !pci->ops->start_link) return -EINVAL; return pci->ops->start_link(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index bcb5bd7ec5ef..0f0d8f477596 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -404,7 +404,7 @@ int dw_pcie_host_init(struct pcie_port *pp) dw_pcie_setup_rc(pp); dw_pcie_msi_init(pp); - if (!dw_pcie_link_up(pci) && pci->ops->start_link) { + if (!dw_pcie_link_up(pci) && pci->ops && pci->ops->start_link) { ret = pci->ops->start_link(pci); if (ret) goto err_free_msi; diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index fb637830bc71..004cb860e266 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -141,7 +141,7 @@ u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size) int ret; u32 val; - if (pci->ops->read_dbi) + if (pci->ops && pci->ops->read_dbi) return pci->ops->read_dbi(pci, pci->dbi_base, reg, size); ret = dw_pcie_read(pci->dbi_base + reg, size, &val); @@ -156,7 +156,7 @@ void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; - if (pci->ops->write_dbi) { + if (pci->ops && pci->ops->write_dbi) { pci->ops->write_dbi(pci, pci->dbi_base, reg, size, val); return; } @@ -171,7 +171,7 @@ void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) { int ret; - if (pci->ops->write_dbi2) { + if (pci->ops && pci->ops->write_dbi2) { pci->ops->write_dbi2(pci, pci->dbi_base2, reg, size, val); return; } @@ -186,7 +186,7 @@ static u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 reg) int ret; u32 val; - if (pci->ops->read_dbi) + if (pci->ops && pci->ops->read_dbi) return pci->ops->read_dbi(pci, pci->atu_base, reg, 4); ret = dw_pcie_read(pci->atu_base + reg, 4, &val); @@ -200,7 +200,7 @@ static void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val) { int ret; - if (pci->ops->write_dbi) { + if (pci->ops && pci->ops->write_dbi) { pci->ops->write_dbi(pci, pci->atu_base, reg, 4, val); return; } @@ -316,7 +316,7 @@ static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, { u32 retries, val; - if (pci->ops->cpu_addr_fixup) + if (pci->ops && pci->ops->cpu_addr_fixup) cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); if (pci->iatu_unroll_enabled) { @@ -531,7 +531,7 @@ int dw_pcie_link_up(struct dw_pcie *pci) { u32 val; - if (pci->ops->link_up) + if (pci->ops && pci->ops->link_up) return pci->ops->link_up(pci); val = readl(pci->dbi_base + PCIE_PORT_DEBUG1); From 2a34b86f9fc8003c02802393c447da876f01dee0 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 28 Jan 2021 14:43:24 +0800 Subject: [PATCH 453/633] PCI: al: Remove useless dw_pcie_ops We have removed the assumption that dw_pcie_ops always exists in the dwc core driver, so we can remove the useless dw_pcie_ops now. Link: https://lore.kernel.org/r/20210128144324.2fa8577c@xhacker.debian Signed-off-by: Jisheng Zhang Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Jonathan Chocron --- drivers/pci/controller/dwc/pcie-al.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c index abf37aa68e51..e8afa50129a8 100644 --- a/drivers/pci/controller/dwc/pcie-al.c +++ b/drivers/pci/controller/dwc/pcie-al.c @@ -314,9 +314,6 @@ static const struct dw_pcie_host_ops al_pcie_host_ops = { .host_init = al_pcie_host_init, }; -static const struct dw_pcie_ops dw_pcie_ops = { -}; - static int al_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -334,7 +331,6 @@ static int al_pcie_probe(struct platform_device *pdev) return -ENOMEM; pci->dev = dev; - pci->ops = &dw_pcie_ops; pci->pp.ops = &al_pcie_host_ops; al_pcie->pci = pci; From c9f04600026f5ea6bbcfd7b06da265604946efd0 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Sun, 17 Jan 2021 04:31:13 +0300 Subject: [PATCH 454/633] dt-bindings: PCI: qcom: Document ddrss_sf_tbu clock for sm8250 On SM8250 additional clock is required for PCIe devices to access NOC. Document this requirement in devicetree bindings. Link: https://lore.kernel.org/r/20210117013114.441973-2-dmitry.baryshkov@linaro.org Fixes: 458168247ccc ("dt-bindings: pci: qcom: Document PCIe bindings for SM8250 SoC") Signed-off-by: Dmitry Baryshkov Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas --- .../devicetree/bindings/pci/qcom,pcie.txt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.txt b/Documentation/devicetree/bindings/pci/qcom,pcie.txt index 3b55310390a0..0da458a051b6 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.txt +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.txt @@ -132,8 +132,8 @@ - "master_bus" AXI Master clock - "slave_bus" AXI Slave clock --clock-names: - Usage: required for sdm845 and sm8250 +- clock-names: + Usage: required for sdm845 Value type: Definition: Should contain the following entries - "aux" Auxiliary clock @@ -144,6 +144,19 @@ - "tbu" PCIe TBU clock - "pipe" PIPE clock +- clock-names: + Usage: required for sm8250 + Value type: + Definition: Should contain the following entries + - "aux" Auxiliary clock + - "cfg" Configuration clock + - "bus_master" Master AXI clock + - "bus_slave" Slave AXI clock + - "slave_q2a" Slave Q2A clock + - "tbu" PCIe TBU clock + - "ddrss_sf_tbu" PCIe SF TBU clock + - "pipe" PIPE clock + - resets: Usage: required Value type: From 7081556f81f78c6397a129bd58ceb7ae64750df9 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Sun, 17 Jan 2021 04:31:14 +0300 Subject: [PATCH 455/633] PCI: qcom: Add support for ddrss_sf_tbu clock On SM8250 additional clock is required for PCIe devices to access NOC. Update PCIe controller driver to control this clock. Link: https://lore.kernel.org/r/20210117013114.441973-3-dmitry.baryshkov@linaro.org Fixes: e1dd639e374a ("PCI: qcom: Add SM8250 SoC support") Signed-off-by: Dmitry Baryshkov Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Reviewed-by: Manivannan Sadhasivam Acked-by: Stanimir Varbanov --- drivers/pci/controller/dwc/pcie-qcom.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index affa2713bf80..ab21aa01c95d 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -159,8 +159,10 @@ struct qcom_pcie_resources_2_3_3 { struct reset_control *rst[7]; }; +/* 6 clocks typically, 7 for sm8250 */ struct qcom_pcie_resources_2_7_0 { - struct clk_bulk_data clks[6]; + struct clk_bulk_data clks[7]; + int num_clks; struct regulator_bulk_data supplies[2]; struct reset_control *pci_reset; struct clk *pipe_clk; @@ -1152,8 +1154,14 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) res->clks[3].id = "bus_slave"; res->clks[4].id = "slave_q2a"; res->clks[5].id = "tbu"; + if (of_device_is_compatible(dev->of_node, "qcom,pcie-sm8250")) { + res->clks[6].id = "ddrss_sf_tbu"; + res->num_clks = 7; + } else { + res->num_clks = 6; + } - ret = devm_clk_bulk_get(dev, ARRAY_SIZE(res->clks), res->clks); + ret = devm_clk_bulk_get(dev, res->num_clks, res->clks); if (ret < 0) return ret; @@ -1175,7 +1183,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) return ret; } - ret = clk_bulk_prepare_enable(ARRAY_SIZE(res->clks), res->clks); + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); if (ret < 0) goto err_disable_regulators; @@ -1227,7 +1235,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) return 0; err_disable_clocks: - clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); + clk_bulk_disable_unprepare(res->num_clks, res->clks); err_disable_regulators: regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); @@ -1238,7 +1246,7 @@ static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) { struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; - clk_bulk_disable_unprepare(ARRAY_SIZE(res->clks), res->clks); + clk_bulk_disable_unprepare(res->num_clks, res->clks); regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); } From 2cfef1971aea6119ee27429181d6cb3383031ac2 Mon Sep 17 00:00:00 2001 From: Ansuel Smith Date: Mon, 19 Oct 2020 18:55:55 +0200 Subject: [PATCH 456/633] PCI: qcom: Use PHY_REFCLK_USE_PAD only for ipq8064 The use of PHY_REFCLK_USE_PAD introduced a regression for apq8064 devices. It was tested that while apq doesn't require the padding, ipq SoC must use it or the kernel hangs on boot. Link: https://lore.kernel.org/r/20201019165555.8269-1-ansuelsmth@gmail.com Fixes: de3c4bf64897 ("PCI: qcom: Add support for tx term offset for rev 2.1.0") Reported-by: Ilia Mirkin Signed-off-by: Ilia Mirkin Signed-off-by: Ansuel Smith Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Stanimir Varbanov Cc: stable@vger.kernel.org # v4.19+ --- drivers/pci/controller/dwc/pcie-qcom.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index ab21aa01c95d..8a7a300163e5 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -400,7 +400,9 @@ static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) /* enable external reference clock */ val = readl(pcie->parf + PCIE20_PARF_PHY_REFCLK); - val &= ~PHY_REFCLK_USE_PAD; + /* USE_PAD is required only for ipq806x */ + if (!of_device_is_compatible(node, "qcom,pcie-apq8064")) + val &= ~PHY_REFCLK_USE_PAD; val |= PHY_REFCLK_SSP_EN; writel(val, pcie->parf + PCIE20_PARF_PHY_REFCLK); From c79c3c34f75d72a066e292b10aa50fc758c97c89 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 24 Feb 2021 12:00:12 -0800 Subject: [PATCH 457/633] hexagon: remove CONFIG_EXPERIMENTAL from defconfigs Since CONFIG_EXPERIMENTAL was removed in 2013, go ahead and drop it from any defconfig files. Link: https://lkml.kernel.org/r/20210115010011.29483-1-rdunlap@infradead.org Fixes: 3d374d09f16f ("final removal of CONFIG_EXPERIMENTAL") Signed-off-by: Randy Dunlap Cc: Kees Cook Cc: Greg Kroah-Hartman Cc: Brian Cain Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/hexagon/configs/comet_defconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/hexagon/configs/comet_defconfig b/arch/hexagon/configs/comet_defconfig index e324f65f41e7..f19ae2ab0aaa 100644 --- a/arch/hexagon/configs/comet_defconfig +++ b/arch/hexagon/configs/comet_defconfig @@ -1,7 +1,6 @@ CONFIG_SMP=y CONFIG_DEFAULT_MMAP_MIN_ADDR=0 CONFIG_HZ_100=y -CONFIG_EXPERIMENTAL=y CONFIG_CROSS_COMPILE="hexagon-" CONFIG_LOCALVERSION="-smp" # CONFIG_LOCALVERSION_AUTO is not set From 6b294bf6b4f6cc4a2cf2029dff31010ab4addffc Mon Sep 17 00:00:00 2001 From: tangchunyou Date: Wed, 24 Feb 2021 12:00:15 -0800 Subject: [PATCH 458/633] scripts/spelling.txt: increase error-prone spell checking Increase maping spelling error check. Link: https://lkml.kernel.org/r/20210121092125.2663-1-tangchunyou@163.com Signed-off-by: WenZhang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- scripts/spelling.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 953f4a2de1e5..ebcb27bda1f5 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -875,6 +875,7 @@ manger||manager manoeuvering||maneuvering manufaucturing||manufacturing mappping||mapping +maping||mapping matchs||matches mathimatical||mathematical mathimatic||mathematic From 02bbbc4b554ed2d971c5f49950244d8f3d0b112e Mon Sep 17 00:00:00 2001 From: zuoqilin Date: Wed, 24 Feb 2021 12:00:18 -0800 Subject: [PATCH 459/633] scripts/spelling.txt: check for "exeeds" Increase exeeds spelling error check. Link: https://lkml.kernel.org/r/20210127060049.915-1-zuoqilin1@163.com Signed-off-by: zuoqilin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- scripts/spelling.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spelling.txt b/scripts/spelling.txt index ebcb27bda1f5..334f932e0b1b 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -566,6 +566,7 @@ estbalishment||establishment etsablishment||establishment etsbalishment||establishment evalution||evaluation +exeeds||exceeds excecutable||executable exceded||exceeded exceds||exceeds From 4945192325708bb5cc5cb5b07f288e118f5f65bd Mon Sep 17 00:00:00 2001 From: dingsenjie Date: Wed, 24 Feb 2021 12:00:21 -0800 Subject: [PATCH 460/633] scripts/spelling.txt: add "allocted" and "exeeds" typo Increase "allocted" and "exeeds" spelling error check. Link: https://lkml.kernel.org/r/20210127081919.1928-1-dingsenjie@163.com Signed-off-by: dingsenjie Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- scripts/spelling.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 334f932e0b1b..a47006de990e 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -103,6 +103,7 @@ alloated||allocated allocatote||allocate allocatrd||allocated allocte||allocate +allocted||allocated allpication||application alocate||allocate alogirhtms||algorithms @@ -575,6 +576,7 @@ excellant||excellent execeeded||exceeded execeeds||exceeds exeed||exceed +exeeds||exceeds exeuction||execution existance||existence existant||existent From 30cdbd53921ff8c39c7c2e7a6318d964a0ae154c Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 24 Feb 2021 12:00:24 -0800 Subject: [PATCH 461/633] scripts/spelling.txt: add more spellings to spelling.txt Here are some of the more common spelling mistakes and typos that I've found while fixing up spelling mistakes in the kernel since September 2020 Link: https://lkml.kernel.org/r/20210210124318.55082-1-colin.king@canonical.com Signed-off-by: Colin Ian King Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- scripts/spelling.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/spelling.txt b/scripts/spelling.txt index a47006de990e..2e3ba91a5072 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -340,6 +340,7 @@ comppatible||compatible compres||compress compresion||compression comression||compression +comunicate||communicate comunication||communication conbination||combination conditionaly||conditionally @@ -467,6 +468,7 @@ developpment||development deveolpment||development devided||divided deviece||device +devision||division diable||disable dicline||decline dictionnary||dictionary @@ -480,6 +482,7 @@ difinition||definition digial||digital dimention||dimension dimesions||dimensions +diconnected||disconnected disgest||digest dispalying||displaying diplay||display @@ -519,6 +522,7 @@ downlads||downloads droped||dropped droput||dropout druing||during +dyanmic||dynamic dynmaic||dynamic eanable||enable eanble||enable @@ -543,6 +547,7 @@ encrupted||encrypted encrypiton||encryption encryptio||encryption endianess||endianness +enpoint||endpoint enhaced||enhanced enlightnment||enlightenment enqueing||enqueuing @@ -644,6 +649,7 @@ forwardig||forwarding frambuffer||framebuffer framming||framing framwork||framework +frequence||frequency frequncy||frequency frequancy||frequency frome||from @@ -686,10 +692,12 @@ handfull||handful hanlde||handle hanled||handled happend||happened +hardare||hardware harware||hardware havind||having heirarchically||hierarchically helpfull||helpful +heterogenous||heterogeneous hexdecimal||hexadecimal hybernate||hibernate hierachy||hierarchy @@ -734,6 +742,7 @@ inconsistant||inconsistent increas||increase incremeted||incremented incrment||increment +incuding||including inculde||include indendation||indentation indended||intended @@ -744,6 +753,7 @@ indiate||indicate indicat||indicate inexpect||inexpected inferface||interface +infinit||infinite infomation||information informatiom||information informations||information @@ -774,6 +784,7 @@ instace||instance instal||install instanciate||instantiate instanciated||instantiated +instuments||instruments insufficent||insufficient inteface||interface integreated||integrated @@ -872,6 +883,7 @@ mailformed||malformed malplaced||misplaced malplace||misplace managable||manageable +managament||management managment||management mangement||management manger||manager @@ -890,6 +902,7 @@ meetign||meeting memeory||memory memmber||member memoery||memory +memroy||memory ment||meant mergable||mergeable mesage||message @@ -1003,6 +1016,7 @@ overlaping||overlapping overide||override overrided||overridden overriden||overridden +overrrun||overrun overun||overrun overwritting||overwriting overwriten||overwritten @@ -1039,6 +1053,7 @@ peforming||performing peice||piece pendantic||pedantic peprocessor||preprocessor +perfomance||performance perfoming||performing perfomring||performing periperal||peripheral @@ -1104,6 +1119,7 @@ prodecure||procedure progamming||programming progams||programs progess||progress +programable||programmable programers||programmers programm||program programms||programs @@ -1148,6 +1164,7 @@ recieved||received recieve||receive reciever||receiver recieves||receives +recieving||receiving recogniced||recognised recognizeable||recognizable recommanded||recommended @@ -1251,6 +1268,7 @@ searchs||searches secquence||sequence secund||second segement||segment +seleted||selected semaphone||semaphore senario||scenario senarios||scenarios @@ -1267,6 +1285,7 @@ seqeunce||sequence seqeuncer||sequencer seqeuencer||sequencer sequece||sequence +sequemce||sequence sequencial||sequential serivce||service serveral||several @@ -1337,6 +1356,7 @@ suble||subtle substract||subtract submited||submitted submition||submission +succeded||succeeded suceed||succeed succesfully||successfully succesful||successful @@ -1357,6 +1377,7 @@ supportin||supporting suppoted||supported suppported||supported suppport||support +supprot||support supress||suppress surpressed||suppressed surpresses||suppresses @@ -1405,6 +1426,7 @@ thresold||threshold throught||through trackling||tracking troughput||throughput +trys||tries thses||these tiggers||triggers tiggered||triggered @@ -1418,7 +1440,9 @@ traking||tracking tramsmitted||transmitted tramsmit||transmit tranasction||transaction +tranceiver||transceiver tranfer||transfer +tranmission||transmission transcevier||transceiver transciever||transceiver transferd||transferred @@ -1472,6 +1496,7 @@ unnecesary||unnecessary unneedingly||unnecessarily unnsupported||unsupported unmached||unmatched +unprecise||imprecise unregester||unregister unresgister||unregister unrgesiter||unregister @@ -1507,6 +1532,7 @@ varient||variant vaule||value verbse||verbose veify||verify +veriosn||version verisons||versions verison||version verson||version From 6bbf29010fa90a7ff22ff14e2875b4e6dea8d576 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 24 Feb 2021 12:00:27 -0800 Subject: [PATCH 462/633] ntfs: layout.h: delete duplicated words Drop the repeated words "the" and "in" in comments. Link: https://lkml.kernel.org/r/20210125194937.24627-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Acked-by: Anton Altaparmakov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ntfs/layout.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/ntfs/layout.h b/fs/ntfs/layout.h index 85422761ff43..5d4bf7a3259f 100644 --- a/fs/ntfs/layout.h +++ b/fs/ntfs/layout.h @@ -703,7 +703,7 @@ typedef struct { /* 14*/ le16 instance; /* The instance of this attribute record. This number is unique within this mft record (see MFT_RECORD/next_attribute_instance notes in - in mft.h for more details). */ + mft.h for more details). */ /* 16*/ union { /* Resident attributes. */ struct { @@ -1838,7 +1838,7 @@ typedef struct { * Also, each security descriptor is stored twice in the $SDS stream with a * fixed offset of 0x40000 bytes (256kib, the Windows cache manager's max size) * between them; i.e. if a SDS_ENTRY specifies an offset of 0x51d0, then the - * the first copy of the security descriptor will be at offset 0x51d0 in the + * first copy of the security descriptor will be at offset 0x51d0 in the * $SDS data stream and the second copy will be at offset 0x451d0. */ typedef struct { From 4dfe6bd94959222e18d512bdf15f6bf9edb9c27c Mon Sep 17 00:00:00 2001 From: Rustam Kovhaev Date: Wed, 24 Feb 2021 12:00:30 -0800 Subject: [PATCH 463/633] ntfs: check for valid standard information attribute Mounting a corrupted filesystem with NTFS resulted in a kernel crash. We should check for valid STANDARD_INFORMATION attribute offset and length before trying to access it Link: https://lkml.kernel.org/r/20210217155930.1506815-1-rkovhaev@gmail.com Link: https://syzkaller.appspot.com/bug?extid=c584225dabdea2f71969 Signed-off-by: Rustam Kovhaev Reported-by: syzbot+c584225dabdea2f71969@syzkaller.appspotmail.com Tested-by: syzbot+c584225dabdea2f71969@syzkaller.appspotmail.com Acked-by: Anton Altaparmakov Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ntfs/inode.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index 4435dbbc0b63..f5c058b3192c 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -629,6 +629,12 @@ static int ntfs_read_locked_inode(struct inode *vi) } a = ctx->attr; /* Get the standard information attribute value. */ + if ((u8 *)a + le16_to_cpu(a->data.resident.value_offset) + + le32_to_cpu(a->data.resident.value_length) > + (u8 *)ctx->mrec + vol->mft_record_size) { + ntfs_error(vi->i_sb, "Corrupt standard information attribute in inode."); + goto unm_err_out; + } si = (STANDARD_INFORMATION*)((u8*)a + le16_to_cpu(a->data.resident.value_offset)); From 6efb59499aff080e6a9f1485ff968918c30c5b0c Mon Sep 17 00:00:00 2001 From: Yi Li Date: Wed, 24 Feb 2021 12:00:34 -0800 Subject: [PATCH 464/633] ocfs2: remove redundant conditional before iput iput handles NULL pointers gracefully, so there's no need to check the pointer before the call. Link: https://lkml.kernel.org/r/20201231040535.4091761-1-yili@winhong.com Signed-off-by: Yi Li Acked-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/super.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 2febc76e9de7..079f8826993e 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -973,8 +973,6 @@ static void ocfs2_disable_quotas(struct ocfs2_super *osb) * quota files */ dquot_disable(sb, type, DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); - if (!inode) - continue; iput(inode); } } From 95e126d650391696f7ba8d318634cc018df10ef9 Mon Sep 17 00:00:00 2001 From: guozh Date: Wed, 24 Feb 2021 12:00:38 -0800 Subject: [PATCH 465/633] ocfs2: clean up some definitions which are not used any more There are some definitions which is not used anymore in OCFS2 module, so as to be removed. Link: https://lkml.kernel.org/r/2021011916182284700534@chinatelecom.cn Signed-off-by: Guozhonghua Acked-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/dlm/dlmast.c | 10 ---------- fs/ocfs2/dlm/dlmcommon.h | 4 ---- 2 files changed, 14 deletions(-) diff --git a/fs/ocfs2/dlm/dlmast.c b/fs/ocfs2/dlm/dlmast.c index 6abaded3ff6b..70a10764f249 100644 --- a/fs/ocfs2/dlm/dlmast.c +++ b/fs/ocfs2/dlm/dlmast.c @@ -165,16 +165,6 @@ void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) spin_unlock(&lock->spinlock); } -void dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) -{ - BUG_ON(!dlm); - BUG_ON(!lock); - - spin_lock(&dlm->ast_lock); - __dlm_queue_bast(dlm, lock); - spin_unlock(&dlm->ast_lock); -} - static void dlm_update_lvb(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, struct dlm_lock *lock) { diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h index c8a444622faa..58d57e25d384 100644 --- a/fs/ocfs2/dlm/dlmcommon.h +++ b/fs/ocfs2/dlm/dlmcommon.h @@ -17,10 +17,7 @@ #define DLM_LOCKID_NAME_MAX 32 -#define DLM_DOMAIN_NAME_MAX_LEN 255 #define DLM_LOCK_RES_OWNER_UNKNOWN O2NM_MAX_NODES -#define DLM_THREAD_SHUFFLE_INTERVAL 5 // flush everything every 5 passes -#define DLM_THREAD_MS 200 // flush at least every 200 ms #define DLM_HASH_SIZE_DEFAULT (1 << 17) #if DLM_HASH_SIZE_DEFAULT < PAGE_SIZE @@ -902,7 +899,6 @@ void __dlm_lockres_grab_inflight_worker(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); void dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); -void dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void __dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); void dlm_do_local_ast(struct dlm_ctxt *dlm, From c57d117f2b2f2a19b570c36f2819ef8d8210af20 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 24 Feb 2021 12:00:41 -0800 Subject: [PATCH 466/633] ocfs2: fix a use after free on error The error handling in this function frees "reg" but it is still on the "o2hb_all_regions" list so it will lead to a use after freew. Joseph Qi points out that we need to clear the bit in the "o2hb_region_bitmap" as well Link: https://lkml.kernel.org/r/YBk4M6HUG8jB/jc7@mwanda Fixes: 1cf257f51191 ("ocfs2: fix memory leak") Signed-off-by: Dan Carpenter Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/cluster/heartbeat.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index 0179a73a3fa2..12a7590601dd 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -2042,7 +2042,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g o2hb_nego_timeout_handler, reg, NULL, ®->hr_handler_list); if (ret) - goto free; + goto remove_item; ret = o2net_register_handler(O2HB_NEGO_APPROVE_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), @@ -2057,6 +2057,12 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g unregister_handler: o2net_unregister_handler_list(®->hr_handler_list); +remove_item: + spin_lock(&o2hb_live_lock); + list_del(®->hr_all_item); + if (o2hb_global_heartbeat_active()) + clear_bit(reg->hr_region_num, o2hb_region_bitmap); + spin_unlock(&o2hb_live_lock); free: kfree(reg); return ERR_PTR(ret); From 7c908aec34733408baa755613141a08b960d8eec Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 24 Feb 2021 12:00:45 -0800 Subject: [PATCH 467/633] ocfs2: simplify the calculation of variables Fix the following coccicheck warnings: fs/ocfs2/refcounttree.c:981:16-18: WARNING !A || A && B is equivalent to !A || B. Link: https://lkml.kernel.org/r/1612235424-80367-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Jiapeng Chong Reported-by: Abaci Robot Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/refcounttree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index c26937824be1..c19a463fac55 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -978,7 +978,7 @@ static int ocfs2_get_refcount_cpos_end(struct ocfs2_caching_info *ci, return 0; } - if (!eb || (eb && !eb->h_next_leaf_blk)) { + if (!eb || !eb->h_next_leaf_blk) { /* * We are the last extent rec, so any high cpos should * be stored in this leaf refcount block. From 3d742d4b6ebb3348e1d478047cfb18b9b337b8df Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Wed, 24 Feb 2021 12:00:48 -0800 Subject: [PATCH 468/633] fs: delete repeated words in comments Delete duplicate words in fs/*.c. The doubled words that are being dropped are: that, be, the, in, and, for Link: https://lkml.kernel.org/r/20201224052810.25315-1-rdunlap@infradead.org Signed-off-by: Randy Dunlap Reviewed-by: Matthew Wilcox (Oracle) Cc: Alexander Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/block_dev.c | 2 +- fs/dcache.c | 4 ++-- fs/direct-io.c | 4 ++-- fs/exec.c | 4 ++-- fs/fhandle.c | 2 +- fs/pipe.c | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/block_dev.c b/fs/block_dev.c index ec26179c8062..91ff9bfd7c1a 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -1270,7 +1270,7 @@ int bdev_disk_changed(struct block_device *bdev, bool invalidate) return ret; } /* - * Only exported for for loop and dasd for historic reasons. Don't use in new + * Only exported for loop and dasd for historic reasons. Don't use in new * code! */ EXPORT_SYMBOL_GPL(bdev_disk_changed); diff --git a/fs/dcache.c b/fs/dcache.c index 799d9e4f0bcd..7d24ff7eb206 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -2176,8 +2176,8 @@ EXPORT_SYMBOL(d_obtain_root); * same inode, only the actual correct case is stored in the dcache for * case-insensitive filesystems. * - * For a case-insensitive lookup match and if the the case-exact dentry - * already exists in in the dcache, use it and return it. + * For a case-insensitive lookup match and if the case-exact dentry + * already exists in the dcache, use it and return it. * * If no entry exists with the exact case name, allocate new dentry with * the exact case, and return the spliced entry. diff --git a/fs/direct-io.c b/fs/direct-io.c index aa1083ecd623..0957e1bb8eb2 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -462,7 +462,7 @@ static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) * Wait for the next BIO to complete. Remove it and return it. NULL is * returned once all BIOs have been completed. This must only be called once * all bios have been issued so that dio->refcount can only decrease. This - * requires that that the caller hold a reference on the dio. + * requires that the caller hold a reference on the dio. */ static struct bio *dio_await_one(struct dio *dio) { @@ -1279,7 +1279,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, if (retval == -ENOTBLK) { /* * The remaining part of the request will be - * be handled by buffered I/O when we return + * handled by buffered I/O when we return */ retval = 0; } diff --git a/fs/exec.c b/fs/exec.c index 6f3c02066ce3..18594f11c31f 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1455,7 +1455,7 @@ EXPORT_SYMBOL(finalize_exec); /* * Prepare credentials and lock ->cred_guard_mutex. * setup_new_exec() commits the new creds and drops the lock. - * Or, if exec fails before, free_bprm() should release ->cred and + * Or, if exec fails before, free_bprm() should release ->cred * and unlock. */ static int prepare_bprm_creds(struct linux_binprm *bprm) @@ -1841,7 +1841,7 @@ static int bprm_execve(struct linux_binprm *bprm, out: /* - * If past the point of no return ensure the the code never + * If past the point of no return ensure the code never * returns to the userspace process. Use an existing fatal * signal if present otherwise terminate the process with * SIGSEGV. diff --git a/fs/fhandle.c b/fs/fhandle.c index 01263ffbc4c0..ec6feeccc276 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -173,7 +173,7 @@ static int handle_to_path(int mountdirfd, struct file_handle __user *ufh, /* * With handle we don't look at the execute bit on the - * the directory. Ideally we would like CAP_DAC_SEARCH. + * directory. Ideally we would like CAP_DAC_SEARCH. * But we don't have that */ if (!capable(CAP_DAC_READ_SEARCH)) { diff --git a/fs/pipe.c b/fs/pipe.c index 39c96845a72f..bfd946a9ad01 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -171,7 +171,7 @@ EXPORT_SYMBOL(generic_pipe_buf_try_steal); * * Description: * This function grabs an extra reference to @buf. It's used in - * in the tee() system call, when we duplicate the buffers in one + * the tee() system call, when we duplicate the buffers in one * pipe into another. */ bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) From 93da400397445f1110b394caab5558d13971378e Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Wed, 24 Feb 2021 12:00:51 -0800 Subject: [PATCH 469/633] ramfs: support O_TMPFILE [akpm@linux-foundation.org: update inode_operations.tmpfile] Link: http://lkml.kernel.org/r/20190206073349.GA15311@avx2 Signed-off-by: Alexey Dobriyan Acked-by: Christian Brauner Cc: Al Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ramfs/inode.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c index 3c2658c8fde0..9ebd17d7befb 100644 --- a/fs/ramfs/inode.c +++ b/fs/ramfs/inode.c @@ -151,6 +151,18 @@ static int ramfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, return error; } +static int ramfs_tmpfile(struct user_namespace *mnt_userns, + struct inode *dir, struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + + inode = ramfs_get_inode(dir->i_sb, dir, mode, 0); + if (!inode) + return -ENOSPC; + d_tmpfile(dentry, inode); + return 0; +} + static const struct inode_operations ramfs_dir_inode_operations = { .create = ramfs_create, .lookup = simple_lookup, @@ -161,6 +173,7 @@ static const struct inode_operations ramfs_dir_inode_operations = { .rmdir = simple_rmdir, .mknod = ramfs_mknod, .rename = simple_rename, + .tmpfile = ramfs_tmpfile, }; /* From 3544de8ee6e4817278b15fe08658de49abf58954 Mon Sep 17 00:00:00 2001 From: Jacob Wen Date: Wed, 24 Feb 2021 12:00:55 -0800 Subject: [PATCH 470/633] mm, tracing: record slab name for kmem_cache_free() Currently, a trace record generated by the RCU core is as below. ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=00000000f3b49a66 It doesn't tell us what the RCU core has freed. This patch adds the slab name to trace_kmem_cache_free(). The new format is as follows. ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=0000000037f79c8d name=dentry ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=00000000f78cb7b5 name=sock_inode_cache ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=0000000018768985 name=pool_workqueue ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=000000006a6cb484 name=radix_tree_node We can use it to understand what the RCU core is going to free. For example, some users maybe interested in when the RCU core starts freeing reclaimable slabs like dentry to reduce memory pressure. Link: https://lkml.kernel.org/r/20201216072804.8838-1-jian.w.wen@oracle.com Signed-off-by: Jacob Wen Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Steven Rostedt Cc: "Paul E. McKenney" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/trace/events/kmem.h | 24 ++++++++++++++++-------- mm/slab.c | 2 +- mm/slob.c | 2 +- mm/slub.c | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index f65b1f6db22d..40845b0d5dad 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -115,7 +115,7 @@ DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ); -DECLARE_EVENT_CLASS(kmem_free, +TRACE_EVENT(kfree, TP_PROTO(unsigned long call_site, const void *ptr), @@ -135,18 +135,26 @@ DECLARE_EVENT_CLASS(kmem_free, (void *)__entry->call_site, __entry->ptr) ); -DEFINE_EVENT(kmem_free, kfree, +TRACE_EVENT(kmem_cache_free, - TP_PROTO(unsigned long call_site, const void *ptr), + TP_PROTO(unsigned long call_site, const void *ptr, const char *name), - TP_ARGS(call_site, ptr) -); + TP_ARGS(call_site, ptr, name), -DEFINE_EVENT(kmem_free, kmem_cache_free, + TP_STRUCT__entry( + __field( unsigned long, call_site ) + __field( const void *, ptr ) + __field( const char *, name ) + ), - TP_PROTO(unsigned long call_site, const void *ptr), + TP_fast_assign( + __entry->call_site = call_site; + __entry->ptr = ptr; + __entry->name = name; + ), - TP_ARGS(call_site, ptr) + TP_printk("call_site=%pS ptr=%p name=%s", + (void *)__entry->call_site, __entry->ptr, __entry->name) ); TRACE_EVENT(mm_page_free, diff --git a/mm/slab.c b/mm/slab.c index dcc55e78f353..2ed27849d1f2 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3717,7 +3717,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) __cache_free(cachep, objp, _RET_IP_); local_irq_restore(flags); - trace_kmem_cache_free(_RET_IP_, objp); + trace_kmem_cache_free(_RET_IP_, objp, cachep->name); } EXPORT_SYMBOL(kmem_cache_free); diff --git a/mm/slob.c b/mm/slob.c index ef87ada8705d..0578429b991b 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -673,7 +673,7 @@ void kmem_cache_free(struct kmem_cache *c, void *b) __kmem_cache_free(b, c->size); } - trace_kmem_cache_free(_RET_IP_, b); + trace_kmem_cache_free(_RET_IP_, b, c->name); } EXPORT_SYMBOL(kmem_cache_free); diff --git a/mm/slub.c b/mm/slub.c index f5baf429654f..f4dc70228861 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3157,7 +3157,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) if (!s) return; slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_); - trace_kmem_cache_free(_RET_IP_, x); + trace_kmem_cache_free(_RET_IP_, x, s->name); } EXPORT_SYMBOL(kmem_cache_free); From 3754000872188e3e4713d9d847fe3c615a47c220 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 24 Feb 2021 12:00:58 -0800 Subject: [PATCH 471/633] mm/sl?b.c: remove ctor argument from kmem_cache_flags This argument hasn't been used since e153362a50a3 ("slub: Remove objsize check in kmem_cache_flags()") so simply remove it. Link: https://lkml.kernel.org/r/20210126095733.974665-1-nborisov@suse.com Signed-off-by: Nikolay Borisov Reviewed-by: Miaohe Lin Reviewed-by: Vlastimil Babka Acked-by: Christoph Lameter Acked-by: David Rientjes Cc: Pekka Enberg Cc: Joonsoo Kim Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slab.c | 3 +-- mm/slab.h | 6 ++---- mm/slab_common.c | 2 +- mm/slub.c | 9 +++------ 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 2ed27849d1f2..9982ff8ff175 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1790,8 +1790,7 @@ static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) } slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name, - void (*ctor)(void *)) + slab_flags_t flags, const char *name) { return flags; } diff --git a/mm/slab.h b/mm/slab.h index ecad9b57bc44..48dc466a3791 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -110,8 +110,7 @@ __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, slab_flags_t flags, void (*ctor)(void *)); slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name, - void (*ctor)(void *)); + slab_flags_t flags, const char *name); #else static inline struct kmem_cache * __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, @@ -119,8 +118,7 @@ __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, { return NULL; } static inline slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name, - void (*ctor)(void *)) + slab_flags_t flags, const char *name) { return flags; } diff --git a/mm/slab_common.c b/mm/slab_common.c index adbace4256ef..580124b7fed8 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -197,7 +197,7 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); size = ALIGN(size, align); - flags = kmem_cache_flags(size, flags, name, NULL); + flags = kmem_cache_flags(size, flags, name); if (flags & SLAB_NEVER_MERGE) return NULL; diff --git a/mm/slub.c b/mm/slub.c index f4dc70228861..5d8d5d21e5d3 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1400,7 +1400,6 @@ __setup("slub_debug", setup_slub_debug); * @object_size: the size of an object without meta data * @flags: flags to set * @name: name of the cache - * @ctor: constructor function * * Debug option(s) are applied to @flags. In addition to the debug * option(s), if a slab name (or multiple) is specified i.e. @@ -1408,8 +1407,7 @@ __setup("slub_debug", setup_slub_debug); * then only the select slabs will receive the debug option(s). */ slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name, - void (*ctor)(void *)) + slab_flags_t flags, const char *name) { char *iter; size_t len; @@ -1474,8 +1472,7 @@ static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) {} slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name, - void (*ctor)(void *)) + slab_flags_t flags, const char *name) { return flags; } @@ -3797,7 +3794,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order) static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) { - s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); + s->flags = kmem_cache_flags(s->size, flags, s->name); #ifdef CONFIG_SLAB_FREELIST_HARDENED s->random = get_random_long(); #endif From 0b41163407e2f3f44d6ed455ebfb1534df23f4a6 Mon Sep 17 00:00:00 2001 From: Zhiyuan Dai Date: Wed, 24 Feb 2021 12:01:01 -0800 Subject: [PATCH 472/633] mm/slab: minor coding style tweaks Fix some coding style issues, improve code reading. Adds whitespace to clearly separate the parameters. Link: https://lkml.kernel.org/r/1612841499-32166-1-git-send-email-daizhiyuan@phytium.com.cn Signed-off-by: Zhiyuan Dai Acked-by: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slab.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 9982ff8ff175..cdad64b2836d 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -272,7 +272,7 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent) #define STATS_DEC_ACTIVE(x) ((x)->num_active--) #define STATS_INC_ALLOCED(x) ((x)->num_allocations++) #define STATS_INC_GROWN(x) ((x)->grown++) -#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y)) +#define STATS_ADD_REAPED(x, y) ((x)->reaped += (y)) #define STATS_SET_HIGH(x) \ do { \ if ((x)->num_active > (x)->high_mark) \ @@ -296,7 +296,7 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent) #define STATS_DEC_ACTIVE(x) do { } while (0) #define STATS_INC_ALLOCED(x) do { } while (0) #define STATS_INC_GROWN(x) do { } while (0) -#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0) +#define STATS_ADD_REAPED(x, y) do { (void)(y); } while (0) #define STATS_SET_HIGH(x) do { } while (0) #define STATS_INC_ERR(x) do { } while (0) #define STATS_INC_NODEALLOCS(x) do { } while (0) @@ -332,7 +332,7 @@ static int obj_offset(struct kmem_cache *cachep) static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) { BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); - return (unsigned long long*) (objp + obj_offset(cachep) - + return (unsigned long long *) (objp + obj_offset(cachep) - sizeof(unsigned long long)); } @@ -580,7 +580,7 @@ static int transfer_objects(struct array_cache *to, if (!nr) return 0; - memcpy(to->entry + to->avail, from->entry + from->avail -nr, + memcpy(to->entry + to->avail, from->entry + from->avail - nr, sizeof(void *) *nr); from->avail -= nr; @@ -2737,7 +2737,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, #else #define kfree_debugcheck(x) do { } while(0) -#define cache_free_debugcheck(x,objp,z) (objp) +#define cache_free_debugcheck(x, objp, z) (objp) #endif static inline void fixup_objfreelist_debug(struct kmem_cache *cachep, @@ -3024,7 +3024,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, return objp; } #else -#define cache_alloc_debugcheck_after(a,b,objp,d) (objp) +#define cache_alloc_debugcheck_after(a, b, objp, d) (objp) #endif static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) From ca220593208d8c433a761738461c31b1bf0be1f9 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Wed, 24 Feb 2021 12:01:04 -0800 Subject: [PATCH 473/633] mm/slub: disable user tracing for kmemleak caches by default If kmemleak is enabled, it uses a kmem cache for its own objects. These objects are used to hold information kmemleak uses, including a stack trace. If slub_debug is also turned on, each of them has *another* stack trace, so the overhead adds up, and on my tests (on ARCH=um, admittedly) 2/3rds of the allocations end up being doing the stack tracing. Turn off SLAB_STORE_USER if SLAB_NOLEAKTRACE was given, to avoid storing the essentially same data twice. Link: https://lkml.kernel.org/r/20210113215114.d94efa13ba30.I117b6764e725b3192318bbcf4269b13b709539ae@changeid Signed-off-by: Johannes Berg Acked-by: David Rientjes Acked-by: Catalin Marinas Acked-by: Vlastimil Babka Cc: Christoph Lameter Cc: Pekka Enberg Cc: Joonsoo Kim Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slub.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 5d8d5d21e5d3..a7c453bf7156 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1413,6 +1413,15 @@ slab_flags_t kmem_cache_flags(unsigned int object_size, size_t len; char *next_block; slab_flags_t block_flags; + slab_flags_t slub_debug_local = slub_debug; + + /* + * If the slab cache is for debugging (e.g. kmemleak) then + * don't store user (stack trace) information by default, + * but let the user enable it via the command line below. + */ + if (flags & SLAB_NOLEAKTRACE) + slub_debug_local &= ~SLAB_STORE_USER; len = strlen(name); next_block = slub_debug_string; @@ -1447,7 +1456,7 @@ slab_flags_t kmem_cache_flags(unsigned int object_size, } } - return flags | slub_debug; + return flags | slub_debug_local; } #else /* !CONFIG_SLUB_DEBUG */ static inline void setup_object_debug(struct kmem_cache *s, From 666716fd267df0007dfbb6480cd79dd5b05da4cc Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:01:08 -0800 Subject: [PATCH 474/633] mm, slub: stop freeing kmem_cache_node structures on node offline Patch series "mm, slab, slub: remove cpu and memory hotplug locks". Some related work caused me to look at how we use get/put_mems_online() and get/put_online_cpus() during kmem cache creation/descruction/shrinking, and realize that it should be actually safe to remove all of that with rather small effort (as e.g. Michal Hocko suspected in some of the past discussions already). This has the benefit to avoid rather heavy locks that have caused locking order issues already in the past. So this is the result, Patches 2 and 3 remove memory hotplug and cpu hotplug locking, respectively. Patch 1 is due to realization that in fact some races exist despite the locks (even if not removed), but the most sane solution is not to introduce more of them, but rather accept some wasted memory in scenarios that should be rare anyway (full memory hot remove), as we do the same in other contexts already. This patch (of 3): Commit e4f8e513c3d3 ("mm/slub: fix a deadlock in show_slab_objects()") has fixed a problematic locking order by removing the memory hotplug lock get/put_online_mems() from show_slab_objects(). During the discussion, it was argued [1] that this is OK, because existing slabs on the node would prevent a hotremove to proceed. That's true, but per-node kmem_cache_node structures are not necessarily allocated on the same node and may exist even without actual slab pages on the same node. Any path that uses get_node() directly or via for_each_kmem_cache_node() (such as show_slab_objects()) can race with freeing of kmem_cache_node even with the !NULL check, resulting in use-after-free. To that end, commit e4f8e513c3d3 argues in a comment that: * We don't really need mem_hotplug_lock (to hold off * slab_mem_going_offline_callback) here because slab's memory hot * unplug code doesn't destroy the kmem_cache->node[] data. While it's true that slab_mem_going_offline_callback() doesn't free the kmem_cache_node, the later callback slab_mem_offline_callback() actually does, so the race and use-after-free exists. Not just for show_slab_objects() after commit e4f8e513c3d3, but also many other places that are not under slab_mutex. And adding slab_mutex locking or other synchronization to SLUB paths such as get_any_partial() would be bad for performance and error-prone. The easiest solution is therefore to make the abovementioned comment true and stop freeing the kmem_cache_node structures, accepting some wasted memory in the full memory node removal scenario. Analogically we also don't free hotremoved pgdat as mentioned in [1], nor the similar per-node structures in SLAB. Importantly this approach will not block the hotremove, as generally such nodes should be movable in order to succeed hotremove in the first place, and thus the GFP_KERNEL allocated kmem_cache_node will come from elsewhere. [1] https://lore.kernel.org/linux-mm/20190924151147.GB23050@dhcp22.suse.cz/ Link: https://lkml.kernel.org/r/20210113131634.3671-1-vbabka@suse.cz Link: https://lkml.kernel.org/r/20210113131634.3671-2-vbabka@suse.cz Signed-off-by: Vlastimil Babka Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Vladimir Davydov Cc: Qian Cai Cc: David Hildenbrand Cc: Michal Hocko Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slub.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index a7c453bf7156..1ecd8a7bd25c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4273,8 +4273,6 @@ static int slab_mem_going_offline_callback(void *arg) static void slab_mem_offline_callback(void *arg) { - struct kmem_cache_node *n; - struct kmem_cache *s; struct memory_notify *marg = arg; int offline_node; @@ -4288,21 +4286,11 @@ static void slab_mem_offline_callback(void *arg) return; mutex_lock(&slab_mutex); - list_for_each_entry(s, &slab_caches, list) { - n = get_node(s, offline_node); - if (n) { - /* - * if n->nr_slabs > 0, slabs still exist on the node - * that is going down. We were unable to free them, - * and offline_pages() function shouldn't call this - * callback. So, we must fail. - */ - BUG_ON(slabs_node(s, offline_node)); - - s->node[offline_node] = NULL; - kmem_cache_free(kmem_cache_node, n); - } - } + /* + * We no longer free kmem_cache_node structures here, as it would be + * racy with all get_node() users, and infeasible to protect them with + * slab_mutex. + */ mutex_unlock(&slab_mutex); } @@ -4328,6 +4316,12 @@ static int slab_mem_going_online_callback(void *arg) */ mutex_lock(&slab_mutex); list_for_each_entry(s, &slab_caches, list) { + /* + * The structure may already exist if the node was previously + * onlined and offlined. + */ + if (get_node(s, nid)) + continue; /* * XXX: kmem_cache_alloc_node will fallback to other nodes * since memory is not yet available from the node that From 7e1fa93deff44677a94dfc323ff629bbf5cf9360 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:01:12 -0800 Subject: [PATCH 475/633] mm, slab, slub: stop taking memory hotplug lock Since commit 03afc0e25f7f ("slab: get_online_mems for kmem_cache_{create,destroy,shrink}") we are taking memory hotplug lock for SLAB and SLUB when creating, destroying or shrinking a cache. It is quite a heavy lock and it's best to avoid it if possible, as we had several issues with lockdep complaining about ordering in the past, see e.g. e4f8e513c3d3 ("mm/slub: fix a deadlock in show_slab_objects()"). The problem scenario in 03afc0e25f7f (solved by the memory hotplug lock) can be summarized as follows: while there's slab_mutex synchronizing new kmem cache creation and SLUB's MEM_GOING_ONLINE callback slab_mem_going_online_callback(), we may miss creation of kmem_cache_node for the hotplugged node in the new kmem cache, because the hotplug callback doesn't yet see the new cache, and cache creation in init_kmem_cache_nodes() only inits kmem_cache_node for nodes in the N_NORMAL_MEMORY nodemask, which however may not yet include the new node, as that happens only later after the MEM_GOING_ONLINE callback. Instead of using get/put_online_mems(), the problem can be solved by SLUB maintaining its own nodemask of nodes for which it has allocated the per-node kmem_cache_node structures. This nodemask would generally mirror the N_NORMAL_MEMORY nodemask, but would be updated only in under SLUB's control in its memory hotplug callbacks under the slab_mutex. This patch adds such nodemask and its handling. Commit 03afc0e25f7f mentiones "issues like [the one above]", but there don't appear to be further issues. All the paths (shared for SLAB and SLUB) taking the memory hotplug locks are also taking the slab_mutex, except kmem_cache_shrink() where 03afc0e25f7f replaced slab_mutex with get/put_online_mems(). We however cannot simply restore slab_mutex in kmem_cache_shrink(), as SLUB can enters the function from a write to sysfs 'shrink' file, thus holding kernfs lock, and in kmem_cache_create() the kernfs lock is nested within slab_mutex. But on closer inspection we don't actually need to protect kmem_cache_shrink() from hotplug callbacks: While SLUB's __kmem_cache_shrink() does for_each_kmem_cache_node(), missing a new node added in parallel hotplug is not fatal, and parallel hotremove does not free kmem_cache_node's anymore after the previous patch, so use-after free cannot happen. The per-node shrinking itself is protected by n->list_lock. Same is true for SLAB, and SLOB is no-op. SLAB also doesn't need the memory hotplug locking, which it only gained by 03afc0e25f7f through the shared paths in slab_common.c. Its memory hotplug callbacks are also protected by slab_mutex against races with these paths. The problem of SLUB relying on N_NORMAL_MEMORY doesn't apply to SLAB, as its setup_kmem_cache_nodes relies on N_ONLINE, and the new node is already set there during the MEM_GOING_ONLINE callback, so no special care is needed for SLAB. As such, this patch removes all get/put_online_mems() usage by the slab subsystem. Link: https://lkml.kernel.org/r/20210113131634.3671-3-vbabka@suse.cz Signed-off-by: Vlastimil Babka Cc: Christoph Lameter Cc: David Hildenbrand Cc: David Rientjes Cc: Joonsoo Kim Cc: Michal Hocko Cc: Pekka Enberg Cc: Qian Cai Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slab_common.c | 8 ++------ mm/slub.c | 28 +++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 580124b7fed8..d73bea53f759 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -310,7 +310,6 @@ kmem_cache_create_usercopy(const char *name, int err; get_online_cpus(); - get_online_mems(); mutex_lock(&slab_mutex); @@ -360,7 +359,6 @@ kmem_cache_create_usercopy(const char *name, out_unlock: mutex_unlock(&slab_mutex); - put_online_mems(); put_online_cpus(); if (err) { @@ -487,7 +485,6 @@ void kmem_cache_destroy(struct kmem_cache *s) return; get_online_cpus(); - get_online_mems(); mutex_lock(&slab_mutex); @@ -504,7 +501,6 @@ void kmem_cache_destroy(struct kmem_cache *s) out_unlock: mutex_unlock(&slab_mutex); - put_online_mems(); put_online_cpus(); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -523,10 +519,10 @@ int kmem_cache_shrink(struct kmem_cache *cachep) int ret; get_online_cpus(); - get_online_mems(); + kasan_cache_shrink(cachep); ret = __kmem_cache_shrink(cachep); - put_online_mems(); + put_online_cpus(); return ret; } diff --git a/mm/slub.c b/mm/slub.c index 1ecd8a7bd25c..175cd905b58a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -235,6 +235,14 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si) #endif } +/* + * Tracks for which NUMA nodes we have kmem_cache_nodes allocated. + * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily + * differ during memory hotplug/hotremove operations. + * Protected by slab_mutex. + */ +static nodemask_t slab_nodes; + /******************************************************************** * Core slab cache functions *******************************************************************/ @@ -2678,7 +2686,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, * ignore the node constraint */ if (unlikely(node != NUMA_NO_NODE && - !node_state(node, N_NORMAL_MEMORY))) + !node_isset(node, slab_nodes))) node = NUMA_NO_NODE; goto new_slab; } @@ -2689,7 +2697,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, * same as above but node_match() being false already * implies node != NUMA_NO_NODE */ - if (!node_state(node, N_NORMAL_MEMORY)) { + if (!node_isset(node, slab_nodes)) { node = NUMA_NO_NODE; goto redo; } else { @@ -3592,7 +3600,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s) { int node; - for_each_node_state(node, N_NORMAL_MEMORY) { + for_each_node_mask(node, slab_nodes) { struct kmem_cache_node *n; if (slab_state == DOWN) { @@ -4286,6 +4294,7 @@ static void slab_mem_offline_callback(void *arg) return; mutex_lock(&slab_mutex); + node_clear(offline_node, slab_nodes); /* * We no longer free kmem_cache_node structures here, as it would be * racy with all get_node() users, and infeasible to protect them with @@ -4335,6 +4344,11 @@ static int slab_mem_going_online_callback(void *arg) init_kmem_cache_node(n); s->node[nid] = n; } + /* + * Any cache created after this point will also have kmem_cache_node + * initialized for the new node. + */ + node_set(nid, slab_nodes); out: mutex_unlock(&slab_mutex); return ret; @@ -4415,6 +4429,7 @@ void __init kmem_cache_init(void) { static __initdata struct kmem_cache boot_kmem_cache, boot_kmem_cache_node; + int node; if (debug_guardpage_minorder()) slub_max_order = 0; @@ -4422,6 +4437,13 @@ void __init kmem_cache_init(void) kmem_cache_node = &boot_kmem_cache_node; kmem_cache = &boot_kmem_cache; + /* + * Initialize the nodemask for which we will allocate per node + * structures. Here we don't need taking slab_mutex yet. + */ + for_each_node_state(node, N_NORMAL_MEMORY) + node_set(node, slab_nodes); + create_boot_cache(kmem_cache_node, "kmem_cache_node", sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0); From 59450bbc12bee1c4e5dd25e6aa5d6a45a7bd6e81 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:01:15 -0800 Subject: [PATCH 476/633] mm, slab, slub: stop taking cpu hotplug lock SLAB has been using get/put_online_cpus() around creating, destroying and shrinking kmem caches since 95402b382901 ("cpu-hotplug: replace per-subsystem mutexes with get_online_cpus()") in 2008, which is supposed to be replacing a private mutex (cache_chain_mutex, called slab_mutex today) with system-wide mechanism, but in case of SLAB it's in fact used in addition to the existing mutex, without explanation why. SLUB appears to have avoided the cpu hotplug lock initially, but gained it due to common code unification, such as 20cea9683ecc ("mm, sl[aou]b: Move kmem_cache_create mutex handling to common code"). Regardless of the history, checking if the hotplug lock is actually needed today suggests that it's not, and therefore it's better to avoid this system-wide lock and the ordering this imposes wrt other locks (such as slab_mutex). Specifically, in SLAB we have for_each_online_cpu() in do_tune_cpucache() protected by slab_mutex, and cpu hotplug callbacks that also take the slab_mutex, which is also taken by the common slab function that currently also take the hotplug lock. Thus the slab_mutex protection should be sufficient. Also per-cpu array caches are allocated for each possible cpu, so not affected by their online/offline state. In SLUB we have for_each_online_cpu() in functions that show statistics and are already unprotected today, as racing with hotplug is not harmful. Otherwise SLUB relies on percpu allocator. The slub_cpu_dead() hotplug callback takes the slab_mutex. To sum up, this patch removes get/put_online_cpus() calls from slab as it should be safe without further adjustments. Link: https://lkml.kernel.org/r/20210113131634.3671-4-vbabka@suse.cz Signed-off-by: Vlastimil Babka Cc: Christoph Lameter Cc: David Hildenbrand Cc: David Rientjes Cc: Joonsoo Kim Cc: Michal Hocko Cc: Pekka Enberg Cc: Qian Cai Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slab_common.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index d73bea53f759..9633fa1bbbfd 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -309,8 +309,6 @@ kmem_cache_create_usercopy(const char *name, const char *cache_name; int err; - get_online_cpus(); - mutex_lock(&slab_mutex); err = kmem_cache_sanity_check(name, size); @@ -359,8 +357,6 @@ kmem_cache_create_usercopy(const char *name, out_unlock: mutex_unlock(&slab_mutex); - put_online_cpus(); - if (err) { if (flags & SLAB_PANIC) panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", @@ -484,8 +480,6 @@ void kmem_cache_destroy(struct kmem_cache *s) if (unlikely(!s)) return; - get_online_cpus(); - mutex_lock(&slab_mutex); s->refcount--; @@ -500,8 +494,6 @@ void kmem_cache_destroy(struct kmem_cache *s) } out_unlock: mutex_unlock(&slab_mutex); - - put_online_cpus(); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -518,12 +510,10 @@ int kmem_cache_shrink(struct kmem_cache *cachep) { int ret; - get_online_cpus(); kasan_cache_shrink(cachep); ret = __kmem_cache_shrink(cachep); - put_online_cpus(); return ret; } EXPORT_SYMBOL(kmem_cache_shrink); From d930ff03c4d12621443f2d1c56d2f80745469021 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:01:19 -0800 Subject: [PATCH 477/633] mm, slub: splice cpu and page freelists in deactivate_slab() In deactivate_slab() we currently move all but one objects on the cpu freelist to the page freelist one by one using the costly cmpxchg_double() operation. Then we unfreeze the page while moving the last object on page freelist, with a final cmpxchg_double(). This can be optimized to avoid the cmpxchg_double() per object. Just count the objects on cpu freelist (to adjust page->inuse properly) and also remember the last object in the chain. Then splice page->freelist to the last object and effectively add the whole cpu freelist to page->freelist while unfreezing the page, with a single cmpxchg_double(). Link: https://lkml.kernel.org/r/20210115183543.15097-1-vbabka@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: Jann Horn Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slub.c | 59 ++++++++++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 175cd905b58a..13e5821ce3e2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2167,9 +2167,9 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, { enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - int lock = 0; + int lock = 0, free_delta = 0; enum slab_modes l = M_NONE, m = M_NONE; - void *nextfree; + void *nextfree, *freelist_iter, *freelist_tail; int tail = DEACTIVATE_TO_HEAD; struct page new; struct page old; @@ -2180,45 +2180,34 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, } /* - * Stage one: Free all available per cpu objects back - * to the page freelist while it is still frozen. Leave the - * last one. - * - * There is no need to take the list->lock because the page - * is still frozen. + * Stage one: Count the objects on cpu's freelist as free_delta and + * remember the last object in freelist_tail for later splicing. */ - while (freelist && (nextfree = get_freepointer(s, freelist))) { - void *prior; - unsigned long counters; + freelist_tail = NULL; + freelist_iter = freelist; + while (freelist_iter) { + nextfree = get_freepointer(s, freelist_iter); /* * If 'nextfree' is invalid, it is possible that the object at - * 'freelist' is already corrupted. So isolate all objects - * starting at 'freelist'. + * 'freelist_iter' is already corrupted. So isolate all objects + * starting at 'freelist_iter' by skipping them. */ - if (freelist_corrupted(s, page, &freelist, nextfree)) + if (freelist_corrupted(s, page, &freelist_iter, nextfree)) break; - do { - prior = page->freelist; - counters = page->counters; - set_freepointer(s, freelist, prior); - new.counters = counters; - new.inuse--; - VM_BUG_ON(!new.frozen); + freelist_tail = freelist_iter; + free_delta++; - } while (!__cmpxchg_double_slab(s, page, - prior, counters, - freelist, new.counters, - "drain percpu freelist")); - - freelist = nextfree; + freelist_iter = nextfree; } /* - * Stage two: Ensure that the page is unfrozen while the - * list presence reflects the actual number of objects - * during unfreeze. + * Stage two: Unfreeze the page while splicing the per-cpu + * freelist to the head of page's freelist. + * + * Ensure that the page is unfrozen while the list presence + * reflects the actual number of objects during unfreeze. * * We setup the list membership and then perform a cmpxchg * with the count. If there is a mismatch then the page @@ -2231,15 +2220,15 @@ static void deactivate_slab(struct kmem_cache *s, struct page *page, */ redo: - old.freelist = page->freelist; - old.counters = page->counters; + old.freelist = READ_ONCE(page->freelist); + old.counters = READ_ONCE(page->counters); VM_BUG_ON(!old.frozen); /* Determine target state of the slab */ new.counters = old.counters; - if (freelist) { - new.inuse--; - set_freepointer(s, freelist, old.freelist); + if (freelist_tail) { + new.inuse -= free_delta; + set_freepointer(s, freelist_tail, old.freelist); new.freelist = freelist; } else new.freelist = old.freelist; From fe2cce15d6821aea1766708a1cf031071cec815f Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:01:22 -0800 Subject: [PATCH 478/633] mm, slub: remove slub_memcg_sysfs boot param and CONFIG_SLUB_MEMCG_SYSFS_ON The boot param and config determine the value of memcg_sysfs_enabled, which is unused since commit 10befea91b61 ("mm: memcg/slab: use a single set of kmem_caches for all allocations") as there are no per-memcg kmem caches anymore. Link: https://lkml.kernel.org/r/20210127124745.7928-1-vbabka@suse.cz Signed-off-by: Vlastimil Babka Reviewed-by: David Hildenbrand Acked-by: Roman Gushchin Acked-by: David Rientjes Reviewed-by: Miaohe Lin Cc: Christoph Lameter Cc: Pekka Enberg Cc: Joonsoo Kim Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/kernel-parameters.txt | 8 -------- init/Kconfig | 14 -------------- mm/slub.c | 16 ---------------- 3 files changed, 38 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 0ac883777318..19cfa8b9eb60 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4899,14 +4899,6 @@ last alloc / free. For more information see Documentation/vm/slub.rst. - slub_memcg_sysfs= [MM, SLUB] - Determines whether to enable sysfs directories for - memory cgroup sub-caches. 1 to enable, 0 to disable. - The default is determined by CONFIG_SLUB_MEMCG_SYSFS_ON. - Enabling this can lead to a very high number of debug - directories and files being created under - /sys/kernel/slub. - slub_max_order= [MM, SLUB] Determines the maximum allowed order for slabs. A high setting may cause OOMs due to memory diff --git a/init/Kconfig b/init/Kconfig index ba8bd5256980..8f3a6c4fc0b4 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1861,20 +1861,6 @@ config SLUB_DEBUG SLUB sysfs support. /sys/slab will not exist and there will be no support for cache validation etc. -config SLUB_MEMCG_SYSFS_ON - default n - bool "Enable memcg SLUB sysfs support by default" if EXPERT - depends on SLUB && SYSFS && MEMCG - help - SLUB creates a directory under /sys/kernel/slab for each - allocation cache to host info and debug files. If memory - cgroup is enabled, each cache can have per memory cgroup - caches. SLUB can create the same sysfs directories for these - caches under /sys/kernel/slab/CACHE/cgroup but it can lead - to a very high number of debug files being created. This is - controlled by slub_memcg_sysfs boot parameter and this - config option determines the parameter's default value. - config COMPAT_BRK bool "Disable heap randomization" default y diff --git a/mm/slub.c b/mm/slub.c index 13e5821ce3e2..49d10e0afb76 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4943,22 +4943,6 @@ enum slab_stat_type { #define SO_OBJECTS (1 << SL_OBJECTS) #define SO_TOTAL (1 << SL_TOTAL) -#ifdef CONFIG_MEMCG -static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON); - -static int __init setup_slub_memcg_sysfs(char *str) -{ - int v; - - if (get_option(&str, &v) > 0) - memcg_sysfs_enabled = v; - - return 1; -} - -__setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs); -#endif - static ssize_t show_slab_objects(struct kmem_cache *s, char *buf, unsigned long flags) { From 457c82c3516d56fc52b6b7518b0bce14b7809a3b Mon Sep 17 00:00:00 2001 From: Zhiyuan Dai Date: Wed, 24 Feb 2021 12:01:26 -0800 Subject: [PATCH 479/633] mm/slub: minor coding style tweaks Add whitespace to fix coding style issues, improve code reading. Link: https://lkml.kernel.org/r/1612847403-5594-1-git-send-email-daizhiyuan@phytium.com.cn Signed-off-by: Zhiyuan Dai Acked-by: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 49d10e0afb76..e0fef8f8d4bf 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3269,7 +3269,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) if (!df.page) continue; - slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_); + slab_free(df.s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); } while (likely(size)); } EXPORT_SYMBOL(kmem_cache_free_bulk); From 91f5345afbc6b58d79b5c5d0bc915fa83e9d238e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:01:29 -0800 Subject: [PATCH 480/633] mm/debug: improve memcg debugging The memcg_data is only valid on the head page, not the tail pages. Change the format and location of the printout within the dump to match the other parts of struct page better. Link: https://lkml.kernel.org/r/20210114190200.1894484-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Zi Yan Cc: Michal Hocko Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/debug.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mm/debug.c b/mm/debug.c index 8a40b3fefbeb..0bdda8407f71 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -110,6 +110,11 @@ void __dump_page(struct page *page, const char *reason) head_compound_mapcount(head)); } } + +#ifdef CONFIG_MEMCG + if (head->memcg_data) + pr_warn("memcg:%lx\n", head->memcg_data); +#endif if (PageKsm(page)) type = "ksm "; else if (PageAnon(page)) @@ -180,11 +185,6 @@ void __dump_page(struct page *page, const char *reason) if (reason) pr_warn("page dumped because: %s\n", reason); - -#ifdef CONFIG_MEMCG - if (!page_poisoned && page->memcg_data) - pr_warn("pages's memcg:%lx\n", page->memcg_data); -#endif } void dump_page(struct page *page, const char *reason) From bb5c47ced46797409f4791d0380db3116d93134c Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 24 Feb 2021 12:01:32 -0800 Subject: [PATCH 481/633] mm/debug_vm_pgtable/basic: add validation for dirtiness after write protect Patch series "mm/debug_vm_pgtable: Some minor updates", v3. This series contains some cleanups and new test suggestions from Catalin from an earlier discussion. https://lore.kernel.org/linux-mm/20201123142237.GF17833@gaia/ This patch (of 2): This adds validation tests for dirtiness after write protect conversion for each page table level. There are two new separate test types involved here. The first test ensures that a given page table entry does not become dirty after pxx_wrprotect(). This is important for platforms like arm64 which transfers and drops the hardware dirty bit (!PTE_RDONLY) to the software dirty bit while making it an write protected one. This test ensures that no fresh page table entry could be created with hardware dirty bit set. The second test ensures that a given page table entry always preserve the dirty information across pxx_wrprotect(). This adds two previously missing PUD level basic tests and while here fixes pxx_wrprotect() related typos in the documentation file. Link: https://lkml.kernel.org/r/1611137241-26220-1-git-send-email-anshuman.khandual@arm.com Link: https://lkml.kernel.org/r/1611137241-26220-2-git-send-email-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Suggested-by: Catalin Marinas Tested-by: Gerald Schaefer [s390] Cc: Christophe Leroy Cc: Gerald Schaefer Cc: Vineet Gupta Cc: Paul Walmsley Cc: Steven Price Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/arch_pgtable_helpers.rst | 8 ++--- mm/debug_vm_pgtable.c | 39 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Documentation/vm/arch_pgtable_helpers.rst b/Documentation/vm/arch_pgtable_helpers.rst index f3591ee3aaa8..552567d863b8 100644 --- a/Documentation/vm/arch_pgtable_helpers.rst +++ b/Documentation/vm/arch_pgtable_helpers.rst @@ -50,7 +50,7 @@ PTE Page Table Helpers +---------------------------+--------------------------------------------------+ | pte_mkwrite | Creates a writable PTE | +---------------------------+--------------------------------------------------+ -| pte_mkwrprotect | Creates a write protected PTE | +| pte_wrprotect | Creates a write protected PTE | +---------------------------+--------------------------------------------------+ | pte_mkspecial | Creates a special PTE | +---------------------------+--------------------------------------------------+ @@ -120,7 +120,7 @@ PMD Page Table Helpers +---------------------------+--------------------------------------------------+ | pmd_mkwrite | Creates a writable PMD | +---------------------------+--------------------------------------------------+ -| pmd_mkwrprotect | Creates a write protected PMD | +| pmd_wrprotect | Creates a write protected PMD | +---------------------------+--------------------------------------------------+ | pmd_mkspecial | Creates a special PMD | +---------------------------+--------------------------------------------------+ @@ -186,7 +186,7 @@ PUD Page Table Helpers +---------------------------+--------------------------------------------------+ | pud_mkwrite | Creates a writable PUD | +---------------------------+--------------------------------------------------+ -| pud_mkwrprotect | Creates a write protected PUD | +| pud_wrprotect | Creates a write protected PUD | +---------------------------+--------------------------------------------------+ | pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | +---------------------------+--------------------------------------------------+ @@ -224,7 +224,7 @@ HugeTLB Page Table Helpers +---------------------------+--------------------------------------------------+ | huge_pte_mkwrite | Creates a writable HugeTLB | +---------------------------+--------------------------------------------------+ -| huge_pte_mkwrprotect | Creates a write protected HugeTLB | +| huge_pte_wrprotect | Creates a write protected HugeTLB | +---------------------------+--------------------------------------------------+ | huge_ptep_get_and_clear | Clears a HugeTLB | +---------------------------+--------------------------------------------------+ diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index c05d9dcf7891..1842d97522bb 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -63,6 +63,16 @@ static void __init pte_basic_tests(unsigned long pfn, pgprot_t prot) pte_t pte = pfn_pte(pfn, prot); pr_debug("Validating PTE basic\n"); + + /* + * This test needs to be executed after the given page table entry + * is created with pfn_pte() to make sure that protection_map[idx] + * does not have the dirty bit enabled from the beginning. This is + * important for platforms like arm64 where (!PTE_RDONLY) indicate + * dirty bit being set. + */ + WARN_ON(pte_dirty(pte_wrprotect(pte))); + WARN_ON(!pte_same(pte, pte)); WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte)))); WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte)))); @@ -70,6 +80,8 @@ static void __init pte_basic_tests(unsigned long pfn, pgprot_t prot) WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte)))); WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte)))); WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte)))); + WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte)))); + WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte)))); } static void __init pte_advanced_tests(struct mm_struct *mm, @@ -137,6 +149,17 @@ static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) return; pr_debug("Validating PMD basic\n"); + + /* + * This test needs to be executed after the given page table entry + * is created with pfn_pmd() to make sure that protection_map[idx] + * does not have the dirty bit enabled from the beginning. This is + * important for platforms like arm64 where (!PTE_RDONLY) indicate + * dirty bit being set. + */ + WARN_ON(pmd_dirty(pmd_wrprotect(pmd))); + + WARN_ON(!pmd_same(pmd, pmd)); WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd)))); WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd)))); @@ -144,6 +167,8 @@ static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd)))); WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd)))); WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd)))); + WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd)))); + WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd)))); /* * A huge page does not point to next level page table * entry. Hence this must qualify as pmd_bad(). @@ -257,11 +282,25 @@ static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) return; pr_debug("Validating PUD basic\n"); + + /* + * This test needs to be executed after the given page table entry + * is created with pfn_pud() to make sure that protection_map[idx] + * does not have the dirty bit enabled from the beginning. This is + * important for platforms like arm64 where (!PTE_RDONLY) indicate + * dirty bit being set. + */ + WARN_ON(pud_dirty(pud_wrprotect(pud))); + WARN_ON(!pud_same(pud, pud)); WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud)))); + WARN_ON(!pud_dirty(pud_mkdirty(pud_mkclean(pud)))); + WARN_ON(pud_dirty(pud_mkclean(pud_mkdirty(pud)))); WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud)))); WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud)))); WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud)))); + WARN_ON(pud_dirty(pud_wrprotect(pud_mkclean(pud)))); + WARN_ON(!pud_dirty(pud_wrprotect(pud_mkdirty(pud)))); if (mm_pmd_folded(mm)) return; From 2e326c07bbe1eabeece4047ab5972ef34b15679b Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 24 Feb 2021 12:01:36 -0800 Subject: [PATCH 482/633] mm/debug_vm_pgtable/basic: iterate over entire protection_map[] Currently the basic tests just validate various page table transformations after starting with vm_get_page_prot(VM_READ|VM_WRITE|VM_EXEC) protection. Instead scan over the entire protection_map[] for better coverage. It also makes sure that all these basic page table tranformations checks hold true irrespective of the starting protection value for the page table entry. There is also a slight change in the debug print format for basic tests to capture the protection value it is being tested with. The modified output looks something like [pte_basic_tests ]: Validating PTE basic () [pte_basic_tests ]: Validating PTE basic (read) [pte_basic_tests ]: Validating PTE basic (write) [pte_basic_tests ]: Validating PTE basic (read|write) [pte_basic_tests ]: Validating PTE basic (exec) [pte_basic_tests ]: Validating PTE basic (read|exec) [pte_basic_tests ]: Validating PTE basic (write|exec) [pte_basic_tests ]: Validating PTE basic (read|write|exec) [pte_basic_tests ]: Validating PTE basic (shared) [pte_basic_tests ]: Validating PTE basic (read|shared) [pte_basic_tests ]: Validating PTE basic (write|shared) [pte_basic_tests ]: Validating PTE basic (read|write|shared) [pte_basic_tests ]: Validating PTE basic (exec|shared) [pte_basic_tests ]: Validating PTE basic (read|exec|shared) [pte_basic_tests ]: Validating PTE basic (write|exec|shared) [pte_basic_tests ]: Validating PTE basic (read|write|exec|shared) This adds a missing argument 'struct mm_struct *' in pud_basic_tests() test . This never got exposed before as PUD based THP is available only on X86 platform where mm_pmd_folded(mm) call gets macro replaced without requiring the mm_struct i.e __is_defined(__PAGETABLE_PMD_FOLDED). Link: https://lkml.kernel.org/r/1611137241-26220-3-git-send-email-anshuman.khandual@arm.com Signed-off-by: Anshuman Khandual Tested-by: Gerald Schaefer [s390] Reviewed-by: Steven Price Suggested-by: Catalin Marinas Cc: Christophe Leroy Cc: Gerald Schaefer Cc: Paul Walmsley Cc: Vineet Gupta Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/debug_vm_pgtable.c | 47 ++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 1842d97522bb..a9bd6ce1ba02 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -58,11 +58,13 @@ #define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK) #define RANDOM_NZVALUE GENMASK(7, 0) -static void __init pte_basic_tests(unsigned long pfn, pgprot_t prot) +static void __init pte_basic_tests(unsigned long pfn, int idx) { + pgprot_t prot = protection_map[idx]; pte_t pte = pfn_pte(pfn, prot); + unsigned long val = idx, *ptr = &val; - pr_debug("Validating PTE basic\n"); + pr_debug("Validating PTE basic (%pGv)\n", ptr); /* * This test needs to be executed after the given page table entry @@ -141,14 +143,16 @@ static void __init pte_savedwrite_tests(unsigned long pfn, pgprot_t prot) } #ifdef CONFIG_TRANSPARENT_HUGEPAGE -static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) +static void __init pmd_basic_tests(unsigned long pfn, int idx) { + pgprot_t prot = protection_map[idx]; pmd_t pmd = pfn_pmd(pfn, prot); + unsigned long val = idx, *ptr = &val; if (!has_transparent_hugepage()) return; - pr_debug("Validating PMD basic\n"); + pr_debug("Validating PMD basic (%pGv)\n", ptr); /* * This test needs to be executed after the given page table entry @@ -274,14 +278,16 @@ static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot) } #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD -static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) +static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx) { + pgprot_t prot = protection_map[idx]; pud_t pud = pfn_pud(pfn, prot); + unsigned long val = idx, *ptr = &val; if (!has_transparent_hugepage()) return; - pr_debug("Validating PUD basic\n"); + pr_debug("Validating PUD basic (%pGv)\n", ptr); /* * This test needs to be executed after the given page table entry @@ -398,7 +404,7 @@ static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot) #endif /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ -static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { } +static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx) { } static void __init pud_advanced_tests(struct mm_struct *mm, struct vm_area_struct *vma, pud_t *pudp, unsigned long pfn, unsigned long vaddr, @@ -411,8 +417,8 @@ static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot) } #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ -static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) { } -static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { } +static void __init pmd_basic_tests(unsigned long pfn, int idx) { } +static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx) { } static void __init pmd_advanced_tests(struct mm_struct *mm, struct vm_area_struct *vma, pmd_t *pmdp, unsigned long pfn, unsigned long vaddr, @@ -938,6 +944,7 @@ static int __init debug_vm_pgtable(void) unsigned long vaddr, pte_aligned, pmd_aligned; unsigned long pud_aligned, p4d_aligned, pgd_aligned; spinlock_t *ptl = NULL; + int idx; pr_info("Validating architecture page table helpers\n"); prot = vm_get_page_prot(VMFLAGS); @@ -1002,9 +1009,25 @@ static int __init debug_vm_pgtable(void) saved_pmdp = pmd_offset(pudp, 0UL); saved_ptep = pmd_pgtable(pmd); - pte_basic_tests(pte_aligned, prot); - pmd_basic_tests(pmd_aligned, prot); - pud_basic_tests(pud_aligned, prot); + /* + * Iterate over the protection_map[] to make sure that all + * the basic page table transformation validations just hold + * true irrespective of the starting protection value for a + * given page table entry. + */ + for (idx = 0; idx < ARRAY_SIZE(protection_map); idx++) { + pte_basic_tests(pte_aligned, idx); + pmd_basic_tests(pmd_aligned, idx); + pud_basic_tests(mm, pud_aligned, idx); + } + + /* + * Both P4D and PGD level tests are very basic which do not + * involve creating page table entries from the protection + * value and the given pfn. Hence just keep them out from + * the above iteration for now to save some test execution + * time. + */ p4d_basic_tests(p4d_aligned, prot); pgd_basic_tests(pgd_aligned, prot); From 1d2cae8ea1cf082df8258fcb5ab35de29821c450 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:01:39 -0800 Subject: [PATCH 483/633] mm/page_owner: use helper function zone_end_pfn() to get end_pfn Commit 108bcc96ef70 ("mm: add & use zone_end_pfn() and zone_spans_pfn()") introduced the helper zone_end_pfn() to calculate the zone end pfn. But pagetypeinfo_showmixedcount_print forgot to use it. And the initialization of local variable pfn is duplicated, remove one. Link: https://lkml.kernel.org/r/20210123070538.5861-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Andrew Morton Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_owner.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index af464bb7fbe7..d15c7c4994f5 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -263,8 +263,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, struct page *page; struct page_ext *page_ext; struct page_owner *page_owner; - unsigned long pfn = zone->zone_start_pfn, block_end_pfn; - unsigned long end_pfn = pfn + zone->spanned_pages; + unsigned long pfn, block_end_pfn; + unsigned long end_pfn = zone_end_pfn(zone); unsigned long count[MIGRATE_TYPES] = { 0, }; int pageblock_mt, page_mt; int i; From 1f7ef657740344541645349a8bece90cbff898f5 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 24 Feb 2021 12:01:42 -0800 Subject: [PATCH 484/633] mm/filemap: remove unused parameter and change to void type for replace_page_cache_page() Since commit 74d609585d8b ("page cache: Add and replace pages using the XArray") was merged, the replace_page_cache_page() can not fail and always return 0, we can remove the redundant return value and void it. Moreover remove the unused gfp_mask. Link: https://lkml.kernel.org/r/609c30e5274ba15d8b90c872fd0d8ac437a9b2bb.1610071401.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Cc: Matthew Wilcox Cc: Miklos Szeredi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fuse/dev.c | 6 +----- include/linux/pagemap.h | 2 +- mm/filemap.c | 7 +------ 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 588f8d1240aa..c6636b4c4ccf 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -844,11 +844,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) if (WARN_ON(PageMlocked(oldpage))) goto out_fallback_unlock; - err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL); - if (err) { - unlock_page(newpage); - goto out_put_old; - } + replace_page_cache_page(oldpage, newpage); get_page(newpage); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index d5570deff400..74e466e5a2ba 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -757,7 +757,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); extern void delete_from_page_cache(struct page *page); extern void __delete_from_page_cache(struct page *page, void *shadow); -int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask); +void replace_page_cache_page(struct page *old, struct page *new); void delete_from_page_cache_batch(struct address_space *mapping, struct pagevec *pvec); diff --git a/mm/filemap.c b/mm/filemap.c index 6ff2a3fb0dc7..7dfed3454a2e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -777,7 +777,6 @@ EXPORT_SYMBOL(file_write_and_wait_range); * replace_page_cache_page - replace a pagecache page with a new one * @old: page to be replaced * @new: page to replace with - * @gfp_mask: allocation mode * * This function replaces a page in the pagecache with a new one. On * success it acquires the pagecache reference for the new page and @@ -786,10 +785,8 @@ EXPORT_SYMBOL(file_write_and_wait_range); * caller must do that. * * The remove + add is atomic. This function cannot fail. - * - * Return: %0 */ -int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) +void replace_page_cache_page(struct page *old, struct page *new) { struct address_space *mapping = old->mapping; void (*freepage)(struct page *) = mapping->a_ops->freepage; @@ -824,8 +821,6 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) if (freepage) freepage(old); put_page(old); - - return 0; } EXPORT_SYMBOL_GPL(replace_page_cache_page); From ab2125df921d991a3c8a4fdcfe617ef6cad6b484 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 24 Feb 2021 12:01:45 -0800 Subject: [PATCH 485/633] mm/filemap: don't revert iter on -EIOCBQUEUED Currently, if I/O is enqueued for async execution direct paths of generic_file_{read,write}_iter() will always revert the iter. There are no users expecting that, and that is also costly. Leave iterators as is on -EIOCBQUEUED. Link: https://lkml.kernel.org/r/f5247b60e7abbd2ff850cd108491f53a2e0c501a.1610207781.git.asml.silence@gmail.com Signed-off-by: Pavel Begunkov Reviewed-by: Christoph Hellwig Cc: Al Viro Cc: Jens Axboe Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 7dfed3454a2e..774a869da106 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2615,7 +2615,8 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) iocb->ki_pos += retval; count -= retval; } - iov_iter_revert(iter, count - iov_iter_count(iter)); + if (retval != -EIOCBQUEUED) + iov_iter_revert(iter, count - iov_iter_count(iter)); /* * Btrfs can have a short DIO read if we encounter @@ -3426,7 +3427,8 @@ generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) } iocb->ki_pos = pos; } - iov_iter_revert(from, write_len - iov_iter_count(from)); + if (written != -EIOCBQUEUED) + iov_iter_revert(from, write_len - iov_iter_count(from)); out: return written; } From 3a6bae48390d25a9937978a6c09ccc400b6efcbd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:01:49 -0800 Subject: [PATCH 486/633] mm/filemap: rename generic_file_buffered_read subfunctions Patch series "Refactor generic_file_buffered_read", v5. This is a combination of Christoph's work to refactor generic_file_buffered_read() and some of my large-page support which was disrupted by Kent's refactoring of generic_file_buffered_read. This patch (of 18): The recent split of generic_file_buffered_read() created some very long function names which are hard to distinguish from each other. Rename as follows: generic_file_buffered_read_readpage -> filemap_read_page generic_file_buffered_read_pagenotuptodate -> filemap_update_page generic_file_buffered_read_no_cached_page -> filemap_create_page generic_file_buffered_read_get_pages -> filemap_get_pages Link: https://lkml.kernel.org/r/20210122160140.223228-1-willy@infradead.org Link: https://lkml.kernel.org/r/20210122160140.223228-2-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Reviewed-by: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 774a869da106..fbfea449f417 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2178,11 +2178,8 @@ static int lock_page_for_iocb(struct kiocb *iocb, struct page *page) return lock_page_killable(page); } -static struct page * -generic_file_buffered_read_readpage(struct kiocb *iocb, - struct file *filp, - struct address_space *mapping, - struct page *page) +static struct page *filemap_read_page(struct kiocb *iocb, struct file *filp, + struct address_space *mapping, struct page *page) { struct file_ra_state *ra = &filp->f_ra; int error; @@ -2233,12 +2230,9 @@ generic_file_buffered_read_readpage(struct kiocb *iocb, return page; } -static struct page * -generic_file_buffered_read_pagenotuptodate(struct kiocb *iocb, - struct file *filp, - struct iov_iter *iter, - struct page *page, - loff_t pos, loff_t count) +static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, + struct iov_iter *iter, struct page *page, loff_t pos, + loff_t count) { struct address_space *mapping = filp->f_mapping; struct inode *inode = mapping->host; @@ -2301,12 +2295,11 @@ generic_file_buffered_read_pagenotuptodate(struct kiocb *iocb, return page; } - return generic_file_buffered_read_readpage(iocb, filp, mapping, page); + return filemap_read_page(iocb, filp, mapping, page); } -static struct page * -generic_file_buffered_read_no_cached_page(struct kiocb *iocb, - struct iov_iter *iter) +static struct page *filemap_create_page(struct kiocb *iocb, + struct iov_iter *iter) { struct file *filp = iocb->ki_filp; struct address_space *mapping = filp->f_mapping; @@ -2317,10 +2310,6 @@ generic_file_buffered_read_no_cached_page(struct kiocb *iocb, if (iocb->ki_flags & IOCB_NOIO) return ERR_PTR(-EAGAIN); - /* - * Ok, it wasn't cached, so we need to create a new - * page.. - */ page = page_cache_alloc(mapping); if (!page) return ERR_PTR(-ENOMEM); @@ -2332,13 +2321,11 @@ generic_file_buffered_read_no_cached_page(struct kiocb *iocb, return error != -EEXIST ? ERR_PTR(error) : NULL; } - return generic_file_buffered_read_readpage(iocb, filp, mapping, page); + return filemap_read_page(iocb, filp, mapping, page); } -static int generic_file_buffered_read_get_pages(struct kiocb *iocb, - struct iov_iter *iter, - struct page **pages, - unsigned int nr) +static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, + struct page **pages, unsigned int nr) { struct file *filp = iocb->ki_filp; struct address_space *mapping = filp->f_mapping; @@ -2365,7 +2352,7 @@ static int generic_file_buffered_read_get_pages(struct kiocb *iocb, if (nr_got) goto got_pages; - pages[0] = generic_file_buffered_read_no_cached_page(iocb, iter); + pages[0] = filemap_create_page(iocb, iter); err = PTR_ERR_OR_ZERO(pages[0]); if (!IS_ERR_OR_NULL(pages[0])) nr_got = 1; @@ -2399,8 +2386,8 @@ static int generic_file_buffered_read_get_pages(struct kiocb *iocb, break; } - page = generic_file_buffered_read_pagenotuptodate(iocb, - filp, iter, page, pg_pos, pg_count); + page = filemap_update_page(iocb, filp, iter, page, + pg_pos, pg_count); if (IS_ERR_OR_NULL(page)) { for (j = i + 1; j < nr_got; j++) put_page(pages[j]); @@ -2479,8 +2466,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, iocb->ki_flags |= IOCB_NOWAIT; i = 0; - pg_nr = generic_file_buffered_read_get_pages(iocb, iter, - pages, nr_pages); + pg_nr = filemap_get_pages(iocb, iter, pages, nr_pages); if (pg_nr < 0) { error = pg_nr; break; From 0c7c575df56b957390206deb018c41acbb412159 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:01:52 -0800 Subject: [PATCH 487/633] mm/filemap: remove dynamically allocated array from filemap_read Increasing the batch size runs into diminishing returns. It's probably better to make, eg, three calls to filemap_get_pages() than it is to call into kmalloc(). Link: https://lkml.kernel.org/r/20210122160140.223228-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Miaohe Lin Cc: Kent Overstreet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index fbfea449f417..f621a7de5833 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2431,8 +2431,8 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, struct file_ra_state *ra = &filp->f_ra; struct address_space *mapping = filp->f_mapping; struct inode *inode = mapping->host; - struct page *pages_onstack[PAGEVEC_SIZE], **pages = NULL; - unsigned int nr_pages = min_t(unsigned int, 512, + struct page *pages[PAGEVEC_SIZE]; + unsigned int nr_pages = min_t(unsigned int, PAGEVEC_SIZE, ((iocb->ki_pos + iter->count + PAGE_SIZE - 1) >> PAGE_SHIFT) - (iocb->ki_pos >> PAGE_SHIFT)); int i, pg_nr, error = 0; @@ -2446,14 +2446,6 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, iov_iter_truncate(iter, inode->i_sb->s_maxbytes); - if (nr_pages > ARRAY_SIZE(pages_onstack)) - pages = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); - - if (!pages) { - pages = pages_onstack; - nr_pages = min_t(unsigned int, nr_pages, ARRAY_SIZE(pages_onstack)); - } - do { cond_resched(); @@ -2538,9 +2530,6 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, file_accessed(filp); - if (pages != pages_onstack) - kfree(pages); - return written ? written : error; } EXPORT_SYMBOL_GPL(generic_file_buffered_read); From ff993ba130009b1b8afb06206887e1e1f5b34591 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:01:55 -0800 Subject: [PATCH 488/633] mm/filemap: convert filemap_get_pages to take a pagevec Using a pagevec lets us keep the pages and the number of pages together which simplifies a lot of the calling conventions. Link: https://lkml.kernel.org/r/20210122160140.223228-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Cc: Kent Overstreet Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 82 ++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index f621a7de5833..aadf9bd340a4 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2325,22 +2325,22 @@ static struct page *filemap_create_page(struct kiocb *iocb, } static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, - struct page **pages, unsigned int nr) + struct pagevec *pvec) { struct file *filp = iocb->ki_filp; struct address_space *mapping = filp->f_mapping; struct file_ra_state *ra = &filp->f_ra; pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; pgoff_t last_index = (iocb->ki_pos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; - int i, j, nr_got, err = 0; + unsigned int nr = min_t(unsigned long, last_index - index, PAGEVEC_SIZE); + int i, j, err = 0; - nr = min_t(unsigned long, last_index - index, nr); find_page: if (fatal_signal_pending(current)) return -EINTR; - nr_got = find_get_pages_contig(mapping, index, nr, pages); - if (nr_got) + pvec->nr = find_get_pages_contig(mapping, index, nr, pvec->pages); + if (pvec->nr) goto got_pages; if (iocb->ki_flags & IOCB_NOIO) @@ -2348,17 +2348,17 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, page_cache_sync_readahead(mapping, ra, filp, index, last_index - index); - nr_got = find_get_pages_contig(mapping, index, nr, pages); - if (nr_got) + pvec->nr = find_get_pages_contig(mapping, index, nr, pvec->pages); + if (pvec->nr) goto got_pages; - pages[0] = filemap_create_page(iocb, iter); - err = PTR_ERR_OR_ZERO(pages[0]); - if (!IS_ERR_OR_NULL(pages[0])) - nr_got = 1; + pvec->pages[0] = filemap_create_page(iocb, iter); + err = PTR_ERR_OR_ZERO(pvec->pages[0]); + if (!IS_ERR_OR_NULL(pvec->pages[0])) + pvec->nr = 1; got_pages: - for (i = 0; i < nr_got; i++) { - struct page *page = pages[i]; + for (i = 0; i < pvec->nr; i++) { + struct page *page = pvec->pages[i]; pgoff_t pg_index = index + i; loff_t pg_pos = max(iocb->ki_pos, (loff_t) pg_index << PAGE_SHIFT); @@ -2366,9 +2366,9 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, if (PageReadahead(page)) { if (iocb->ki_flags & IOCB_NOIO) { - for (j = i; j < nr_got; j++) - put_page(pages[j]); - nr_got = i; + for (j = i; j < pvec->nr; j++) + put_page(pvec->pages[j]); + pvec->nr = i; err = -EAGAIN; break; } @@ -2379,9 +2379,9 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, if (!PageUptodate(page)) { if ((iocb->ki_flags & IOCB_NOWAIT) || ((iocb->ki_flags & IOCB_WAITQ) && i)) { - for (j = i; j < nr_got; j++) - put_page(pages[j]); - nr_got = i; + for (j = i; j < pvec->nr; j++) + put_page(pvec->pages[j]); + pvec->nr = i; err = -EAGAIN; break; } @@ -2389,17 +2389,17 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, page = filemap_update_page(iocb, filp, iter, page, pg_pos, pg_count); if (IS_ERR_OR_NULL(page)) { - for (j = i + 1; j < nr_got; j++) - put_page(pages[j]); - nr_got = i; + for (j = i + 1; j < pvec->nr; j++) + put_page(pvec->pages[j]); + pvec->nr = i; err = PTR_ERR_OR_ZERO(page); break; } } } - if (likely(nr_got)) - return nr_got; + if (likely(pvec->nr)) + return 0; if (err) return err; /* @@ -2431,11 +2431,8 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, struct file_ra_state *ra = &filp->f_ra; struct address_space *mapping = filp->f_mapping; struct inode *inode = mapping->host; - struct page *pages[PAGEVEC_SIZE]; - unsigned int nr_pages = min_t(unsigned int, PAGEVEC_SIZE, - ((iocb->ki_pos + iter->count + PAGE_SIZE - 1) >> PAGE_SHIFT) - - (iocb->ki_pos >> PAGE_SHIFT)); - int i, pg_nr, error = 0; + struct pagevec pvec; + int i, error = 0; bool writably_mapped; loff_t isize, end_offset; @@ -2457,12 +2454,9 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, if ((iocb->ki_flags & IOCB_WAITQ) && written) iocb->ki_flags |= IOCB_NOWAIT; - i = 0; - pg_nr = filemap_get_pages(iocb, iter, pages, nr_pages); - if (pg_nr < 0) { - error = pg_nr; + error = filemap_get_pages(iocb, iter, &pvec); + if (error < 0) break; - } /* * i_size must be checked after we know the pages are Uptodate. @@ -2478,9 +2472,9 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); - while ((iocb->ki_pos >> PAGE_SHIFT) + pg_nr > + while ((iocb->ki_pos >> PAGE_SHIFT) + pvec.nr > (end_offset + PAGE_SIZE - 1) >> PAGE_SHIFT) - put_page(pages[--pg_nr]); + put_page(pvec.pages[--pvec.nr]); /* * Once we start copying data, we don't want to be touching any @@ -2494,11 +2488,11 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, */ if (iocb->ki_pos >> PAGE_SHIFT != ra->prev_pos >> PAGE_SHIFT) - mark_page_accessed(pages[0]); - for (i = 1; i < pg_nr; i++) - mark_page_accessed(pages[i]); + mark_page_accessed(pvec.pages[0]); + for (i = 1; i < pagevec_count(&pvec); i++) + mark_page_accessed(pvec.pages[i]); - for (i = 0; i < pg_nr; i++) { + for (i = 0; i < pagevec_count(&pvec); i++) { unsigned int offset = iocb->ki_pos & ~PAGE_MASK; unsigned int bytes = min_t(loff_t, end_offset - iocb->ki_pos, PAGE_SIZE - offset); @@ -2510,9 +2504,9 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, * before reading the page on the kernel side. */ if (writably_mapped) - flush_dcache_page(pages[i]); + flush_dcache_page(pvec.pages[i]); - copied = copy_page_to_iter(pages[i], offset, bytes, iter); + copied = copy_page_to_iter(pvec.pages[i], offset, bytes, iter); written += copied; iocb->ki_pos += copied; @@ -2524,8 +2518,8 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, } } put_pages: - for (i = 0; i < pg_nr; i++) - put_page(pages[i]); + for (i = 0; i < pagevec_count(&pvec); i++) + put_page(pvec.pages[i]); } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); file_accessed(filp); From cbd59c48ae2bcadc4a7599c29cf32fd3f9b78251 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:01:59 -0800 Subject: [PATCH 489/633] mm/filemap: use head pages in generic_file_buffered_read Add filemap_get_read_batch() which returns the head pages which represent a contiguous array of bytes in the file. It also stops when encountering a page marked as Readahead or !Uptodate (but does return that page) so it can be handled appropriately by filemap_get_pages(). That lets us remove the loop in filemap_get_pages() and check only the last page. Link: https://lkml.kernel.org/r/20210122160140.223228-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 122 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index aadf9bd340a4..4fce9735e172 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2178,6 +2178,51 @@ static int lock_page_for_iocb(struct kiocb *iocb, struct page *page) return lock_page_killable(page); } +/* + * filemap_get_read_batch - Get a batch of pages for read + * + * Get a batch of pages which represent a contiguous range of bytes + * in the file. No tail pages will be returned. If @index is in the + * middle of a THP, the entire THP will be returned. The last page in + * the batch may have Readahead set or be not Uptodate so that the + * caller can take the appropriate action. + */ +static void filemap_get_read_batch(struct address_space *mapping, + pgoff_t index, pgoff_t max, struct pagevec *pvec) +{ + XA_STATE(xas, &mapping->i_pages, index); + struct page *head; + + rcu_read_lock(); + for (head = xas_load(&xas); head; head = xas_next(&xas)) { + if (xas_retry(&xas, head)) + continue; + if (xas.xa_index > max || xa_is_value(head)) + break; + if (!page_cache_get_speculative(head)) + goto retry; + + /* Has the page moved or been split? */ + if (unlikely(head != xas_reload(&xas))) + goto put_page; + + if (!pagevec_add(pvec, head)) + break; + if (!PageUptodate(head)) + break; + if (PageReadahead(head)) + break; + xas.xa_index = head->index + thp_nr_pages(head) - 1; + xas.xa_offset = (xas.xa_index >> xas.xa_shift) & XA_CHUNK_MASK; + continue; +put_page: + put_page(head); +retry: + xas_reset(&xas); + } + rcu_read_unlock(); +} + static struct page *filemap_read_page(struct kiocb *iocb, struct file *filp, struct address_space *mapping, struct page *page) { @@ -2331,15 +2376,15 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, struct address_space *mapping = filp->f_mapping; struct file_ra_state *ra = &filp->f_ra; pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; - pgoff_t last_index = (iocb->ki_pos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; - unsigned int nr = min_t(unsigned long, last_index - index, PAGEVEC_SIZE); - int i, j, err = 0; + pgoff_t last_index; + int err = 0; + last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE); find_page: if (fatal_signal_pending(current)) return -EINTR; - pvec->nr = find_get_pages_contig(mapping, index, nr, pvec->pages); + filemap_get_read_batch(mapping, index, last_index, pvec); if (pvec->nr) goto got_pages; @@ -2348,29 +2393,30 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, page_cache_sync_readahead(mapping, ra, filp, index, last_index - index); - pvec->nr = find_get_pages_contig(mapping, index, nr, pvec->pages); + filemap_get_read_batch(mapping, index, last_index, pvec); if (pvec->nr) goto got_pages; pvec->pages[0] = filemap_create_page(iocb, iter); err = PTR_ERR_OR_ZERO(pvec->pages[0]); - if (!IS_ERR_OR_NULL(pvec->pages[0])) - pvec->nr = 1; + if (IS_ERR_OR_NULL(pvec->pages[0])) + goto err; + pvec->nr = 1; + return 0; got_pages: - for (i = 0; i < pvec->nr; i++) { - struct page *page = pvec->pages[i]; - pgoff_t pg_index = index + i; + { + struct page *page = pvec->pages[pvec->nr - 1]; + pgoff_t pg_index = page->index; loff_t pg_pos = max(iocb->ki_pos, (loff_t) pg_index << PAGE_SHIFT); loff_t pg_count = iocb->ki_pos + iter->count - pg_pos; if (PageReadahead(page)) { if (iocb->ki_flags & IOCB_NOIO) { - for (j = i; j < pvec->nr; j++) - put_page(pvec->pages[j]); - pvec->nr = i; + put_page(page); + pvec->nr--; err = -EAGAIN; - break; + goto err; } page_cache_async_readahead(mapping, ra, filp, page, pg_index, last_index - pg_index); @@ -2378,26 +2424,23 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, if (!PageUptodate(page)) { if ((iocb->ki_flags & IOCB_NOWAIT) || - ((iocb->ki_flags & IOCB_WAITQ) && i)) { - for (j = i; j < pvec->nr; j++) - put_page(pvec->pages[j]); - pvec->nr = i; + ((iocb->ki_flags & IOCB_WAITQ) && pvec->nr > 1)) { + put_page(page); + pvec->nr--; err = -EAGAIN; - break; + goto err; } page = filemap_update_page(iocb, filp, iter, page, pg_pos, pg_count); if (IS_ERR_OR_NULL(page)) { - for (j = i + 1; j < pvec->nr; j++) - put_page(pvec->pages[j]); - pvec->nr = i; + pvec->nr--; err = PTR_ERR_OR_ZERO(page); - break; } } } +err: if (likely(pvec->nr)) return 0; if (err) @@ -2442,6 +2485,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, return 0; iov_iter_truncate(iter, inode->i_sb->s_maxbytes); + pagevec_init(&pvec); do { cond_resched(); @@ -2469,13 +2513,8 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, isize = i_size_read(inode); if (unlikely(iocb->ki_pos >= isize)) goto put_pages; - end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); - while ((iocb->ki_pos >> PAGE_SHIFT) + pvec.nr > - (end_offset + PAGE_SIZE - 1) >> PAGE_SHIFT) - put_page(pvec.pages[--pvec.nr]); - /* * Once we start copying data, we don't want to be touching any * cachelines that might be contended: @@ -2489,24 +2528,32 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, if (iocb->ki_pos >> PAGE_SHIFT != ra->prev_pos >> PAGE_SHIFT) mark_page_accessed(pvec.pages[0]); - for (i = 1; i < pagevec_count(&pvec); i++) - mark_page_accessed(pvec.pages[i]); for (i = 0; i < pagevec_count(&pvec); i++) { - unsigned int offset = iocb->ki_pos & ~PAGE_MASK; - unsigned int bytes = min_t(loff_t, end_offset - iocb->ki_pos, - PAGE_SIZE - offset); - unsigned int copied; + struct page *page = pvec.pages[i]; + size_t page_size = thp_size(page); + size_t offset = iocb->ki_pos & (page_size - 1); + size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, + page_size - offset); + size_t copied; + if (end_offset < page_offset(page)) + break; + if (i > 0) + mark_page_accessed(page); /* * If users can be writing to this page using arbitrary * virtual addresses, take care about potential aliasing * before reading the page on the kernel side. */ - if (writably_mapped) - flush_dcache_page(pvec.pages[i]); + if (writably_mapped) { + int j; - copied = copy_page_to_iter(pvec.pages[i], offset, bytes, iter); + for (j = 0; j < thp_nr_pages(page); j++) + flush_dcache_page(page + j); + } + + copied = copy_page_to_iter(page, offset, bytes, iter); written += copied; iocb->ki_pos += copied; @@ -2520,6 +2567,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, put_pages: for (i = 0; i < pagevec_count(&pvec); i++) put_page(pvec.pages[i]); + pagevec_reinit(&pvec); } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); file_accessed(filp); From 4805462598113f350838d612d0895db2dbb3992b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:02 -0800 Subject: [PATCH 490/633] mm/filemap: pass a sleep state to put_and_wait_on_page_locked This is prep work for the next patch, but I think at least one of the current callers would prefer a killable sleep to an uninterruptible one. Link: https://lkml.kernel.org/r/20210122160140.223228-6-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/pagemap.h | 3 +-- mm/filemap.c | 7 +++++-- mm/huge_memory.c | 4 ++-- mm/migrate.c | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 74e466e5a2ba..bd629d676a27 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -681,8 +681,7 @@ static inline int wait_on_page_locked_killable(struct page *page) return wait_on_page_bit_killable(compound_head(page), PG_locked); } -extern void put_and_wait_on_page_locked(struct page *page); - +int put_and_wait_on_page_locked(struct page *page, int state); void wait_on_page_writeback(struct page *page); extern void end_page_writeback(struct page *page); void wait_for_stable_page(struct page *page); diff --git a/mm/filemap.c b/mm/filemap.c index 4fce9735e172..389a20eec9b8 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1384,20 +1384,23 @@ static int wait_on_page_locked_async(struct page *page, /** * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked * @page: The page to wait for. + * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). * * The caller should hold a reference on @page. They expect the page to * become unlocked relatively soon, but do not wish to hold up migration * (for example) by holding the reference while waiting for the page to * come unlocked. After this function returns, the caller should not * dereference @page. + * + * Return: 0 if the page was unlocked or -EINTR if interrupted by a signal. */ -void put_and_wait_on_page_locked(struct page *page) +int put_and_wait_on_page_locked(struct page *page, int state) { wait_queue_head_t *q; page = compound_head(page); q = page_waitqueue(page); - wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, DROP); + return wait_on_page_bit_common(q, page, PG_locked, state, DROP); } /** diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 91ca9b103ee5..f655137536eb 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1439,7 +1439,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) if (!get_page_unless_zero(page)) goto out_unlock; spin_unlock(vmf->ptl); - put_and_wait_on_page_locked(page); + put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); goto out; } @@ -1475,7 +1475,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) if (!get_page_unless_zero(page)) goto out_unlock; spin_unlock(vmf->ptl); - put_and_wait_on_page_locked(page); + put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); goto out; } diff --git a/mm/migrate.c b/mm/migrate.c index 20ca887ea769..4ec727adfaa2 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -331,7 +331,7 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); - put_and_wait_on_page_locked(page); + put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); return; out: pte_unmap_unlock(ptep, ptl); @@ -365,7 +365,7 @@ void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) if (!get_page_unless_zero(page)) goto unlock; spin_unlock(ptl); - put_and_wait_on_page_locked(page); + put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE); return; unlock: spin_unlock(ptl); From bd8a1f3655a704b9a1924fb3feffa3ecd6e5f8ae Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:05 -0800 Subject: [PATCH 491/633] mm/filemap: support readpage splitting a page For page splitting to succeed, the thread asking to split the page has to be the only one with a reference to the page. Calling wait_on_page_locked() while holding a reference to the page will effectively prevent this from happening with sufficient threads waiting on the same page. Use put_and_wait_on_page_locked() to sleep without holding a reference to the page, then retry the page lookup after the page is unlocked. Since we now get the page lock a little earlier in filemap_update_page(), we can eliminate a number of duplicate checks. The original intent (commit ebded02788b5 ("avoid unnecessary calls to lock_page when waiting for IO to complete during a read")) behind getting the page lock later was to avoid re-locking the page after it has been brought uptodate by another thread. We still avoid that because we go through the normal lookup path again after the winning thread has brought the page uptodate. Link: https://lkml.kernel.org/r/20210122160140.223228-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 76 ++++++++++++++++------------------------------------ 1 file changed, 23 insertions(+), 53 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 389a20eec9b8..f62b589cbfe4 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1373,14 +1373,6 @@ static int __wait_on_page_locked_async(struct page *page, return ret; } -static int wait_on_page_locked_async(struct page *page, - struct wait_page_queue *wait) -{ - if (!PageLocked(page)) - return 0; - return __wait_on_page_locked_async(compound_head(page), wait, false); -} - /** * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked * @page: The page to wait for. @@ -2286,64 +2278,42 @@ static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, struct inode *inode = mapping->host; int error; - /* - * See comment in do_read_cache_page on why - * wait_on_page_locked is used to avoid unnecessarily - * serialisations and why it's safe. - */ if (iocb->ki_flags & IOCB_WAITQ) { - error = wait_on_page_locked_async(page, - iocb->ki_waitq); + error = lock_page_async(page, iocb->ki_waitq); + if (error) { + put_page(page); + return ERR_PTR(error); + } } else { - error = wait_on_page_locked_killable(page); + if (!trylock_page(page)) { + put_and_wait_on_page_locked(page, TASK_KILLABLE); + return NULL; + } } - if (unlikely(error)) { - put_page(page); - return ERR_PTR(error); - } - if (PageUptodate(page)) - return page; + if (!page->mapping) + goto truncated; + if (PageUptodate(page)) + goto uptodate; if (inode->i_blkbits == PAGE_SHIFT || !mapping->a_ops->is_partially_uptodate) - goto page_not_up_to_date; + goto readpage; /* pipes can't handle partially uptodate pages */ if (unlikely(iov_iter_is_pipe(iter))) - goto page_not_up_to_date; - if (!trylock_page(page)) - goto page_not_up_to_date; - /* Did it get truncated before we got the lock? */ - if (!page->mapping) - goto page_not_up_to_date_locked; + goto readpage; if (!mapping->a_ops->is_partially_uptodate(page, - pos & ~PAGE_MASK, count)) - goto page_not_up_to_date_locked; + pos & (thp_size(page) - 1), count)) + goto readpage; +uptodate: unlock_page(page); return page; -page_not_up_to_date: - /* Get exclusive access to the page ... */ - error = lock_page_for_iocb(iocb, page); - if (unlikely(error)) { - put_page(page); - return ERR_PTR(error); - } - -page_not_up_to_date_locked: - /* Did it get truncated before we got the lock? */ - if (!page->mapping) { - unlock_page(page); - put_page(page); - return NULL; - } - - /* Did somebody else fill it already? */ - if (PageUptodate(page)) { - unlock_page(page); - return page; - } - +readpage: return filemap_read_page(iocb, filp, mapping, page); +truncated: + unlock_page(page); + put_page(page); + return NULL; } static struct page *filemap_create_page(struct kiocb *iocb, From f32b5dd721fb8861f3c1b8e7c06ac978236d0236 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:09 -0800 Subject: [PATCH 492/633] mm/filemap: inline __wait_on_page_locked_async into caller The previous patch removed wait_on_page_locked_async(), so inline __wait_on_page_locked_async into __lock_page_async(). Link: https://lkml.kernel.org/r/20210122160140.223228-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 53 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index f62b589cbfe4..43448d4d66f3 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1343,36 +1343,6 @@ int wait_on_page_bit_killable(struct page *page, int bit_nr) } EXPORT_SYMBOL(wait_on_page_bit_killable); -static int __wait_on_page_locked_async(struct page *page, - struct wait_page_queue *wait, bool set) -{ - struct wait_queue_head *q = page_waitqueue(page); - int ret = 0; - - wait->page = page; - wait->bit_nr = PG_locked; - - spin_lock_irq(&q->lock); - __add_wait_queue_entry_tail(q, &wait->wait); - SetPageWaiters(page); - if (set) - ret = !trylock_page(page); - else - ret = PageLocked(page); - /* - * If we were successful now, we know we're still on the - * waitqueue as we're still under the lock. This means it's - * safe to remove and return success, we know the callback - * isn't going to trigger. - */ - if (!ret) - __remove_wait_queue(q, &wait->wait); - else - ret = -EIOCBQUEUED; - spin_unlock_irq(&q->lock); - return ret; -} - /** * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked * @page: The page to wait for. @@ -1548,7 +1518,28 @@ EXPORT_SYMBOL_GPL(__lock_page_killable); int __lock_page_async(struct page *page, struct wait_page_queue *wait) { - return __wait_on_page_locked_async(page, wait, true); + struct wait_queue_head *q = page_waitqueue(page); + int ret = 0; + + wait->page = page; + wait->bit_nr = PG_locked; + + spin_lock_irq(&q->lock); + __add_wait_queue_entry_tail(q, &wait->wait); + SetPageWaiters(page); + ret = !trylock_page(page); + /* + * If we were successful now, we know we're still on the + * waitqueue as we're still under the lock. This means it's + * safe to remove and return success, we know the callback + * isn't going to trigger. + */ + if (!ret) + __remove_wait_queue(q, &wait->wait); + else + ret = -EIOCBQUEUED; + spin_unlock_irq(&q->lock); + return ret; } /* From 33a0f5c6b34f58e632f1855ff29228d49bc23bcc Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:12 -0800 Subject: [PATCH 493/633] mm/filemap: don't call ->readpage if IOCB_WAITQ is set The readpage operation can block in many (most?) filesystems, so we should punt to a work queue instead of calling it. This was the last caller of lock_page_for_iocb(), so remove it. Link: https://lkml.kernel.org/r/20210122160140.223228-9-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 43448d4d66f3..8bd1bb375d07 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2154,16 +2154,6 @@ static void shrink_readahead_size_eio(struct file_ra_state *ra) ra->ra_pages /= 4; } -static int lock_page_for_iocb(struct kiocb *iocb, struct page *page) -{ - if (iocb->ki_flags & IOCB_WAITQ) - return lock_page_async(page, iocb->ki_waitq); - else if (iocb->ki_flags & IOCB_NOWAIT) - return trylock_page(page) ? 0 : -EAGAIN; - else - return lock_page_killable(page); -} - /* * filemap_get_read_batch - Get a batch of pages for read * @@ -2215,7 +2205,7 @@ static struct page *filemap_read_page(struct kiocb *iocb, struct file *filp, struct file_ra_state *ra = &filp->f_ra; int error; - if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) { + if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) { unlock_page(page); put_page(page); return ERR_PTR(-EAGAIN); @@ -2236,7 +2226,7 @@ static struct page *filemap_read_page(struct kiocb *iocb, struct file *filp, } if (!PageUptodate(page)) { - error = lock_page_for_iocb(iocb, page); + error = lock_page_killable(page); if (unlikely(error)) { put_page(page); return ERR_PTR(error); From 68430303c84e1fd457a05f424b02ea8393708552 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:15 -0800 Subject: [PATCH 494/633] mm/filemap: change filemap_read_page calling conventions Make this function more generic by passing the file instead of the iocb. Check in the callers whether we should call readpage or not. Also make it return an errno / 0 / AOP_TRUNCATED_PAGE, and make calling put_page() the caller's responsibility. Link: https://lkml.kernel.org/r/20210122160140.223228-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 89 +++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 8bd1bb375d07..0b7310971678 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2199,56 +2199,38 @@ static void filemap_get_read_batch(struct address_space *mapping, rcu_read_unlock(); } -static struct page *filemap_read_page(struct kiocb *iocb, struct file *filp, - struct address_space *mapping, struct page *page) +static int filemap_read_page(struct file *file, struct address_space *mapping, + struct page *page) { - struct file_ra_state *ra = &filp->f_ra; int error; - if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) { - unlock_page(page); - put_page(page); - return ERR_PTR(-EAGAIN); - } - /* - * A previous I/O error may have been due to temporary - * failures, eg. multipath errors. - * PG_error will be set again if readpage fails. + * A previous I/O error may have been due to temporary failures, + * eg. multipath errors. PG_error will be set again if readpage + * fails. */ ClearPageError(page); /* Start the actual read. The read will unlock the page. */ - error = mapping->a_ops->readpage(filp, page); - - if (unlikely(error)) { - put_page(page); - return error != AOP_TRUNCATED_PAGE ? ERR_PTR(error) : NULL; - } + error = mapping->a_ops->readpage(file, page); + if (error) + return error; + if (PageUptodate(page)) + return 0; + error = lock_page_killable(page); + if (error) + return error; if (!PageUptodate(page)) { - error = lock_page_killable(page); - if (unlikely(error)) { - put_page(page); - return ERR_PTR(error); + if (page->mapping == NULL) { + /* page truncated */ + error = AOP_TRUNCATED_PAGE; + } else { + shrink_readahead_size_eio(&file->f_ra); + error = -EIO; } - if (!PageUptodate(page)) { - if (page->mapping == NULL) { - /* - * invalidate_mapping_pages got it - */ - unlock_page(page); - put_page(page); - return NULL; - } - unlock_page(page); - shrink_readahead_size_eio(ra); - put_page(page); - return ERR_PTR(-EIO); - } - unlock_page(page); } - - return page; + unlock_page(page); + return error; } static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, @@ -2290,7 +2272,18 @@ static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, return page; readpage: - return filemap_read_page(iocb, filp, mapping, page); + if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) { + unlock_page(page); + put_page(page); + return ERR_PTR(-EAGAIN); + } + error = filemap_read_page(iocb->ki_filp, mapping, page); + if (!error) + return page; + put_page(page); + if (error == AOP_TRUNCATED_PAGE) + return NULL; + return ERR_PTR(error); truncated: unlock_page(page); put_page(page); @@ -2306,7 +2299,7 @@ static struct page *filemap_create_page(struct kiocb *iocb, struct page *page; int error; - if (iocb->ki_flags & IOCB_NOIO) + if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) return ERR_PTR(-EAGAIN); page = page_cache_alloc(mapping); @@ -2315,12 +2308,14 @@ static struct page *filemap_create_page(struct kiocb *iocb, error = add_to_page_cache_lru(page, mapping, index, mapping_gfp_constraint(mapping, GFP_KERNEL)); - if (error) { - put_page(page); - return error != -EEXIST ? ERR_PTR(error) : NULL; - } - - return filemap_read_page(iocb, filp, mapping, page); + if (!error) + error = filemap_read_page(iocb->ki_filp, mapping, page); + if (!error) + return page; + put_page(page); + if (error == -EEXIST || error == AOP_TRUNCATED_PAGE) + return NULL; + return ERR_PTR(error); } static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, From f253e1854ce8441eefe98f193def2c477a017d81 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:18 -0800 Subject: [PATCH 495/633] mm/filemap: change filemap_create_page calling conventions By moving the iocb flag checks to the caller, we can pass the file and the page index instead of the iocb. It never needed the iter. By passing the pagevec, we can return an errno (or AOP_TRUNCATED_PAGE) instead of an ERR_PTR. Link: https://lkml.kernel.org/r/20210122160140.223228-11-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 53 ++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 0b7310971678..c0e5aff274f3 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2290,32 +2290,33 @@ static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, return NULL; } -static struct page *filemap_create_page(struct kiocb *iocb, - struct iov_iter *iter) +static int filemap_create_page(struct file *file, + struct address_space *mapping, pgoff_t index, + struct pagevec *pvec) { - struct file *filp = iocb->ki_filp; - struct address_space *mapping = filp->f_mapping; - pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; struct page *page; int error; - if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) - return ERR_PTR(-EAGAIN); - page = page_cache_alloc(mapping); if (!page) - return ERR_PTR(-ENOMEM); + return -ENOMEM; error = add_to_page_cache_lru(page, mapping, index, - mapping_gfp_constraint(mapping, GFP_KERNEL)); - if (!error) - error = filemap_read_page(iocb->ki_filp, mapping, page); - if (!error) - return page; + mapping_gfp_constraint(mapping, GFP_KERNEL)); + if (error == -EEXIST) + error = AOP_TRUNCATED_PAGE; + if (error) + goto error; + + error = filemap_read_page(file, mapping, page); + if (error) + goto error; + + pagevec_add(pvec, page); + return 0; +error: put_page(page); - if (error == -EEXIST || error == AOP_TRUNCATED_PAGE) - return NULL; - return ERR_PTR(error); + return error; } static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, @@ -2343,15 +2344,15 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, page_cache_sync_readahead(mapping, ra, filp, index, last_index - index); filemap_get_read_batch(mapping, index, last_index, pvec); - if (pvec->nr) - goto got_pages; - - pvec->pages[0] = filemap_create_page(iocb, iter); - err = PTR_ERR_OR_ZERO(pvec->pages[0]); - if (IS_ERR_OR_NULL(pvec->pages[0])) - goto err; - pvec->nr = 1; - return 0; + if (!pagevec_count(pvec)) { + if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) + return -EAGAIN; + err = filemap_create_page(filp, mapping, + iocb->ki_pos >> PAGE_SHIFT, pvec); + if (err == AOP_TRUNCATED_PAGE) + goto find_page; + return err; + } got_pages: { struct page *page = pvec->pages[pvec->nr - 1]; From 4612aeef09ec492ca5877e06f0dbac5383da5e88 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:22 -0800 Subject: [PATCH 496/633] mm/filemap: convert filemap_update_page to return an errno Use AOP_TRUNCATED_PAGE to indicate that no error occurred, but the page we looked up is no longer valid. In this case, the reference to the page will have been removed; if we hit any other error, the caller will release the reference. Link: https://lkml.kernel.org/r/20210122160140.223228-12-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index c0e5aff274f3..67c4df82aa41 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2233,24 +2233,21 @@ static int filemap_read_page(struct file *file, struct address_space *mapping, return error; } -static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, - struct iov_iter *iter, struct page *page, loff_t pos, - loff_t count) +static int filemap_update_page(struct kiocb *iocb, + struct address_space *mapping, struct iov_iter *iter, + struct page *page, loff_t pos, loff_t count) { - struct address_space *mapping = filp->f_mapping; struct inode *inode = mapping->host; int error; if (iocb->ki_flags & IOCB_WAITQ) { error = lock_page_async(page, iocb->ki_waitq); - if (error) { - put_page(page); - return ERR_PTR(error); - } + if (error) + return error; } else { if (!trylock_page(page)) { put_and_wait_on_page_locked(page, TASK_KILLABLE); - return NULL; + return AOP_TRUNCATED_PAGE; } } @@ -2269,25 +2266,21 @@ static struct page *filemap_update_page(struct kiocb *iocb, struct file *filp, goto readpage; uptodate: unlock_page(page); - return page; + return 0; readpage: if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) { unlock_page(page); - put_page(page); - return ERR_PTR(-EAGAIN); + return -EAGAIN; } error = filemap_read_page(iocb->ki_filp, mapping, page); - if (!error) - return page; - put_page(page); if (error == AOP_TRUNCATED_PAGE) - return NULL; - return ERR_PTR(error); + put_page(page); + return error; truncated: unlock_page(page); put_page(page); - return NULL; + return AOP_TRUNCATED_PAGE; } static int filemap_create_page(struct file *file, @@ -2381,11 +2374,12 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, goto err; } - page = filemap_update_page(iocb, filp, iter, page, + err = filemap_update_page(iocb, mapping, iter, page, pg_pos, pg_count); - if (IS_ERR_OR_NULL(page)) { + if (err) { + if (err < 0) + put_page(page); pvec->nr--; - err = PTR_ERR_OR_ZERO(page); } } } @@ -2393,6 +2387,8 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, err: if (likely(pvec->nr)) return 0; + if (err == AOP_TRUNCATED_PAGE) + goto find_page; if (err) return err; /* From 87d1d7b688319ae1580f057faa460d7f0b381430 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:25 -0800 Subject: [PATCH 497/633] mm/filemap: move the iocb checks into filemap_update_page We don't need to give up when a non-blocking request sees a !Uptodate page. We may be able to satisfy the read from a partially-uptodate page. Link: https://lkml.kernel.org/r/20210122160140.223228-13-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 67c4df82aa41..cd863c4ff9ae 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2240,15 +2240,16 @@ static int filemap_update_page(struct kiocb *iocb, struct inode *inode = mapping->host; int error; - if (iocb->ki_flags & IOCB_WAITQ) { - error = lock_page_async(page, iocb->ki_waitq); - if (error) - return error; - } else { - if (!trylock_page(page)) { + if (!trylock_page(page)) { + if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) + return -EAGAIN; + if (!(iocb->ki_flags & IOCB_WAITQ)) { put_and_wait_on_page_locked(page, TASK_KILLABLE); return AOP_TRUNCATED_PAGE; } + error = __lock_page_async(page, iocb->ki_waitq); + if (error) + return error; } if (!page->mapping) @@ -2366,14 +2367,9 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, } if (!PageUptodate(page)) { - if ((iocb->ki_flags & IOCB_NOWAIT) || - ((iocb->ki_flags & IOCB_WAITQ) && pvec->nr > 1)) { - put_page(page); - pvec->nr--; - err = -EAGAIN; - goto err; - } - + if ((iocb->ki_flags & IOCB_WAITQ) && + pagevec_count(pvec) > 1) + iocb->ki_flags |= IOCB_NOWAIT; err = filemap_update_page(iocb, mapping, iter, page, pg_pos, pg_count); if (err) { From fce70da3a80fcd0a9c0192dedd6bf86a43845ac9 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:28 -0800 Subject: [PATCH 498/633] mm/filemap: add filemap_range_uptodate Move the complicated condition and the calculations out of filemap_update_page() into its own function. [willy@infradead.org: unlock page before dropping its refcount] Link: https://lkml.kernel.org/r/20210201125229.GO308988@casper.infradead.org Link: https://lkml.kernel.org/r/20210122160140.223228-14-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 65 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index cd863c4ff9ae..1eaafc9402e0 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2233,11 +2233,36 @@ static int filemap_read_page(struct file *file, struct address_space *mapping, return error; } +static bool filemap_range_uptodate(struct address_space *mapping, + loff_t pos, struct iov_iter *iter, struct page *page) +{ + int count; + + if (PageUptodate(page)) + return true; + /* pipes can't handle partially uptodate pages */ + if (iov_iter_is_pipe(iter)) + return false; + if (!mapping->a_ops->is_partially_uptodate) + return false; + if (mapping->host->i_blkbits >= (PAGE_SHIFT + thp_order(page))) + return false; + + count = iter->count; + if (page_offset(page) > pos) { + count -= page_offset(page) - pos; + pos = 0; + } else { + pos -= page_offset(page); + } + + return mapping->a_ops->is_partially_uptodate(page, pos, count); +} + static int filemap_update_page(struct kiocb *iocb, struct address_space *mapping, struct iov_iter *iter, - struct page *page, loff_t pos, loff_t count) + struct page *page) { - struct inode *inode = mapping->host; int error; if (!trylock_page(page)) { @@ -2254,26 +2279,15 @@ static int filemap_update_page(struct kiocb *iocb, if (!page->mapping) goto truncated; - if (PageUptodate(page)) - goto uptodate; - if (inode->i_blkbits == PAGE_SHIFT || - !mapping->a_ops->is_partially_uptodate) - goto readpage; - /* pipes can't handle partially uptodate pages */ - if (unlikely(iov_iter_is_pipe(iter))) - goto readpage; - if (!mapping->a_ops->is_partially_uptodate(page, - pos & (thp_size(page) - 1), count)) - goto readpage; -uptodate: - unlock_page(page); - return 0; -readpage: - if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) { - unlock_page(page); - return -EAGAIN; - } + error = 0; + if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, page)) + goto unlock; + + error = -EAGAIN; + if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) + goto unlock; + error = filemap_read_page(iocb->ki_filp, mapping, page); if (error == AOP_TRUNCATED_PAGE) put_page(page); @@ -2282,6 +2296,9 @@ static int filemap_update_page(struct kiocb *iocb, unlock_page(page); put_page(page); return AOP_TRUNCATED_PAGE; +unlock: + unlock_page(page); + return error; } static int filemap_create_page(struct file *file, @@ -2351,9 +2368,6 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, { struct page *page = pvec->pages[pvec->nr - 1]; pgoff_t pg_index = page->index; - loff_t pg_pos = max(iocb->ki_pos, - (loff_t) pg_index << PAGE_SHIFT); - loff_t pg_count = iocb->ki_pos + iter->count - pg_pos; if (PageReadahead(page)) { if (iocb->ki_flags & IOCB_NOIO) { @@ -2370,8 +2384,7 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, if ((iocb->ki_flags & IOCB_WAITQ) && pagevec_count(pvec) > 1) iocb->ki_flags |= IOCB_NOWAIT; - err = filemap_update_page(iocb, mapping, iter, page, - pg_pos, pg_count); + err = filemap_update_page(iocb, mapping, iter, page); if (err) { if (err < 0) put_page(page); From 5963fe031638bb812c49ddf5adcdc783a57430f7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:32 -0800 Subject: [PATCH 499/633] mm/filemap: split filemap_readahead out of filemap_get_pages This simplifies the error handling. Link: https://lkml.kernel.org/r/20210122160140.223228-15-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 1eaafc9402e0..f8181beded38 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2330,6 +2330,17 @@ static int filemap_create_page(struct file *file, return error; } +static int filemap_readahead(struct kiocb *iocb, struct file *file, + struct address_space *mapping, struct page *page, + pgoff_t last_index) +{ + if (iocb->ki_flags & IOCB_NOIO) + return -EAGAIN; + page_cache_async_readahead(mapping, &file->f_ra, file, page, + page->index, last_index - page->index); + return 0; +} + static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, struct pagevec *pvec) { @@ -2367,17 +2378,15 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, got_pages: { struct page *page = pvec->pages[pvec->nr - 1]; - pgoff_t pg_index = page->index; if (PageReadahead(page)) { - if (iocb->ki_flags & IOCB_NOIO) { + err = filemap_readahead(iocb, filp, mapping, page, + last_index); + if (err) { put_page(page); pvec->nr--; - err = -EAGAIN; goto err; } - page_cache_async_readahead(mapping, ra, filp, page, - pg_index, last_index - pg_index); } if (!PageUptodate(page)) { From 2642fca647257210bf6127297748d472c22702cd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:35 -0800 Subject: [PATCH 500/633] mm/filemap: restructure filemap_get_pages Remove the got_pages label, remove indentation, rename find_page to retry, simplify error handling. Link: https://lkml.kernel.org/r/20210122160140.223228-16-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 71 +++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 43 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index f8181beded38..3311b2af9061 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2349,70 +2349,55 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, struct file_ra_state *ra = &filp->f_ra; pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; pgoff_t last_index; + struct page *page; int err = 0; last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE); -find_page: +retry: if (fatal_signal_pending(current)) return -EINTR; filemap_get_read_batch(mapping, index, last_index, pvec); - if (pvec->nr) - goto got_pages; - - if (iocb->ki_flags & IOCB_NOIO) - return -EAGAIN; - - page_cache_sync_readahead(mapping, ra, filp, index, last_index - index); - - filemap_get_read_batch(mapping, index, last_index, pvec); + if (!pagevec_count(pvec)) { + if (iocb->ki_flags & IOCB_NOIO) + return -EAGAIN; + page_cache_sync_readahead(mapping, ra, filp, index, + last_index - index); + filemap_get_read_batch(mapping, index, last_index, pvec); + } if (!pagevec_count(pvec)) { if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) return -EAGAIN; err = filemap_create_page(filp, mapping, iocb->ki_pos >> PAGE_SHIFT, pvec); if (err == AOP_TRUNCATED_PAGE) - goto find_page; + goto retry; return err; } -got_pages: - { - struct page *page = pvec->pages[pvec->nr - 1]; - if (PageReadahead(page)) { - err = filemap_readahead(iocb, filp, mapping, page, - last_index); - if (err) { - put_page(page); - pvec->nr--; - goto err; - } - } - - if (!PageUptodate(page)) { - if ((iocb->ki_flags & IOCB_WAITQ) && - pagevec_count(pvec) > 1) - iocb->ki_flags |= IOCB_NOWAIT; - err = filemap_update_page(iocb, mapping, iter, page); - if (err) { - if (err < 0) - put_page(page); - pvec->nr--; - } - } + page = pvec->pages[pagevec_count(pvec) - 1]; + if (PageReadahead(page)) { + err = filemap_readahead(iocb, filp, mapping, page, last_index); + if (err) + goto err; + } + if (!PageUptodate(page)) { + if ((iocb->ki_flags & IOCB_WAITQ) && pagevec_count(pvec) > 1) + iocb->ki_flags |= IOCB_NOWAIT; + err = filemap_update_page(iocb, mapping, iter, page); + if (err) + goto err; } + return 0; err: - if (likely(pvec->nr)) + if (err < 0) + put_page(page); + if (likely(--pvec->nr)) return 0; if (err == AOP_TRUNCATED_PAGE) - goto find_page; - if (err) - return err; - /* - * No pages and no error means we raced and should retry: - */ - goto find_page; + goto retry; + return err; } /** From aa1ec2f69780c5b9590143162101b6dc3dc1de5f Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:02:38 -0800 Subject: [PATCH 501/633] mm/filemap: don't relock the page after calling readpage We don't need to get the page lock again; we just need to wait for the I/O to finish, so use wait_on_page_locked_killable() like the other callers of ->readpage. Link: https://lkml.kernel.org/r/20210122160140.223228-17-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Reviewed-by: Christoph Hellwig Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 3311b2af9061..1358cb061fd6 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2214,23 +2214,16 @@ static int filemap_read_page(struct file *file, struct address_space *mapping, error = mapping->a_ops->readpage(file, page); if (error) return error; - if (PageUptodate(page)) - return 0; - error = lock_page_killable(page); + error = wait_on_page_locked_killable(page); if (error) return error; - if (!PageUptodate(page)) { - if (page->mapping == NULL) { - /* page truncated */ - error = AOP_TRUNCATED_PAGE; - } else { - shrink_readahead_size_eio(&file->f_ra); - error = -EIO; - } - } - unlock_page(page); - return error; + if (PageUptodate(page)) + return 0; + if (!page->mapping) /* page truncated */ + return AOP_TRUNCATED_PAGE; + shrink_readahead_size_eio(&file->f_ra); + return -EIO; } static bool filemap_range_uptodate(struct address_space *mapping, From 87fa0f3eb267eed966ee194907bc15376c1b758f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Feb 2021 12:02:42 -0800 Subject: [PATCH 502/633] mm/filemap: rename generic_file_buffered_read to filemap_read Rename generic_file_buffered_read to match the naming of filemap_fault, also update the written parameter to a more descriptive name and improve the kerneldoc comment. Link: https://lkml.kernel.org/r/20210122160140.223228-18-willy@infradead.org Signed-off-by: Christoph Hellwig Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/btrfs/file.c | 2 +- include/linux/fs.h | 4 ++-- mm/filemap.c | 35 ++++++++++++++++------------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index be9e3900cce8..bf2c51a9607a 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -3634,7 +3634,7 @@ static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return ret; } - return generic_file_buffered_read(iocb, to, ret); + return filemap_read(iocb, to, ret); } const struct file_operations btrfs_file_operations = { diff --git a/include/linux/fs.h b/include/linux/fs.h index 418b772d6eca..ec8f3ddf4a6a 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3080,8 +3080,8 @@ extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *); extern int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count); extern int generic_file_rw_checks(struct file *file_in, struct file *file_out); -extern ssize_t generic_file_buffered_read(struct kiocb *iocb, - struct iov_iter *to, ssize_t already_read); +ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *to, + ssize_t already_read); extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *); extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); diff --git a/mm/filemap.c b/mm/filemap.c index 1358cb061fd6..28c290da22c1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2394,23 +2394,20 @@ static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter, } /** - * generic_file_buffered_read - generic file read routine - * @iocb: the iocb to read - * @iter: data destination - * @written: already copied + * filemap_read - Read data from the page cache. + * @iocb: The iocb to read. + * @iter: Destination for the data. + * @already_read: Number of bytes already read by the caller. * - * This is a generic file read routine, and uses the - * mapping->a_ops->readpage() function for the actual low-level stuff. + * Copies data from the page cache. If the data is not currently present, + * uses the readahead and readpage address_space operations to fetch it. * - * This is really ugly. But the goto's actually try to clarify some - * of the logic when it comes to error handling etc. - * - * Return: - * * total number of bytes copied, including those the were already @written - * * negative error code if nothing was copied + * Return: Total number of bytes copied, including those already read by + * the caller. If an error happens before any bytes are copied, returns + * a negative error number. */ -ssize_t generic_file_buffered_read(struct kiocb *iocb, - struct iov_iter *iter, ssize_t written) +ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, + ssize_t already_read) { struct file *filp = iocb->ki_filp; struct file_ra_state *ra = &filp->f_ra; @@ -2437,7 +2434,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, * can no longer safely return -EIOCBQUEUED. Hence mark * an async read NOWAIT at that point. */ - if ((iocb->ki_flags & IOCB_WAITQ) && written) + if ((iocb->ki_flags & IOCB_WAITQ) && already_read) iocb->ki_flags |= IOCB_NOWAIT; error = filemap_get_pages(iocb, iter, &pvec); @@ -2497,7 +2494,7 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, copied = copy_page_to_iter(page, offset, bytes, iter); - written += copied; + already_read += copied; iocb->ki_pos += copied; ra->prev_pos = iocb->ki_pos; @@ -2514,9 +2511,9 @@ ssize_t generic_file_buffered_read(struct kiocb *iocb, file_accessed(filp); - return written ? written : error; + return already_read ? already_read : error; } -EXPORT_SYMBOL_GPL(generic_file_buffered_read); +EXPORT_SYMBOL_GPL(filemap_read); /** * generic_file_read_iter - generic filesystem read routine @@ -2591,7 +2588,7 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) goto out; } - retval = generic_file_buffered_read(iocb, iter, retval); + retval = filemap_read(iocb, iter, retval); out: return retval; } From 826ea860bc4d119731026655c383c7773c9d2dad Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Feb 2021 12:02:45 -0800 Subject: [PATCH 503/633] mm/filemap: simplify generic_file_read_iter Avoid the pointless goto out just for returning retval. Link: https://lkml.kernel.org/r/20210122160140.223228-19-willy@infradead.org Signed-off-by: Christoph Hellwig Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Kent Overstreet Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/filemap.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 28c290da22c1..820d81901eae 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2543,7 +2543,7 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) ssize_t retval = 0; if (!count) - goto out; /* skip atime */ + return 0; /* skip atime */ if (iocb->ki_flags & IOCB_DIRECT) { struct file *file = iocb->ki_filp; @@ -2561,7 +2561,7 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) iocb->ki_pos, iocb->ki_pos + count - 1); if (retval < 0) - goto out; + return retval; } file_accessed(file); @@ -2585,12 +2585,10 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) */ if (retval < 0 || !count || iocb->ki_pos >= size || IS_DAX(inode)) - goto out; + return retval; } - retval = filemap_read(iocb, iter, retval); -out: - return retval; + return filemap_read(iocb, iter, retval); } EXPORT_SYMBOL(generic_file_read_iter); From 4ebd3aec3842662300979dacd6fb38e3e8edf7f4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 24 Feb 2021 12:02:48 -0800 Subject: [PATCH 504/633] fs/buffer.c: add checking buffer head stat before clear clear_buffer_new() is used to clear buffer new stat. When PAGE_SIZE is 64K, most buffer heads in the list are not needed to clear. clear_buffer_new() has an enpensive atomic modification operation, Let's add checking buffer head before clear it as __block_write_begin_int does which is good for performance. Link: https://lkml.kernel.org/r/1612332890-57918-1-git-send-email-zhangshaokun@hisilicon.com Signed-off-by: Yang Guo Signed-off-by: Shaokun Zhang Cc: Alexander Viro Cc: Nick Piggin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/buffer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/buffer.c b/fs/buffer.c index 32647d2011df..f1c3a5b27a90 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2083,7 +2083,8 @@ static int __block_commit_write(struct inode *inode, struct page *page, set_buffer_uptodate(bh); mark_buffer_dirty(bh); } - clear_buffer_new(bh); + if (buffer_new(bh)) + clear_buffer_new(bh); block_start = block_end; bh = bh->b_this_page; From 6986c3e2b19505e9b2112fc2e548e9f99fa3021f Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Wed, 24 Feb 2021 12:02:52 -0800 Subject: [PATCH 505/633] mm: backing-dev: Remove duplicated macro definition Move the K() macro a little forward to remove the same macro definition. Link: https://lkml.kernel.org/r/d1ccdf2d3116dce9814f2bcc1f0415ecb4c76ea5.1612862230.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: Andrew Morton Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/backing-dev.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/backing-dev.c b/mm/backing-dev.c index e33797579338..eca555f658d9 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -32,6 +32,8 @@ LIST_HEAD(bdi_list); /* bdi_wq serves all asynchronous writeback tasks */ struct workqueue_struct *bdi_wq; +#define K(x) ((x) << (PAGE_SHIFT - 10)) + #ifdef CONFIG_DEBUG_FS #include #include @@ -69,7 +71,6 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v) global_dirty_limits(&background_thresh, &dirty_thresh); wb_thresh = wb_calc_thresh(wb, dirty_thresh); -#define K(x) ((x) << (PAGE_SHIFT - 10)) seq_printf(m, "BdiWriteback: %10lu kB\n" "BdiReclaimable: %10lu kB\n" @@ -98,7 +99,6 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v) nr_more_io, nr_dirty_time, !list_empty(&bdi->bdi_list), bdi->wb.state); -#undef K return 0; } @@ -146,8 +146,6 @@ static ssize_t read_ahead_kb_store(struct device *dev, return count; } -#define K(pages) ((pages) << (PAGE_SHIFT - 10)) - #define BDI_SHOW(name, expr) \ static ssize_t name##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ From 191a7221b70d7fa7005404f508e1802f6556ba78 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 24 Feb 2021 12:02:55 -0800 Subject: [PATCH 506/633] mm/swap_slots.c: remove redundant NULL check Fix below warnings reported by coccicheck: mm/swap_slots.c:197:3-9: WARNING: NULL check before some freeing functions is not needed. Link: https://lkml.kernel.org/r/1611214229-11225-1-git-send-email-abaci-bugfix@linux.alibaba.com Signed-off-by: Yang Li Reported-by: Abaci Robot Reviewed-by: Andrew Morton Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swap_slots.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 0357fbe70645..be9de6d5b516 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -193,8 +193,7 @@ static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type, cache->slots_ret = NULL; } spin_unlock_irq(&cache->free_lock); - if (slots) - kvfree(slots); + kvfree(slots); } } From cf532faa41c55ad39fcff211132c58b0acf35c62 Mon Sep 17 00:00:00 2001 From: Stephen Zhang Date: Wed, 24 Feb 2021 12:02:58 -0800 Subject: [PATCH 507/633] mm/swapfile.c: fix debugging information problem Once the function name is changed, it may be easy to forget to modify the corresponding code here. Link: https://lkml.kernel.org/r/1611369120-2276-1-git-send-email-stephenzhangzsd@gmail.com Signed-off-by: Stephen Zhang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swapfile.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 96799a2f6957..f039745989d2 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1157,13 +1157,13 @@ static struct swap_info_struct *__swap_info_get(swp_entry_t entry) return p; bad_offset: - pr_err("swap_info_get: %s%08lx\n", Bad_offset, entry.val); + pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); goto out; bad_device: - pr_err("swap_info_get: %s%08lx\n", Unused_file, entry.val); + pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val); goto out; bad_nofile: - pr_err("swap_info_get: %s%08lx\n", Bad_file, entry.val); + pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); out: return NULL; } @@ -1180,7 +1180,7 @@ static struct swap_info_struct *_swap_info_get(swp_entry_t entry) return p; bad_free: - pr_err("swap_info_get: %s%08lx\n", Unused_offset, entry.val); + pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val); out: return NULL; } From 25eaab438dd58092c5f0c62118d933bf8b2fcc76 Mon Sep 17 00:00:00 2001 From: Georgi Djakov Date: Wed, 24 Feb 2021 12:03:01 -0800 Subject: [PATCH 508/633] mm/page_io: use pr_alert_ratelimited for swap read/write errors If there are errors during swap read or write, they can easily fill the log buffer and remove any previous messages that might be useful for debugging, especially on systems that rely for logging only on the kernel ring-buffer. For example, on a systems using zram as swap, we are more likely to see any page allocation errors preceding the swap write errors if the alerts are ratelimited. Link: https://lkml.kernel.org/r/20210201142055.29068-1-georgi.djakov@linaro.org Signed-off-by: Georgi Djakov Acked-by: Minchan Kim Reviewed-by: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_io.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 92f7941c6d01..485fa5cca4a2 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -41,9 +41,9 @@ void end_swap_bio_write(struct bio *bio) * Also clear PG_reclaim to avoid rotate_reclaimable_page() */ set_page_dirty(page); - pr_alert("Write-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); + pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); ClearPageReclaim(page); } end_page_writeback(page); @@ -106,9 +106,9 @@ static void end_swap_bio_read(struct bio *bio) if (bio->bi_status) { SetPageError(page); ClearPageUptodate(page); - pr_alert("Read-error on swap-device (%u:%u:%llu)\n", - MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), - (unsigned long long)bio->bi_iter.bi_sector); + pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", + MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), + (unsigned long long)bio->bi_iter.bi_sector); goto out; } From e48333b660d57898ad6240570084ffa734f64368 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Wed, 24 Feb 2021 12:03:05 -0800 Subject: [PATCH 509/633] mm/swap_state: constify static struct attribute_group The only usage of swap_attr_group is to pass its address to sysfs_create_group() which takes a pointer to const attribute_group. Make it const to allow the compiler to put it in read-only memory. Link: https://lkml.kernel.org/r/20210201233254.91809-1-rikard.falkeborn@gmail.com Signed-off-by: Rikard Falkeborn Reviewed-by: Amy Parker Acked-by: "Huang, Ying" Reviewed-by: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swap_state.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/swap_state.c b/mm/swap_state.c index 751c1ef2fe0e..3507c82b2ef1 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -927,7 +927,7 @@ static struct attribute *swap_attrs[] = { NULL, }; -static struct attribute_group swap_attr_group = { +static const struct attribute_group swap_attr_group = { .attrs = swap_attrs, }; From cad8320b4b395702e49578580c70026c8271ea88 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:03:08 -0800 Subject: [PATCH 510/633] mm/swap: don't SetPageWorkingset unconditionally during swapin We are capable of SetPageWorkingset based on refault distances after commit aae466b0052e ("mm/swap: implement workingset detection for anonymous LRU"). This is done by workingset_refault(), which is right above the unconditional SetPageWorkingset deleted by this patch. The unconditional SetPageWorkingset miscategorizes pages that are read ahead or never belonged to the working set (e.g., tmpfs pages accessed only once by fd). When those pages are swapped in (after they were swapped out) for the first time, they skew PSI (when using async swap). When this happens again, depending on their refault distances, they might skew workingset_restore_anon counter in addition to PSI because their shadows indicate they were part of the working set. Historically, SetPageWorkingset was added as part of the PSI series, and Johannes said: "It was meant to mark incoming pages under IO with SetPageWorkingset when waiting for them constituted a memory stall. On the page cache side, because we HAVE workingset detection, this was specific to recently evicted pages that had been active in their previous life. On the anon side, the aging algorithm had no distinction between workingset and sporadically used pages. Given the choice between a) no swapin stalls are pressure and b) all swapin stalls are pressure, I went with the latter in order to detect swap storms. The false positive case - high rate of swapin without severe memory pressure - was relatively unlikely, because we tried to avoid swapping until everything was completely on fire in the first place." Link: https://lkml.kernel.org/r/20201209012400.1771150-1-yuzhao@google.com Link: https://lkml.kernel.org/r/20201214231253.62313-1-yuzhao@google.com Signed-off-by: Yu Zhao Acked-by: Vlastimil Babka Acked-by: Johannes Weiner Acked-by: Joonsoo Kim Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/swap_state.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/swap_state.c b/mm/swap_state.c index 3507c82b2ef1..807b00eae969 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -537,7 +537,6 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, workingset_refault(page, shadow); /* Caller will initiate read into locked page */ - SetPageWorkingset(page); lru_cache_add(page); *new_page_allocated = true; return page; From 2e9bd483159939ed2c0704b914294653c8341d25 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 24 Feb 2021 12:03:11 -0800 Subject: [PATCH 511/633] mm: memcg/slab: pre-allocate obj_cgroups for slab caches with SLAB_ACCOUNT In general it's unknown in advance if a slab page will contain accounted objects or not. In order to avoid memory waste, an obj_cgroup vector is allocated dynamically when a need to account of a new object arises. Such approach is memory efficient, but requires an expensive cmpxchg() to set up the memcg/objcgs pointer, because an allocation can race with a different allocation on another cpu. But in some common cases it's known for sure that a slab page will contain accounted objects: if the page belongs to a slab cache with a SLAB_ACCOUNT flag set. It includes such popular objects like vm_area_struct, anon_vma, task_struct, etc. In such cases we can pre-allocate the objcgs vector and simple assign it to the page without any atomic operations, because at this early stage the page is not visible to anyone else. A very simplistic benchmark (allocating 10000000 64-bytes objects in a row) shows ~15% win. In the real life it seems that most workloads are not very sensitive to the speed of (accounted) slab allocations. [guro@fb.com: open-code set_page_objcgs() and add some comments, by Johannes] Link: https://lkml.kernel.org/r/20201113001926.GA2934489@carbon.dhcp.thefacebook.com [akpm@linux-foundation.org: fix it for mm-slub-call-account_slab_page-after-slab-page-initialization-fix.patch] Link: https://lkml.kernel.org/r/20201110195753.530157-2-guro@fb.com Signed-off-by: Roman Gushchin Acked-by: Johannes Weiner Reviewed-by: Shakeel Butt Cc: Michal Hocko Cc: Christoph Lameter Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memcontrol.h | 19 ------------------- mm/memcontrol.c | 23 +++++++++++++++++++---- mm/slab.c | 2 +- mm/slab.h | 14 ++++++++++---- mm/slub.c | 2 +- 5 files changed, 31 insertions(+), 29 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index eeb0b52203e9..7a4dd1cb19fe 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -475,19 +475,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) return (struct obj_cgroup **)(memcg_data & ~MEMCG_DATA_FLAGS_MASK); } -/* - * set_page_objcgs - associate a page with a object cgroups vector - * @page: a pointer to the page struct - * @objcgs: a pointer to the object cgroups vector - * - * Atomically associates a page with a vector of object cgroups. - */ -static inline bool set_page_objcgs(struct page *page, - struct obj_cgroup **objcgs) -{ - return !cmpxchg(&page->memcg_data, 0, (unsigned long)objcgs | - MEMCG_DATA_OBJCGS); -} #else static inline struct obj_cgroup **page_objcgs(struct page *page) { @@ -498,12 +485,6 @@ static inline struct obj_cgroup **page_objcgs_check(struct page *page) { return NULL; } - -static inline bool set_page_objcgs(struct page *page, - struct obj_cgroup **objcgs) -{ - return true; -} #endif static __always_inline bool memcg_stat_item_in_bytes(int idx) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 0b9bd354e97e..60ce452e42e6 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2935,9 +2935,10 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg) #ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, - gfp_t gfp) + gfp_t gfp, bool new_page) { unsigned int objects = objs_per_slab_page(s, page); + unsigned long memcg_data; void *vec; vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp, @@ -2945,11 +2946,25 @@ int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, if (!vec) return -ENOMEM; - if (!set_page_objcgs(page, vec)) + memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS; + if (new_page) { + /* + * If the slab page is brand new and nobody can yet access + * it's memcg_data, no synchronization is required and + * memcg_data can be simply assigned. + */ + page->memcg_data = memcg_data; + } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) { + /* + * If the slab page is already in use, somebody can allocate + * and assign obj_cgroups in parallel. In this case the existing + * objcg vector should be reused. + */ kfree(vec); - else - kmemleak_not_leak(vec); + return 0; + } + kmemleak_not_leak(vec); return 0; } diff --git a/mm/slab.c b/mm/slab.c index cdad64b2836d..9c9e0c255c4b 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1379,7 +1379,7 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, return NULL; } - account_slab_page(page, cachep->gfporder, cachep); + account_slab_page(page, cachep->gfporder, cachep, flags); __SetPageSlab(page); /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ if (sk_memalloc_socks() && page_is_pfmemalloc(page)) diff --git a/mm/slab.h b/mm/slab.h index 48dc466a3791..076582f58f68 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -238,7 +238,7 @@ static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t fla #ifdef CONFIG_MEMCG_KMEM int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s, - gfp_t gfp); + gfp_t gfp, bool new_page); static inline void memcg_free_page_obj_cgroups(struct page *page) { @@ -315,7 +315,8 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, page = virt_to_head_page(p[i]); if (!page_objcgs(page) && - memcg_alloc_page_obj_cgroups(page, s, flags)) { + memcg_alloc_page_obj_cgroups(page, s, flags, + false)) { obj_cgroup_uncharge(objcg, obj_full_size(s)); continue; } @@ -379,7 +380,8 @@ static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) } static inline int memcg_alloc_page_obj_cgroups(struct page *page, - struct kmem_cache *s, gfp_t gfp) + struct kmem_cache *s, gfp_t gfp, + bool new_page) { return 0; } @@ -420,8 +422,12 @@ static inline struct kmem_cache *virt_to_cache(const void *obj) } static __always_inline void account_slab_page(struct page *page, int order, - struct kmem_cache *s) + struct kmem_cache *s, + gfp_t gfp) { + if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT)) + memcg_alloc_page_obj_cgroups(page, s, gfp, true); + mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s), PAGE_SIZE << order); } diff --git a/mm/slub.c b/mm/slub.c index e0fef8f8d4bf..ca566361561e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1785,7 +1785,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) page->objects = oo_objects(oo); - account_slab_page(page, oo_order(oo), s); + account_slab_page(page, oo_order(oo), s, flags); page->slab_cache = s; __SetPageSlab(page); From f3344adf38bdb3107d40483dd9501215ad40edce Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:15 -0800 Subject: [PATCH 512/633] mm: memcontrol: optimize per-lruvec stats counter memory usage The vmstat threshold is 32 (MEMCG_CHARGE_BATCH), Actually the threshold can be as big as MEMCG_CHARGE_BATCH * PAGE_SIZE. It still fits into s32. So introduce struct batched_lruvec_stat to optimize memory usage. The size of struct lruvec_stat is 304 bytes on 64 bit systems. As it is a per-cpu structure. So with this patch, we can save 304 / 2 * ncpu bytes per-memcg per-node where ncpu is the number of the possible CPU. If there are c memory cgroup (include dying cgroup) and n NUMA node in the system. Finally, we can save (152 * ncpu * c * n) bytes. [akpm@linux-foundation.org: fix typo in comment] Link: https://lkml.kernel.org/r/20201210042121.39665-1-songmuchun@bytedance.com Signed-off-by: Muchun Song Reviewed-by: Shakeel Butt Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Cc: Shakeel Butt Cc: Roman Gushchin Cc: Stephen Rothwell Cc: Chris Down Cc: Yafang Shao Cc: Wei Yang Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memcontrol.h | 14 ++++++++++++-- mm/memcontrol.c | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7a4dd1cb19fe..41bbf71edd9f 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -92,6 +92,10 @@ struct lruvec_stat { long count[NR_VM_NODE_STAT_ITEMS]; }; +struct batched_lruvec_stat { + s32 count[NR_VM_NODE_STAT_ITEMS]; +}; + /* * Bitmap of shrinker::id corresponding to memcg-aware shrinkers, * which have elements charged to this memcg. @@ -107,11 +111,17 @@ struct memcg_shrinker_map { struct mem_cgroup_per_node { struct lruvec lruvec; - /* Legacy local VM stats */ + /* + * Legacy local VM stats. This should be struct lruvec_stat and + * cannot be optimized to struct batched_lruvec_stat. Because + * the threshold of the lruvec_stat_cpu can be as big as + * MEMCG_CHARGE_BATCH * PAGE_SIZE. It can fit into s32. But this + * filed has no upper limit. + */ struct lruvec_stat __percpu *lruvec_stat_local; /* Subtree VM stats (batched updates) */ - struct lruvec_stat __percpu *lruvec_stat_cpu; + struct batched_lruvec_stat __percpu *lruvec_stat_cpu; atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS]; unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 60ce452e42e6..b259e7d8ce41 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5208,7 +5208,7 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node) return 1; } - pn->lruvec_stat_cpu = alloc_percpu_gfp(struct lruvec_stat, + pn->lruvec_stat_cpu = alloc_percpu_gfp(struct batched_lruvec_stat, GFP_KERNEL_ACCOUNT); if (!pn->lruvec_stat_cpu) { free_percpu(pn->lruvec_stat_local); @@ -7093,6 +7093,14 @@ static int __init mem_cgroup_init(void) { int cpu, node; + /* + * Currently s32 type (can refer to struct batched_lruvec_stat) is + * used for per-memcg-per-cpu caching of per-node statistics. In order + * to work fine, we should make sure that the overfill threshold can't + * exceed S32_MAX / PAGE_SIZE. + */ + BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE); + cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead); From b0ba3bff3e7bb6b58bb248bdd2f3d8ad52fd10c3 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:19 -0800 Subject: [PATCH 513/633] mm: memcontrol: fix NR_ANON_THPS accounting in charge moving Patch series "Convert all THP vmstat counters to pages", v6. This patch series is aimed to convert all THP vmstat counters to pages. The unit of some vmstat counters are pages, some are bytes, some are HPAGE_PMD_NR, and some are KiB. When we want to expose these vmstat counters to the userspace, we have to know the unit of the vmstat counters is which one. When the unit is bytes or kB, both clearly distinguishable by the B/KB suffix. But for the THP vmstat counters, we may make mistakes. For example, the below is some bug fix for the THP vmstat counters: - 7de2e9f195b9 ("mm: memcontrol: correct the NR_ANON_THPS counter of hierarchical memcg") - The first commit in this series ("fix NR_ANON_THPS accounting in charge moving") This patch series can make the code clear. And make all the unit of the THP vmstat counters in pages. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. In this series, I changed the following vmstat counters unit from HPAGE_PMD_NR to pages. However, there is no change to the print format of output to user space. - NR_ANON_THPS - NR_FILE_THPS - NR_SHMEM_THPS - NR_SHMEM_PMDMAPPED - NR_FILE_PMDMAPPED Doing this also can make the statistics more accuracy for the THP vmstat counters. This series is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Because we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. From this point of view, I think that do this converting is reasonable. Thanks Hugh for mentioning this. This was inspired by Johannes and Roman. Thanks to them. This patch (of 7): The unit of NR_ANON_THPS is HPAGE_PMD_NR already. So it should inc/dec by one rather than nr_pages. Link: https://lkml.kernel.org/r/20201228164110.2838-1-songmuchun@bytedance.com Link: https://lkml.kernel.org/r/20201228164110.2838-2-songmuchun@bytedance.com Fixes: 468c398233da ("mm: memcontrol: switch to native NR_ANON_THPS counter") Signed-off-by: Muchun Song Acked-by: Michal Hocko Acked-by: Johannes Weiner Acked-by: Pankaj Gupta Reviewed-by: Roman Gushchin Reviewed-by: Shakeel Butt Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Joonsoo Kim Cc: NeilBrown Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Sami Tolvanen Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b259e7d8ce41..e31e47e7bab2 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5652,10 +5652,8 @@ static int mem_cgroup_move_account(struct page *page, __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); if (PageTransHuge(page)) { - __mod_lruvec_state(from_vec, NR_ANON_THPS, - -nr_pages); - __mod_lruvec_state(to_vec, NR_ANON_THPS, - nr_pages); + __dec_lruvec_state(from_vec, NR_ANON_THPS); + __inc_lruvec_state(to_vec, NR_ANON_THPS); } } From 69473e5de87389be6c0fa4a5d574a50c8f904fb3 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:23 -0800 Subject: [PATCH 514/633] mm: memcontrol: convert NR_ANON_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_ANON_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-3-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Greg Kroah-Hartman Cc: Rafael. J. Wysocki Cc: Alexey Dobriyan Cc: Johannes Weiner Cc: Vladimir Davydov Cc: Hugh Dickins Cc: Shakeel Butt Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Feng Tang Cc: NeilBrown Cc: Joonsoo Kim Cc: Randy Dunlap Cc: Michal Hocko Cc: Pankaj Gupta Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 15 +++++++++------ fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 13 +++++++++++++ mm/huge_memory.c | 3 ++- mm/memcontrol.c | 20 ++++++-------------- mm/page_alloc.c | 2 +- mm/rmap.c | 6 +++--- mm/vmstat.c | 11 +++++++++-- 8 files changed, 44 insertions(+), 28 deletions(-) diff --git a/drivers/base/node.c b/drivers/base/node.c index 04f71c7bc3f8..6da0c3508bc9 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -461,8 +461,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE , - nid, K(node_page_state(pgdat, NR_ANON_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_ANON_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * @@ -519,10 +518,14 @@ static ssize_t node_read_vmstat(struct device *dev, sum_zone_numa_state(nid, i)); #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) - len += sysfs_emit_at(buf, len, "%s %lu\n", - node_stat_name(i), - node_page_state_pages(pgdat, i)); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } return len; } diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index d6fc74619625..a635c8a84ddf 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -129,7 +129,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) #ifdef CONFIG_TRANSPARENT_HUGEPAGE show_val_kb(m, "AnonHugePages: ", - global_node_page_state(NR_ANON_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_ANON_THPS)); show_val_kb(m, "ShmemHugePages: ", global_node_page_state(NR_SHMEM_THPS) * HPAGE_PMD_NR); show_val_kb(m, "ShmemPmdMapped: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index b593316bff3d..67d50ef5dd20 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -209,6 +209,19 @@ enum node_stat_item { NR_VM_NODE_STAT_ITEMS }; +/* + * Returns true if the item should be printed in THPs (/proc/vmstat + * currently prints number of anon, file and shmem THPs. But the item + * is charged in pages). + */ +static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) +{ + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + return false; + + return item == NR_ANON_THPS; +} + /* * Returns true if the value is measured in bytes (most vmstat values are * measured in pages). This defines the API part, the internal representation diff --git a/mm/huge_memory.c b/mm/huge_memory.c index f655137536eb..3691e863070a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2176,7 +2176,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, lock_page_memcg(page); if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { /* Last compound_mapcount is gone. */ - __dec_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, + -HPAGE_PMD_NR); if (TestClearPageDoubleMap(page)) { /* No need in mapcount reference anymore */ for (i = 0; i < HPAGE_PMD_NR; i++) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e31e47e7bab2..b2405f049006 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1533,7 +1533,7 @@ static struct memory_stat memory_stats[] = { * on some architectures, the macro of HPAGE_PMD_SIZE is not * constant(e.g. powerpc). */ - { "anon_thp", 0, NR_ANON_THPS }, + { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, { "file_thp", 0, NR_FILE_THPS }, { "shmem_thp", 0, NR_SHMEM_THPS }, #endif @@ -1566,8 +1566,7 @@ static int __init memory_stats_init(void) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_ANON_THPS || - memory_stats[i].idx == NR_FILE_THPS || + if (memory_stats[i].idx == NR_FILE_THPS || memory_stats[i].idx == NR_SHMEM_THPS) memory_stats[i].ratio = HPAGE_PMD_SIZE; #endif @@ -4087,10 +4086,6 @@ static int memcg_stat_show(struct seq_file *m, void *v) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) continue; nr = memcg_page_state_local(memcg, memcg1_stats[i]); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memcg1_stats[i] == NR_ANON_THPS) - nr *= HPAGE_PMD_NR; -#endif seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE); } @@ -4121,10 +4116,6 @@ static int memcg_stat_show(struct seq_file *m, void *v) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) continue; nr = memcg_page_state(memcg, memcg1_stats[i]); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memcg1_stats[i] == NR_ANON_THPS) - nr *= HPAGE_PMD_NR; -#endif seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], (u64)nr * PAGE_SIZE); } @@ -5652,10 +5643,11 @@ static int mem_cgroup_move_account(struct page *page, __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages); __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages); if (PageTransHuge(page)) { - __dec_lruvec_state(from_vec, NR_ANON_THPS); - __inc_lruvec_state(to_vec, NR_ANON_THPS); + __mod_lruvec_state(from_vec, NR_ANON_THPS, + -nr_pages); + __mod_lruvec_state(to_vec, NR_ANON_THPS, + nr_pages); } - } } else { __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ef5070fed76b..2e2e47f8714b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5587,7 +5587,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), - K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_ANON_THPS)), #endif K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), node_page_state(pgdat, NR_KERNEL_STACK_KB), diff --git a/mm/rmap.c b/mm/rmap.c index 08c56aaf72eb..c4d5c63cfd29 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1144,7 +1144,7 @@ void do_page_add_anon_rmap(struct page *page, * disabled. */ if (compound) - __inc_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, nr); __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr); } @@ -1186,7 +1186,7 @@ void page_add_new_anon_rmap(struct page *page, if (hpage_pincount_available(page)) atomic_set(compound_pincount_ptr(page), 0); - __inc_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, nr); } else { /* Anon THP always mapped first with PMD */ VM_BUG_ON_PAGE(PageTransCompound(page), page); @@ -1292,7 +1292,7 @@ static void page_remove_anon_compound_rmap(struct page *page) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return; - __dec_lruvec_page_state(page, NR_ANON_THPS); + __mod_lruvec_page_state(page, NR_ANON_THPS, -thp_nr_pages(page)); if (TestClearPageDoubleMap(page)) { /* diff --git a/mm/vmstat.c b/mm/vmstat.c index f8942160fc95..07dc0af50cf0 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1619,8 +1619,12 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, if (is_zone_first_populated(pgdat, zone)) { seq_printf(m, "\n per-node stats"); for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; seq_printf(m, "\n %-12s %lu", node_stat_name(i), - node_page_state_pages(pgdat, i)); + pages); } } seq_printf(m, @@ -1740,8 +1744,11 @@ static void *vmstat_start(struct seq_file *m, loff_t *pos) v += NR_VM_NUMA_STAT_ITEMS; #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { v[i] = global_node_page_state_pages(i); + if (vmstat_item_print_in_thp(i)) + v[i] /= HPAGE_PMD_NR; + } v += NR_VM_NODE_STAT_ITEMS; global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, From bf9ecead53c89d3d2cf60acbc460174ebbcf0027 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:27 -0800 Subject: [PATCH 515/633] mm: memcontrol: convert NR_FILE_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with if hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_FILE_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-4-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/filemap.c | 2 +- mm/huge_memory.c | 5 ++++- mm/khugepaged.c | 4 +++- mm/memcontrol.c | 5 ++--- 7 files changed, 14 insertions(+), 10 deletions(-) diff --git a/drivers/base/node.c b/drivers/base/node.c index 6da0c3508bc9..d5952f754911 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -466,8 +466,7 @@ static ssize_t node_read_meminfo(struct device *dev, HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * HPAGE_PMD_NR) #endif diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index a635c8a84ddf..7ea4679880c8 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -135,7 +135,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "ShmemPmdMapped: ", global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); show_val_kb(m, "FileHugePages: ", - global_node_page_state(NR_FILE_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", global_node_page_state(NR_FILE_PMDMAPPED) * HPAGE_PMD_NR); #endif diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 67d50ef5dd20..b751a9898bb6 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -219,7 +219,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return false; - return item == NR_ANON_THPS; + return item == NR_ANON_THPS || + item == NR_FILE_THPS; } /* diff --git a/mm/filemap.c b/mm/filemap.c index 820d81901eae..9efe059e2b58 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -208,7 +208,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, if (PageTransHuge(page)) __dec_lruvec_page_state(page, NR_SHMEM_THPS); } else if (PageTransHuge(page)) { - __dec_lruvec_page_state(page, NR_FILE_THPS); + __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 3691e863070a..77181faa2340 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2752,10 +2752,13 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) } spin_unlock(&ds_queue->split_queue_lock); if (mapping) { + int nr = thp_nr_pages(head); + if (PageSwapBacked(head)) __dec_lruvec_page_state(head, NR_SHMEM_THPS); else - __dec_lruvec_page_state(head, NR_FILE_THPS); + __mod_lruvec_page_state(head, NR_FILE_THPS, + -nr); } __split_huge_page(page, list, end); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index fb0fdaec34d5..c427fe2ca7ff 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1643,6 +1643,7 @@ static void collapse_file(struct mm_struct *mm, XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); int nr_none = 0, result = SCAN_SUCCEED; bool is_shmem = shmem_file(file); + int nr; VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); @@ -1854,11 +1855,12 @@ static void collapse_file(struct mm_struct *mm, put_page(page); goto xa_unlocked; } + nr = thp_nr_pages(new_page); if (is_shmem) __inc_lruvec_page_state(new_page, NR_SHMEM_THPS); else { - __inc_lruvec_page_state(new_page, NR_FILE_THPS); + __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); filemap_nr_thps_inc(mapping); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b2405f049006..fc552f57d3fb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1534,7 +1534,7 @@ static struct memory_stat memory_stats[] = { * constant(e.g. powerpc). */ { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, - { "file_thp", 0, NR_FILE_THPS }, + { "file_thp", PAGE_SIZE, NR_FILE_THPS }, { "shmem_thp", 0, NR_SHMEM_THPS }, #endif { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON }, @@ -1566,8 +1566,7 @@ static int __init memory_stats_init(void) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_FILE_THPS || - memory_stats[i].idx == NR_SHMEM_THPS) + if (memory_stats[i].idx == NR_SHMEM_THPS) memory_stats[i].ratio = HPAGE_PMD_SIZE; #endif VM_BUG_ON(!memory_stats[i].ratio); From 57b2847d3c1dc154923578efb47a12302a57d700 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:31 -0800 Subject: [PATCH 516/633] mm: memcontrol: convert NR_SHMEM_THPS account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_SHMEM_THPS account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-5-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/filemap.c | 2 +- mm/huge_memory.c | 3 ++- mm/khugepaged.c | 2 +- mm/memcontrol.c | 26 ++------------------------ mm/page_alloc.c | 2 +- mm/shmem.c | 2 +- 9 files changed, 12 insertions(+), 33 deletions(-) diff --git a/drivers/base/node.c b/drivers/base/node.c index d5952f754911..6d5ac6ffb6e1 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -462,8 +462,7 @@ static ssize_t node_read_meminfo(struct device *dev, #ifdef CONFIG_TRANSPARENT_HUGEPAGE , nid, K(node_page_state(pgdat, NR_ANON_THPS)), - nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), nid, K(node_page_state(pgdat, NR_FILE_THPS)), diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 7ea4679880c8..cfb107eaa3e6 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -131,7 +131,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "AnonHugePages: ", global_node_page_state(NR_ANON_THPS)); show_val_kb(m, "ShmemHugePages: ", - global_node_page_state(NR_SHMEM_THPS) * HPAGE_PMD_NR); + global_node_page_state(NR_SHMEM_THPS)); show_val_kb(m, "ShmemPmdMapped: ", global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); show_val_kb(m, "FileHugePages: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index b751a9898bb6..788837f40b38 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -220,7 +220,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return false; return item == NR_ANON_THPS || - item == NR_FILE_THPS; + item == NR_FILE_THPS || + item == NR_SHMEM_THPS; } /* diff --git a/mm/filemap.c b/mm/filemap.c index 9efe059e2b58..46a8b9e82434 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -206,7 +206,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, if (PageSwapBacked(page)) { __mod_lruvec_page_state(page, NR_SHMEM, -nr); if (PageTransHuge(page)) - __dec_lruvec_page_state(page, NR_SHMEM_THPS); + __mod_lruvec_page_state(page, NR_SHMEM_THPS, -nr); } else if (PageTransHuge(page)) { __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 77181faa2340..86a3015ba54f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2755,7 +2755,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) int nr = thp_nr_pages(head); if (PageSwapBacked(head)) - __dec_lruvec_page_state(head, NR_SHMEM_THPS); + __mod_lruvec_page_state(head, NR_SHMEM_THPS, + -nr); else __mod_lruvec_page_state(head, NR_FILE_THPS, -nr); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index c427fe2ca7ff..75e246f680f4 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1858,7 +1858,7 @@ static void collapse_file(struct mm_struct *mm, nr = thp_nr_pages(new_page); if (is_shmem) - __inc_lruvec_page_state(new_page, NR_SHMEM_THPS); + __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); else { __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); filemap_nr_thps_inc(mapping); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index fc552f57d3fb..d3a0c59210e7 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1516,7 +1516,7 @@ struct memory_stat { unsigned int idx; }; -static struct memory_stat memory_stats[] = { +static const struct memory_stat memory_stats[] = { { "anon", PAGE_SIZE, NR_ANON_MAPPED }, { "file", PAGE_SIZE, NR_FILE_PAGES }, { "kernel_stack", 1024, NR_KERNEL_STACK_KB }, @@ -1528,14 +1528,9 @@ static struct memory_stat memory_stats[] = { { "file_dirty", PAGE_SIZE, NR_FILE_DIRTY }, { "file_writeback", PAGE_SIZE, NR_WRITEBACK }, #ifdef CONFIG_TRANSPARENT_HUGEPAGE - /* - * The ratio will be initialized in memory_stats_init(). Because - * on some architectures, the macro of HPAGE_PMD_SIZE is not - * constant(e.g. powerpc). - */ { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, { "file_thp", PAGE_SIZE, NR_FILE_THPS }, - { "shmem_thp", 0, NR_SHMEM_THPS }, + { "shmem_thp", PAGE_SIZE, NR_SHMEM_THPS }, #endif { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON }, { "active_anon", PAGE_SIZE, NR_ACTIVE_ANON }, @@ -1560,23 +1555,6 @@ static struct memory_stat memory_stats[] = { { "workingset_nodereclaim", 1, WORKINGSET_NODERECLAIM }, }; -static int __init memory_stats_init(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (memory_stats[i].idx == NR_SHMEM_THPS) - memory_stats[i].ratio = HPAGE_PMD_SIZE; -#endif - VM_BUG_ON(!memory_stats[i].ratio); - VM_BUG_ON(memory_stats[i].idx >= MEMCG_NR_STAT); - } - - return 0; -} -pure_initcall(memory_stats_init); - static char *memory_stat_format(struct mem_cgroup *memcg) { struct seq_buf s; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2e2e47f8714b..df292d8e659b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5584,7 +5584,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_WRITEBACK)), K(node_page_state(pgdat, NR_SHMEM)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE - K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_SHMEM_THPS)), K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR), K(node_page_state(pgdat, NR_ANON_THPS)), diff --git a/mm/shmem.c b/mm/shmem.c index 7924b3bf46fb..ff741d229701 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -713,7 +713,7 @@ static int shmem_add_to_page_cache(struct page *page, } if (PageTransHuge(page)) { count_vm_event(THP_FILE_ALLOC); - __inc_lruvec_page_state(page, NR_SHMEM_THPS); + __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr); } mapping->nrpages += nr; __mod_lruvec_page_state(page, NR_FILE_PAGES, nr); From a1528e21f8915e16252cda1137fe29672c918361 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:35 -0800 Subject: [PATCH 517/633] mm: memcontrol: convert NR_SHMEM_PMDMAPPED account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_SHMEM_PMDMAPPED account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-6-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/page_alloc.c | 3 +-- mm/rmap.c | 14 ++++++++++---- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/drivers/base/node.c b/drivers/base/node.c index 6d5ac6ffb6e1..7a66aefe4e46 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -463,8 +463,7 @@ static ssize_t node_read_meminfo(struct device *dev, , nid, K(node_page_state(pgdat, NR_ANON_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), - nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * - HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), nid, K(node_page_state(pgdat, NR_FILE_THPS)), nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * HPAGE_PMD_NR) diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index cfb107eaa3e6..c61f440570f9 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -133,7 +133,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "ShmemHugePages: ", global_node_page_state(NR_SHMEM_THPS)); show_val_kb(m, "ShmemPmdMapped: ", - global_node_page_state(NR_SHMEM_PMDMAPPED) * HPAGE_PMD_NR); + global_node_page_state(NR_SHMEM_PMDMAPPED)); show_val_kb(m, "FileHugePages: ", global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 788837f40b38..7bdbfeeb5c8c 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -221,7 +221,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return item == NR_ANON_THPS || item == NR_FILE_THPS || - item == NR_SHMEM_THPS; + item == NR_SHMEM_THPS || + item == NR_SHMEM_PMDMAPPED; } /* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index df292d8e659b..069561aadc7b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5585,8 +5585,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(node_page_state(pgdat, NR_SHMEM)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE K(node_page_state(pgdat, NR_SHMEM_THPS)), - K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) - * HPAGE_PMD_NR), + K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), K(node_page_state(pgdat, NR_ANON_THPS)), #endif K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), diff --git a/mm/rmap.c b/mm/rmap.c index c4d5c63cfd29..1c1b576c0627 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1211,14 +1211,17 @@ void page_add_file_rmap(struct page *page, bool compound) VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); lock_page_memcg(page); if (compound && PageTransHuge(page)) { - for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { + int nr_pages = thp_nr_pages(page); + + for (i = 0, nr = 0; i < nr_pages; i++) { if (atomic_inc_and_test(&page[i]._mapcount)) nr++; } if (!atomic_inc_and_test(compound_mapcount_ptr(page))) goto out; if (PageSwapBacked(page)) - __inc_node_page_state(page, NR_SHMEM_PMDMAPPED); + __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, + nr_pages); else __inc_node_page_state(page, NR_FILE_PMDMAPPED); } else { @@ -1252,14 +1255,17 @@ static void page_remove_file_rmap(struct page *page, bool compound) /* page still mapped by someone else? */ if (compound && PageTransHuge(page)) { - for (i = 0, nr = 0; i < thp_nr_pages(page); i++) { + int nr_pages = thp_nr_pages(page); + + for (i = 0, nr = 0; i < nr_pages; i++) { if (atomic_add_negative(-1, &page[i]._mapcount)) nr++; } if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) return; if (PageSwapBacked(page)) - __dec_node_page_state(page, NR_SHMEM_PMDMAPPED); + __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, + -nr_pages); else __dec_node_page_state(page, NR_FILE_PMDMAPPED); } else { From 380780e71895ae301505ffcec8f954ab3666a4c7 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:39 -0800 Subject: [PATCH 518/633] mm: memcontrol: convert NR_FILE_PMDMAPPED account to pages Currently we use struct per_cpu_nodestat to cache the vmstat counters, which leads to inaccurate statistics especially THP vmstat counters. In the systems with hundreds of processors it can be GBs of memory. For example, for a 96 CPUs system, the threshold is the maximum number of 125. And the per cpu counters can cache 23.4375 GB in total. The THP page is already a form of batched addition (it will add 512 worth of memory in one go) so skipping the batching seems like sensible. Although every THP stats update overflows the per-cpu counter, resorting to atomic global updates. But it can make the statistics more accuracy for the THP vmstat counters. So we convert the NR_FILE_PMDMAPPED account to pages. This patch is consistent with 8f182270dfec ("mm/swap.c: flush lru pvecs on compound page arrival"). Doing this also can make the unit of vmstat counters more unified. Finally, the unit of the vmstat counters are pages, kB and bytes. The B/KB suffix can tell us that the unit is bytes or kB. The rest which is without suffix are pages. Link: https://lkml.kernel.org/r/20201228164110.2838-7-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/base/node.c | 3 +-- fs/proc/meminfo.c | 2 +- include/linux/mmzone.h | 3 ++- mm/rmap.c | 6 ++++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/base/node.c b/drivers/base/node.c index 7a66aefe4e46..d02d86aec19f 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -465,8 +465,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), nid, K(node_page_state(pgdat, NR_FILE_THPS)), - nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * - HPAGE_PMD_NR) + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif ); len += hugetlb_report_node_meminfo(buf, len, nid); diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index c61f440570f9..6fa761c9cc78 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -137,7 +137,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "FileHugePages: ", global_node_page_state(NR_FILE_THPS)); show_val_kb(m, "FilePmdMapped: ", - global_node_page_state(NR_FILE_PMDMAPPED) * HPAGE_PMD_NR); + global_node_page_state(NR_FILE_PMDMAPPED)); #endif #ifdef CONFIG_CMA diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 7bdbfeeb5c8c..66d68e5d5a0f 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -222,7 +222,8 @@ static __always_inline bool vmstat_item_print_in_thp(enum node_stat_item item) return item == NR_ANON_THPS || item == NR_FILE_THPS || item == NR_SHMEM_THPS || - item == NR_SHMEM_PMDMAPPED; + item == NR_SHMEM_PMDMAPPED || + item == NR_FILE_PMDMAPPED; } /* diff --git a/mm/rmap.c b/mm/rmap.c index 1c1b576c0627..5ebf16fae4b9 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1223,7 +1223,8 @@ void page_add_file_rmap(struct page *page, bool compound) __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, nr_pages); else - __inc_node_page_state(page, NR_FILE_PMDMAPPED); + __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, + nr_pages); } else { if (PageTransCompound(page) && page_mapping(page)) { VM_WARN_ON_ONCE(!PageLocked(page)); @@ -1267,7 +1268,8 @@ static void page_remove_file_rmap(struct page *page, bool compound) __mod_lruvec_page_state(page, NR_SHMEM_PMDMAPPED, -nr_pages); else - __dec_node_page_state(page, NR_FILE_PMDMAPPED); + __mod_lruvec_page_state(page, NR_FILE_PMDMAPPED, + -nr_pages); } else { if (!atomic_add_negative(-1, &page->_mapcount)) return; From fff66b79a19c9b3f2aa02b0a32fe598977c89eea Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:03:43 -0800 Subject: [PATCH 519/633] mm: memcontrol: make the slab calculation consistent Although the ratio of the slab is one, we also should read the ratio from the related memory_stats instead of hard-coding. And the local variable of size is already the value of slab_unreclaimable. So we do not need to read again. To do this we need some code like below: if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { - size = memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) + - memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B); + VM_BUG_ON(i < 1); + VM_BUG_ON(memory_stats[i - 1].idx != NR_SLAB_RECLAIMABLE_B); + size += memcg_page_state(memcg, memory_stats[i - 1].idx) * + memory_stats[i - 1].ratio; It requires a series of VM_BUG_ONs or comments to ensure these two items are actually adjacent and in the right order. So it would probably be easier to implement this using a wrapper that has a big switch() for unit conversion. More details about this discussion can refer to: https://lore.kernel.org/patchwork/patch/1348611/ This would fix the ratio inconsistency and get rid of the order guarantee. Link: https://lkml.kernel.org/r/20201228164110.2838-8-songmuchun@bytedance.com Signed-off-by: Muchun Song Cc: Alexey Dobriyan Cc: Feng Tang Cc: Greg Kroah-Hartman Cc: Hugh Dickins Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Michal Hocko Cc: NeilBrown Cc: Pankaj Gupta Cc: Rafael. J. Wysocki Cc: Randy Dunlap Cc: Roman Gushchin Cc: Sami Tolvanen Cc: Shakeel Butt Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 105 ++++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d3a0c59210e7..f27e862a6845 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1512,49 +1512,71 @@ static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) struct memory_stat { const char *name; - unsigned int ratio; unsigned int idx; }; static const struct memory_stat memory_stats[] = { - { "anon", PAGE_SIZE, NR_ANON_MAPPED }, - { "file", PAGE_SIZE, NR_FILE_PAGES }, - { "kernel_stack", 1024, NR_KERNEL_STACK_KB }, - { "pagetables", PAGE_SIZE, NR_PAGETABLE }, - { "percpu", 1, MEMCG_PERCPU_B }, - { "sock", PAGE_SIZE, MEMCG_SOCK }, - { "shmem", PAGE_SIZE, NR_SHMEM }, - { "file_mapped", PAGE_SIZE, NR_FILE_MAPPED }, - { "file_dirty", PAGE_SIZE, NR_FILE_DIRTY }, - { "file_writeback", PAGE_SIZE, NR_WRITEBACK }, + { "anon", NR_ANON_MAPPED }, + { "file", NR_FILE_PAGES }, + { "kernel_stack", NR_KERNEL_STACK_KB }, + { "pagetables", NR_PAGETABLE }, + { "percpu", MEMCG_PERCPU_B }, + { "sock", MEMCG_SOCK }, + { "shmem", NR_SHMEM }, + { "file_mapped", NR_FILE_MAPPED }, + { "file_dirty", NR_FILE_DIRTY }, + { "file_writeback", NR_WRITEBACK }, #ifdef CONFIG_TRANSPARENT_HUGEPAGE - { "anon_thp", PAGE_SIZE, NR_ANON_THPS }, - { "file_thp", PAGE_SIZE, NR_FILE_THPS }, - { "shmem_thp", PAGE_SIZE, NR_SHMEM_THPS }, + { "anon_thp", NR_ANON_THPS }, + { "file_thp", NR_FILE_THPS }, + { "shmem_thp", NR_SHMEM_THPS }, #endif - { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON }, - { "active_anon", PAGE_SIZE, NR_ACTIVE_ANON }, - { "inactive_file", PAGE_SIZE, NR_INACTIVE_FILE }, - { "active_file", PAGE_SIZE, NR_ACTIVE_FILE }, - { "unevictable", PAGE_SIZE, NR_UNEVICTABLE }, - - /* - * Note: The slab_reclaimable and slab_unreclaimable must be - * together and slab_reclaimable must be in front. - */ - { "slab_reclaimable", 1, NR_SLAB_RECLAIMABLE_B }, - { "slab_unreclaimable", 1, NR_SLAB_UNRECLAIMABLE_B }, + { "inactive_anon", NR_INACTIVE_ANON }, + { "active_anon", NR_ACTIVE_ANON }, + { "inactive_file", NR_INACTIVE_FILE }, + { "active_file", NR_ACTIVE_FILE }, + { "unevictable", NR_UNEVICTABLE }, + { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B }, + { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B }, /* The memory events */ - { "workingset_refault_anon", 1, WORKINGSET_REFAULT_ANON }, - { "workingset_refault_file", 1, WORKINGSET_REFAULT_FILE }, - { "workingset_activate_anon", 1, WORKINGSET_ACTIVATE_ANON }, - { "workingset_activate_file", 1, WORKINGSET_ACTIVATE_FILE }, - { "workingset_restore_anon", 1, WORKINGSET_RESTORE_ANON }, - { "workingset_restore_file", 1, WORKINGSET_RESTORE_FILE }, - { "workingset_nodereclaim", 1, WORKINGSET_NODERECLAIM }, + { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, + { "workingset_refault_file", WORKINGSET_REFAULT_FILE }, + { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON }, + { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE }, + { "workingset_restore_anon", WORKINGSET_RESTORE_ANON }, + { "workingset_restore_file", WORKINGSET_RESTORE_FILE }, + { "workingset_nodereclaim", WORKINGSET_NODERECLAIM }, }; +/* Translate stat items to the correct unit for memory.stat output */ +static int memcg_page_state_unit(int item) +{ + switch (item) { + case MEMCG_PERCPU_B: + case NR_SLAB_RECLAIMABLE_B: + case NR_SLAB_UNRECLAIMABLE_B: + case WORKINGSET_REFAULT_ANON: + case WORKINGSET_REFAULT_FILE: + case WORKINGSET_ACTIVATE_ANON: + case WORKINGSET_ACTIVATE_FILE: + case WORKINGSET_RESTORE_ANON: + case WORKINGSET_RESTORE_FILE: + case WORKINGSET_NODERECLAIM: + return 1; + case NR_KERNEL_STACK_KB: + return SZ_1K; + default: + return PAGE_SIZE; + } +} + +static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, + int item) +{ + return memcg_page_state(memcg, item) * memcg_page_state_unit(item); +} + static char *memory_stat_format(struct mem_cgroup *memcg) { struct seq_buf s; @@ -1578,13 +1600,12 @@ static char *memory_stat_format(struct mem_cgroup *memcg) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) { u64 size; - size = memcg_page_state(memcg, memory_stats[i].idx); - size *= memory_stats[i].ratio; + size = memcg_page_state_output(memcg, memory_stats[i].idx); seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size); if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) { - size = memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) + - memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B); + size += memcg_page_state_output(memcg, + NR_SLAB_RECLAIMABLE_B); seq_buf_printf(&s, "slab %llu\n", size); } } @@ -6375,6 +6396,12 @@ static int memory_stat_show(struct seq_file *m, void *v) } #ifdef CONFIG_NUMA +static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec, + int item) +{ + return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item); +} + static int memory_numa_stat_show(struct seq_file *m, void *v) { int i; @@ -6392,8 +6419,8 @@ static int memory_numa_stat_show(struct seq_file *m, void *v) struct lruvec *lruvec; lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); - size = lruvec_page_state(lruvec, memory_stats[i].idx); - size *= memory_stats[i].ratio; + size = lruvec_page_state_output(lruvec, + memory_stats[i].idx); seq_printf(m, " N%d=%llu", nid, size); } seq_putc(m, '\n'); From d7e3aba583e6d13a81932597c5ee8da3c8b6af04 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Wed, 24 Feb 2021 12:03:47 -0800 Subject: [PATCH 520/633] mm/memcg: revise the using condition of lock_page_lruvec function series lock_page_lruvec() and its variants are safe to use under the same conditions as commit_charge(): add lock_page_memcg() to the comment. Polished with Hugh Dickins' suggestions, thanks! Link: https://lkml.kernel.org/r/1608614453-10739-1-git-send-email-alex.shi@linux.alibaba.com Signed-off-by: Alex Shi Acked-by: Hugh Dickins Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index f27e862a6845..2b13b2cf5dcb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1346,10 +1346,11 @@ void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page) * lock_page_lruvec - lock and return lruvec for a given page. * @page: the page * - * This series functions should be used in either conditions: - * PageLRU is cleared or unset - * or page->_refcount is zero - * or page is locked. + * These functions are safe to use under any of the following conditions: + * - page locked + * - PageLRU cleared + * - lock_page_memcg() + * - page->_refcount is zero */ struct lruvec *lock_page_lruvec(struct page *page) { From f9b1038ebccad354256cf84749cbc321b5347497 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Wed, 24 Feb 2021 12:03:51 -0800 Subject: [PATCH 521/633] mm/memcg: remove rcu locking for lock_page_lruvec function series lock_page_lruvec() and its variants used rcu_read_lock() with the intention of safeguarding against the mem_cgroup being destroyed concurrently; but so long as they are called under the specified conditions (as they are), there is no way for the page's mem_cgroup to be destroyed. Delete the unnecessary rcu_read_lock() and _unlock(). Hugh Dickins polished the commit log. Thanks a lot! Link: https://lkml.kernel.org/r/1608614453-10739-2-git-send-email-alex.shi@linux.alibaba.com Signed-off-by: Alex Shi Acked-by: Hugh Dickins Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2b13b2cf5dcb..5435b370c929 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1357,10 +1357,8 @@ struct lruvec *lock_page_lruvec(struct page *page) struct lruvec *lruvec; struct pglist_data *pgdat = page_pgdat(page); - rcu_read_lock(); lruvec = mem_cgroup_page_lruvec(page, pgdat); spin_lock(&lruvec->lru_lock); - rcu_read_unlock(); lruvec_memcg_debug(lruvec, page); @@ -1372,10 +1370,8 @@ struct lruvec *lock_page_lruvec_irq(struct page *page) struct lruvec *lruvec; struct pglist_data *pgdat = page_pgdat(page); - rcu_read_lock(); lruvec = mem_cgroup_page_lruvec(page, pgdat); spin_lock_irq(&lruvec->lru_lock); - rcu_read_unlock(); lruvec_memcg_debug(lruvec, page); @@ -1387,10 +1383,8 @@ struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags) struct lruvec *lruvec; struct pglist_data *pgdat = page_pgdat(page); - rcu_read_lock(); lruvec = mem_cgroup_page_lruvec(page, pgdat); spin_lock_irqsave(&lruvec->lru_lock, *flags); - rcu_read_unlock(); lruvec_memcg_debug(lruvec, page); From b6038942480e574c697ea1a80019bbe586c1d654 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 24 Feb 2021 12:03:55 -0800 Subject: [PATCH 522/633] mm: memcg: add swapcache stat for memcg v2 This patch adds swapcache stat for the cgroup v2. The swapcache represents the memory that is accounted against both the memory and the swap limit of the cgroup. The main motivation behind exposing the swapcache stat is for enabling users to gracefully migrate from cgroup v1's memsw counter to cgroup v2's memory and swap counters. Cgroup v1's memsw limit allows users to limit the memory+swap usage of a workload but without control on the exact proportion of memory and swap. Cgroup v2 provides separate limits for memory and swap which enables more control on the exact usage of memory and swap individually for the workload. With some little subtleties, the v1's memsw limit can be switched with the sum of the v2's memory and swap limits. However the alternative for memsw usage is not yet available in cgroup v2. Exposing per-cgroup swapcache stat enables that alternative. Adding the memory usage and swap usage and subtracting the swapcache will approximate the memsw usage. This will help in the transparent migration of the workloads depending on memsw usage and limit to v2' memory and swap counters. The reasons these applications are still interested in this approximate memsw usage are: (1) these applications are not really interested in two separate memory and swap usage metrics. A single usage metric is more simple to use and reason about for them. (2) The memsw usage metric hides the underlying system's swap setup from the applications. Applications with multiple instances running in a datacenter with heterogeneous systems (some have swap and some don't) will keep seeing a consistent view of their usage. [akpm@linux-foundation.org: fix CONFIG_SWAP=n build] Link: https://lkml.kernel.org/r/20210108155813.2914586-3-shakeelb@google.com Signed-off-by: Shakeel Butt Acked-by: Michal Hocko Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Muchun Song Cc: Yang Shi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/cgroup-v2.rst | 4 ++++ drivers/base/node.c | 6 ++++++ include/linux/mmzone.h | 3 +++ include/linux/swap.h | 6 +++++- mm/memcontrol.c | 3 +++ mm/migrate.c | 6 ++++++ mm/swap_state.c | 28 ++----------------------- mm/vmstat.c | 3 +++ 8 files changed, 32 insertions(+), 27 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index c513eafaddea..9acc57f68f31 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1299,6 +1299,10 @@ PAGE_SIZE multiple when read back. Amount of cached filesystem data that was modified and is currently being written back to disk + swapcached + Amount of swap cached in memory. The swapcache is accounted + against both memory and swap usage. + anon_thp Amount of memory used in anonymous mappings backed by transparent hugepages diff --git a/drivers/base/node.c b/drivers/base/node.c index d02d86aec19f..f449dbb2c746 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -372,14 +372,19 @@ static ssize_t node_read_meminfo(struct device *dev, struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; unsigned long sreclaimable, sunreclaimable; + unsigned long swapcached = 0; si_meminfo_node(&i, nid); sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); +#ifdef CONFIG_SWAP + swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); +#endif len = sysfs_emit_at(buf, len, "Node %d MemTotal: %8lu kB\n" "Node %d MemFree: %8lu kB\n" "Node %d MemUsed: %8lu kB\n" + "Node %d SwapCached: %8lu kB\n" "Node %d Active: %8lu kB\n" "Node %d Inactive: %8lu kB\n" "Node %d Active(anon): %8lu kB\n" @@ -391,6 +396,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(i.totalram), nid, K(i.freeram), nid, K(i.totalram - i.freeram), + nid, K(swapcached), nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + node_page_state(pgdat, NR_ACTIVE_FILE)), nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 66d68e5d5a0f..fc99e9241846 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -206,6 +206,9 @@ enum node_stat_item { NR_KERNEL_SCS_KB, /* measured in KiB */ #endif NR_PAGETABLE, /* used for pagetables */ +#ifdef CONFIG_SWAP + NR_SWAPCACHE, +#endif NR_VM_NODE_STAT_ITEMS }; diff --git a/include/linux/swap.h b/include/linux/swap.h index 3f1f7ae0fbe9..0d64a2bc7e00 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -408,7 +408,11 @@ extern struct address_space *swapper_spaces[]; #define swap_address_space(entry) \ (&swapper_spaces[swp_type(entry)][swp_offset(entry) \ >> SWAP_ADDRESS_SPACE_SHIFT]) -extern unsigned long total_swapcache_pages(void); +static inline unsigned long total_swapcache_pages(void) +{ + return global_node_page_state(NR_SWAPCACHE); +} + extern void show_swap_cache_info(void); extern int add_to_swap(struct page *page); extern void *get_shadow_from_swap_cache(swp_entry_t entry); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 5435b370c929..58edfb6614b8 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1521,6 +1521,9 @@ static const struct memory_stat memory_stats[] = { { "file_mapped", NR_FILE_MAPPED }, { "file_dirty", NR_FILE_DIRTY }, { "file_writeback", NR_WRITEBACK }, +#ifdef CONFIG_SWAP + { "swapcached", NR_SWAPCACHE }, +#endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE { "anon_thp", NR_ANON_THPS }, { "file_thp", NR_FILE_THPS }, diff --git a/mm/migrate.c b/mm/migrate.c index 4ec727adfaa2..62b81d5257aa 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -500,6 +500,12 @@ int migrate_page_move_mapping(struct address_space *mapping, __mod_lruvec_state(old_lruvec, NR_SHMEM, -nr); __mod_lruvec_state(new_lruvec, NR_SHMEM, nr); } +#ifdef CONFIG_SWAP + if (PageSwapCache(page)) { + __mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr); + __mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr); + } +#endif if (dirty && mapping_can_writeback(mapping)) { __mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr); __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr); diff --git a/mm/swap_state.c b/mm/swap_state.c index 807b00eae969..c1a648d9092b 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -68,32 +68,6 @@ static struct { unsigned long find_total; } swap_cache_info; -unsigned long total_swapcache_pages(void) -{ - unsigned int i, j, nr; - unsigned long ret = 0; - struct address_space *spaces; - struct swap_info_struct *si; - - for (i = 0; i < MAX_SWAPFILES; i++) { - swp_entry_t entry = swp_entry(i, 1); - - /* Avoid get_swap_device() to warn for bad swap entry */ - if (!swp_swap_info(entry)) - continue; - /* Prevent swapoff to free swapper_spaces */ - si = get_swap_device(entry); - if (!si) - continue; - nr = nr_swapper_spaces[i]; - spaces = swapper_spaces[i]; - for (j = 0; j < nr; j++) - ret += spaces[j].nrpages; - put_swap_device(si); - } - return ret; -} - static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); void show_swap_cache_info(void) @@ -163,6 +137,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, address_space->nrexceptional -= nr_shadows; address_space->nrpages += nr; __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); + __mod_lruvec_page_state(page, NR_SWAPCACHE, nr); ADD_CACHE_INFO(add_total, nr); unlock: xas_unlock_irq(&xas); @@ -203,6 +178,7 @@ void __delete_from_swap_cache(struct page *page, address_space->nrexceptional += nr; address_space->nrpages -= nr; __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); + __mod_lruvec_page_state(page, NR_SWAPCACHE, -nr); ADD_CACHE_INFO(del_total, nr); } diff --git a/mm/vmstat.c b/mm/vmstat.c index 07dc0af50cf0..a0e949542204 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1215,6 +1215,9 @@ const char * const vmstat_text[] = { "nr_shadow_call_stack", #endif "nr_page_table_pages", +#ifdef CONFIG_SWAP + "nr_swapcached", +#endif /* enum writeback_stat_item counters */ "nr_dirty_threshold", From c1a660dea3fa616420606f1e206e6d22f7e05c30 Mon Sep 17 00:00:00 2001 From: Roman Gushchin Date: Wed, 24 Feb 2021 12:03:58 -0800 Subject: [PATCH 523/633] mm: kmem: make __memcg_kmem_(un)charge static I've noticed that __memcg_kmem_charge() and __memcg_kmem_uncharge() are not used anywhere except memcontrol.c. Yet they are not declared as non-static and are declared in memcontrol.h. This patch makes them static. Link: https://lkml.kernel.org/r/20210108020332.4096911-1-guro@fb.com Signed-off-by: Roman Gushchin Reviewed-by: Shakeel Butt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memcontrol.h | 3 --- mm/memcontrol.c | 11 ++++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 41bbf71edd9f..7a38a1517a05 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1592,9 +1592,6 @@ static inline void memcg_set_shrinker_bit(struct mem_cgroup *memcg, #endif #ifdef CONFIG_MEMCG_KMEM -int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, - unsigned int nr_pages); -void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order); void __memcg_kmem_uncharge_page(struct page *page, int order); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 58edfb6614b8..dce5291525a5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -255,6 +255,11 @@ struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) #ifdef CONFIG_MEMCG_KMEM extern spinlock_t css_set_lock; +static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages); +static void __memcg_kmem_uncharge(struct mem_cgroup *memcg, + unsigned int nr_pages); + static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); @@ -3087,8 +3092,8 @@ static void memcg_free_cache_id(int id) * * Returns 0 on success, an error code on failure. */ -int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, - unsigned int nr_pages) +static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages) { struct page_counter *counter; int ret; @@ -3120,7 +3125,7 @@ int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp, * @memcg: memcg to uncharge * @nr_pages: number of pages to uncharge */ -void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) +static void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) { if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) page_counter_uncharge(&memcg->kmem, nr_pages); From 802f1d522d5fdaefc2b935141bc8fe03d43a99ab Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Wed, 24 Feb 2021 12:04:01 -0800 Subject: [PATCH 524/633] mm: page_counter: re-layout structure to reduce false sharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When checking a memory cgroup related performance regression [1], from the perf c2c profiling data, we found high false sharing for accessing 'usage' and 'parent'. On 64 bit system, the 'usage' and 'parent' are close to each other, and easy to be in one cacheline (for cacheline size == 64+ B). 'usage' is usally written, while 'parent' is usually read as the cgroup's hierarchical counting nature. So move the 'parent' to the end of the structure to make sure they are in different cache lines. Following are some performance data with the patch, against v5.11-rc1. [ In the data, A means a platform with 2 sockets 48C/96T, B is a platform of 4 sockests 72C/144T, and if a %stddev will be shown bigger than 2%, P100/P50 means number of test tasks equals to 100%/50% of nr_cpu] will-it-scale/malloc1 --------------------- v5.11-rc1 v5.11-rc1+patch A-P100 15782 ± 2% -0.1% 15765 ± 3% will-it-scale.per_process_ops A-P50 21511 +8.9% 23432 will-it-scale.per_process_ops B-P100 9155 +2.2% 9357 will-it-scale.per_process_ops B-P50 10967 +7.1% 11751 ± 2% will-it-scale.per_process_ops will-it-scale/pagefault2 ------------------------ v5.11-rc1 v5.11-rc1+patch A-P100 79028 +3.0% 81411 will-it-scale.per_process_ops A-P50 183960 ± 2% +4.4% 192078 ± 2% will-it-scale.per_process_ops B-P100 85966 +9.9% 94467 ± 3% will-it-scale.per_process_ops B-P50 198195 +9.8% 217526 will-it-scale.per_process_ops fio (4k/1M is block size) ------------------------- v5.11-rc1 v5.11-rc1+patch A-P50-r-4k 16881 ± 2% +1.2% 17081 ± 2% fio.read_bw_MBps A-P50-w-4k 3931 +4.5% 4111 ± 2% fio.write_bw_MBps A-P50-r-1M 15178 -0.2% 15154 fio.read_bw_MBps A-P50-w-1M 3924 +0.1% 3929 fio.write_bw_MBps [1].https://lore.kernel.org/lkml/20201102091543.GM31092@shao2-debian/ Link: https://lkml.kernel.org/r/1611040814-33449-1-git-send-email-feng.tang@intel.com Signed-off-by: Feng Tang Reviewed-by: Roman Gushchin Reviewed-by: Shakeel Butt Acked-by: Johannes Weiner Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/page_counter.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 85bd413e784e..679591301994 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -12,7 +12,6 @@ struct page_counter { unsigned long low; unsigned long high; unsigned long max; - struct page_counter *parent; /* effective memory.min and memory.min usage tracking */ unsigned long emin; @@ -27,6 +26,14 @@ struct page_counter { /* legacy */ unsigned long watermark; unsigned long failcnt; + + /* + * 'parent' is placed here to be far from 'usage' to reduce + * cache false sharing, as 'usage' is written mostly while + * parent is frequently read for cgroup's hierarchical + * counting nature. + */ + struct page_counter *parent; }; #if BITS_PER_LONG == 32 From 8a260162f9a0634db9a1ee7b8db276e7a00ee1d9 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 24 Feb 2021 12:04:05 -0800 Subject: [PATCH 525/633] mm/memcontrol: remove redundant NULL check Fix below warnings reported by coccicheck: mm/memcontrol.c:451:3-9: WARNING: NULL check before some freeing functions is not needed. Link: https://lkml.kernel.org/r/1611216029-34397-1-git-send-email-abaci-bugfix@linux.alibaba.com Signed-off-by: Yang Li Reported-by: Abaci Robot Reviewed-by: Andrew Morton Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index dce5291525a5..ed5cc78a8dbf 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -452,8 +452,7 @@ static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) for_each_node(nid) { pn = mem_cgroup_nodeinfo(memcg, nid); map = rcu_dereference_protected(pn->shrinker_map, true); - if (map) - kvfree(map); + kvfree(map); rcu_assign_pointer(pn->shrinker_map, NULL); } } From c41a40b6baf732ca1d519ff558fb0082c0c04e9a Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:04:08 -0800 Subject: [PATCH 526/633] mm: memcontrol: replace the loop with a list_for_each_entry() The rule of list walk has gone since commit a9d5adeeb4b2 ("mm/memcontrol: allow to uncharge page without using page->lru field") So remove the strange comment and replace the loop with a list_for_each_entry(). There is only one caller of the uncharge_list(). So just fold it into mem_cgroup_uncharge_list() and remove it. Link: https://lkml.kernel.org/r/20210204163055.56080-1-songmuchun@bytedance.com Signed-off-by: Muchun Song Acked-by: Johannes Weiner Acked-by: Roman Gushchin Reviewed-by: Miaohe Lin Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ed5cc78a8dbf..8c035846c7a4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6862,31 +6862,6 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) css_put(&ug->memcg->css); } -static void uncharge_list(struct list_head *page_list) -{ - struct uncharge_gather ug; - struct list_head *next; - - uncharge_gather_clear(&ug); - - /* - * Note that the list can be a single page->lru; hence the - * do-while loop instead of a simple list_for_each_entry(). - */ - next = page_list->next; - do { - struct page *page; - - page = list_entry(next, struct page, lru); - next = page->lru.next; - - uncharge_page(page, &ug); - } while (next != page_list); - - if (ug.memcg) - uncharge_batch(&ug); -} - /** * mem_cgroup_uncharge - uncharge a page * @page: page to uncharge @@ -6918,11 +6893,17 @@ void mem_cgroup_uncharge(struct page *page) */ void mem_cgroup_uncharge_list(struct list_head *page_list) { + struct uncharge_gather ug; + struct page *page; + if (mem_cgroup_disabled()) return; - if (!list_empty(page_list)) - uncharge_list(page_list); + uncharge_gather_clear(&ug); + list_for_each_entry(page, page_list, lru) + uncharge_page(page, &ug); + if (ug.memcg) + uncharge_batch(&ug); } /** From a7b7e1df892457935ec4f35ef9e9aa344758dbc9 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Wed, 24 Feb 2021 12:04:12 -0800 Subject: [PATCH 527/633] mm/list_lru.c: remove kvfree_rcu_local() The list_lru file used to have local kvfree_rcu() which was renamed by commit e0feed08ab41 ("mm/list_lru.c: Rename kvfree_rcu() to local variant") to introduce the globally visible kvfree_rcu(). Now we have global kvfree_rcu(), so remove the local kvfree_rcu_local() and just use the global one. Link: https://lkml.kernel.org/r/20210207152148.1285842-1-shakeelb@google.com Signed-off-by: Shakeel Butt Reviewed-by: Uladzislau Rezki Reviewed-by: Kirill Tkhai Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/list_lru.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mm/list_lru.c b/mm/list_lru.c index fe230081690b..6f067b6b935f 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -373,21 +373,13 @@ static void memcg_destroy_list_lru_node(struct list_lru_node *nlru) struct list_lru_memcg *memcg_lrus; /* * This is called when shrinker has already been unregistered, - * and nobody can use it. So, there is no need to use kvfree_rcu_local(). + * and nobody can use it. So, there is no need to use kvfree_rcu(). */ memcg_lrus = rcu_dereference_protected(nlru->memcg_lrus, true); __memcg_destroy_list_lru_node(memcg_lrus, 0, memcg_nr_cache_ids); kvfree(memcg_lrus); } -static void kvfree_rcu_local(struct rcu_head *head) -{ - struct list_lru_memcg *mlru; - - mlru = container_of(head, struct list_lru_memcg, rcu); - kvfree(mlru); -} - static int memcg_update_list_lru_node(struct list_lru_node *nlru, int old_size, int new_size) { @@ -419,7 +411,7 @@ static int memcg_update_list_lru_node(struct list_lru_node *nlru, rcu_assign_pointer(nlru->memcg_lrus, new); spin_unlock_irq(&nlru->lock); - call_rcu(&old->rcu, kvfree_rcu_local); + kvfree_rcu(old, rcu); return 0; } From 6eeb104e114cb6b7391c2d69ff873403858c1f35 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 24 Feb 2021 12:04:15 -0800 Subject: [PATCH 528/633] fs: buffer: use raw page_memcg() on locked page alloc_page_buffers() currently uses get_mem_cgroup_from_page() for charging the buffers to the page owner, which does an rcu-protected page->memcg lookup and acquires a reference. But buffer allocation has the page lock held throughout, which pins the page to the memcg and thereby the memcg - neither rcu nor holding an extra reference during the allocation are necessary. Use a raw page_memcg() instead. This was the last user of get_mem_cgroup_from_page(), delete it. Link: https://lkml.kernel.org/r/20210209190126.97842-1-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Reported-by: Muchun Song Reviewed-by: Shakeel Butt Acked-by: Roman Gushchin Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/buffer.c | 4 ++-- include/linux/memcontrol.h | 7 ------- mm/memcontrol.c | 23 ----------------------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index f1c3a5b27a90..0cb7ffd4977c 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -847,7 +847,8 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, if (retry) gfp |= __GFP_NOFAIL; - memcg = get_mem_cgroup_from_page(page); + /* The page lock pins the memcg */ + memcg = page_memcg(page); old_memcg = set_active_memcg(memcg); head = NULL; @@ -868,7 +869,6 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, } out: set_active_memcg(old_memcg); - mem_cgroup_put(memcg); return head; /* * In case anything failed, we just free everything we got. diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7a38a1517a05..e6dc793d587d 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -680,8 +680,6 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); -struct mem_cgroup *get_mem_cgroup_from_page(struct page *page); - struct lruvec *lock_page_lruvec(struct page *page); struct lruvec *lock_page_lruvec_irq(struct page *page); struct lruvec *lock_page_lruvec_irqsave(struct page *page, @@ -1191,11 +1189,6 @@ static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) -{ - return NULL; -} - static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8c035846c7a4..7fdc001ce15f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1047,29 +1047,6 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm) } EXPORT_SYMBOL(get_mem_cgroup_from_mm); -/** - * get_mem_cgroup_from_page: Obtain a reference on given page's memcg. - * @page: page from which memcg should be extracted. - * - * Obtain a reference on page->memcg and returns it if successful. Otherwise - * root_mem_cgroup is returned. - */ -struct mem_cgroup *get_mem_cgroup_from_page(struct page *page) -{ - struct mem_cgroup *memcg = page_memcg(page); - - if (mem_cgroup_disabled()) - return NULL; - - rcu_read_lock(); - /* Page should not get uncharged and freed memcg under us. */ - if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css))) - memcg = root_mem_cgroup; - rcu_read_unlock(); - return memcg; -} -EXPORT_SYMBOL(get_mem_cgroup_from_page); - static __always_inline struct mem_cgroup *active_memcg(void) { if (in_interrupt()) From cae3af62b33aa931427a0f211e04347b22180b36 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:04:19 -0800 Subject: [PATCH 529/633] mm: memcontrol: fix swap undercounting in cgroup2 When pages are swapped in, the VM may retain the swap copy to avoid repeated writes in the future. It's also retained if shared pages are faulted back in some processes, but not in others. During that time we have an in-memory copy of the page, as well as an on-swap copy. Cgroup1 and cgroup2 handle these overlapping lifetimes slightly differently due to the nature of how they account memory and swap: Cgroup1 has a unified memory+swap counter that tracks a data page regardless whether it's in-core or swapped out. On swapin, we transfer the charge from the swap entry to the newly allocated swapcache page, even though the swap entry might stick around for a while. That's why we have a mem_cgroup_uncharge_swap() call inside mem_cgroup_charge(). Cgroup2 tracks memory and swap as separate, independent resources and thus has split memory and swap counters. On swapin, we charge the newly allocated swapcache page as memory, while the swap slot in turn must remain charged to the swap counter as long as its allocated too. The cgroup2 logic was broken by commit 2d1c498072de ("mm: memcontrol: make swap tracking an integral part of memory control"), because it accidentally removed the do_memsw_account() check in the branch inside mem_cgroup_uncharge() that was supposed to tell the difference between the charge transfer in cgroup1 and the separate counters in cgroup2. As a result, cgroup2 currently undercounts retained swap to varying degrees: swap slots are cached up to 50% of the configured limit or total available swap space; partially faulted back shared pages are only limited by physical capacity. This in turn allows cgroups to significantly overconsume their alloted swap space. Add the do_memsw_account() check back to fix this problem. Link: https://lkml.kernel.org/r/20210217153237.92484-1-songmuchun@bytedance.com Fixes: 2d1c498072de ("mm: memcontrol: make swap tracking an integral part of memory control") Signed-off-by: Muchun Song Acked-by: Johannes Weiner Reviewed-by: Shakeel Butt Acked-by: Michal Hocko Cc: Vladimir Davydov Cc: [5.8+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7fdc001ce15f..2db2aeac8a9e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6748,7 +6748,19 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) memcg_check_events(memcg, page); local_irq_enable(); - if (PageSwapCache(page)) { + /* + * Cgroup1's unified memory+swap counter has been charged with the + * new swapcache page, finish the transfer by uncharging the swap + * slot. The swap slot would also get uncharged when it dies, but + * it can stick around indefinitely and we'd count the page twice + * the entire time. + * + * Cgroup2 has separate resource counters for memory and swap, + * so this is a non-issue here. Memory and swap charge lifetimes + * correspond 1:1 to page and swap slot lifetimes: we charge the + * page to memory here, and uncharge swap when the slot is freed. + */ + if (do_memsw_account() && PageSwapCache(page)) { swp_entry_t entry = { .val = page_private(page) }; /* * The swap entry might not get freed for a long time, From 1685bde6b9af55923180a76152036c7fb7176db0 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:04:22 -0800 Subject: [PATCH 530/633] mm: memcontrol: fix get_active_memcg return value We use a global percpu int_active_memcg variable to store the remote memcg when we are in the interrupt context. But get_active_memcg always return the current->active_memcg or root_mem_cgroup. The remote memcg (set in the interrupt context) is ignored. This is not what we want. So fix it. Link: https://lkml.kernel.org/r/20210223091101.42150-1-songmuchun@bytedance.com Fixes: 37d5985c003d ("mm: kmem: prepare remote memcg charging infra for interrupt contexts") Signed-off-by: Muchun Song Reviewed-by: Shakeel Butt Reviewed-by: Roman Gushchin Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2db2aeac8a9e..845eec01ef9d 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1061,13 +1061,9 @@ static __always_inline struct mem_cgroup *get_active_memcg(void) rcu_read_lock(); memcg = active_memcg(); - if (memcg) { - /* current->active_memcg must hold a ref. */ - if (WARN_ON_ONCE(!css_tryget(&memcg->css))) - memcg = root_mem_cgroup; - else - memcg = current->active_memcg; - } + /* remote memcg must hold a ref. */ + if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css))) + memcg = root_mem_cgroup; rcu_read_unlock(); return memcg; From 96403bfe50c344b587ea53894954a9d152af1c9d Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Wed, 24 Feb 2021 12:04:26 -0800 Subject: [PATCH 531/633] mm: memcontrol: fix slub memory accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SLUB currently account kmalloc() and kmalloc_node() allocations larger than order-1 page per-node. But it forget to update the per-memcg vmstats. So it can lead to inaccurate statistics of "slab_unreclaimable" which is from memory.stat. Fix it by using mod_lruvec_page_state instead of mod_node_page_state. Link: https://lkml.kernel.org/r/20210223092423.42420-1-songmuchun@bytedance.com Fixes: 6a486c0ad4dc ("mm, sl[ou]b: improve memory accounting") Signed-off-by: Muchun Song Reviewed-by: Shakeel Butt Reviewed-by: Roman Gushchin Reviewed-by: Michal Koutný Cc: Johannes Weiner Cc: Michal Hocko Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slab_common.c | 4 ++-- mm/slub.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 9633fa1bbbfd..5be7825ad3ce 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -898,8 +898,8 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) page = alloc_pages(flags, order); if (likely(page)) { ret = page_address(page); - mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, - PAGE_SIZE << order); + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, + PAGE_SIZE << order); } ret = kasan_kmalloc_large(ret, size, flags); /* As ret might get tagged, call kmemleak hook after KASAN. */ diff --git a/mm/slub.c b/mm/slub.c index ca566361561e..046d7194acaf 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4042,8 +4042,8 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node) page = alloc_pages_node(node, flags, order); if (page) { ptr = page_address(page); - mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, - PAGE_SIZE << order); + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, + PAGE_SIZE << order); } return kmalloc_large_node_hook(ptr, size, flags); @@ -4174,8 +4174,8 @@ void kfree(const void *x) BUG_ON(!PageCompound(page)); kfree_hook(object); - mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, - -(PAGE_SIZE << order)); + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, + -(PAGE_SIZE << order)); __free_pages(page, order); return; } From b7204006c8602f43793ee1b285e963084bdb1a26 Mon Sep 17 00:00:00 2001 From: Adrian Huang Date: Wed, 24 Feb 2021 12:04:29 -0800 Subject: [PATCH 532/633] mm/mmap.c: remove unnecessary local variable The local variable 'retval' is assigned just for once in __do_sys_brk(), and the function returns the value of the local variable right after the assignment. Remove unnecessary assignment and local variable declaration. Link: https://lkml.kernel.org/r/20201222103249.30683-1-adrianhuang0701@gmail.com Signed-off-by: Adrian Huang Acked-by: Souptick Joarder Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/mmap.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index 90673febce6a..3f287599a7a3 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -189,7 +189,6 @@ static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long struct list_head *uf); SYSCALL_DEFINE1(brk, unsigned long, brk) { - unsigned long retval; unsigned long newbrk, oldbrk, origbrk; struct mm_struct *mm = current->mm; struct vm_area_struct *next; @@ -281,9 +280,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) return brk; out: - retval = origbrk; mmap_write_unlock(mm); - return retval; + return origbrk; } static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) From 90a3e375d324b2255b83e3dd29e99e2b05d82aaf Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:04:33 -0800 Subject: [PATCH 533/633] mm/memory.c: fix potential pte_unmap_unlock pte error Since commit 42e4089c7890 ("x86/speculation/l1tf: Disallow non privileged high MMIO PROT_NONE mappings"), when the first pfn modify is not allowed, we would break the loop with pte unchanged. Then the wrong pte - 1 would be passed to pte_unmap_unlock. Andi said: "While the fix is correct, I'm not sure if it actually is a real bug. Is there any architecture that would do something else than unlocking the underlying page? If it's just the underlying page then it should be always the same page, so no bug" Link: https://lkml.kernel.org/r/20210109080118.20885-1-linmiaohe@huawei.com Fixes: 42e4089c789 ("x86/speculation/l1tf: Disallow non privileged high MMIO PROT_NONE mappings") Signed-off-by: Hongxiang Lou Signed-off-by: Miaohe Lin Cc: Thomas Gleixner Cc: Dave Hansen Cc: Andi Kleen Cc: Josh Poimboeuf Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memory.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 5da964079678..83e1bcf7a669 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2177,11 +2177,11 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot) { - pte_t *pte; + pte_t *pte, *mapped_pte; spinlock_t *ptl; int err = 0; - pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; arch_enter_lazy_mmu_mode(); @@ -2195,7 +2195,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, pfn++; } while (pte++, addr += PAGE_SIZE, addr != end); arch_leave_lazy_mmu_mode(); - pte_unmap_unlock(pte - 1, ptl); + pte_unmap_unlock(mapped_pte, ptl); return err; } From c045c72ccde3a267963f8e85f388db4c40dea3b3 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:04:36 -0800 Subject: [PATCH 534/633] mm/pgtable-generic.c: simplify the VM_BUG_ON condition in pmdp_huge_clear_flush() The condition (A && !C && !D) || !A is equivalent to !A || (A && !C && !D) and can be further simplified to !A || (!C && !D). Link: https://lkml.kernel.org/r/20210201114319.34720-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Andrew Morton Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/pgtable-generic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 9578db83e312..fa1375f3e3b2 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -135,8 +135,8 @@ pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, { pmd_t pmd; VM_BUG_ON(address & ~HPAGE_PMD_MASK); - VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && - !pmd_devmap(*pmdp)) || !pmd_present(*pmdp)); + VM_BUG_ON(!pmd_present(*pmdp) || (!pmd_trans_huge(*pmdp) && + !pmd_devmap(*pmdp))); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return pmd; From 374437a274e24e8e3ccd19f704e80d325f75f254 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:04:39 -0800 Subject: [PATCH 535/633] mm/pgtable-generic.c: optimize the VM_BUG_ON condition in pmdp_huge_clear_flush() The developer will have trouble figuring out why the BUG actually triggered when there is a complex expression in the VM_BUG_ON. Because we can only identify the condition triggered BUG via line number provided by VM_BUG_ON. Optimize this by spliting such a complex expression into two simple conditions. Link: https://lkml.kernel.org/r/20210203084137.25522-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Suggested-by: Andrew Morton Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/pgtable-generic.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index fa1375f3e3b2..c2210e1cdb51 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -135,8 +135,9 @@ pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, { pmd_t pmd; VM_BUG_ON(address & ~HPAGE_PMD_MASK); - VM_BUG_ON(!pmd_present(*pmdp) || (!pmd_trans_huge(*pmdp) && - !pmd_devmap(*pmdp))); + VM_BUG_ON(!pmd_present(*pmdp)); + /* Below assumes pmd_present() is true */ + VM_BUG_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return pmd; From 8abb50c76b484e8d8dc355c092170c37b5f832f5 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:04:42 -0800 Subject: [PATCH 536/633] mm/memory.c: fix potential pte_unmap_unlock pte error If all pte entry is none in 'non-create' case, we would break the loop with pte unchanged. Then the wrong pte - 1 would be passed to pte_unmap_unlock. This is a theoretical issue which may not be a real bug. So it's not worth cc stable. Link: https://lkml.kernel.org/r/20210205081925.59809-1-linmiaohe@huawei.com Fixes: aee16b3cee27 ("Add apply_to_page_range() which applies a function to a pte range") Signed-off-by: Miaohe Lin Reviewed-by: Andrew Morton Cc: Ingo Molnar Cc: Ian Pratt Cc: Chris Wright Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memory.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 83e1bcf7a669..ef4e10c60b5f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2394,18 +2394,18 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask) { - pte_t *pte; + pte_t *pte, *mapped_pte; int err = 0; spinlock_t *ptl; if (create) { - pte = (mm == &init_mm) ? + mapped_pte = pte = (mm == &init_mm) ? pte_alloc_kernel_track(pmd, addr, mask) : pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; } else { - pte = (mm == &init_mm) ? + mapped_pte = pte = (mm == &init_mm) ? pte_offset_kernel(pmd, addr) : pte_offset_map_lock(mm, pmd, addr, &ptl); } @@ -2428,7 +2428,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, arch_leave_lazy_mmu_mode(); if (mm != &init_mm) - pte_unmap_unlock(pte-1, ptl); + pte_unmap_unlock(mapped_pte, ptl); return err; } From dbf53f7597be11ffc18b16444a1ffc7d7b76746e Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Wed, 24 Feb 2021 12:04:46 -0800 Subject: [PATCH 537/633] mm/mprotect.c: optimize error detection in do_mprotect_pkey() Obviously, the error variable detection of the if statement is for the mprotect callback function, so it is also put into the scope of calling callbck. This is a cleanup which makes this site consistent with the rest of this function's error handling. Link: https://lkml.kernel.org/r/20210118133310.98375-1-tianjia.zhang@linux.alibaba.com Signed-off-by: Tianjia Zhang Reported-by: Jia Zhang Reviewed-by: Andrew Morton Cc: Jarkko Sakkinen Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/mprotect.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/mprotect.c b/mm/mprotect.c index ab709023e9aa..94188df1ee55 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -617,10 +617,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (tmp > end) tmp = end; - if (vma->vm_ops && vma->vm_ops->mprotect) + if (vma->vm_ops && vma->vm_ops->mprotect) { error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags); - if (error) - goto out; + if (error) + goto out; + } error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) From ee8ab1903e3d912d8f10bedbf96c3b6a1c8cbede Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Wed, 24 Feb 2021 12:04:49 -0800 Subject: [PATCH 538/633] mm: rmap: explicitly reset vma->anon_vma in unlink_anon_vmas() In case the vma will continue to be used after unlink its relevant anon_vma, we need to reset the vma->anon_vma pointer to NULL. So, later when fault happen within this vma again, a new anon_vma will be prepared. By this way, the vma will only be checked for reverse mapping of pages which been fault in after the unlink_anon_vmas call. Currently, the mremap with MREMAP_DONTUNMAP scenario will continue use the vma after moved its page table entries to a new vma. For other scenarios, the vma itself will be freed after call unlink_anon_vmas. Link: https://lkml.kernel.org/r/20210119075126.3513154-1-lixinhai.lxh@gmail.com Signed-off-by: Li Xinhai Cc: Andrea Arcangeli Cc: Brian Geffon Cc: Kirill A. Shutemov Cc: Lokesh Gidra Cc: Minchan Kim Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/rmap.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/rmap.c b/mm/rmap.c index 5ebf16fae4b9..e26ae119a131 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -413,8 +413,15 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_del(&avc->same_vma); anon_vma_chain_free(avc); } - if (vma->anon_vma) + if (vma->anon_vma) { vma->anon_vma->degree--; + + /* + * vma would still be needed after unlink, and anon_vma will be prepared + * when handle fault. + */ + vma->anon_vma = NULL; + } unlock_anon_vma_root(root); /* From 1583aa278f5f6a58b6ff8e9e703d0cca2b953d97 Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Wed, 24 Feb 2021 12:04:53 -0800 Subject: [PATCH 539/633] mm: mremap: unlink anon_vmas when mremap with MREMAP_DONTUNMAP success mremap with MREMAP_DONTUNMAP can move all page table entries to new vma, which means all pages allocated for the old vma are not relevant to it anymore, and the relevant anon_vma links needs to be unlinked, in nature the old vma is much like been freshly created and have no pages been fault in. But we should not do unlink, if the new vma has effectively merged with the old one. [lixinhai.lxh@gmail.com: v2] Link: https://lkml.kernel.org/r/20210127083917.309264-2-lixinhai.lxh@gmail.com Link: https://lkml.kernel.org/r/20210119075126.3513154-2-lixinhai.lxh@gmail.com Signed-off-by: Li Xinhai Cc: Brian Geffon Cc: Lokesh Gidra Cc: Minchan Kim Cc: Kirill A. Shutemov Cc: Vlastimil Babka Cc: Andrea Arcangeli Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/mremap.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mm/mremap.c b/mm/mremap.c index 47192691fe32..ec8f840399ed 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -593,6 +593,14 @@ static unsigned long move_vma(struct vm_area_struct *vma, /* We always clear VM_LOCKED[ONFAULT] on the old vma */ vma->vm_flags &= VM_LOCKED_CLEAR_MASK; + /* + * anon_vma links of the old vma is no longer needed after its page + * table has been moved. + */ + if (new_vma != vma && vma->vm_start == old_addr && + vma->vm_end == (old_addr + old_len)) + unlink_anon_vmas(vma); + /* Because we won't unmap we don't need to touch locked_vm */ return new_addr; } From 5df6d792011b0b221f0a3a7ba5a732230cd71b4f Mon Sep 17 00:00:00 2001 From: "sh_def@163.com" Date: Wed, 24 Feb 2021 12:04:57 -0800 Subject: [PATCH 540/633] mm/page_reporting: use list_entry_is_head() in page_reporting_cycle() Replace '&next->lru != list' with list_entry_is_head(). No functional change. Link: https://lkml.kernel.org/r/20201222182735.GA1257912@ubuntu-A520I-AC Signed-off-by: sh Reviewed-by: Andrew Morton Cc: Alexander Duyck Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_reporting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_reporting.c b/mm/page_reporting.c index cd8e13d41df4..c50d93ffa252 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -211,7 +211,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone, } /* Rotate any leftover pages to the head of the freelist */ - if (&next->lru != list && !list_is_first(&next->lru, list)) + if (!list_entry_is_head(next, list, lru) && !list_is_first(&next->lru, list)) list_rotate_to_front(&next->lru, list); spin_unlock_irq(&zone->lock); From fb9bf0484af4770240342f4d1b3dd054889cc31e Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 24 Feb 2021 12:05:00 -0800 Subject: [PATCH 541/633] vmalloc: remove redundant NULL check Fix below warnings reported by coccicheck: fs/proc/vmcore.c:1503:2-7: WARNING: NULL check before some freeing functions is not needed. Link: https://lkml.kernel.org/r/1611216753-44598-1-git-send-email-abaci-bugfix@linux.alibaba.com Signed-off-by: Yang Li Reported-by: Abaci Robot Acked-by: Baoquan He Cc: Dave Young Cc: Vivek Goyal Cc: Alexey Dobriyan Cc: "Uladzislau Rezki (Sony)" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/vmcore.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c index c3a345c28a93..9a15334da208 100644 --- a/fs/proc/vmcore.c +++ b/fs/proc/vmcore.c @@ -1503,11 +1503,8 @@ int vmcore_add_device_dump(struct vmcoredd_data *data) return 0; out_err: - if (buf) - vfree(buf); - - if (dump) - vfree(dump); + vfree(buf); + vfree(dump); return ret; } From f00748bfa0246c428bf93f45267b8f1aa1816098 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:05 -0800 Subject: [PATCH 542/633] kasan: prefix global functions with kasan_ Patch series "kasan: HW_TAGS tests support and fixes", v4. This patchset adds support for running KASAN-KUnit tests with the hardware tag-based mode and also contains a few fixes. This patch (of 15): There's a number of internal KASAN functions that are used across multiple source code files and therefore aren't marked as static inline. To avoid littering the kernel function names list with generic function names, prefix all such KASAN functions with kasan_. As a part of this change: - Rename internal (un)poison_range() to kasan_(un)poison() (no _range) to avoid name collision with a public kasan_unpoison_range(). - Rename check_memory_region() to kasan_check_range(), as it's a more fitting name. Link: https://lkml.kernel.org/r/cover.1610733117.git.andreyknvl@google.com Link: https://linux-review.googlesource.com/id/I719cc93483d4ba288a634dba80ee6b7f2809cd26 Link: https://lkml.kernel.org/r/13777aedf8d3ebbf35891136e1f2287e2f34aaba.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Suggested-by: Marco Elver Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Catalin Marinas Cc: Vincenzo Frascino Cc: Dmitry Vyukov Cc: Will Deacon Cc: Andrey Ryabinin Cc: Peter Collingbourne Cc: Evgenii Stepanov Cc: Branislav Rankov Cc: Kevin Brodsky Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/kasan/common.c | 47 +++++++++++++++++++------------------- mm/kasan/generic.c | 36 ++++++++++++++--------------- mm/kasan/kasan.h | 48 +++++++++++++++++++-------------------- mm/kasan/quarantine.c | 22 +++++++++--------- mm/kasan/report.c | 13 ++++++----- mm/kasan/report_generic.c | 8 +++---- mm/kasan/report_hw_tags.c | 8 +++---- mm/kasan/report_sw_tags.c | 8 +++---- mm/kasan/shadow.c | 26 ++++++++++----------- mm/kasan/sw_tags.c | 16 ++++++------- tools/objtool/check.c | 2 +- 11 files changed, 117 insertions(+), 117 deletions(-) diff --git a/mm/kasan/common.c b/mm/kasan/common.c index b25167664ead..eedc3e0fe365 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -60,7 +60,7 @@ void kasan_disable_current(void) void __kasan_unpoison_range(const void *address, size_t size) { - unpoison_range(address, size); + kasan_unpoison(address, size); } #if CONFIG_KASAN_STACK @@ -69,7 +69,7 @@ void kasan_unpoison_task_stack(struct task_struct *task) { void *base = task_stack_page(task); - unpoison_range(base, THREAD_SIZE); + kasan_unpoison(base, THREAD_SIZE); } /* Unpoison the stack for the current task beyond a watermark sp value. */ @@ -82,7 +82,7 @@ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) */ void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1)); - unpoison_range(base, watermark - base); + kasan_unpoison(base, watermark - base); } #endif /* CONFIG_KASAN_STACK */ @@ -105,18 +105,17 @@ void __kasan_alloc_pages(struct page *page, unsigned int order) if (unlikely(PageHighMem(page))) return; - tag = random_tag(); + tag = kasan_random_tag(); for (i = 0; i < (1 << order); i++) page_kasan_tag_set(page + i, tag); - unpoison_range(page_address(page), PAGE_SIZE << order); + kasan_unpoison(page_address(page), PAGE_SIZE << order); } void __kasan_free_pages(struct page *page, unsigned int order) { if (likely(!PageHighMem(page))) - poison_range(page_address(page), - PAGE_SIZE << order, - KASAN_FREE_PAGE); + kasan_poison(page_address(page), PAGE_SIZE << order, + KASAN_FREE_PAGE); } /* @@ -246,18 +245,18 @@ void __kasan_poison_slab(struct page *page) for (i = 0; i < compound_nr(page); i++) page_kasan_tag_reset(page + i); - poison_range(page_address(page), page_size(page), + kasan_poison(page_address(page), page_size(page), KASAN_KMALLOC_REDZONE); } void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object) { - unpoison_range(object, cache->object_size); + kasan_unpoison(object, cache->object_size); } void __kasan_poison_object_data(struct kmem_cache *cache, void *object) { - poison_range(object, cache->object_size, KASAN_KMALLOC_REDZONE); + kasan_poison(object, cache->object_size, KASAN_KMALLOC_REDZONE); } /* @@ -294,7 +293,7 @@ static u8 assign_tag(struct kmem_cache *cache, const void *object, * set, assign a tag when the object is being allocated (init == false). */ if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU)) - return init ? KASAN_TAG_KERNEL : random_tag(); + return init ? KASAN_TAG_KERNEL : kasan_random_tag(); /* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */ #ifdef CONFIG_SLAB @@ -305,7 +304,7 @@ static u8 assign_tag(struct kmem_cache *cache, const void *object, * For SLUB assign a random tag during slab creation, otherwise reuse * the already assigned tag. */ - return init ? random_tag() : get_tag(object); + return init ? kasan_random_tag() : get_tag(object); #endif } @@ -346,12 +345,12 @@ static bool ____kasan_slab_free(struct kmem_cache *cache, void *object, if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU)) return false; - if (check_invalid_free(tagged_object)) { + if (kasan_check_invalid_free(tagged_object)) { kasan_report_invalid_free(tagged_object, ip); return true; } - poison_range(object, cache->object_size, KASAN_KMALLOC_FREE); + kasan_poison(object, cache->object_size, KASAN_KMALLOC_FREE); if (!kasan_stack_collection_enabled()) return false; @@ -361,7 +360,7 @@ static bool ____kasan_slab_free(struct kmem_cache *cache, void *object, kasan_set_free_info(cache, object, tag); - return quarantine_put(cache, object); + return kasan_quarantine_put(cache, object); } bool __kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip) @@ -386,7 +385,7 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip) kasan_report_invalid_free(ptr, ip); return; } - poison_range(ptr, page_size(page), KASAN_FREE_PAGE); + kasan_poison(ptr, page_size(page), KASAN_FREE_PAGE); } else { ____kasan_slab_free(page->slab_cache, ptr, ip, false); } @@ -409,7 +408,7 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object, u8 tag; if (gfpflags_allow_blocking(flags)) - quarantine_reduce(); + kasan_quarantine_reduce(); if (unlikely(object == NULL)) return NULL; @@ -421,9 +420,9 @@ static void *____kasan_kmalloc(struct kmem_cache *cache, const void *object, tag = assign_tag(cache, object, false, keep_tag); /* Tag is ignored in set_tag without CONFIG_KASAN_SW/HW_TAGS */ - unpoison_range(set_tag(object, tag), size); - poison_range((void *)redzone_start, redzone_end - redzone_start, - KASAN_KMALLOC_REDZONE); + kasan_unpoison(set_tag(object, tag), size); + kasan_poison((void *)redzone_start, redzone_end - redzone_start, + KASAN_KMALLOC_REDZONE); if (kasan_stack_collection_enabled()) set_alloc_info(cache, (void *)object, flags); @@ -452,7 +451,7 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, unsigned long redzone_end; if (gfpflags_allow_blocking(flags)) - quarantine_reduce(); + kasan_quarantine_reduce(); if (unlikely(ptr == NULL)) return NULL; @@ -462,8 +461,8 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, KASAN_GRANULE_SIZE); redzone_end = (unsigned long)ptr + page_size(page); - unpoison_range(ptr, size); - poison_range((void *)redzone_start, redzone_end - redzone_start, + kasan_unpoison(ptr, size); + kasan_poison((void *)redzone_start, redzone_end - redzone_start, KASAN_PAGE_REDZONE); return (void *)ptr; diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 5106b84b07d4..acab8862dc67 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -158,7 +158,7 @@ static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size) return memory_is_poisoned_n(addr, size); } -static __always_inline bool check_memory_region_inline(unsigned long addr, +static __always_inline bool check_region_inline(unsigned long addr, size_t size, bool write, unsigned long ret_ip) { @@ -179,13 +179,13 @@ static __always_inline bool check_memory_region_inline(unsigned long addr, return !kasan_report(addr, size, write, ret_ip); } -bool check_memory_region(unsigned long addr, size_t size, bool write, - unsigned long ret_ip) +bool kasan_check_range(unsigned long addr, size_t size, bool write, + unsigned long ret_ip) { - return check_memory_region_inline(addr, size, write, ret_ip); + return check_region_inline(addr, size, write, ret_ip); } -bool check_invalid_free(void *addr) +bool kasan_check_invalid_free(void *addr) { s8 shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(addr)); @@ -194,22 +194,22 @@ bool check_invalid_free(void *addr) void kasan_cache_shrink(struct kmem_cache *cache) { - quarantine_remove_cache(cache); + kasan_quarantine_remove_cache(cache); } void kasan_cache_shutdown(struct kmem_cache *cache) { if (!__kmem_cache_empty(cache)) - quarantine_remove_cache(cache); + kasan_quarantine_remove_cache(cache); } static void register_global(struct kasan_global *global) { size_t aligned_size = round_up(global->size, KASAN_GRANULE_SIZE); - unpoison_range(global->beg, global->size); + kasan_unpoison(global->beg, global->size); - poison_range(global->beg + aligned_size, + kasan_poison(global->beg + aligned_size, global->size_with_redzone - aligned_size, KASAN_GLOBAL_REDZONE); } @@ -231,7 +231,7 @@ EXPORT_SYMBOL(__asan_unregister_globals); #define DEFINE_ASAN_LOAD_STORE(size) \ void __asan_load##size(unsigned long addr) \ { \ - check_memory_region_inline(addr, size, false, _RET_IP_);\ + check_region_inline(addr, size, false, _RET_IP_); \ } \ EXPORT_SYMBOL(__asan_load##size); \ __alias(__asan_load##size) \ @@ -239,7 +239,7 @@ EXPORT_SYMBOL(__asan_unregister_globals); EXPORT_SYMBOL(__asan_load##size##_noabort); \ void __asan_store##size(unsigned long addr) \ { \ - check_memory_region_inline(addr, size, true, _RET_IP_); \ + check_region_inline(addr, size, true, _RET_IP_); \ } \ EXPORT_SYMBOL(__asan_store##size); \ __alias(__asan_store##size) \ @@ -254,7 +254,7 @@ DEFINE_ASAN_LOAD_STORE(16); void __asan_loadN(unsigned long addr, size_t size) { - check_memory_region(addr, size, false, _RET_IP_); + kasan_check_range(addr, size, false, _RET_IP_); } EXPORT_SYMBOL(__asan_loadN); @@ -264,7 +264,7 @@ EXPORT_SYMBOL(__asan_loadN_noabort); void __asan_storeN(unsigned long addr, size_t size) { - check_memory_region(addr, size, true, _RET_IP_); + kasan_check_range(addr, size, true, _RET_IP_); } EXPORT_SYMBOL(__asan_storeN); @@ -290,11 +290,11 @@ void __asan_alloca_poison(unsigned long addr, size_t size) WARN_ON(!IS_ALIGNED(addr, KASAN_ALLOCA_REDZONE_SIZE)); - unpoison_range((const void *)(addr + rounded_down_size), - size - rounded_down_size); - poison_range(left_redzone, KASAN_ALLOCA_REDZONE_SIZE, + kasan_unpoison((const void *)(addr + rounded_down_size), + size - rounded_down_size); + kasan_poison(left_redzone, KASAN_ALLOCA_REDZONE_SIZE, KASAN_ALLOCA_LEFT); - poison_range(right_redzone, padding_size + KASAN_ALLOCA_REDZONE_SIZE, + kasan_poison(right_redzone, padding_size + KASAN_ALLOCA_REDZONE_SIZE, KASAN_ALLOCA_RIGHT); } EXPORT_SYMBOL(__asan_alloca_poison); @@ -305,7 +305,7 @@ void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom) if (unlikely(!stack_top || stack_top > stack_bottom)) return; - unpoison_range(stack_top, stack_bottom - stack_top); + kasan_unpoison(stack_top, stack_bottom - stack_top); } EXPORT_SYMBOL(__asan_allocas_unpoison); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 8c706e7652f2..3810e75b8eea 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -195,14 +195,14 @@ static inline bool addr_has_metadata(const void *addr) } /** - * check_memory_region - Check memory region, and report if invalid access. + * kasan_check_range - Check memory region, and report if invalid access. * @addr: the accessed address * @size: the accessed size * @write: true if access is a write access * @ret_ip: return address * @return: true if access was valid, false if invalid */ -bool check_memory_region(unsigned long addr, size_t size, bool write, +bool kasan_check_range(unsigned long addr, size_t size, bool write, unsigned long ret_ip); #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ @@ -215,19 +215,19 @@ static inline bool addr_has_metadata(const void *addr) #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) -void print_tags(u8 addr_tag, const void *addr); +void kasan_print_tags(u8 addr_tag, const void *addr); #else -static inline void print_tags(u8 addr_tag, const void *addr) { } +static inline void kasan_print_tags(u8 addr_tag, const void *addr) { } #endif -void *find_first_bad_addr(void *addr, size_t size); -const char *get_bug_type(struct kasan_access_info *info); -void metadata_fetch_row(char *buffer, void *row); +void *kasan_find_first_bad_addr(void *addr, size_t size); +const char *kasan_get_bug_type(struct kasan_access_info *info); +void kasan_metadata_fetch_row(char *buffer, void *row); #if defined(CONFIG_KASAN_GENERIC) && CONFIG_KASAN_STACK -void print_address_stack_frame(const void *addr); +void kasan_print_address_stack_frame(const void *addr); #else -static inline void print_address_stack_frame(const void *addr) { } +static inline void kasan_print_address_stack_frame(const void *addr) { } #endif bool kasan_report(unsigned long addr, size_t size, @@ -244,13 +244,13 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, #if defined(CONFIG_KASAN_GENERIC) && \ (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) -bool quarantine_put(struct kmem_cache *cache, void *object); -void quarantine_reduce(void); -void quarantine_remove_cache(struct kmem_cache *cache); +bool kasan_quarantine_put(struct kmem_cache *cache, void *object); +void kasan_quarantine_reduce(void); +void kasan_quarantine_remove_cache(struct kmem_cache *cache); #else -static inline bool quarantine_put(struct kmem_cache *cache, void *object) { return false; } -static inline void quarantine_reduce(void) { } -static inline void quarantine_remove_cache(struct kmem_cache *cache) { } +static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; } +static inline void kasan_quarantine_reduce(void) { } +static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { } #endif #ifndef arch_kasan_set_tag @@ -293,28 +293,28 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) #endif /* CONFIG_KASAN_HW_TAGS */ #ifdef CONFIG_KASAN_SW_TAGS -u8 random_tag(void); +u8 kasan_random_tag(void); #elif defined(CONFIG_KASAN_HW_TAGS) -static inline u8 random_tag(void) { return hw_get_random_tag(); } +static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); } #else -static inline u8 random_tag(void) { return 0; } +static inline u8 kasan_random_tag(void) { return 0; } #endif #ifdef CONFIG_KASAN_HW_TAGS -static inline void poison_range(const void *address, size_t size, u8 value) +static inline void kasan_poison(const void *address, size_t size, u8 value) { hw_set_mem_tag_range(kasan_reset_tag(address), round_up(size, KASAN_GRANULE_SIZE), value); } -static inline void unpoison_range(const void *address, size_t size) +static inline void kasan_unpoison(const void *address, size_t size) { hw_set_mem_tag_range(kasan_reset_tag(address), round_up(size, KASAN_GRANULE_SIZE), get_tag(address)); } -static inline bool check_invalid_free(void *addr) +static inline bool kasan_check_invalid_free(void *addr) { u8 ptr_tag = get_tag(addr); u8 mem_tag = hw_get_mem_tag(addr); @@ -325,9 +325,9 @@ static inline bool check_invalid_free(void *addr) #else /* CONFIG_KASAN_HW_TAGS */ -void poison_range(const void *address, size_t size, u8 value); -void unpoison_range(const void *address, size_t size); -bool check_invalid_free(void *addr); +void kasan_poison(const void *address, size_t size, u8 value); +void kasan_unpoison(const void *address, size_t size); +bool kasan_check_invalid_free(void *addr); #endif /* CONFIG_KASAN_HW_TAGS */ diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index 55783125a767..728fb24c5683 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -168,7 +168,7 @@ static void qlist_free_all(struct qlist_head *q, struct kmem_cache *cache) qlist_init(q); } -bool quarantine_put(struct kmem_cache *cache, void *object) +bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { unsigned long flags; struct qlist_head *q; @@ -184,11 +184,11 @@ bool quarantine_put(struct kmem_cache *cache, void *object) /* * Note: irq must be disabled until after we move the batch to the - * global quarantine. Otherwise quarantine_remove_cache() can miss - * some objects belonging to the cache if they are in our local temp - * list. quarantine_remove_cache() executes on_each_cpu() at the - * beginning which ensures that it either sees the objects in per-cpu - * lists or in the global quarantine. + * global quarantine. Otherwise kasan_quarantine_remove_cache() can + * miss some objects belonging to the cache if they are in our local + * temp list. kasan_quarantine_remove_cache() executes on_each_cpu() + * at the beginning which ensures that it either sees the objects in + * per-cpu lists or in the global quarantine. */ local_irq_save(flags); @@ -222,7 +222,7 @@ bool quarantine_put(struct kmem_cache *cache, void *object) return true; } -void quarantine_reduce(void) +void kasan_quarantine_reduce(void) { size_t total_size, new_quarantine_size, percpu_quarantines; unsigned long flags; @@ -234,7 +234,7 @@ void quarantine_reduce(void) return; /* - * srcu critical section ensures that quarantine_remove_cache() + * srcu critical section ensures that kasan_quarantine_remove_cache() * will not miss objects belonging to the cache while they are in our * local to_free list. srcu is chosen because (1) it gives us private * grace period domain that does not interfere with anything else, @@ -309,15 +309,15 @@ static void per_cpu_remove_cache(void *arg) } /* Free all quarantined objects belonging to cache. */ -void quarantine_remove_cache(struct kmem_cache *cache) +void kasan_quarantine_remove_cache(struct kmem_cache *cache) { unsigned long flags, i; struct qlist_head to_free = QLIST_INIT; /* * Must be careful to not miss any objects that are being moved from - * per-cpu list to the global quarantine in quarantine_put(), - * nor objects being freed in quarantine_reduce(). on_each_cpu() + * per-cpu list to the global quarantine in kasan_quarantine_put(), + * nor objects being freed in kasan_quarantine_reduce(). on_each_cpu() * achieves the first goal, while synchronize_srcu() achieves the * second. */ diff --git a/mm/kasan/report.c b/mm/kasan/report.c index c0fb21797550..e93d7973792e 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -61,7 +61,7 @@ __setup("kasan_multi_shot", kasan_set_multi_shot); static void print_error_description(struct kasan_access_info *info) { pr_err("BUG: KASAN: %s in %pS\n", - get_bug_type(info), (void *)info->ip); + kasan_get_bug_type(info), (void *)info->ip); if (info->access_size) pr_err("%s of size %zu at addr %px by task %s/%d\n", info->is_write ? "Write" : "Read", info->access_size, @@ -247,7 +247,7 @@ static void print_address_description(void *addr, u8 tag) dump_page(page, "kasan: bad access detected"); } - print_address_stack_frame(addr); + kasan_print_address_stack_frame(addr); } static bool meta_row_is_guilty(const void *row, const void *addr) @@ -293,7 +293,7 @@ static void print_memory_metadata(const void *addr) * function, because generic functions may try to * access kasan mapping for the passed address. */ - metadata_fetch_row(&metadata[0], row); + kasan_metadata_fetch_row(&metadata[0], row); print_hex_dump(KERN_ERR, buffer, DUMP_PREFIX_NONE, META_BYTES_PER_ROW, 1, @@ -350,7 +350,7 @@ void kasan_report_invalid_free(void *object, unsigned long ip) start_report(&flags); pr_err("BUG: KASAN: double-free or invalid-free in %pS\n", (void *)ip); - print_tags(tag, object); + kasan_print_tags(tag, object); pr_err("\n"); print_address_description(object, tag); pr_err("\n"); @@ -378,7 +378,8 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write, info.access_addr = tagged_addr; if (addr_has_metadata(untagged_addr)) - info.first_bad_addr = find_first_bad_addr(tagged_addr, size); + info.first_bad_addr = + kasan_find_first_bad_addr(tagged_addr, size); else info.first_bad_addr = untagged_addr; info.access_size = size; @@ -389,7 +390,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write, print_error_description(&info); if (addr_has_metadata(untagged_addr)) - print_tags(get_tag(tagged_addr), info.first_bad_addr); + kasan_print_tags(get_tag(tagged_addr), info.first_bad_addr); pr_err("\n"); if (addr_has_metadata(untagged_addr)) { diff --git a/mm/kasan/report_generic.c b/mm/kasan/report_generic.c index 8a9c889872da..41f374585144 100644 --- a/mm/kasan/report_generic.c +++ b/mm/kasan/report_generic.c @@ -30,7 +30,7 @@ #include "kasan.h" #include "../slab.h" -void *find_first_bad_addr(void *addr, size_t size) +void *kasan_find_first_bad_addr(void *addr, size_t size) { void *p = addr; @@ -105,7 +105,7 @@ static const char *get_wild_bug_type(struct kasan_access_info *info) return bug_type; } -const char *get_bug_type(struct kasan_access_info *info) +const char *kasan_get_bug_type(struct kasan_access_info *info) { /* * If access_size is a negative number, then it has reason to be @@ -123,7 +123,7 @@ const char *get_bug_type(struct kasan_access_info *info) return get_wild_bug_type(info); } -void metadata_fetch_row(char *buffer, void *row) +void kasan_metadata_fetch_row(char *buffer, void *row) { memcpy(buffer, kasan_mem_to_shadow(row), META_BYTES_PER_ROW); } @@ -263,7 +263,7 @@ static bool __must_check get_address_stack_frame_info(const void *addr, return true; } -void print_address_stack_frame(const void *addr) +void kasan_print_address_stack_frame(const void *addr) { unsigned long offset; const char *frame_descr; diff --git a/mm/kasan/report_hw_tags.c b/mm/kasan/report_hw_tags.c index 57114f0e14d1..42b2168755d6 100644 --- a/mm/kasan/report_hw_tags.c +++ b/mm/kasan/report_hw_tags.c @@ -15,17 +15,17 @@ #include "kasan.h" -const char *get_bug_type(struct kasan_access_info *info) +const char *kasan_get_bug_type(struct kasan_access_info *info) { return "invalid-access"; } -void *find_first_bad_addr(void *addr, size_t size) +void *kasan_find_first_bad_addr(void *addr, size_t size) { return kasan_reset_tag(addr); } -void metadata_fetch_row(char *buffer, void *row) +void kasan_metadata_fetch_row(char *buffer, void *row) { int i; @@ -33,7 +33,7 @@ void metadata_fetch_row(char *buffer, void *row) buffer[i] = hw_get_mem_tag(row + i * KASAN_GRANULE_SIZE); } -void print_tags(u8 addr_tag, const void *addr) +void kasan_print_tags(u8 addr_tag, const void *addr) { u8 memory_tag = hw_get_mem_tag((void *)addr); diff --git a/mm/kasan/report_sw_tags.c b/mm/kasan/report_sw_tags.c index 1b026793ad57..3d20d3451d9e 100644 --- a/mm/kasan/report_sw_tags.c +++ b/mm/kasan/report_sw_tags.c @@ -29,7 +29,7 @@ #include "kasan.h" #include "../slab.h" -const char *get_bug_type(struct kasan_access_info *info) +const char *kasan_get_bug_type(struct kasan_access_info *info) { #ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY struct kasan_alloc_meta *alloc_meta; @@ -72,7 +72,7 @@ const char *get_bug_type(struct kasan_access_info *info) return "invalid-access"; } -void *find_first_bad_addr(void *addr, size_t size) +void *kasan_find_first_bad_addr(void *addr, size_t size) { u8 tag = get_tag(addr); void *p = kasan_reset_tag(addr); @@ -83,12 +83,12 @@ void *find_first_bad_addr(void *addr, size_t size) return p; } -void metadata_fetch_row(char *buffer, void *row) +void kasan_metadata_fetch_row(char *buffer, void *row) { memcpy(buffer, kasan_mem_to_shadow(row), META_BYTES_PER_ROW); } -void print_tags(u8 addr_tag, const void *addr) +void kasan_print_tags(u8 addr_tag, const void *addr) { u8 *shadow = (u8 *)kasan_mem_to_shadow(addr); diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 7c2c08c55f32..38958eb0d653 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -27,20 +27,20 @@ bool __kasan_check_read(const volatile void *p, unsigned int size) { - return check_memory_region((unsigned long)p, size, false, _RET_IP_); + return kasan_check_range((unsigned long)p, size, false, _RET_IP_); } EXPORT_SYMBOL(__kasan_check_read); bool __kasan_check_write(const volatile void *p, unsigned int size) { - return check_memory_region((unsigned long)p, size, true, _RET_IP_); + return kasan_check_range((unsigned long)p, size, true, _RET_IP_); } EXPORT_SYMBOL(__kasan_check_write); #undef memset void *memset(void *addr, int c, size_t len) { - if (!check_memory_region((unsigned long)addr, len, true, _RET_IP_)) + if (!kasan_check_range((unsigned long)addr, len, true, _RET_IP_)) return NULL; return __memset(addr, c, len); @@ -50,8 +50,8 @@ void *memset(void *addr, int c, size_t len) #undef memmove void *memmove(void *dest, const void *src, size_t len) { - if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || - !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) + if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) || + !kasan_check_range((unsigned long)dest, len, true, _RET_IP_)) return NULL; return __memmove(dest, src, len); @@ -61,8 +61,8 @@ void *memmove(void *dest, const void *src, size_t len) #undef memcpy void *memcpy(void *dest, const void *src, size_t len) { - if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) || - !check_memory_region((unsigned long)dest, len, true, _RET_IP_)) + if (!kasan_check_range((unsigned long)src, len, false, _RET_IP_) || + !kasan_check_range((unsigned long)dest, len, true, _RET_IP_)) return NULL; return __memcpy(dest, src, len); @@ -72,7 +72,7 @@ void *memcpy(void *dest, const void *src, size_t len) * Poisons the shadow memory for 'size' bytes starting from 'addr'. * Memory addresses should be aligned to KASAN_GRANULE_SIZE. */ -void poison_range(const void *address, size_t size, u8 value) +void kasan_poison(const void *address, size_t size, u8 value) { void *shadow_start, *shadow_end; @@ -90,7 +90,7 @@ void poison_range(const void *address, size_t size, u8 value) __memset(shadow_start, value, shadow_end - shadow_start); } -void unpoison_range(const void *address, size_t size) +void kasan_unpoison(const void *address, size_t size) { u8 tag = get_tag(address); @@ -101,7 +101,7 @@ void unpoison_range(const void *address, size_t size) */ address = kasan_reset_tag(address); - poison_range(address, size, tag); + kasan_poison(address, size, tag); if (size & KASAN_GRANULE_MASK) { u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); @@ -286,7 +286,7 @@ int kasan_populate_vmalloc(unsigned long addr, unsigned long size) * // vmalloc() allocates memory * // let a = area->addr * // we reach kasan_populate_vmalloc - * // and call unpoison_range: + * // and call kasan_unpoison: * STORE shadow(a), unpoison_val * ... * STORE shadow(a+99), unpoison_val x = LOAD p @@ -321,7 +321,7 @@ void kasan_poison_vmalloc(const void *start, unsigned long size) return; size = round_up(size, KASAN_GRANULE_SIZE); - poison_range(start, size, KASAN_VMALLOC_INVALID); + kasan_poison(start, size, KASAN_VMALLOC_INVALID); } void kasan_unpoison_vmalloc(const void *start, unsigned long size) @@ -329,7 +329,7 @@ void kasan_unpoison_vmalloc(const void *start, unsigned long size) if (!is_vmalloc_or_module_addr(start)) return; - unpoison_range(start, size); + kasan_unpoison(start, size); } static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr, diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index 5dcd830805b2..cc271fceb5d5 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -57,7 +57,7 @@ void __init kasan_init_sw_tags(void) * sequence has in fact positive effect, since interrupts that randomly skew * PRNG at unpredictable points do only good. */ -u8 random_tag(void) +u8 kasan_random_tag(void) { u32 state = this_cpu_read(prng_state); @@ -67,7 +67,7 @@ u8 random_tag(void) return (u8)(state % (KASAN_TAG_MAX + 1)); } -bool check_memory_region(unsigned long addr, size_t size, bool write, +bool kasan_check_range(unsigned long addr, size_t size, bool write, unsigned long ret_ip) { u8 tag; @@ -118,7 +118,7 @@ bool check_memory_region(unsigned long addr, size_t size, bool write, return true; } -bool check_invalid_free(void *addr) +bool kasan_check_invalid_free(void *addr) { u8 tag = get_tag(addr); u8 shadow_byte = READ_ONCE(*(u8 *)kasan_mem_to_shadow(kasan_reset_tag(addr))); @@ -130,12 +130,12 @@ bool check_invalid_free(void *addr) #define DEFINE_HWASAN_LOAD_STORE(size) \ void __hwasan_load##size##_noabort(unsigned long addr) \ { \ - check_memory_region(addr, size, false, _RET_IP_); \ + kasan_check_range(addr, size, false, _RET_IP_); \ } \ EXPORT_SYMBOL(__hwasan_load##size##_noabort); \ void __hwasan_store##size##_noabort(unsigned long addr) \ { \ - check_memory_region(addr, size, true, _RET_IP_); \ + kasan_check_range(addr, size, true, _RET_IP_); \ } \ EXPORT_SYMBOL(__hwasan_store##size##_noabort) @@ -147,19 +147,19 @@ DEFINE_HWASAN_LOAD_STORE(16); void __hwasan_loadN_noabort(unsigned long addr, unsigned long size) { - check_memory_region(addr, size, false, _RET_IP_); + kasan_check_range(addr, size, false, _RET_IP_); } EXPORT_SYMBOL(__hwasan_loadN_noabort); void __hwasan_storeN_noabort(unsigned long addr, unsigned long size) { - check_memory_region(addr, size, true, _RET_IP_); + kasan_check_range(addr, size, true, _RET_IP_); } EXPORT_SYMBOL(__hwasan_storeN_noabort); void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size) { - poison_range((void *)addr, size, tag); + kasan_poison((void *)addr, size, tag); } EXPORT_SYMBOL(__hwasan_tag_memory); diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 080a3d6cbd75..80e7b822a432 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -666,7 +666,7 @@ static void add_ignores(struct objtool_file *file) static const char *uaccess_safe_builtin[] = { /* KASAN */ "kasan_report", - "check_memory_region", + "kasan_check_range", /* KASAN out-of-line */ "__asan_loadN_noabort", "__asan_load1_noabort", From 4062c24598b160f929d93a4a6fbf50b0b0b13d11 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:09 -0800 Subject: [PATCH 543/633] kasan: clarify HW_TAGS impact on TBI Mention in the documentation that enabling CONFIG_KASAN_HW_TAGS always results in in-kernel TBI (Top Byte Ignore) being enabled. Also do a few minor documentation cleanups. Link: https://linux-review.googlesource.com/id/Iba2a6697e3c6304cb53f89ec61dedc77fa29e3ae Link: https://lkml.kernel.org/r/3b4ea6875bb14d312092ad14ac55cb456c83c08e.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/dev-tools/kasan.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst index a248ac3941be..0ae0efe82e8e 100644 --- a/Documentation/dev-tools/kasan.rst +++ b/Documentation/dev-tools/kasan.rst @@ -147,15 +147,14 @@ negative values to distinguish between different kinds of inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). In the report above the arrows point to the shadow byte 03, which means that -the accessed address is partially accessible. - -For tag-based KASAN this last report section shows the memory tags around the -accessed address (see `Implementation details`_ section). +the accessed address is partially accessible. For tag-based KASAN modes this +last report section shows the memory tags around the accessed address +(see the `Implementation details`_ section). Boot parameters ~~~~~~~~~~~~~~~ -Hardware tag-based KASAN mode (see the section about different mode below) is +Hardware tag-based KASAN mode (see the section about various modes below) is intended for use in production as a security mitigation. Therefore it supports boot parameters that allow to disable KASAN competely or otherwise control particular KASAN features. @@ -289,6 +288,13 @@ reserved to tag freed memory regions. Hardware tag-based KASAN currently only supports tagging of kmem_cache_alloc/kmalloc and page_alloc memory. +If the hardware doesn't support MTE (pre ARMv8.5), hardware tag-based KASAN +won't be enabled. In this case all boot parameters are ignored. + +Note, that enabling CONFIG_KASAN_HW_TAGS always results in in-kernel TBI being +enabled. Even when kasan.mode=off is provided, or when the hardware doesn't +support MTE (but supports TBI). + What memory accesses are sanitised by KASAN? -------------------------------------------- From 0fd379253691e7bb7c0285a7b87525e1ff6e2fd2 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:13 -0800 Subject: [PATCH 544/633] kasan: clean up comments in tests Clarify and update comments in KASAN tests. Link: https://linux-review.googlesource.com/id/I6c816c51fa1e0eb7aa3dead6bda1f339d2af46c8 Link: https://lkml.kernel.org/r/ba6db104d53ae0e3796f80ef395f6873c1c1282f.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 59 +++++++++++++++++++++++++---------------- lib/test_kasan_module.c | 5 ++-- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 2947274cc2d3..6f46e27c2af7 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -28,10 +28,9 @@ #define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE) /* - * We assign some test results to these globals to make sure the tests - * are not eliminated as dead code. + * Some tests use these global variables to store return values from function + * calls that could otherwise be eliminated by the compiler as dead code. */ - void *kasan_ptr_result; int kasan_int_result; @@ -39,14 +38,13 @@ static struct kunit_resource resource; static struct kunit_kasan_expectation fail_data; static bool multishot; +/* + * Temporarily enable multi-shot mode. Otherwise, KASAN would only report the + * first detected bug and panic the kernel if panic_on_warn is enabled. + */ static int kasan_test_init(struct kunit *test) { - /* - * Temporarily enable multi-shot mode and set panic_on_warn=0. - * Otherwise, we'd only get a report for the first case. - */ multishot = kasan_save_enable_multi_shot(); - return 0; } @@ -56,12 +54,12 @@ static void kasan_test_exit(struct kunit *test) } /** - * KUNIT_EXPECT_KASAN_FAIL() - Causes a test failure when the expression does - * not cause a KASAN error. This uses a KUnit resource named "kasan_data." Do - * Do not use this name for a KUnit resource outside here. - * + * KUNIT_EXPECT_KASAN_FAIL() - check that the executed expression produces a + * KASAN report; causes a test failure otherwise. This relies on a KUnit + * resource named "kasan_data". Do not use this name for KUnit resources + * outside of KASAN tests. */ -#define KUNIT_EXPECT_KASAN_FAIL(test, condition) do { \ +#define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ fail_data.report_expected = true; \ fail_data.report_found = false; \ kunit_add_named_resource(test, \ @@ -69,7 +67,7 @@ static void kasan_test_exit(struct kunit *test) NULL, \ &resource, \ "kasan_data", &fail_data); \ - condition; \ + expression; \ KUNIT_EXPECT_EQ(test, \ fail_data.report_expected, \ fail_data.report_found); \ @@ -121,7 +119,8 @@ static void kmalloc_pagealloc_oob_right(struct kunit *test) return; } - /* Allocate a chunk that does not fit into a SLUB cache to trigger + /* + * Allocate a chunk that does not fit into a SLUB cache to trigger * the page allocator fallback. */ ptr = kmalloc(size, GFP_KERNEL); @@ -168,7 +167,9 @@ static void kmalloc_large_oob_right(struct kunit *test) { char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE - 256; - /* Allocate a chunk that is large enough, but still fits into a slab + + /* + * Allocate a chunk that is large enough, but still fits into a slab * and does not trigger the page allocator fallback in SLUB. */ ptr = kmalloc(size, GFP_KERNEL); @@ -469,10 +470,13 @@ static void ksize_unpoisons_memory(struct kunit *test) ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); real_size = ksize(ptr); - /* This access doesn't trigger an error. */ + + /* This access shouldn't trigger a KASAN report. */ ptr[size] = 'x'; - /* This one does. */ + + /* This one must. */ KUNIT_EXPECT_KASAN_FAIL(test, ptr[real_size] = 'y'); + kfree(ptr); } @@ -568,7 +572,7 @@ static void kmem_cache_invalid_free(struct kunit *test) return; } - /* Trigger invalid free, the object doesn't get freed */ + /* Trigger invalid free, the object doesn't get freed. */ KUNIT_EXPECT_KASAN_FAIL(test, kmem_cache_free(cache, p + 1)); /* @@ -585,7 +589,10 @@ static void kasan_memchr(struct kunit *test) char *ptr; size_t size = 24; - /* See https://bugzilla.kernel.org/show_bug.cgi?id=206337 */ + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { kunit_info(test, "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); @@ -610,7 +617,10 @@ static void kasan_memcmp(struct kunit *test) size_t size = 24; int arr[9]; - /* See https://bugzilla.kernel.org/show_bug.cgi?id=206337 */ + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { kunit_info(test, "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); @@ -634,7 +644,10 @@ static void kasan_strings(struct kunit *test) char *ptr; size_t size = 24; - /* See https://bugzilla.kernel.org/show_bug.cgi?id=206337 */ + /* + * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. + * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. + */ if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { kunit_info(test, "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); @@ -706,7 +719,7 @@ static void kasan_bitops_generic(struct kunit *test) } /* - * Allocate 1 more byte, which causes kzalloc to round up to 16-bytes; + * Allocate 1 more byte, which causes kzalloc to round up to 16 bytes; * this way we do not actually corrupt other memory. */ bits = kzalloc(sizeof(*bits) + 1, GFP_KERNEL); diff --git a/lib/test_kasan_module.c b/lib/test_kasan_module.c index 3b4cc77992d2..eee017ff8980 100644 --- a/lib/test_kasan_module.c +++ b/lib/test_kasan_module.c @@ -123,8 +123,9 @@ static noinline void __init kasan_workqueue_uaf(void) static int __init test_kasan_module_init(void) { /* - * Temporarily enable multi-shot mode. Otherwise, we'd only get a - * report for the first case. + * Temporarily enable multi-shot mode. Otherwise, KASAN would only + * report the first detected bug and panic the kernel if panic_on_warn + * is enabled. */ bool multishot = kasan_save_enable_multi_shot(); From da17e377723f50c7acd019e39cfeeca342415714 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:17 -0800 Subject: [PATCH 545/633] kasan: add macros to simplify checking test constraints Some KASAN tests require specific kernel configs to be enabled. Instead of copy-pasting the checks for these configs add a few helper macros and use them. Link: https://linux-review.googlesource.com/id/I237484a7fddfedf4a4aae9cc61ecbcdbe85a0a63 Link: https://lkml.kernel.org/r/6a0fcdb9676b7e869cfc415893ede12d916c246c.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Suggested-by: Alexander Potapenko Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 101 +++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 70 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 6f46e27c2af7..714ea27fcc3e 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -73,6 +73,20 @@ static void kasan_test_exit(struct kunit *test) fail_data.report_found); \ } while (0) +#define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ + if (!IS_ENABLED(config)) { \ + kunit_info((test), "skipping, " #config " required"); \ + return; \ + } \ +} while (0) + +#define KASAN_TEST_NEEDS_CONFIG_OFF(test, config) do { \ + if (IS_ENABLED(config)) { \ + kunit_info((test), "skipping, " #config " enabled"); \ + return; \ + } \ +} while (0) + static void kmalloc_oob_right(struct kunit *test) { char *ptr; @@ -114,10 +128,7 @@ static void kmalloc_pagealloc_oob_right(struct kunit *test) char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - if (!IS_ENABLED(CONFIG_SLUB)) { - kunit_info(test, "CONFIG_SLUB is not enabled."); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); /* * Allocate a chunk that does not fit into a SLUB cache to trigger @@ -135,10 +146,7 @@ static void kmalloc_pagealloc_uaf(struct kunit *test) char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - if (!IS_ENABLED(CONFIG_SLUB)) { - kunit_info(test, "CONFIG_SLUB is not enabled."); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); @@ -152,10 +160,7 @@ static void kmalloc_pagealloc_invalid_free(struct kunit *test) char *ptr; size_t size = KMALLOC_MAX_CACHE_SIZE + 10; - if (!IS_ENABLED(CONFIG_SLUB)) { - kunit_info(test, "CONFIG_SLUB is not enabled."); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); @@ -218,10 +223,7 @@ static void kmalloc_oob_16(struct kunit *test) } *ptr1, *ptr2; /* This test is specifically crafted for the generic mode. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_GENERIC required\n"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); ptr1 = kmalloc(sizeof(*ptr1) - 3, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); @@ -454,10 +456,7 @@ static void kasan_global_oob(struct kunit *test) char *p = &global_array[ARRAY_SIZE(global_array) + i]; /* Only generic mode instruments globals. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_GENERIC required"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } @@ -486,10 +485,7 @@ static void kasan_stack_oob(struct kunit *test) volatile int i = OOB_TAG_OFF; char *p = &stack_array[ARRAY_SIZE(stack_array) + i]; - if (!IS_ENABLED(CONFIG_KASAN_STACK)) { - kunit_info(test, "CONFIG_KASAN_STACK is not enabled"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } @@ -501,15 +497,8 @@ static void kasan_alloca_oob_left(struct kunit *test) char *p = alloca_array - 1; /* Only generic mode instruments dynamic allocas. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_GENERIC required"); - return; - } - - if (!IS_ENABLED(CONFIG_KASAN_STACK)) { - kunit_info(test, "CONFIG_KASAN_STACK is not enabled"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } @@ -521,15 +510,8 @@ static void kasan_alloca_oob_right(struct kunit *test) char *p = alloca_array + i; /* Only generic mode instruments dynamic allocas. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_GENERIC required"); - return; - } - - if (!IS_ENABLED(CONFIG_KASAN_STACK)) { - kunit_info(test, "CONFIG_KASAN_STACK is not enabled"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_STACK); KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } @@ -593,11 +575,7 @@ static void kasan_memchr(struct kunit *test) * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. */ - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - kunit_info(test, - "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); - return; - } + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); if (OOB_TAG_OFF) size = round_up(size, OOB_TAG_OFF); @@ -621,11 +599,7 @@ static void kasan_memcmp(struct kunit *test) * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. */ - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - kunit_info(test, - "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); - return; - } + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); if (OOB_TAG_OFF) size = round_up(size, OOB_TAG_OFF); @@ -648,11 +622,7 @@ static void kasan_strings(struct kunit *test) * str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT. * See https://bugzilla.kernel.org/show_bug.cgi?id=206337 for details. */ - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - kunit_info(test, - "str* functions are not instrumented with CONFIG_AMD_MEM_ENCRYPT"); - return; - } + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_AMD_MEM_ENCRYPT); ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); @@ -713,10 +683,7 @@ static void kasan_bitops_generic(struct kunit *test) long *bits; /* This test is specifically crafted for the generic mode. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_GENERIC required\n"); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); /* * Allocate 1 more byte, which causes kzalloc to round up to 16 bytes; @@ -744,11 +711,8 @@ static void kasan_bitops_tags(struct kunit *test) { long *bits; - /* This test is specifically crafted for the tag-based mode. */ - if (IS_ENABLED(CONFIG_KASAN_GENERIC)) { - kunit_info(test, "CONFIG_KASAN_SW_TAGS required\n"); - return; - } + /* This test is specifically crafted for tag-based modes. */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); /* Allocation size will be rounded to up granule size, which is 16. */ bits = kzalloc(sizeof(*bits), GFP_KERNEL); @@ -777,10 +741,7 @@ static void vmalloc_oob(struct kunit *test) { void *area; - if (!IS_ENABLED(CONFIG_KASAN_VMALLOC)) { - kunit_info(test, "CONFIG_KASAN_VMALLOC is not enabled."); - return; - } + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_VMALLOC); /* * We have to be careful not to hit the guard page. From 573a48092313dec7b254d9dbcc2db62167f00456 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:21 -0800 Subject: [PATCH 546/633] kasan: add match-all tag tests Add 3 new tests for tag-based KASAN modes: 1. Check that match-all pointer tag is not assigned randomly. 2. Check that 0xff works as a match-all pointer tag. 3. Check that there are no match-all memory tags. Note, that test #3 causes a significant number (255) of KASAN reports to be printed during execution for the SW_TAGS mode. [arnd@arndb.de: export kasan_poison] Link: https://lkml.kernel.org/r/20210125112831.2156212-1-arnd@kernel.org [akpm@linux-foundation.org: s/EXPORT_SYMBOL_GPL/EXPORT_SYMBOL/, per Andrey] Link: https://linux-review.googlesource.com/id/I78f1375efafa162b37f3abcb2c5bc2f3955dfd8e Link: https://lkml.kernel.org/r/da841a5408e2204bf25f3b23f70540a65844e8a4.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Signed-off-by: Arnd Bergmann Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++ mm/kasan/kasan.h | 6 ++++ mm/kasan/shadow.c | 1 + 3 files changed, 99 insertions(+) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 714ea27fcc3e..c344fe506ffc 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -754,6 +755,94 @@ static void vmalloc_oob(struct kunit *test) vfree(area); } +/* + * Check that the assigned pointer tag falls within the [KASAN_TAG_MIN, + * KASAN_TAG_KERNEL) range (note: excluding the match-all tag) for tag-based + * modes. + */ +static void match_all_not_assigned(struct kunit *test) +{ + char *ptr; + struct page *pages; + int i, size, order; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + for (i = 0; i < 256; i++) { + size = (get_random_int() % 1024) + 1; + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + kfree(ptr); + } + + for (i = 0; i < 256; i++) { + order = (get_random_int() % 4) + 1; + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + free_pages((unsigned long)ptr, order); + } +} + +/* Check that 0xff works as a match-all pointer tag for tag-based modes. */ +static void match_all_ptr_tag(struct kunit *test) +{ + char *ptr; + u8 tag; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + ptr = kmalloc(128, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + /* Backup the assigned tag. */ + tag = get_tag(ptr); + KUNIT_EXPECT_NE(test, tag, (u8)KASAN_TAG_KERNEL); + + /* Reset the tag to 0xff.*/ + ptr = set_tag(ptr, KASAN_TAG_KERNEL); + + /* This access shouldn't trigger a KASAN report. */ + *ptr = 0; + + /* Recover the pointer tag and free. */ + ptr = set_tag(ptr, tag); + kfree(ptr); +} + +/* Check that there are no match-all memory tags for tag-based modes. */ +static void match_all_mem_tag(struct kunit *test) +{ + char *ptr; + int tag; + + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + ptr = kmalloc(128, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_NE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + + /* For each possible tag value not matching the pointer tag. */ + for (tag = KASAN_TAG_MIN; tag <= KASAN_TAG_KERNEL; tag++) { + if (tag == get_tag(ptr)) + continue; + + /* Mark the first memory granule with the chosen memory tag. */ + kasan_poison(ptr, KASAN_GRANULE_SIZE, (u8)tag); + + /* This access must cause a KASAN report. */ + KUNIT_EXPECT_KASAN_FAIL(test, *ptr = 0); + } + + /* Recover the memory tag and free. */ + kasan_poison(ptr, KASAN_GRANULE_SIZE, get_tag(ptr)); + kfree(ptr); +} + static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kmalloc_oob_right), KUNIT_CASE(kmalloc_oob_left), @@ -793,6 +882,9 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kasan_bitops_tags), KUNIT_CASE(kmalloc_double_kzfree), KUNIT_CASE(vmalloc_oob), + KUNIT_CASE(match_all_not_assigned), + KUNIT_CASE(match_all_ptr_tag), + KUNIT_CASE(match_all_mem_tag), {} }; diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 3810e75b8eea..12932186a7f9 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -36,6 +36,12 @@ extern bool kasan_flag_panic __ro_after_init; #define KASAN_TAG_INVALID 0xFE /* inaccessible memory tag */ #define KASAN_TAG_MAX 0xFD /* maximum value for random tags */ +#ifdef CONFIG_KASAN_HW_TAGS +#define KASAN_TAG_MIN 0xF0 /* mimimum value for random tags */ +#else +#define KASAN_TAG_MIN 0x00 /* mimimum value for random tags */ +#endif + #ifdef CONFIG_KASAN_GENERIC #define KASAN_FREE_PAGE 0xFF /* page was freed */ #define KASAN_PAGE_REDZONE 0xFE /* redzone for kmalloc_large allocations */ diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 38958eb0d653..80adc85d0393 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -89,6 +89,7 @@ void kasan_poison(const void *address, size_t size, u8 value) __memset(shadow_start, value, shadow_end - shadow_start); } +EXPORT_SYMBOL(kasan_poison); void kasan_unpoison(const void *address, size_t size) { From f05842cfb9ae25b5e78c618429c4716d9e4d5fc8 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:26 -0800 Subject: [PATCH 547/633] kasan, arm64: allow using KUnit tests with HW_TAGS mode On a high level, this patch allows running KUnit KASAN tests with the hardware tag-based KASAN mode. Internally, this change reenables tag checking at the end of each KASAN test that triggers a tag fault and leads to tag checking being disabled. Also simplify is_write calculation in report_tag_fault. With this patch KASAN tests are still failing for the hardware tag-based mode; fixes come in the next few patches. [andreyknvl@google.com: export HW_TAGS symbols for KUnit tests] Link: https://lkml.kernel.org/r/e7eeb252da408b08f0c81b950a55fb852f92000b.1613155970.git.andreyknvl@google.com Link: https://linux-review.googlesource.com/id/Id94dc9eccd33b23cda4950be408c27f879e474c8 Link: https://lkml.kernel.org/r/51b23112cf3fd62b8f8e9df81026fa2b15870501.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Catalin Marinas Reviewed-by: Vincenzo Frascino Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Marco Elver Cc: Peter Collingbourne Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm64/include/asm/memory.h | 1 + arch/arm64/include/asm/mte-kasan.h | 12 +++++++++ arch/arm64/kernel/mte.c | 12 +++++++++ arch/arm64/mm/fault.c | 20 +++++++++----- lib/Kconfig.kasan | 4 +-- lib/test_kasan.c | 42 +++++++++++++++++++++--------- mm/kasan/hw_tags.c | 16 ++++++++++++ mm/kasan/kasan.h | 21 +++++++++++++++ 8 files changed, 107 insertions(+), 21 deletions(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index bc09af26c1b8..c759faf7a1ff 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -244,6 +244,7 @@ static inline const void *__tag_set(const void *addr, u8 tag) #ifdef CONFIG_KASAN_HW_TAGS #define arch_enable_tagging() mte_enable_kernel() +#define arch_set_tagging_report_once(state) mte_set_report_once(state) #define arch_init_tags(max_tag) mte_init_tags(max_tag) #define arch_get_random_tag() mte_get_random_tag() #define arch_get_mem_tag(addr) mte_get_mem_tag(addr) diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index 26349a4b5e2e..3748d5bb88c0 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -32,6 +32,9 @@ void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag); void mte_enable_kernel(void); void mte_init_tags(u64 max_tag); +void mte_set_report_once(bool state); +bool mte_report_once(void); + #else /* CONFIG_ARM64_MTE */ static inline u8 mte_get_ptr_tag(void *ptr) @@ -60,6 +63,15 @@ static inline void mte_init_tags(u64 max_tag) { } +static inline void mte_set_report_once(bool state) +{ +} + +static inline bool mte_report_once(void) +{ + return false; +} + #endif /* CONFIG_ARM64_MTE */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 80b62fe49dcf..2cfc850809ce 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -25,6 +25,8 @@ u64 gcr_kernel_excl __ro_after_init; +static bool report_fault_once = true; + static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) { pte_t old_pte = READ_ONCE(*ptep); @@ -158,6 +160,16 @@ void mte_enable_kernel(void) isb(); } +void mte_set_report_once(bool state) +{ + WRITE_ONCE(report_fault_once, state); +} + +bool mte_report_once(void) +{ + return READ_ONCE(report_fault_once); +} + static void update_sctlr_el1_tcf0(u64 tcf0) { /* ISB required for the kernel uaccess routines */ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2e339f0bd958..dc9f96442edc 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -302,12 +302,24 @@ static void die_kernel_fault(const char *msg, unsigned long addr, static void report_tag_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - bool is_write = ((esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT) != 0; + static bool reported; + bool is_write; + + if (READ_ONCE(reported)) + return; + + /* + * This is used for KASAN tests and assumes that no MTE faults + * happened before running the tests. + */ + if (mte_report_once()) + WRITE_ONCE(reported, true); /* * SAS bits aren't set for all faults reported in EL1, so we can't * find out access size. */ + is_write = !!(esr & ESR_ELx_WNR); kasan_report(addr, 0, is_write, regs->pc); } #else @@ -319,12 +331,8 @@ static inline void report_tag_fault(unsigned long addr, unsigned int esr, static void do_tag_recovery(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - static bool reported; - if (!READ_ONCE(reported)) { - report_tag_fault(addr, esr, regs); - WRITE_ONCE(reported, true); - } + report_tag_fault(addr, esr, regs); /* * Disable MTE Tag Checking on the local CPU for the current EL. diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index f5fa4ba126bf..3091432acb0a 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -190,11 +190,11 @@ config KASAN_KUNIT_TEST kernel debugging features like KASAN. For more information on KUnit and unit tests in general, please refer - to the KUnit documentation in Documentation/dev-tools/kunit + to the KUnit documentation in Documentation/dev-tools/kunit. config TEST_KASAN_MODULE tristate "KUnit-incompatible tests of KASAN bug detection capabilities" - depends on m && KASAN + depends on m && KASAN && !KASAN_HW_TAGS help This is a part of the KASAN test suite that is incompatible with KUnit. Currently includes tests that do bad copy_from/to_user diff --git a/lib/test_kasan.c b/lib/test_kasan.c index c344fe506ffc..502709db41c0 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -41,16 +41,20 @@ static bool multishot; /* * Temporarily enable multi-shot mode. Otherwise, KASAN would only report the - * first detected bug and panic the kernel if panic_on_warn is enabled. + * first detected bug and panic the kernel if panic_on_warn is enabled. For + * hardware tag-based KASAN also allow tag checking to be reenabled for each + * test, see the comment for KUNIT_EXPECT_KASAN_FAIL(). */ static int kasan_test_init(struct kunit *test) { multishot = kasan_save_enable_multi_shot(); + kasan_set_tagging_report_once(false); return 0; } static void kasan_test_exit(struct kunit *test) { + kasan_set_tagging_report_once(true); kasan_restore_multi_shot(multishot); } @@ -59,19 +63,31 @@ static void kasan_test_exit(struct kunit *test) * KASAN report; causes a test failure otherwise. This relies on a KUnit * resource named "kasan_data". Do not use this name for KUnit resources * outside of KASAN tests. + * + * For hardware tag-based KASAN, when a tag fault happens, tag checking is + * normally auto-disabled. When this happens, this test handler reenables + * tag checking. As tag checking can be only disabled or enabled per CPU, this + * handler disables migration (preemption). */ -#define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ - fail_data.report_expected = true; \ - fail_data.report_found = false; \ - kunit_add_named_resource(test, \ - NULL, \ - NULL, \ - &resource, \ - "kasan_data", &fail_data); \ - expression; \ - KUNIT_EXPECT_EQ(test, \ - fail_data.report_expected, \ - fail_data.report_found); \ +#define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ + if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) \ + migrate_disable(); \ + fail_data.report_expected = true; \ + fail_data.report_found = false; \ + kunit_add_named_resource(test, \ + NULL, \ + NULL, \ + &resource, \ + "kasan_data", &fail_data); \ + expression; \ + KUNIT_EXPECT_EQ(test, \ + fail_data.report_expected, \ + fail_data.report_found); \ + if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) { \ + if (fail_data.report_found) \ + kasan_enable_tagging(); \ + migrate_enable(); \ + } \ } while (0) #define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index d558799b25b3..b31aeef505dd 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -185,3 +185,19 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, return &alloc_meta->free_track[0]; } + +#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) + +void kasan_set_tagging_report_once(bool state) +{ + hw_set_tagging_report_once(state); +} +EXPORT_SYMBOL_GPL(kasan_set_tagging_report_once); + +void kasan_enable_tagging(void) +{ + hw_enable_tagging(); +} +EXPORT_SYMBOL_GPL(kasan_enable_tagging); + +#endif diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 12932186a7f9..1298b79f9518 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -280,6 +280,9 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) #ifndef arch_init_tags #define arch_init_tags(max_tag) #endif +#ifndef arch_set_tagging_report_once +#define arch_set_tagging_report_once(state) +#endif #ifndef arch_get_random_tag #define arch_get_random_tag() (0xFF) #endif @@ -292,12 +295,30 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) #define hw_enable_tagging() arch_enable_tagging() #define hw_init_tags(max_tag) arch_init_tags(max_tag) +#define hw_set_tagging_report_once(state) arch_set_tagging_report_once(state) #define hw_get_random_tag() arch_get_random_tag() #define hw_get_mem_tag(addr) arch_get_mem_tag(addr) #define hw_set_mem_tag_range(addr, size, tag) arch_set_mem_tag_range((addr), (size), (tag)) +#else /* CONFIG_KASAN_HW_TAGS */ + +#define hw_enable_tagging() +#define hw_set_tagging_report_once(state) + #endif /* CONFIG_KASAN_HW_TAGS */ +#if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) + +void kasan_set_tagging_report_once(bool state); +void kasan_enable_tagging(void); + +#else /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ + +static inline void kasan_set_tagging_report_once(bool state) { } +static inline void kasan_enable_tagging(void) { } + +#endif /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */ + #ifdef CONFIG_KASAN_SW_TAGS u8 kasan_random_tag(void); #elif defined(CONFIG_KASAN_HW_TAGS) From 5d92bdffd2d53f98de683229c0ad7d028703fdba Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:29 -0800 Subject: [PATCH 548/633] kasan: rename CONFIG_TEST_KASAN_MODULE Rename CONFIG_TEST_KASAN_MODULE to CONFIG_KASAN_MODULE_TEST. This naming is more consistent with the existing CONFIG_KASAN_KUNIT_TEST. Link: https://linux-review.googlesource.com/id/Id347dfa5fe8788b7a1a189863e039f409da0ae5f Link: https://lkml.kernel.org/r/f08250246683981bcf8a094fbba7c361995624d2.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/dev-tools/kasan.rst | 8 ++++---- lib/Kconfig.kasan | 2 +- lib/Makefile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst index 0ae0efe82e8e..cde14aeefca7 100644 --- a/Documentation/dev-tools/kasan.rst +++ b/Documentation/dev-tools/kasan.rst @@ -358,17 +358,17 @@ unmapped. This will require changes in arch-specific code. This allows ``VMAP_STACK`` support on x86, and can simplify support of architectures that do not have a fixed module region. -CONFIG_KASAN_KUNIT_TEST & CONFIG_TEST_KASAN_MODULE --------------------------------------------------- +CONFIG_KASAN_KUNIT_TEST and CONFIG_KASAN_MODULE_TEST +---------------------------------------------------- -KASAN tests consist on two parts: +KASAN tests consist of two parts: 1. Tests that are integrated with the KUnit Test Framework. Enabled with ``CONFIG_KASAN_KUNIT_TEST``. These tests can be run and partially verified automatically in a few different ways, see the instructions below. 2. Tests that are currently incompatible with KUnit. Enabled with -``CONFIG_TEST_KASAN_MODULE`` and can only be run as a module. These tests can +``CONFIG_KASAN_MODULE_TEST`` and can only be run as a module. These tests can only be verified manually, by loading the kernel module and inspecting the kernel log for KASAN reports. diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 3091432acb0a..624ae1df7984 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -192,7 +192,7 @@ config KASAN_KUNIT_TEST For more information on KUnit and unit tests in general, please refer to the KUnit documentation in Documentation/dev-tools/kunit. -config TEST_KASAN_MODULE +config KASAN_MODULE_TEST tristate "KUnit-incompatible tests of KASAN bug detection capabilities" depends on m && KASAN && !KASAN_HW_TAGS help diff --git a/lib/Makefile b/lib/Makefile index fb7d946bb8c3..b5307d3eec1a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -66,7 +66,7 @@ obj-$(CONFIG_TEST_IDA) += test_ida.o obj-$(CONFIG_KASAN_KUNIT_TEST) += test_kasan.o CFLAGS_test_kasan.o += -fno-builtin CFLAGS_test_kasan.o += $(call cc-disable-warning, vla) -obj-$(CONFIG_TEST_KASAN_MODULE) += test_kasan_module.o +obj-$(CONFIG_KASAN_MODULE_TEST) += test_kasan_module.o CFLAGS_test_kasan_module.o += -fno-builtin obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o CFLAGS_test_ubsan.o += $(call cc-disable-warning, vla) From 2e4bde6a1e3a3feb8511685b8c97be668728eefb Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:34 -0800 Subject: [PATCH 549/633] kasan: add compiler barriers to KUNIT_EXPECT_KASAN_FAIL It might not be obvious to the compiler that the expression must be executed between writing and reading to fail_data. In this case, the compiler might reorder or optimize away some of the accesses, and the tests will fail. Add compiler barriers around the expression in KUNIT_EXPECT_KASAN_FAIL and use READ/WRITE_ONCE() for accessing fail_data fields. Link: https://linux-review.googlesource.com/id/I046079f48641a1d36fe627fc8827a9249102fd50 Link: https://lkml.kernel.org/r/6f11596f367d8ae8f71d800351e9a5d91eda19f6.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 17 ++++++++++++----- mm/kasan/report.c | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 502709db41c0..603fd7937b94 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -68,23 +68,30 @@ static void kasan_test_exit(struct kunit *test) * normally auto-disabled. When this happens, this test handler reenables * tag checking. As tag checking can be only disabled or enabled per CPU, this * handler disables migration (preemption). + * + * Since the compiler doesn't see that the expression can change the fail_data + * fields, it can reorder or optimize away the accesses to those fields. + * Use READ/WRITE_ONCE() for the accesses and compiler barriers around the + * expression to prevent that. */ #define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) \ migrate_disable(); \ - fail_data.report_expected = true; \ - fail_data.report_found = false; \ + WRITE_ONCE(fail_data.report_expected, true); \ + WRITE_ONCE(fail_data.report_found, false); \ kunit_add_named_resource(test, \ NULL, \ NULL, \ &resource, \ "kasan_data", &fail_data); \ + barrier(); \ expression; \ + barrier(); \ KUNIT_EXPECT_EQ(test, \ - fail_data.report_expected, \ - fail_data.report_found); \ + READ_ONCE(fail_data.report_expected), \ + READ_ONCE(fail_data.report_found)); \ if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) { \ - if (fail_data.report_found) \ + if (READ_ONCE(fail_data.report_found)) \ kasan_enable_tagging(); \ migrate_enable(); \ } \ diff --git a/mm/kasan/report.c b/mm/kasan/report.c index e93d7973792e..234f35a84f19 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -331,7 +331,7 @@ static void kasan_update_kunit_status(struct kunit *cur_test) } kasan_data = (struct kunit_kasan_expectation *)resource->data; - kasan_data->report_found = true; + WRITE_ONCE(kasan_data->report_found, true); kunit_put_resource(resource); } #endif /* IS_ENABLED(CONFIG_KUNIT) */ From 1b1df4c4e2576f6b9c5b1f5f1fc9435e3f6c6b47 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:38 -0800 Subject: [PATCH 550/633] kasan: adapt kmalloc_uaf2 test to HW_TAGS mode In the kmalloc_uaf2() test, the pointers to the two allocated memory blocks might happen to be the same, and the test will fail. With the software tag-based mode, the probability of the that is 1/254, so it's hard to observe the failure. For the hardware tag-based mode though, the probablity is 1/14, which is quite noticable. Allow up to 16 attempts at generating different tags for the tag-based modes. Link: https://linux-review.googlesource.com/id/Ibfa458ef2804ff465d8eb07434a300bf36388d55 Link: https://lkml.kernel.org/r/9cd5cf2f633dcbf55cab801cd26845d2b075cec7.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 603fd7937b94..9a227d7e06d6 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -382,7 +382,9 @@ static void kmalloc_uaf2(struct kunit *test) { char *ptr1, *ptr2; size_t size = 43; + int counter = 0; +again: ptr1 = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); @@ -391,6 +393,15 @@ static void kmalloc_uaf2(struct kunit *test) ptr2 = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + /* + * For tag-based KASAN ptr1 and ptr2 tags might happen to be the same. + * Allow up to 16 attempts at generating different tags. + */ + if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && ptr1 == ptr2 && counter++ < 16) { + kfree(ptr2); + goto again; + } + KUNIT_EXPECT_KASAN_FAIL(test, ptr1[40] = 'x'); KUNIT_EXPECT_PTR_NE(test, ptr1, ptr2); From e66e1799a76621003e5b04c9c057826a2152e103 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:42 -0800 Subject: [PATCH 551/633] kasan: fix memory corruption in kasan_bitops_tags test Since the hardware tag-based KASAN mode might not have a redzone that comes after an allocated object (when kasan.mode=prod is enabled), the kasan_bitops_tags() test ends up corrupting the next object in memory. Change the test so it always accesses the redzone that lies within the allocated object's boundaries. Link: https://linux-review.googlesource.com/id/I67f51d1ee48f0a8d0fe2658c2a39e4879fe0832a Link: https://lkml.kernel.org/r/7d452ce4ae35bb1988d2c9244dfea56cf2cc9315.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 9a227d7e06d6..e59f185b8075 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -749,13 +749,13 @@ static void kasan_bitops_tags(struct kunit *test) /* This test is specifically crafted for tag-based modes. */ KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); - /* Allocation size will be rounded to up granule size, which is 16. */ - bits = kzalloc(sizeof(*bits), GFP_KERNEL); + /* kmalloc-64 cache will be used and the last 16 bytes will be the redzone. */ + bits = kzalloc(48, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bits); - /* Do the accesses past the 16 allocated bytes. */ - kasan_bitops_modify(test, BITS_PER_LONG, &bits[1]); - kasan_bitops_test_and_modify(test, BITS_PER_LONG + BITS_PER_BYTE, &bits[1]); + /* Do the accesses past the 48 allocated bytes, but within the redone. */ + kasan_bitops_modify(test, BITS_PER_LONG, (void *)bits + 48); + kasan_bitops_test_and_modify(test, BITS_PER_LONG + BITS_PER_BYTE, (void *)bits + 48); kfree(bits); } From 027b37b552f326aa94ef06c7ea77088b16c41e6e Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:46 -0800 Subject: [PATCH 552/633] kasan: move _RET_IP_ to inline wrappers Generic mm functions that call KASAN annotations that might report a bug pass _RET_IP_ to them as an argument. This allows KASAN to include the name of the function that called the mm function in its report's header. Now that KASAN has inline wrappers for all of its annotations, move _RET_IP_ to those wrappers to simplify annotation call sites. Link: https://linux-review.googlesource.com/id/I8fb3c06d49671305ee184175a39591bc26647a67 Link: https://lkml.kernel.org/r/5c1490eddf20b436b8c4eeea83fce47687d5e4a4.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kasan.h | 20 +++++++++----------- mm/mempool.c | 2 +- mm/slab.c | 2 +- mm/slub.c | 4 ++-- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/include/linux/kasan.h b/include/linux/kasan.h index 0aea9e2a2a01..a7254186558a 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -185,19 +185,18 @@ static __always_inline void * __must_check kasan_init_slab_obj( } bool __kasan_slab_free(struct kmem_cache *s, void *object, unsigned long ip); -static __always_inline bool kasan_slab_free(struct kmem_cache *s, void *object, - unsigned long ip) +static __always_inline bool kasan_slab_free(struct kmem_cache *s, void *object) { if (kasan_enabled()) - return __kasan_slab_free(s, object, ip); + return __kasan_slab_free(s, object, _RET_IP_); return false; } void __kasan_slab_free_mempool(void *ptr, unsigned long ip); -static __always_inline void kasan_slab_free_mempool(void *ptr, unsigned long ip) +static __always_inline void kasan_slab_free_mempool(void *ptr) { if (kasan_enabled()) - __kasan_slab_free_mempool(ptr, ip); + __kasan_slab_free_mempool(ptr, _RET_IP_); } void * __must_check __kasan_slab_alloc(struct kmem_cache *s, @@ -241,10 +240,10 @@ static __always_inline void * __must_check kasan_krealloc(const void *object, } void __kasan_kfree_large(void *ptr, unsigned long ip); -static __always_inline void kasan_kfree_large(void *ptr, unsigned long ip) +static __always_inline void kasan_kfree_large(void *ptr) { if (kasan_enabled()) - __kasan_kfree_large(ptr, ip); + __kasan_kfree_large(ptr, _RET_IP_); } bool kasan_save_enable_multi_shot(void); @@ -277,12 +276,11 @@ static inline void *kasan_init_slab_obj(struct kmem_cache *cache, { return (void *)object; } -static inline bool kasan_slab_free(struct kmem_cache *s, void *object, - unsigned long ip) +static inline bool kasan_slab_free(struct kmem_cache *s, void *object) { return false; } -static inline void kasan_slab_free_mempool(void *ptr, unsigned long ip) {} +static inline void kasan_slab_free_mempool(void *ptr) {} static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags) { @@ -302,7 +300,7 @@ static inline void *kasan_krealloc(const void *object, size_t new_size, { return (void *)object; } -static inline void kasan_kfree_large(void *ptr, unsigned long ip) {} +static inline void kasan_kfree_large(void *ptr) {} #endif /* CONFIG_KASAN */ diff --git a/mm/mempool.c b/mm/mempool.c index 624ed51b060f..79959fac27d7 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -104,7 +104,7 @@ static inline void poison_element(mempool_t *pool, void *element) static __always_inline void kasan_poison_element(mempool_t *pool, void *element) { if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) - kasan_slab_free_mempool(element, _RET_IP_); + kasan_slab_free_mempool(element); else if (pool->alloc == mempool_alloc_pages) kasan_free_pages(element, (unsigned long)pool->pool_data); } diff --git a/mm/slab.c b/mm/slab.c index 9c9e0c255c4b..35c68d99d460 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3420,7 +3420,7 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, memset(objp, 0, cachep->object_size); /* Put the object into the quarantine, don't touch it for now. */ - if (kasan_slab_free(cachep, objp, _RET_IP_)) + if (kasan_slab_free(cachep, objp)) return; /* Use KCSAN to help debug racy use-after-free. */ diff --git a/mm/slub.c b/mm/slub.c index 046d7194acaf..b2833ce85c92 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1528,7 +1528,7 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) static __always_inline void kfree_hook(void *x) { kmemleak_free(x); - kasan_kfree_large(x, _RET_IP_); + kasan_kfree_large(x); } static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x) @@ -1558,7 +1558,7 @@ static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x) KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT); /* KASAN might put x into memory quarantine, delaying its reuse */ - return kasan_slab_free(s, x, _RET_IP_); + return kasan_slab_free(s, x); } static inline bool slab_free_freelist_hook(struct kmem_cache *s, From 611806b4bf8dd97a4f3d73f5cf3c2c7730c51eb2 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:50 -0800 Subject: [PATCH 553/633] kasan: fix bug detection via ksize for HW_TAGS mode The currently existing kasan_check_read/write() annotations are intended to be used for kernel modules that have KASAN compiler instrumentation disabled. Thus, they are only relevant for the software KASAN modes that rely on compiler instrumentation. However there's another use case for these annotations: ksize() checks that the object passed to it is indeed accessible before unpoisoning the whole object. This is currently done via __kasan_check_read(), which is compiled away for the hardware tag-based mode that doesn't rely on compiler instrumentation. This leads to KASAN missing detecting some memory corruptions. Provide another annotation called kasan_check_byte() that is available for all KASAN modes. As the implementation rename and reuse kasan_check_invalid_free(). Use this new annotation in ksize(). To avoid having ksize() as the top frame in the reported stack trace pass _RET_IP_ to __kasan_check_byte(). Also add a new ksize_uaf() test that checks that a use-after-free is detected via ksize() itself, and via plain accesses that happen later. Link: https://linux-review.googlesource.com/id/Iaabf771881d0f9ce1b969f2a62938e99d3308ec5 Link: https://lkml.kernel.org/r/f32ad74a60b28d8402482a38476f02bb7600f620.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kasan-checks.h | 6 ++++++ include/linux/kasan.h | 17 +++++++++++++++++ lib/test_kasan.c | 20 ++++++++++++++++++++ mm/kasan/common.c | 11 ++++++++++- mm/kasan/generic.c | 4 ++-- mm/kasan/kasan.h | 10 +++++----- mm/kasan/sw_tags.c | 6 +++--- mm/slab_common.c | 16 +++++++++------- 8 files changed, 72 insertions(+), 18 deletions(-) diff --git a/include/linux/kasan-checks.h b/include/linux/kasan-checks.h index ca5e89fb10d3..3d6d22a25bdc 100644 --- a/include/linux/kasan-checks.h +++ b/include/linux/kasan-checks.h @@ -4,6 +4,12 @@ #include +/* + * The annotations present in this file are only relevant for the software + * KASAN modes that rely on compiler instrumentation, and will be optimized + * away for the hardware tag-based KASAN mode. Use kasan_check_byte() instead. + */ + /* * __kasan_check_*: Always available when KASAN is enabled. This may be used * even in compilation units that selectively disable KASAN, but must use KASAN diff --git a/include/linux/kasan.h b/include/linux/kasan.h index a7254186558a..7eaf2d9effb4 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -246,6 +246,19 @@ static __always_inline void kasan_kfree_large(void *ptr) __kasan_kfree_large(ptr, _RET_IP_); } +/* + * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for + * the hardware tag-based mode that doesn't rely on compiler instrumentation. + */ +bool __kasan_check_byte(const void *addr, unsigned long ip); +static __always_inline bool kasan_check_byte(const void *addr) +{ + if (kasan_enabled()) + return __kasan_check_byte(addr, _RET_IP_); + return true; +} + + bool kasan_save_enable_multi_shot(void); void kasan_restore_multi_shot(bool enabled); @@ -301,6 +314,10 @@ static inline void *kasan_krealloc(const void *object, size_t new_size, return (void *)object; } static inline void kasan_kfree_large(void *ptr) {} +static inline bool kasan_check_byte(const void *address) +{ + return true; +} #endif /* CONFIG_KASAN */ diff --git a/lib/test_kasan.c b/lib/test_kasan.c index e59f185b8075..3f771fabd0ec 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -496,6 +496,7 @@ static void kasan_global_oob(struct kunit *test) KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)p); } +/* Check that ksize() makes the whole object accessible. */ static void ksize_unpoisons_memory(struct kunit *test) { char *ptr; @@ -514,6 +515,24 @@ static void ksize_unpoisons_memory(struct kunit *test) kfree(ptr); } +/* + * Check that a use-after-free is detected by ksize() and via normal accesses + * after it. + */ +static void ksize_uaf(struct kunit *test) +{ + char *ptr; + int size = 128 - KASAN_GRANULE_SIZE; + + ptr = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + kfree(ptr); + + KUNIT_EXPECT_KASAN_FAIL(test, ksize(ptr)); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = *ptr); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = *(ptr + size)); +} + static void kasan_stack_oob(struct kunit *test) { char stack_array[10]; @@ -907,6 +926,7 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kasan_alloca_oob_left), KUNIT_CASE(kasan_alloca_oob_right), KUNIT_CASE(ksize_unpoisons_memory), + KUNIT_CASE(ksize_uaf), KUNIT_CASE(kmem_cache_double_free), KUNIT_CASE(kmem_cache_invalid_free), KUNIT_CASE(kasan_memchr), diff --git a/mm/kasan/common.c b/mm/kasan/common.c index eedc3e0fe365..b18189ef3a92 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -345,7 +345,7 @@ static bool ____kasan_slab_free(struct kmem_cache *cache, void *object, if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU)) return false; - if (kasan_check_invalid_free(tagged_object)) { + if (!kasan_byte_accessible(tagged_object)) { kasan_report_invalid_free(tagged_object, ip); return true; } @@ -490,3 +490,12 @@ void __kasan_kfree_large(void *ptr, unsigned long ip) kasan_report_invalid_free(ptr, ip); /* The object will be poisoned by kasan_free_pages(). */ } + +bool __kasan_check_byte(const void *address, unsigned long ip) +{ + if (!kasan_byte_accessible(address)) { + kasan_report((unsigned long)address, 1, false, ip); + return false; + } + return true; +} diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index acab8862dc67..3f17a1218055 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -185,11 +185,11 @@ bool kasan_check_range(unsigned long addr, size_t size, bool write, return check_region_inline(addr, size, write, ret_ip); } -bool kasan_check_invalid_free(void *addr) +bool kasan_byte_accessible(const void *addr) { s8 shadow_byte = READ_ONCE(*(s8 *)kasan_mem_to_shadow(addr)); - return shadow_byte < 0 || shadow_byte >= KASAN_GRANULE_SIZE; + return shadow_byte >= 0 && shadow_byte < KASAN_GRANULE_SIZE; } void kasan_cache_shrink(struct kmem_cache *cache) diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 1298b79f9518..cc14b6e6c14c 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -341,20 +341,20 @@ static inline void kasan_unpoison(const void *address, size_t size) round_up(size, KASAN_GRANULE_SIZE), get_tag(address)); } -static inline bool kasan_check_invalid_free(void *addr) +static inline bool kasan_byte_accessible(const void *addr) { u8 ptr_tag = get_tag(addr); - u8 mem_tag = hw_get_mem_tag(addr); + u8 mem_tag = hw_get_mem_tag((void *)addr); - return (mem_tag == KASAN_TAG_INVALID) || - (ptr_tag != KASAN_TAG_KERNEL && ptr_tag != mem_tag); + return (mem_tag != KASAN_TAG_INVALID) && + (ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag); } #else /* CONFIG_KASAN_HW_TAGS */ void kasan_poison(const void *address, size_t size, u8 value); void kasan_unpoison(const void *address, size_t size); -bool kasan_check_invalid_free(void *addr); +bool kasan_byte_accessible(const void *addr); #endif /* CONFIG_KASAN_HW_TAGS */ diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index cc271fceb5d5..94c2d33be333 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -118,13 +118,13 @@ bool kasan_check_range(unsigned long addr, size_t size, bool write, return true; } -bool kasan_check_invalid_free(void *addr) +bool kasan_byte_accessible(const void *addr) { u8 tag = get_tag(addr); u8 shadow_byte = READ_ONCE(*(u8 *)kasan_mem_to_shadow(kasan_reset_tag(addr))); - return (shadow_byte == KASAN_TAG_INVALID) || - (tag != KASAN_TAG_KERNEL && tag != shadow_byte); + return (shadow_byte != KASAN_TAG_INVALID) && + (tag == KASAN_TAG_KERNEL || tag == shadow_byte); } #define DEFINE_HWASAN_LOAD_STORE(size) \ diff --git a/mm/slab_common.c b/mm/slab_common.c index 5be7825ad3ce..7c8298c17145 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1218,19 +1218,21 @@ size_t ksize(const void *objp) size_t size; /* - * We need to check that the pointed to object is valid, and only then - * unpoison the shadow memory below. We use __kasan_check_read(), to - * generate a more useful report at the time ksize() is called (rather - * than later where behaviour is undefined due to potential - * use-after-free or double-free). + * We need to first check that the pointer to the object is valid, and + * only then unpoison the memory. The report printed from ksize() is + * more useful, then when it's printed later when the behaviour could + * be undefined due to a potential use-after-free or double-free. * - * If the pointed to memory is invalid we return 0, to avoid users of + * We use kasan_check_byte(), which is supported for the hardware + * tag-based KASAN mode, unlike kasan_check_read/write(). + * + * If the pointed to memory is invalid, we return 0 to avoid users of * ksize() writing to and potentially corrupting the memory region. * * We want to perform the check before __ksize(), to avoid potentially * crashing in __ksize() due to accessing invalid metadata. */ - if (unlikely(ZERO_OR_NULL_PTR(objp)) || !__kasan_check_read(objp, 1)) + if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp)) return 0; size = __ksize(objp); From 858bdeb046f6dc7a79039d577d03e4d2b39272b7 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:55 -0800 Subject: [PATCH 554/633] kasan: add proper page allocator tests The currently existing page allocator tests rely on kmalloc fallback with large sizes that is only present for SLUB. Add proper tests that use alloc/free_pages(). Link: https://linux-review.googlesource.com/id/Ia173d5a1b215fe6b2548d814ef0f4433cf983570 Link: https://lkml.kernel.org/r/a2648930e55ff75b8e700f2e0d905c2b55a67483.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 51 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 3f771fabd0ec..acbc7d54d067 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -147,6 +147,12 @@ static void kmalloc_node_oob_right(struct kunit *test) kfree(ptr); } +/* + * These kmalloc_pagealloc_* tests try allocating a memory chunk that doesn't + * fit into a slab cache and therefore is allocated via the page allocator + * fallback. Since this kind of fallback is only implemented for SLUB, these + * tests are limited to that allocator. + */ static void kmalloc_pagealloc_oob_right(struct kunit *test) { char *ptr; @@ -154,14 +160,11 @@ static void kmalloc_pagealloc_oob_right(struct kunit *test) KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_SLUB); - /* - * Allocate a chunk that does not fit into a SLUB cache to trigger - * the page allocator fallback. - */ ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); KUNIT_EXPECT_KASAN_FAIL(test, ptr[size + OOB_TAG_OFF] = 0); + kfree(ptr); } @@ -174,8 +177,8 @@ static void kmalloc_pagealloc_uaf(struct kunit *test) ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - kfree(ptr); + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = 0); } @@ -192,6 +195,42 @@ static void kmalloc_pagealloc_invalid_free(struct kunit *test) KUNIT_EXPECT_KASAN_FAIL(test, kfree(ptr + 1)); } +static void pagealloc_oob_right(struct kunit *test) +{ + char *ptr; + struct page *pages; + size_t order = 4; + size_t size = (1UL << (PAGE_SHIFT + order)); + + /* + * With generic KASAN page allocations have no redzones, thus + * out-of-bounds detection is not guaranteed. + * See https://bugzilla.kernel.org/show_bug.cgi?id=210503. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 0); + free_pages((unsigned long)ptr, order); +} + +static void pagealloc_uaf(struct kunit *test) +{ + char *ptr; + struct page *pages; + size_t order = 4; + + pages = alloc_pages(GFP_KERNEL, order); + ptr = page_address(pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + free_pages((unsigned long)ptr, order); + + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = 0); +} + static void kmalloc_large_oob_right(struct kunit *test) { char *ptr; @@ -903,6 +942,8 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kmalloc_pagealloc_oob_right), KUNIT_CASE(kmalloc_pagealloc_uaf), KUNIT_CASE(kmalloc_pagealloc_invalid_free), + KUNIT_CASE(pagealloc_oob_right), + KUNIT_CASE(pagealloc_uaf), KUNIT_CASE(kmalloc_large_oob_right), KUNIT_CASE(kmalloc_oob_krealloc_more), KUNIT_CASE(kmalloc_oob_krealloc_less), From 115161354d0e0af6fc07dcbbf0fc4e7574d32cd6 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:05:59 -0800 Subject: [PATCH 555/633] kasan: add a test for kmem_cache_alloc/free_bulk Add a test for kmem_cache_alloc/free_bulk to make sure there are no false-positives when these functions are used. Link: https://linux-review.googlesource.com/id/I2a8bf797aecf81baeac61380c567308f319e263d Link: https://lkml.kernel.org/r/418122ebe4600771ac81e9ca6eab6740cf8dcfa1.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index acbc7d54d067..b04729b61d1d 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -479,10 +479,11 @@ static void kmem_cache_oob(struct kunit *test) { char *p; size_t size = 200; - struct kmem_cache *cache = kmem_cache_create("test_cache", - size, 0, - 0, NULL); + struct kmem_cache *cache; + + cache = kmem_cache_create("test_cache", size, 0, 0, NULL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + p = kmem_cache_alloc(cache, GFP_KERNEL); if (!p) { kunit_err(test, "Allocation failed: %s\n", __func__); @@ -491,11 +492,12 @@ static void kmem_cache_oob(struct kunit *test) } KUNIT_EXPECT_KASAN_FAIL(test, *p = p[size + OOB_TAG_OFF]); + kmem_cache_free(cache, p); kmem_cache_destroy(cache); } -static void memcg_accounted_kmem_cache(struct kunit *test) +static void kmem_cache_accounted(struct kunit *test) { int i; char *p; @@ -522,6 +524,31 @@ static void memcg_accounted_kmem_cache(struct kunit *test) kmem_cache_destroy(cache); } +static void kmem_cache_bulk(struct kunit *test) +{ + struct kmem_cache *cache; + size_t size = 200; + char *p[10]; + bool ret; + int i; + + cache = kmem_cache_create("test_cache", size, 0, 0, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + + ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p); + if (!ret) { + kunit_err(test, "Allocation failed: %s\n", __func__); + kmem_cache_destroy(cache); + return; + } + + for (i = 0; i < ARRAY_SIZE(p); i++) + p[i][0] = p[i][size - 1] = 42; + + kmem_cache_free_bulk(cache, ARRAY_SIZE(p), (void **)&p); + kmem_cache_destroy(cache); +} + static char global_array[10]; static void kasan_global_oob(struct kunit *test) @@ -961,7 +988,8 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kfree_via_page), KUNIT_CASE(kfree_via_phys), KUNIT_CASE(kmem_cache_oob), - KUNIT_CASE(memcg_accounted_kmem_cache), + KUNIT_CASE(kmem_cache_accounted), + KUNIT_CASE(kmem_cache_bulk), KUNIT_CASE(kasan_global_oob), KUNIT_CASE(kasan_stack_oob), KUNIT_CASE(kasan_alloca_oob_left), From d82dc3a40d12c6eea15c18d24c0bdbc887d0e7c6 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 24 Feb 2021 12:06:02 -0800 Subject: [PATCH 556/633] kasan: don't run tests when KASAN is not enabled Don't run KASAN tests when it's disabled with kasan.mode=off to avoid corrupting kernel memory. Link: https://linux-review.googlesource.com/id/I6447af436a69a94bfc35477f6bf4e2122948355e Link: https://lkml.kernel.org/r/25bd4fb5cae7b421d806a1f33fb633edd313f0c7.1610733117.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Reviewed-by: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kasan.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/test_kasan.c b/lib/test_kasan.c index b04729b61d1d..25576303897b 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -47,6 +47,11 @@ static bool multishot; */ static int kasan_test_init(struct kunit *test) { + if (!kasan_enabled()) { + kunit_err(test, "can't run KASAN tests with KASAN disabled"); + return -1; + } + multishot = kasan_save_enable_multi_shot(); kasan_set_tagging_report_once(false); return 0; From 93f503c3fcd168a43e4a6c875fe2cfafaf8439dc Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Feb 2021 12:06:10 -0800 Subject: [PATCH 557/633] mm: fix prototype warning from kernel test robot Patch series "mm: clean up names and parameters of memmap_init_xxxx functions", v5. This patchset corrects inappropriate function names of memmap_init_xxx, and simplify parameters of functions in the code flow. And also fix a prototype warning reported by lkp. This patch (of 5); Kernel test robot calling make with 'W=1' is triggering warning like below for memmap_init_zone() function. mm/page_alloc.c:6259:23: warning: no previous prototype for 'memmap_init_zone' [-Wmissing-prototypes] 6259 | void __meminit __weak memmap_init_zone(unsigned long size, int nid, | ^~~~~~~~~~~~~~~~ Fix it by adding the function declaration in include/linux/mm.h. Since memmap_init_zone() has a generic version with '__weak', the declaratoin in ia64 header file can be simply removed. Link: https://lkml.kernel.org/r/20210122135956.5946-1-bhe@redhat.com Link: https://lkml.kernel.org/r/20210122135956.5946-2-bhe@redhat.com Signed-off-by: Baoquan He Reported-by: kernel test robot Reviewed-by: Mike Rapoport Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/ia64/include/asm/pgtable.h | 6 ------ include/linux/mm.h | 2 ++ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 779b6972aa84..9b4efe89e62d 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -517,12 +517,6 @@ extern struct page *zero_page_memmap_ptr; __changed; \ }) #endif - -# ifdef CONFIG_VIRTUAL_MEM_MAP - /* arch mem_map init routine is needed due to holes in a virtual mem_map */ - extern void memmap_init (unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn); -# endif /* CONFIG_VIRTUAL_MEM_MAP */ # endif /* !__ASSEMBLY__ */ /* diff --git a/include/linux/mm.h b/include/linux/mm.h index 7deb7168104d..1c1eabdf112c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2408,6 +2408,8 @@ extern void set_dma_reserve(unsigned long new_dma_reserve); extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, unsigned long, enum meminit_context, struct vmem_altmap *, int migratetype); +extern void memmap_init(unsigned long size, int nid, + unsigned long zone, unsigned long range_start_pfn); extern void setup_per_zone_wmarks(void); extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); From ab28cb6e1e5e59eb8bf3ad399133617414301d3a Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Feb 2021 12:06:14 -0800 Subject: [PATCH 558/633] mm: rename memmap_init() and memmap_init_zone() The current memmap_init_zone() only handles memory region inside one zone, actually memmap_init() does the memmap init of one zone. So rename both of them accordingly. Link: https://lkml.kernel.org/r/20210122135956.5946-3-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/ia64/mm/init.c | 6 +++--- include/linux/mm.h | 4 ++-- mm/memory_hotplug.c | 2 +- mm/page_alloc.c | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index b19f47a5a305..39b782f62d7d 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -536,18 +536,18 @@ virtual_memmap_init(u64 start, u64 end, void *arg) / sizeof(struct page)); if (map_start < map_end) - memmap_init_zone((unsigned long)(map_end - map_start), + memmap_init_range((unsigned long)(map_end - map_start), args->nid, args->zone, page_to_pfn(map_start), page_to_pfn(map_end), MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); return 0; } void __meminit -memmap_init (unsigned long size, int nid, unsigned long zone, +memmap_init_zone(unsigned long size, int nid, unsigned long zone, unsigned long start_pfn) { if (!vmem_map) { - memmap_init_zone(size, nid, zone, start_pfn, start_pfn + size, + memmap_init_range(size, nid, zone, start_pfn, start_pfn + size, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } else { struct page *start; diff --git a/include/linux/mm.h b/include/linux/mm.h index 1c1eabdf112c..56d0eeae0ce3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2405,10 +2405,10 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); #endif extern void set_dma_reserve(unsigned long new_dma_reserve); -extern void memmap_init_zone(unsigned long, int, unsigned long, +extern void memmap_init_range(unsigned long, int, unsigned long, unsigned long, unsigned long, enum meminit_context, struct vmem_altmap *, int migratetype); -extern void memmap_init(unsigned long size, int nid, +extern void memmap_init_zone(unsigned long size, int nid, unsigned long zone, unsigned long range_start_pfn); extern void setup_per_zone_wmarks(void); extern int __meminit init_per_zone_wmark_min(void); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index f9d57b9be8c7..ddcb1cd24c60 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -713,7 +713,7 @@ void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, * expects the zone spans the pfn range. All the pages in the range * are reserved so nobody should be touching them so we should be safe */ - memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 0, + memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0, MEMINIT_HOTPLUG, altmap, migratetype); set_zone_contiguous(zone); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 069561aadc7b..519cf52963eb 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6121,7 +6121,7 @@ overlap_memmap_init(unsigned long zone, unsigned long *pfn) * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related * zone stats (e.g., nr_isolate_pageblock) are touched. */ -void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, +void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, unsigned long start_pfn, unsigned long zone_end_pfn, enum meminit_context context, struct vmem_altmap *altmap, int migratetype) @@ -6258,7 +6258,7 @@ static void __meminit zone_init_free_lists(struct zone *zone) } } -void __meminit __weak memmap_init(unsigned long size, int nid, +void __meminit __weak memmap_init_zone(unsigned long size, int nid, unsigned long zone, unsigned long range_start_pfn) { @@ -6272,7 +6272,7 @@ void __meminit __weak memmap_init(unsigned long size, int nid, if (end_pfn > start_pfn) { size = end_pfn - start_pfn; - memmap_init_zone(size, nid, zone, start_pfn, range_end_pfn, + memmap_init_range(size, nid, zone, start_pfn, range_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } } @@ -6982,7 +6982,7 @@ static void __init free_area_init_core(struct pglist_data *pgdat) set_pageblock_order(); setup_usemap(pgdat, zone, zone_start_pfn, size); init_currently_empty_zone(zone, zone_start_pfn, size); - memmap_init(size, nid, j, zone_start_pfn); + memmap_init_zone(size, nid, j, zone_start_pfn); } } From 3256ff83c566235e812498ee1dc806c45a5d5af7 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Feb 2021 12:06:17 -0800 Subject: [PATCH 559/633] mm: simplify parater of function memmap_init_zone() As David suggested, simply passing 'struct zone *zone' is enough. We can get all needed information from 'struct zone*' easily. Link: https://lkml.kernel.org/r/20210122135956.5946-4-bhe@redhat.com Signed-off-by: Baoquan He Suggested-by: David Hildenbrand Reviewed-by: Mike Rapoport Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/ia64/mm/init.c | 12 +++++++----- include/linux/mm.h | 3 +-- mm/page_alloc.c | 24 +++++++++++------------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 39b782f62d7d..16d0d7d22657 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -542,12 +542,14 @@ virtual_memmap_init(u64 start, u64 end, void *arg) return 0; } -void __meminit -memmap_init_zone(unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn) +void __meminit memmap_init_zone(struct zone *zone) { + int nid = zone_to_nid(zone), zone_id = zone_idx(zone); + unsigned long start_pfn = zone->zone_start_pfn; + unsigned long size = zone->spanned_pages; + if (!vmem_map) { - memmap_init_range(size, nid, zone, start_pfn, start_pfn + size, + memmap_init_range(size, nid, zone_id, start_pfn, start_pfn + size, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } else { struct page *start; @@ -557,7 +559,7 @@ memmap_init_zone(unsigned long size, int nid, unsigned long zone, args.start = start; args.end = start + size; args.nid = nid; - args.zone = zone; + args.zone = zone_id; efi_memmap_walk(virtual_memmap_init, &args); } diff --git a/include/linux/mm.h b/include/linux/mm.h index 56d0eeae0ce3..2601a5cce0ef 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2408,8 +2408,7 @@ extern void set_dma_reserve(unsigned long new_dma_reserve); extern void memmap_init_range(unsigned long, int, unsigned long, unsigned long, unsigned long, enum meminit_context, struct vmem_altmap *, int migratetype); -extern void memmap_init_zone(unsigned long size, int nid, - unsigned long zone, unsigned long range_start_pfn); +extern void memmap_init_zone(struct zone *zone); extern void setup_per_zone_wmarks(void); extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 519cf52963eb..aa04c54ad47c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6258,23 +6258,21 @@ static void __meminit zone_init_free_lists(struct zone *zone) } } -void __meminit __weak memmap_init_zone(unsigned long size, int nid, - unsigned long zone, - unsigned long range_start_pfn) +void __meminit __weak memmap_init_zone(struct zone *zone) { + unsigned long zone_start_pfn = zone->zone_start_pfn; + unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; + int i, nid = zone_to_nid(zone), zone_id = zone_idx(zone); unsigned long start_pfn, end_pfn; - unsigned long range_end_pfn = range_start_pfn + size; - int i; for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { - start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); - end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); + start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); + end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); - if (end_pfn > start_pfn) { - size = end_pfn - start_pfn; - memmap_init_range(size, nid, zone, start_pfn, range_end_pfn, - MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); - } + if (end_pfn > start_pfn) + memmap_init_range(end_pfn - start_pfn, nid, + zone_id, start_pfn, zone_end_pfn, + MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } } @@ -6982,7 +6980,7 @@ static void __init free_area_init_core(struct pglist_data *pgdat) set_pageblock_order(); setup_usemap(pgdat, zone, zone_start_pfn, size); init_currently_empty_zone(zone, zone_start_pfn, size); - memmap_init_zone(size, nid, j, zone_start_pfn); + memmap_init_zone(zone); } } From 7010a6eca49fc4a5a50f491342f08ddaa087ff07 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Feb 2021 12:06:20 -0800 Subject: [PATCH 560/633] mm: simplify parameter of setup_usemap() Parameter 'zone' has got needed information, let's remove other unnecessary parameters. Link: https://lkml.kernel.org/r/20210122135956.5946-5-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index aa04c54ad47c..56a59f5b4fc2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6765,25 +6765,22 @@ static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned l return usemapsize / 8; } -static void __ref setup_usemap(struct pglist_data *pgdat, - struct zone *zone, - unsigned long zone_start_pfn, - unsigned long zonesize) +static void __ref setup_usemap(struct zone *zone) { - unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); + unsigned long usemapsize = usemap_size(zone->zone_start_pfn, + zone->spanned_pages); zone->pageblock_flags = NULL; if (usemapsize) { zone->pageblock_flags = memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, - pgdat->node_id); + zone_to_nid(zone)); if (!zone->pageblock_flags) panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", - usemapsize, zone->name, pgdat->node_id); + usemapsize, zone->name, zone_to_nid(zone)); } } #else -static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, - unsigned long zone_start_pfn, unsigned long zonesize) {} +static inline void setup_usemap(struct zone *zone) {} #endif /* CONFIG_SPARSEMEM */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -6978,7 +6975,7 @@ static void __init free_area_init_core(struct pglist_data *pgdat) continue; set_pageblock_order(); - setup_usemap(pgdat, zone, zone_start_pfn, size); + setup_usemap(zone); init_currently_empty_zone(zone, zone_start_pfn, size); memmap_init_zone(zone); } From 9699ee7b2984c612ec3b45c1f7b963daebec3d6c Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Wed, 24 Feb 2021 12:06:24 -0800 Subject: [PATCH 561/633] mm: remove unneeded local variable in free_area_init_core Local variable 'zone_start_pfn' is not needed since there's only one call site in free_area_init_core(). Let's remove it and pass zone->zone_start_pfn directly to init_currently_empty_zone(). Link: https://lkml.kernel.org/r/20210122135956.5946-6-bhe@redhat.com Signed-off-by: Baoquan He Reviewed-by: Mike Rapoport Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 56a59f5b4fc2..35e230cadb94 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6927,7 +6927,6 @@ static void __init free_area_init_core(struct pglist_data *pgdat) for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = pgdat->node_zones + j; unsigned long size, freesize, memmap_pages; - unsigned long zone_start_pfn = zone->zone_start_pfn; size = zone->spanned_pages; freesize = zone->present_pages; @@ -6976,7 +6975,7 @@ static void __init free_area_init_core(struct pglist_data *pgdat) set_pageblock_order(); setup_usemap(zone); - init_currently_empty_zone(zone, zone_start_pfn, size); + init_currently_empty_zone(zone, zone->zone_start_pfn, size); memmap_init_zone(zone); } } From b3880c690beb7f3abf50f333bd8f3ea7040aaf89 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 24 Feb 2021 12:06:28 -0800 Subject: [PATCH 562/633] video: fbdev: acornfb: remove free_unused_pages() Patch series "mm: simplify free_highmem_page() and free_reserved_page()". Let's simplify and unify free_highmem_page() and free_reserved_page(). This patch (of 2): This function is never used and it is one of the last remaining user of __free_reserved_page(). Let's just drop it. Link: https://lkml.kernel.org/r/20210126182113.19892-1-david@redhat.com Link: https://lkml.kernel.org/r/20210126182113.19892-2-david@redhat.com Fixes: ffd29195ed720188 ("drivers/video/acornfb.c: remove dead code") Signed-off-by: David Hildenbrand Reviewed-by: Oscar Salvador Reviewed-by: Anshuman Khandual Cc: Thomas Gleixner Cc: "Peter Zijlstra (Intel)" Cc: Mike Rapoport Cc: Michal Hocko Cc: Wei Yang Cc: "Gustavo A. R. Silva" Cc: Sam Ravnborg Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/video/fbdev/acornfb.c | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/drivers/video/fbdev/acornfb.c b/drivers/video/fbdev/acornfb.c index bcc92aecf666..1b72edc01cfb 100644 --- a/drivers/video/fbdev/acornfb.c +++ b/drivers/video/fbdev/acornfb.c @@ -921,40 +921,6 @@ static int acornfb_detect_monitortype(void) return 4; } -/* - * This enables the unused memory to be freed on older Acorn machines. - * We are freeing memory on behalf of the architecture initialisation - * code here. - */ -static inline void -free_unused_pages(unsigned int virtual_start, unsigned int virtual_end) -{ - int mb_freed = 0; - - /* - * Align addresses - */ - virtual_start = PAGE_ALIGN(virtual_start); - virtual_end = PAGE_ALIGN(virtual_end); - - while (virtual_start < virtual_end) { - struct page *page; - - /* - * Clear page reserved bit, - * set count to 1, and free - * the page. - */ - page = virt_to_page(virtual_start); - __free_reserved_page(page); - - virtual_start += PAGE_SIZE; - mb_freed += PAGE_SIZE / 1024; - } - - printk("acornfb: freed %dK memory\n", mb_freed); -} - static int acornfb_probe(struct platform_device *dev) { unsigned long size; From a0cd7a7c4bc004587d1f4785a320f58e72d880eb Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 24 Feb 2021 12:06:32 -0800 Subject: [PATCH 563/633] mm: simplify free_highmem_page() and free_reserved_page() adjust_managed_page_count() as called by free_reserved_page() properly handles pages in a highmem zone, so we can reuse it for free_highmem_page(). We can now get rid of totalhigh_pages_inc() and simplify free_reserved_page(). Link: https://lkml.kernel.org/r/20210126182113.19892-3-david@redhat.com Signed-off-by: David Hildenbrand Reviewed-by: Oscar Salvador Reviewed-by: Anshuman Khandual Cc: Thomas Gleixner Cc: "Peter Zijlstra (Intel)" Cc: Mike Rapoport Cc: Michal Hocko Cc: Wei Yang Cc: "Gustavo A. R. Silva" Cc: Sam Ravnborg Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/highmem-internal.h | 5 ----- include/linux/mm.h | 16 ++-------------- mm/page_alloc.c | 11 ----------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/include/linux/highmem-internal.h b/include/linux/highmem-internal.h index 1bbe96dc8be6..7902c7d8b55f 100644 --- a/include/linux/highmem-internal.h +++ b/include/linux/highmem-internal.h @@ -127,11 +127,6 @@ static inline unsigned long totalhigh_pages(void) return (unsigned long)atomic_long_read(&_totalhigh_pages); } -static inline void totalhigh_pages_inc(void) -{ - atomic_long_inc(&_totalhigh_pages); -} - static inline void totalhigh_pages_add(long count) { atomic_long_add(count, &_totalhigh_pages); diff --git a/include/linux/mm.h b/include/linux/mm.h index 2601a5cce0ef..76cab132c295 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2310,32 +2310,20 @@ extern void free_initmem(void); extern unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); -#ifdef CONFIG_HIGHMEM -/* - * Free a highmem page into the buddy system, adjusting totalhigh_pages - * and totalram_pages. - */ -extern void free_highmem_page(struct page *page); -#endif - extern void adjust_managed_page_count(struct page *page, long count); extern void mem_init_print_info(const char *str); extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end); /* Free the reserved page into the buddy system, so it gets managed. */ -static inline void __free_reserved_page(struct page *page) +static inline void free_reserved_page(struct page *page) { ClearPageReserved(page); init_page_count(page); __free_page(page); -} - -static inline void free_reserved_page(struct page *page) -{ - __free_reserved_page(page); adjust_managed_page_count(page, 1); } +#define free_highmem_page(page) free_reserved_page(page) static inline void mark_page_reserved(struct page *page) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 35e230cadb94..ddccc59f2f72 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -7691,17 +7691,6 @@ unsigned long free_reserved_area(void *start, void *end, int poison, const char return pages; } -#ifdef CONFIG_HIGHMEM -void free_highmem_page(struct page *page) -{ - __free_reserved_page(page); - totalram_pages_inc(); - atomic_long_inc(&page_zone(page)->managed_pages); - totalhigh_pages_inc(); -} -#endif - - void __init mem_init_print_info(const char *str) { unsigned long physpages, codesize, datasize, rosize, bss_size; From 3b2ebeaf98a028d5dd4ec63095855ef507920276 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 24 Feb 2021 12:06:35 -0800 Subject: [PATCH 564/633] mm/gfp: add kernel-doc for gfp_t The generated html will link to the definition of the gfp_t automatically once we define it. Move the one-paragraph overview of GFP flags from the documentation directory into gfp.h and pull gfp.h into the documentation. This generates warnings with clang (https://lkml.kernel.org/r/20210219195509.GA59987@24bbad8f3778), so use a #if 0 to hide it from the compiler for now. Link: https://lkml.kernel.org/r/20210215204909.3824509-1-willy@infradead.org Link: https://lkml.kernel.org/r/20210220003049.GZ2858050@casper.infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Mike Rapoport Cc: Nathan Chancellor Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/core-api/mm-api.rst | 7 ++----- include/linux/gfp.h | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 2adffb3f7914..201b5423303b 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -19,11 +19,8 @@ User Space Memory Access Memory Allocation Controls ========================== -Functions which need to allocate memory often use GFP flags to express -how that memory should be allocated. The GFP acronym stands for "get -free pages", the underlying memory allocation function. Not every GFP -flag is allowed to every function which may allocate memory. Most -users will want to use a plain ``GFP_KERNEL``. +.. kernel-doc:: include/linux/gfp.h + :internal: .. kernel-doc:: include/linux/gfp.h :doc: Page mobility and placement hints diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 80544d5c08e7..220cd553a9e7 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -8,6 +8,20 @@ #include #include +/* The typedef is in types.h but we want the documentation here */ +#if 0 +/** + * typedef gfp_t - Memory allocation flags. + * + * GFP flags are commonly used throughout Linux to indicate how memory + * should be allocated. The GFP acronym stands for get_free_pages(), + * the underlying memory allocation function. Not every GFP flag is + * supported by every function which may allocate memory. Most users + * will want to use a plain ``GFP_KERNEL``. + */ +typedef unsigned int __bitwise gfp_t; +#endif + struct vm_area_struct; /* From 30c9cf49270423f8cb0d2c152486e248f375cccb Mon Sep 17 00:00:00 2001 From: Aili Yao Date: Wed, 24 Feb 2021 12:06:39 -0800 Subject: [PATCH 565/633] mm,hwpoison: send SIGBUS to PF_MCE_EARLY processes on action required events When a memory uncorrected error is triggered by process who accessed the address with error, It's Action Required Case for only current process which triggered this; This Action Required case means Action optional to other process who share the same page. Usually killing current process will be sufficient, other processes sharing the same page will get be signaled when they really touch the poisoned page. But there is another scenario that other processes sharing the same page want to be signaled early with PF_MCE_EARLY set. In this case, we should get them into kill list and signal BUS_MCEERR_AO to them. So in this patch, task_early_kill will check current process if force_early is set, and if not current,the code will fallback to find_early_kill_thread() to check if there is PF_MCE_EARLY process who cares the error. In kill_proc(), BUS_MCEERR_AR is only send to current, other processes in kill list will be signaled with BUS_MCEERR_AO. Link: https://lkml.kernel.org/r/20210122132424.313c8f5f.yaoaili@kingsoft.com Signed-off-by: Aili Yao Reviewed-by: Oscar Salvador Acked-by: Naoya Horiguchi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memory-failure.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index e9481632fcd1..55c671904aac 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -243,9 +243,13 @@ static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) pfn, t->comm, t->pid); if (flags & MF_ACTION_REQUIRED) { - WARN_ON_ONCE(t != current); - ret = force_sig_mceerr(BUS_MCEERR_AR, + if (t == current) + ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb); + else + /* Signal other processes sharing the page if they have PF_MCE_EARLY set. */ + ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, + addr_lsb, t); } else { /* * Don't use force here, it's convenient if the signal @@ -440,26 +444,26 @@ static struct task_struct *find_early_kill_thread(struct task_struct *tsk) * Determine whether a given process is "early kill" process which expects * to be signaled when some page under the process is hwpoisoned. * Return task_struct of the dedicated thread (main thread unless explicitly - * specified) if the process is "early kill," and otherwise returns NULL. + * specified) if the process is "early kill" and otherwise returns NULL. * - * Note that the above is true for Action Optional case, but not for Action - * Required case where SIGBUS should sent only to the current thread. + * Note that the above is true for Action Optional case. For Action Required + * case, it's only meaningful to the current thread which need to be signaled + * with SIGBUS, this error is Action Optional for other non current + * processes sharing the same error page,if the process is "early kill", the + * task_struct of the dedicated thread will also be returned. */ static struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) { if (!tsk->mm) return NULL; - if (force_early) { - /* - * Comparing ->mm here because current task might represent - * a subthread, while tsk always points to the main thread. - */ - if (tsk->mm == current->mm) - return current; - else - return NULL; - } + /* + * Comparing ->mm here because current task might represent + * a subthread, while tsk always points to the main thread. + */ + if (force_early && tsk->mm == current->mm) + return current; + return find_early_kill_thread(tsk); } From fca40573e0f742dfd81cf20b8a7c6ce0e543b8b6 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 24 Feb 2021 12:06:42 -0800 Subject: [PATCH 566/633] mm/huge_memory.c: update tlb entry if pmd is changed When set_pmd_at is called in function do_huge_pmd_anonymous_page, new tlb entry can be added by software on MIPS platform. Here add update_mmu_cache_pmd when pmd entry is set, and update_mmu_cache_pmd is defined as empty excepts arc/mips platform. This patch has no negative effect on other platforms except arc/mips system. Link: http://lkml.kernel.org/r/1592990792-1923-2-git-send-email-maobibo@loongson.cn Signed-off-by: Bibo Mao Cc: Anshuman Khandual Cc: Daniel Silsby Cc: "Kirill A. Shutemov" Cc: Mike Kravetz Cc: Mike Rapoport Cc: Paul Burton Cc: Ralf Baechle Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/huge_memory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 86a3015ba54f..080b471c4fbf 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -636,6 +636,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, lru_cache_add_inactive_or_unevictable(page, vma); pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(vma->vm_mm); spin_unlock(vmf->ptl); @@ -749,6 +750,7 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) } else { set_huge_zero_page(pgtable, vma->vm_mm, vma, haddr, vmf->pmd, zero_page); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); spin_unlock(vmf->ptl); } } else { From aba677f94b7d1004a9477464d78111d9082546f9 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 24 Feb 2021 12:06:46 -0800 Subject: [PATCH 567/633] MIPS: do not call flush_tlb_all when setting pmd entry Function set_pmd_at is to set pmd entry, if tlb entry need to be flushed, there exists pmdp_huge_clear_flush alike function before set_pmd_at is called. So it is not necessary to call flush_tlb_all in this function. In these scenarios, tlb for the pmd range needs to be flushed: - privilege degrade such as wrprotect is set on the pmd entry - pmd entry is cleared - there is exception if set_pmd_at is issued by dup_mmap, since flush_tlb_mm is called for parent process, it is not necessary to flush tlb in function copy_huge_pmd. Link: http://lkml.kernel.org/r/1592990792-1923-3-git-send-email-maobibo@loongson.cn Signed-off-by: Bibo Mao Cc: Anshuman Khandual Cc: Daniel Silsby Cc: "Kirill A. Shutemov" Cc: Mike Kravetz Cc: Mike Rapoport Cc: Paul Burton Cc: Ralf Baechle Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/mips/mm/pgtable-32.c | 1 - arch/mips/mm/pgtable-64.c | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c index bd4b0656add3..61891af25019 100644 --- a/arch/mips/mm/pgtable-32.c +++ b/arch/mips/mm/pgtable-32.c @@ -45,7 +45,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } #endif /* defined(CONFIG_TRANSPARENT_HUGEPAGE) */ diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c index 183ff9f9c026..7536f7804c44 100644 --- a/arch/mips/mm/pgtable-64.c +++ b/arch/mips/mm/pgtable-64.c @@ -100,7 +100,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } void __init pagetable_init(void) From cc2205a67dec5a700227a693fc113441e73e4641 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:06:50 -0800 Subject: [PATCH 568/633] mm/hugetlb: fix potential double free in hugetlb_register_node() error path In hugetlb_sysfs_add_hstate(), we would do kobject_put() on hstate_kobjs when failed to create sysfs group but forget to set hstate_kobjs to NULL. Then in hugetlb_register_node() error path, we may free it again via hugetlb_unregister_node(). Link: https://lkml.kernel.org/r/20210107123249.36964-1-linmiaohe@huawei.com Fixes: a3437870160c ("hugetlb: new sysfs interface") Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Reviewed-by: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 905a7d549b00..dd56e205e7ab 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2988,8 +2988,10 @@ static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, return -ENOMEM; retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); - if (retval) + if (retval) { kobject_put(hstate_kobjs[hi]); + hstate_kobjs[hi] = NULL; + } return retval; } From a1ba9da8f0f9a37d900ff7eff66482cf7de8015e Mon Sep 17 00:00:00 2001 From: Li Xinhai Date: Wed, 24 Feb 2021 12:06:54 -0800 Subject: [PATCH 569/633] mm/hugetlb.c: fix unnecessary address expansion of pmd sharing The current code would unnecessarily expand the address range. Consider one example, (start, end) = (1G-2M, 3G+2M), and (vm_start, vm_end) = (1G-4M, 3G+4M), the expected adjustment should be keep (1G-2M, 3G+2M) without expand. But the current result will be (1G-4M, 3G+4M). Actually, the range (1G-4M, 1G) and (3G, 3G+4M) would never been involved in pmd sharing. After this patch, we will check that the vma span at least one PUD aligned size and the start,end range overlap the aligned range of vma. With above example, the aligned vma range is (1G, 3G), so if (start, end) range is within (1G-4M, 1G), or within (3G, 3G+4M), then no adjustment to both start and end. Otherwise, we will have chance to adjust start downwards or end upwards without exceeding (vm_start, vm_end). Mike: : The 'adjusted range' is used for calls to mmu notifiers and cache(tlb) : flushing. Since the current code unnecessarily expands the range in some : cases, more entries than necessary would be flushed. This would/could : result in performance degradation. However, this is highly dependent on : the user runtime. Is there a combination of vma layout and calls to : actually hit this issue? If the issue is hit, will those entries : unnecessarily flushed be used again and need to be unnecessarily reloaded? Link: https://lkml.kernel.org/r/20210104081631.2921415-1-lixinhai.lxh@gmail.com Fixes: 75802ca66354 ("mm/hugetlb: fix calculation of adjust_range_if_pmd_sharing_possible") Signed-off-by: Li Xinhai Suggested-by: Mike Kravetz Reviewed-by: Mike Kravetz Cc: Peter Xu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index dd56e205e7ab..66edef56a070 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5288,21 +5288,23 @@ static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { - unsigned long a_start, a_end; + unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE), + v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE); - if (!(vma->vm_flags & VM_MAYSHARE)) + /* + * vma need span at least one aligned PUD size and the start,end range + * must at least partialy within it. + */ + if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) || + (*end <= v_start) || (*start >= v_end)) return; /* Extend the range to be PUD aligned for a worst case scenario */ - a_start = ALIGN_DOWN(*start, PUD_SIZE); - a_end = ALIGN(*end, PUD_SIZE); + if (*start > v_start) + *start = ALIGN_DOWN(*start, PUD_SIZE); - /* - * Intersect the range with the vma range, since pmd sharing won't be - * across vma after all - */ - *start = max(vma->vm_start, a_start); - *end = min(vma->vm_end, a_end); + if (*end < v_end) + *end = ALIGN(*end, PUD_SIZE); } /* From 0aa7f3544aaa02a7df5095dc1bc338bcd73b7872 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:06:57 -0800 Subject: [PATCH 570/633] mm/hugetlb: avoid unnecessary hugetlb_acct_memory() call When reservation accounting remains unchanged, hugetlb_acct_memory() will do nothing except holding and releasing hugetlb_lock. We should avoid this unnecessary hugetlb_lock lock/unlock cycle which is happening on 'most' hugetlb munmap operations by check delta against 0 at the beginning of hugetlb_acct_memory. Link: https://lkml.kernel.org/r/20210115092013.61012-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Oscar Salvador Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 66edef56a070..0173f9f2fcc6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3591,6 +3591,9 @@ static int hugetlb_acct_memory(struct hstate *h, long delta) { int ret = -ENOMEM; + if (!delta) + return 0; + spin_lock(&hugetlb_lock); /* * When cpuset is configured, it breaks the strict hugetlb page From c78a7f3639932c48b4e1d329fc80fd26aa1a2fa3 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:01 -0800 Subject: [PATCH 571/633] mm/hugetlb: use helper huge_page_order and pages_per_huge_page Since commit a5516438959d ("hugetlb: modular state for hugetlb page size"), we can use huge_page_order to access hstate->order and pages_per_huge_page to fetch the pages per huge page. But gather_bootmem_prealloc() forgot to use it. Link: https://lkml.kernel.org/r/20210114114435.40075-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 0173f9f2fcc6..9b7fa72fa8fd 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2476,7 +2476,7 @@ static void __init gather_bootmem_prealloc(void) struct hstate *h = m->hstate; WARN_ON(page_count(page) != 1); - prep_compound_huge_page(page, h->order); + prep_compound_huge_page(page, huge_page_order(h)); WARN_ON(PageReserved(page)); prep_new_huge_page(h, page, page_to_nid(page)); put_page(page); /* free it into the hugepage allocator */ @@ -2488,7 +2488,7 @@ static void __init gather_bootmem_prealloc(void) * side-effects, like CommitLimit going negative. */ if (hstate_is_gigantic(h)) - adjust_managed_page_count(page, 1 << h->order); + adjust_managed_page_count(page, pages_per_huge_page(h)); cond_resched(); } } From 1d88433bb00853bed0c776b6ad9156855c127da0 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:05 -0800 Subject: [PATCH 572/633] mm/hugetlb: fix use after free when subpool max_hpages accounting is not enabled If a hugetlbfs filesystem is created with the min_size option and without the size option, used_hpages is always 0 and might lead to release subpool prematurely because it indicates no pages are used now while there might be. In order to fix this issue, we should check used_hpages == 0 iff max_hpages accounting is enabled. As max_hpages accounting should be enabled in most common case, this is not worth a Cc stable. [mike.kravetz@oracle.com: new changelog] Link: https://lkml.kernel.org/r/20210126115510.53374-1-linmiaohe@huawei.com Signed-off-by: Hongxiang Lou Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 9b7fa72fa8fd..e8d39b0fe4bd 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -97,16 +97,26 @@ static inline void ClearPageHugeFreed(struct page *head) /* Forward declaration */ static int hugetlb_acct_memory(struct hstate *h, long delta); +static inline bool subpool_is_free(struct hugepage_subpool *spool) +{ + if (spool->count) + return false; + if (spool->max_hpages != -1) + return spool->used_hpages == 0; + if (spool->min_hpages != -1) + return spool->rsv_hpages == spool->min_hpages; + + return true; +} + static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) { - bool free = (spool->count == 0) && (spool->used_hpages == 0); - spin_unlock(&spool->lock); /* If no pages are used, and no other handles to the subpool * remain, give up any reservations based on minimum size and * free the subpool */ - if (free) { + if (subpool_is_free(spool)) { if (spool->min_hpages != -1) hugetlb_acct_memory(spool->hstate, -spool->min_hpages); From c93b0a99260741a4fe39c0a8b73f45f34a5b7868 Mon Sep 17 00:00:00 2001 From: Jiapeng Zhong Date: Wed, 24 Feb 2021 12:07:09 -0800 Subject: [PATCH 573/633] mm/hugetlb: simplify the calculation of variables Fix the following coccicheck warnings: mm/hugetlb.c:3372:20-22: WARNING !A || A && B is equivalent to !A || B. Link: https://lkml.kernel.org/r/1611643468-52233-1-git-send-email-abaci-bugfix@linux.alibaba.com Signed-off-by: Jiapeng Zhong Reported-by: Abaci Robot Reviewed-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e8d39b0fe4bd..6920c71d7e0d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3420,8 +3420,7 @@ static unsigned int allowed_mems_nr(struct hstate *h) mpol_allowed = policy_nodemask_current(gfp_mask); for_each_node_mask(node, cpuset_current_mems_allowed) { - if (!mpol_allowed || - (mpol_allowed && node_isset(node, *mpol_allowed))) + if (!mpol_allowed || node_isset(node, *mpol_allowed)) nr += array[node]; } From 0fa5bc4023c188082024833b3deffd5543b93bc9 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Wed, 24 Feb 2021 12:07:12 -0800 Subject: [PATCH 574/633] mm/hugetlb: grab head page refcount once for group of subpages Patch series "mm/hugetlb: follow_hugetlb_page() improvements", v2. While looking at ZONE_DEVICE struct page reuse particularly the last patch[0], I found two possible improvements for follow_hugetlb_page() which is solely used for get_user_pages()/pin_user_pages(). The first patch batches page refcount updates while the second tidies up storing the subpages/vmas. Both together bring the cost of slow variant of gup() cost from ~87.6k usecs to ~5.8k usecs. libhugetlbfs tests seem to pass as well gup_test benchmarks with hugetlbfs vmas. This patch (of 2): follow_hugetlb_page() once it locks the pmd/pud, checks all its N subpages in a huge page and grabs a reference for each one. Similar to gup-fast, have follow_hugetlb_page() grab the head page refcount only after counting all its subpages that are part of the just faulted huge page. Consequently we reduce the number of atomics necessary to pin said huge page, which improves non-fast gup() considerably: - 16G with 1G huge page size gup_test -f /mnt/huge/file -m 16384 -r 10 -L -S -n 512 -w PIN_LONGTERM_BENCHMARK: ~87.6k us -> ~12.8k us Link: https://lkml.kernel.org/r/20210128182632.24562-1-joao.m.martins@oracle.com Link: https://lkml.kernel.org/r/20210128182632.24562-2-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 3 +++ mm/gup.c | 5 ++--- mm/hugetlb.c | 43 ++++++++++++++++++++++++------------------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 76cab132c295..77e64e3eac80 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1187,6 +1187,9 @@ static inline void get_page(struct page *page) } bool __must_check try_grab_page(struct page *page, unsigned int flags); +__maybe_unused struct page *try_grab_compound_head(struct page *page, int refs, + unsigned int flags); + static inline __must_check bool try_get_page(struct page *page) { diff --git a/mm/gup.c b/mm/gup.c index e4c224cd9661..e40579624f10 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -78,9 +78,8 @@ static inline struct page *try_get_compound_head(struct page *page, int refs) * considered failure, and furthermore, a likely bug in the caller, so a warning * is also emitted. */ -static __maybe_unused struct page *try_grab_compound_head(struct page *page, - int refs, - unsigned int flags) +__maybe_unused struct page *try_grab_compound_head(struct page *page, + int refs, unsigned int flags) { if (flags & FOLL_GET) return try_get_compound_head(page, refs); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6920c71d7e0d..d4fc5db6bc32 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4796,7 +4796,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long vaddr = *position; unsigned long remainder = *nr_pages; struct hstate *h = hstate_vma(vma); - int err = -EFAULT; + int err = -EFAULT, refs; while (vaddr < vma->vm_end && remainder) { pte_t *pte; @@ -4916,26 +4916,11 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, continue; } + refs = 0; + same_page: - if (pages) { + if (pages) pages[i] = mem_map_offset(page, pfn_offset); - /* - * try_grab_page() should always succeed here, because: - * a) we hold the ptl lock, and b) we've just checked - * that the huge page is present in the page tables. If - * the huge page is present, then the tail pages must - * also be present. The ptl prevents the head page and - * tail pages from being rearranged in any way. So this - * page must be available at this point, unless the page - * refcount overflowed: - */ - if (WARN_ON_ONCE(!try_grab_page(pages[i], flags))) { - spin_unlock(ptl); - remainder = 0; - err = -ENOMEM; - break; - } - } if (vmas) vmas[i] = vma; @@ -4944,6 +4929,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, ++pfn_offset; --remainder; ++i; + ++refs; if (vaddr < vma->vm_end && remainder && pfn_offset < pages_per_huge_page(h)) { /* @@ -4951,6 +4937,25 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, * of this compound page. */ goto same_page; + } else if (pages) { + /* + * try_grab_compound_head() should always succeed here, + * because: a) we hold the ptl lock, and b) we've just + * checked that the huge page is present in the page + * tables. If the huge page is present, then the tail + * pages must also be present. The ptl prevents the + * head page and tail pages from being rearranged in + * any way. So this page must be available at this + * point, unless the page refcount overflowed: + */ + if (WARN_ON_ONCE(!try_grab_compound_head(pages[i-1], + refs, + flags))) { + spin_unlock(ptl); + remainder = 0; + err = -ENOMEM; + break; + } } spin_unlock(ptl); } From 82e5d378b0e4736899e7f8f9f0f03138228f9a45 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Wed, 24 Feb 2021 12:07:16 -0800 Subject: [PATCH 575/633] mm/hugetlb: refactor subpage recording For a given hugepage backing a VA, there's a rather ineficient loop which is solely responsible for storing subpages in GUP @pages/@vmas array. For each subpage we check whether it's within range or size of @pages and keep increment @pfn_offset and a couple other variables per subpage iteration. Simplify this logic and minimize the cost of each iteration to just store the output page/vma. Instead of incrementing number of @refs iteratively, we do it through pre-calculation of @refs and only with a tight loop for storing pinned subpages/vmas. Additionally, retain existing behaviour with using mem_map_offset() when recording the subpages for configurations that don't have a contiguous mem_map. pinning consequently improves bringing us close to {pin,get}_user_pages_fast: - 16G with 1G huge page size gup_test -f /mnt/huge/file -m 16384 -r 30 -L -S -n 512 -w PIN_LONGTERM_BENCHMARK: ~12.8k us -> ~5.8k us PIN_FAST_BENCHMARK: ~3.7k us Link: https://lkml.kernel.org/r/20210128182632.24562-3-joao.m.martins@oracle.com Signed-off-by: Joao Martins Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d4fc5db6bc32..cf6653879bb4 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4787,6 +4787,20 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, goto out; } +static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma, + int refs, struct page **pages, + struct vm_area_struct **vmas) +{ + int nr; + + for (nr = 0; nr < refs; nr++) { + if (likely(pages)) + pages[nr] = mem_map_offset(page, nr); + if (vmas) + vmas[nr] = vma; + } +} + long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, unsigned long *nr_pages, @@ -4916,28 +4930,16 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, continue; } - refs = 0; + refs = min3(pages_per_huge_page(h) - pfn_offset, + (vma->vm_end - vaddr) >> PAGE_SHIFT, remainder); -same_page: - if (pages) - pages[i] = mem_map_offset(page, pfn_offset); + if (pages || vmas) + record_subpages_vmas(mem_map_offset(page, pfn_offset), + vma, refs, + likely(pages) ? pages + i : NULL, + vmas ? vmas + i : NULL); - if (vmas) - vmas[i] = vma; - - vaddr += PAGE_SIZE; - ++pfn_offset; - --remainder; - ++i; - ++refs; - if (vaddr < vma->vm_end && remainder && - pfn_offset < pages_per_huge_page(h)) { - /* - * We use pfn_offset to avoid touching the pageframes - * of this compound page. - */ - goto same_page; - } else if (pages) { + if (pages) { /* * try_grab_compound_head() should always succeed here, * because: a) we hold the ptl lock, and b) we've just @@ -4948,7 +4950,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, * any way. So this page must be available at this * point, unless the page refcount overflowed: */ - if (WARN_ON_ONCE(!try_grab_compound_head(pages[i-1], + if (WARN_ON_ONCE(!try_grab_compound_head(pages[i], refs, flags))) { spin_unlock(ptl); @@ -4957,6 +4959,11 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, break; } } + + vaddr += (refs << PAGE_SHIFT); + remainder -= refs; + i += refs; + spin_unlock(ptl); } *nr_pages = remainder; From 6c26d3108393211ecfd44d89404cfb744027bafd Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:19 -0800 Subject: [PATCH 576/633] mm/hugetlb: fix some comment typos Fix typos sasitfy to satisfy, reservtion to reservation, hugegpage to hugepage and uniprocesor to uniprocessor in comments. Link: https://lkml.kernel.org/r/20210128112028.64831-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Souptick Joarder Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index b5807f23caf8..ac3cb239a7ec 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -37,7 +37,7 @@ struct hugepage_subpool { struct hstate *hstate; long min_hpages; /* Minimum huge pages or -1 if no minimum. */ long rsv_hpages; /* Pages reserved against global pool to */ - /* sasitfy minimum size. */ + /* satisfy minimum size. */ }; struct resv_map { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cf6653879bb4..fc895e25920c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1434,7 +1434,7 @@ static void __free_huge_page(struct page *page) * reservation. If the page was associated with a subpool, there * would have been a page reserved in the subpool before allocation * via hugepage_subpool_get_pages(). Since we are 'restoring' the - * reservtion, do not call hugepage_subpool_put_pages() as this will + * reservation, do not call hugepage_subpool_put_pages() as this will * remove the reserved page from the subpool. */ if (!restore_reserve) { @@ -3707,7 +3707,7 @@ static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) /* * We cannot handle pagefaults against hugetlb pages at all. They cause * handle_mm_fault() to try to instantiate regular-sized pages in the - * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get + * hugepage VMA. do_page_fault() is supposed to trap this, so BUG is we get * this far. */ static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) @@ -4491,7 +4491,7 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) } #else /* - * For uniprocesor systems we always use a single mutex, so just + * For uniprocessor systems we always use a single mutex, so just * return 0 and avoid the hashing overhead. */ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) From 5291c09b3edb657f23c1939750c702ba2d74932f Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Wed, 24 Feb 2021 12:07:22 -0800 Subject: [PATCH 577/633] mm/hugetlb: remove redundant check in preparing and destroying gigantic page Gigantic page is a compound page and its order is more than 1. Thus it must be available for hpage_pincount. Let's remove the redundant check for gigantic page. Link: https://lkml.kernel.org/r/20210202112002.73170-1-yanfei.xu@windriver.com Signed-off-by: Yanfei Xu Reviewed-by: Mike Kravetz Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index fc895e25920c..527e2b89e209 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1234,8 +1234,7 @@ static void destroy_compound_gigantic_page(struct page *page, struct page *p = page + 1; atomic_set(compound_mapcount_ptr(page), 0); - if (hpage_pincount_available(page)) - atomic_set(compound_pincount_ptr(page), 0); + atomic_set(compound_pincount_ptr(page), 0); for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { clear_compound_head(p); @@ -1563,9 +1562,7 @@ static void prep_compound_gigantic_page(struct page *page, unsigned int order) set_compound_head(p, page); } atomic_set(compound_mapcount_ptr(page), -1); - - if (hpage_pincount_available(page)) - atomic_set(compound_pincount_ptr(page), 0); + atomic_set(compound_pincount_ptr(page), 0); } /* From 578b7725d4bde8eca23218278d1d8103dd0c3dde Mon Sep 17 00:00:00 2001 From: Zhiyuan Dai Date: Wed, 24 Feb 2021 12:07:26 -0800 Subject: [PATCH 578/633] mm/hugetlb.c: fix typos in comments Fix typo in comment. Link: https://lkml.kernel.org/r/1612256106-9436-1-git-send-email-daizhiyuan@phytium.com.cn Signed-off-by: Zhiyuan Dai Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 527e2b89e209..9d3a1416fc2f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4028,7 +4028,7 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, /* * This is called when the original mapper is failing to COW a MAP_PRIVATE - * mappping it owns the reserve page for. The intention is to unmap the page + * mapping it owns the reserve page for. The intention is to unmap the page * from other VMAs and let the children be SIGKILLed if they are faulting the * same region. */ From 2efeb8da992b955fa7705259e4b2f5937979deff Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:29 -0800 Subject: [PATCH 579/633] mm/huge_memory.c: remove unused return value of set_huge_zero_page() The return value of set_huge_zero_page() is always ignored. So we should drop such return value. Link: https://lkml.kernel.org/r/20210203084816.46307-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/huge_memory.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 080b471c4fbf..4cdcc4da36e8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -691,20 +691,19 @@ static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma) } /* Caller must hold page table lock. */ -static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, +static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, struct page *zero_page) { pmd_t entry; if (!pmd_none(*pmd)) - return false; + return; entry = mk_pmd(zero_page, vma->vm_page_prot); entry = pmd_mkhuge(entry); if (pgtable) pgtable_trans_huge_deposit(mm, pmd, pgtable); set_pmd_at(mm, haddr, pmd, entry); mm_inc_nr_ptes(mm); - return true; } vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) From bae84953815793f68ddd8edeadd3f4e32676a2c8 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Wed, 24 Feb 2021 12:07:32 -0800 Subject: [PATCH 580/633] mm/pmem: avoid inserting hugepage PTE entry with fsdax if hugepage support is disabled Differentiate between hardware not supporting hugepages and user disabling THP via 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' For the devdax namespace, the kernel handles the above via the supported_alignment attribute and failing to initialize the namespace if the namespace align value is not supported on the platform. For the fsdax namespace, the kernel will continue to initialize the namespace. This can result in the kernel creating a huge pte entry even though the hardware don't support the same. We do want hugepage support with pmem even if the end-user disabled THP via sysfs file (/sys/kernel/mm/transparent_hugepage/enabled). Hence differentiate between hardware/firmware lacking support vs user-controlled disable of THP and prevent a huge fault if the hardware lacks hugepage support. Link: https://lkml.kernel.org/r/20210205023956.417587-1-aneesh.kumar@linux.ibm.com Signed-off-by: Aneesh Kumar K.V Reviewed-by: Dan Williams Cc: "Kirill A . Shutemov" Cc: Jan Kara Cc: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/huge_mm.h | 15 +++++++++------ mm/huge_memory.c | 6 +++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 6a19f35f836b..ba973efcd369 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -78,6 +78,7 @@ static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, } enum transparent_hugepage_flag { + TRANSPARENT_HUGEPAGE_NEVER_DAX, TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, @@ -123,6 +124,13 @@ extern unsigned long transparent_hugepage_flags; */ static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) { + + /* + * If the hardware/firmware marked hugepage support disabled. + */ + if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX)) + return false; + if (vma->vm_flags & VM_NOHUGEPAGE) return false; @@ -134,12 +142,7 @@ static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma) if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG)) return true; - /* - * For dax vmas, try to always use hugepage mappings. If the kernel does - * not support hugepages, fsdax mappings will fallback to PAGE_SIZE - * mappings, and device-dax namespaces, that try to guarantee a given - * mapping size, will fail to enable - */ + if (vma_is_dax(vma)) return true; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 4cdcc4da36e8..d77605c30f2e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -386,7 +386,11 @@ static int __init hugepage_init(void) struct kobject *hugepage_kobj; if (!has_transparent_hugepage()) { - transparent_hugepage_flags = 0; + /* + * Hardware doesn't support hugepages, hence disable + * DAX PMD support. + */ + transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_NEVER_DAX; return -EINVAL; } From 8938494c8567ebd9ebf2a230e1707ee1f9805342 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:36 -0800 Subject: [PATCH 581/633] hugetlb_cgroup: use helper pages_per_huge_page() in hugetlb_cgroup We could use helper function pages_per_huge_page() to get the number of pages in a hstate to simplify the code slightly. Link: https://lkml.kernel.org/r/20210205084513.29624-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb_cgroup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 9182848dda3e..f68b51fcda3d 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -113,7 +113,7 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, rsvd_parent); limit = round_down(PAGE_COUNTER_MAX, - 1 << huge_page_order(&hstates[idx])); + pages_per_huge_page(&hstates[idx])); ret = page_counter_set_max( hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx), @@ -460,7 +460,7 @@ static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) counter = &h_cg->hugepage[idx]; limit = round_down(PAGE_COUNTER_MAX, - 1 << huge_page_order(&hstates[idx])); + pages_per_huge_page(&hstates[idx])); switch (MEMFILE_ATTR(cft->private)) { case RES_RSVD_USAGE: @@ -507,7 +507,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, return ret; idx = MEMFILE_IDX(of_cft(of)->private); - nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx])); + nr_pages = round_down(nr_pages, pages_per_huge_page(&hstates[idx])); switch (MEMFILE_ATTR(of_cft(of)->private)) { case RES_RSVD_LIMIT: From 07e51edf839ab85187acf013384ceecbbba40b0b Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:39 -0800 Subject: [PATCH 582/633] mm/hugetlb: use helper function range_in_vma() in page_table_shareable() We could use helper function range_in_vma() to check whether the vma is in the desired range to simplify the code. Link: https://lkml.kernel.org/r/20210204112949.43051-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 9d3a1416fc2f..4c956e64a10c 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5282,7 +5282,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, */ if (pmd_index(addr) != pmd_index(saddr) || vm_flags != svm_flags || - sbase < svma->vm_start || svma->vm_end < s_end) + !range_in_vma(svma, sbase, s_end)) return 0; return saddr; From 3f1b0162f6f6ae8a9012819b07d433bd0ec37d25 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:43 -0800 Subject: [PATCH 583/633] mm/hugetlb: remove unnecessary VM_BUG_ON_PAGE on putback_active_hugepage() All callers know they are operating on a hugetlb head page. So this VM_BUG_ON_PAGE can not catch anything useful. Link: https://lkml.kernel.org/r/20210209071151.44731-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Oscar Salvador Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4c956e64a10c..ddfd1f99c884 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5622,7 +5622,6 @@ bool isolate_huge_page(struct page *page, struct list_head *list) void putback_active_hugepage(struct page *page) { - VM_BUG_ON_PAGE(!PageHead(page), page); spin_lock(&hugetlb_lock); set_page_huge_active(page); list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); From aca78307bfdaf3f99e040616f41aab7f8a566dfc Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:07:46 -0800 Subject: [PATCH 584/633] mm/hugetlb: use helper huge_page_size() to get hugepage size We can use helper huge_page_size() to get the hugepage size directly to simplify the code slightly. [linmiaohe@huawei.com: use helper huge_page_size() to get hugepage size] Link: https://lkml.kernel.org/r/20210209021803.49211-1-linmiaohe@huawei.com Link: https://lkml.kernel.org/r/20210208082450.15716-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ddfd1f99c884..6786b313e6ac 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3248,7 +3248,7 @@ void __init hugetlb_add_hstate(unsigned int order) BUG_ON(order == 0); h = &hstates[hugetlb_max_hstate++]; h->order = order; - h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); + h->mask = ~(huge_page_size(h) - 1); for (i = 0; i < MAX_NUMNODES; ++i) INIT_LIST_HEAD(&h->hugepage_freelists[i]); INIT_LIST_HEAD(&h->hugepage_activelist); @@ -3523,7 +3523,7 @@ void hugetlb_report_meminfo(struct seq_file *m) for_each_hstate(h) { unsigned long count = h->nr_huge_pages; - total += (PAGE_SIZE << huge_page_order(h)) * count; + total += huge_page_size(h) * count; if (h == &default_hstate) seq_printf(m, @@ -3536,10 +3536,10 @@ void hugetlb_report_meminfo(struct seq_file *m) h->free_huge_pages, h->resv_huge_pages, h->surplus_huge_pages, - (PAGE_SIZE << huge_page_order(h)) / 1024); + huge_page_size(h) / SZ_1K); } - seq_printf(m, "Hugetlb: %8lu kB\n", total / 1024); + seq_printf(m, "Hugetlb: %8lu kB\n", total / SZ_1K); } int hugetlb_report_node_meminfo(char *buf, int len, int nid) @@ -3573,7 +3573,7 @@ void hugetlb_show_meminfo(void) h->nr_huge_pages_node[nid], h->free_huge_pages_node[nid], h->surplus_huge_pages_node[nid], - 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); + huge_page_size(h) / SZ_1K); } void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm) @@ -3696,9 +3696,7 @@ static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr) static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma) { - struct hstate *hstate = hstate_vma(vma); - - return 1UL << huge_page_shift(hstate); + return huge_page_size(hstate_vma(vma)); } /* From dbfee5aee7e54f83d96ceb8e3e80717fac62ad63 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:07:50 -0800 Subject: [PATCH 585/633] hugetlb: fix update_and_free_page contig page struct assumption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit page structs are not guaranteed to be contiguous for gigantic pages. The routine update_and_free_page can encounter a gigantic page, yet it assumes page structs are contiguous when setting page flags in subpages. If update_and_free_page encounters non-contiguous page structs, we can see “BUG: Bad page state in process …” errors. Non-contiguous page structs are generally not an issue. However, they can exist with a specific kernel configuration and hotplug operations. For example: Configure the kernel with CONFIG_SPARSEMEM and !CONFIG_SPARSEMEM_VMEMMAP. Then, hotplug add memory for the area where the gigantic page will be allocated. Zi Yan outlined steps to reproduce here [1]. [1] https://lore.kernel.org/linux-mm/16F7C58B-4D79-41C5-9B64-A1A1628F4AF2@nvidia.com/ Link: https://lkml.kernel.org/r/20210217184926.33567-1-mike.kravetz@oracle.com Fixes: 944d9fec8d7a ("hugetlb: add support for gigantic page allocation at runtime") Signed-off-by: Zi Yan Signed-off-by: Mike Kravetz Cc: Zi Yan Cc: Davidlohr Bueso Cc: "Kirill A . Shutemov" Cc: Andrea Arcangeli Cc: Matthew Wilcox Cc: Oscar Salvador Cc: Joao Martins Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6786b313e6ac..3f6a50373d4f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1321,14 +1321,16 @@ static inline void destroy_compound_gigantic_page(struct page *page, static void update_and_free_page(struct hstate *h, struct page *page) { int i; + struct page *subpage = page; if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) return; h->nr_huge_pages--; h->nr_huge_pages_node[page_to_nid(page)]--; - for (i = 0; i < pages_per_huge_page(h); i++) { - page[i].flags &= ~(1 << PG_locked | 1 << PG_error | + for (i = 0; i < pages_per_huge_page(h); + i++, subpage = mem_map_next(subpage, page, i)) { + subpage->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | 1 << PG_dirty | 1 << PG_active | 1 << PG_private | 1 << PG_writeback); From 3272cfc2525b3a2810a59312d7a1e6f04a0ca3ef Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:07:54 -0800 Subject: [PATCH 586/633] hugetlb: fix copy_huge_page_from_user contig page struct assumption page structs are not guaranteed to be contiguous for gigantic pages. The routine copy_huge_page_from_user can encounter gigantic pages, yet it assumes page structs are contiguous when copying pages from user space. Since page structs for the target gigantic page are not contiguous, the data copied from user space could overwrite other pages not associated with the gigantic page and cause data corruption. Non-contiguous page structs are generally not an issue. However, they can exist with a specific kernel configuration and hotplug operations. For example: Configure the kernel with CONFIG_SPARSEMEM and !CONFIG_SPARSEMEM_VMEMMAP. Then, hotplug add memory for the area where the gigantic page will be allocated. Link: https://lkml.kernel.org/r/20210217184926.33567-2-mike.kravetz@oracle.com Fixes: 8fb5debc5fcd ("userfaultfd: hugetlbfs: add hugetlb_mcopy_atomic_pte for userfaultfd support") Signed-off-by: Mike Kravetz Cc: Zi Yan Cc: Davidlohr Bueso Cc: "Kirill A . Shutemov" Cc: Andrea Arcangeli Cc: Matthew Wilcox Cc: Oscar Salvador Cc: Joao Martins Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memory.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index ef4e10c60b5f..784249f3307b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5177,17 +5177,19 @@ long copy_huge_page_from_user(struct page *dst_page, void *page_kaddr; unsigned long i, rc = 0; unsigned long ret_val = pages_per_huge_page * PAGE_SIZE; + struct page *subpage = dst_page; - for (i = 0; i < pages_per_huge_page; i++) { + for (i = 0; i < pages_per_huge_page; + i++, subpage = mem_map_next(subpage, dst_page, i)) { if (allow_pagefault) - page_kaddr = kmap(dst_page + i); + page_kaddr = kmap(subpage); else - page_kaddr = kmap_atomic(dst_page + i); + page_kaddr = kmap_atomic(subpage); rc = copy_from_user(page_kaddr, (const void __user *)(src + i * PAGE_SIZE), PAGE_SIZE); if (allow_pagefault) - kunmap(dst_page + i); + kunmap(subpage); else kunmap_atomic(page_kaddr); From 7ecc956551f8a66618f71838c790a9b0b4f9ca10 Mon Sep 17 00:00:00 2001 From: Chen Wandun Date: Wed, 24 Feb 2021 12:07:58 -0800 Subject: [PATCH 587/633] mm/hugetlb: suppress wrong warning info when alloc gigantic page If hugetlb_cma is enabled, it will skip boot time allocation when allocating gigantic page, that doesn't means allocation failure, so suppress this warning info. Link: https://lkml.kernel.org/r/20210219123909.13130-1-chenwandun@huawei.com Fixes: cf11e85fc08c ("mm: hugetlb: optionally allocate gigantic hugepages using cma") Signed-off-by: Chen Wandun Reviewed-by: Mike Kravetz Cc: Roman Gushchin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 3f6a50373d4f..8a1d1f85e21e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2529,7 +2529,7 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) if (hstate_is_gigantic(h)) { if (hugetlb_cma_size) { pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n"); - break; + goto free; } if (!alloc_bootmem_huge_page(h)) break; @@ -2547,7 +2547,7 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) h->max_huge_pages, buf, i); h->max_huge_pages = i; } - +free: kfree(node_alloc_noretry); } From c2135f7c570bc274035834848d9bf46ea89ba763 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Wed, 24 Feb 2021 12:08:01 -0800 Subject: [PATCH 588/633] mm/vmscan: __isolate_lru_page_prepare() cleanup The function just returns 2 results, so using a 'switch' to deal with its result is unnecessary. Also simplify it to a bool func as Vlastimil suggested. Also remove 'goto' by reusing list_move(), and take Matthew Wilcox's suggestion to update comments in function. Link: https://lkml.kernel.org/r/728874d7-2d93-4049-68c1-dcc3b2d52ccd@linux.alibaba.com Signed-off-by: Alex Shi Reviewed-by: Andrew Morton Acked-by: Vlastimil Babka Cc: Matthew Wilcox Cc: Hugh Dickins Cc: Yu Zhao Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/swap.h | 2 +- mm/compaction.c | 2 +- mm/vmscan.c | 70 ++++++++++++++++++++------------------------ 3 files changed, 34 insertions(+), 40 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index 0d64a2bc7e00..32f665b1ee85 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -356,7 +356,7 @@ extern void lru_cache_add_inactive_or_unevictable(struct page *page, extern unsigned long zone_reclaimable_pages(struct zone *zone); extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, gfp_t gfp_mask, nodemask_t *mask); -extern int __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode); +extern bool __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode); extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, gfp_t gfp_mask, diff --git a/mm/compaction.c b/mm/compaction.c index 190ccdaa6c19..8f52532675a9 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -988,7 +988,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (unlikely(!get_page_unless_zero(page))) goto isolate_fail; - if (__isolate_lru_page_prepare(page, isolate_mode) != 0) + if (!__isolate_lru_page_prepare(page, isolate_mode)) goto isolate_fail_put; /* Try isolate the page */ diff --git a/mm/vmscan.c b/mm/vmscan.c index b1b574ad199d..04509994aed4 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1539,19 +1539,17 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, * page: page to consider * mode: one of the LRU isolation modes defined above * - * returns 0 on success, -ve errno on failure. + * returns true on success, false on failure. */ -int __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode) +bool __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode) { - int ret = -EBUSY; - /* Only take pages on the LRU. */ if (!PageLRU(page)) - return ret; + return false; /* Compaction should not handle unevictable pages but CMA can do so */ if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE)) - return ret; + return false; /* * To minimise LRU disruption, the caller can indicate that it only @@ -1564,7 +1562,7 @@ int __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode) if (mode & ISOLATE_ASYNC_MIGRATE) { /* All the caller can do on PageWriteback is block */ if (PageWriteback(page)) - return ret; + return false; if (PageDirty(page)) { struct address_space *mapping; @@ -1580,20 +1578,20 @@ int __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode) * from the page cache. */ if (!trylock_page(page)) - return ret; + return false; mapping = page_mapping(page); migrate_dirty = !mapping || mapping->a_ops->migratepage; unlock_page(page); if (!migrate_dirty) - return ret; + return false; } } if ((mode & ISOLATE_UNMAPPED) && page_mapped(page)) - return ret; + return false; - return 0; + return true; } /* @@ -1677,35 +1675,31 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, * only when the page is being freed somewhere else. */ scan += nr_pages; - switch (__isolate_lru_page_prepare(page, mode)) { - case 0: - /* - * Be careful not to clear PageLRU until after we're - * sure the page is not being freed elsewhere -- the - * page release code relies on it. - */ - if (unlikely(!get_page_unless_zero(page))) - goto busy; - - if (!TestClearPageLRU(page)) { - /* - * This page may in other isolation path, - * but we still hold lru_lock. - */ - put_page(page); - goto busy; - } - - nr_taken += nr_pages; - nr_zone_taken[page_zonenum(page)] += nr_pages; - list_move(&page->lru, dst); - break; - - default: -busy: - /* else it is being freed elsewhere */ + if (!__isolate_lru_page_prepare(page, mode)) { + /* It is being freed elsewhere */ list_move(&page->lru, src); + continue; } + /* + * Be careful not to clear PageLRU until after we're + * sure the page is not being freed elsewhere -- the + * page release code relies on it. + */ + if (unlikely(!get_page_unless_zero(page))) { + list_move(&page->lru, src); + continue; + } + + if (!TestClearPageLRU(page)) { + /* Another thread is already isolating this page */ + put_page(page); + list_move(&page->lru, src); + continue; + } + + nr_taken += nr_pages; + nr_zone_taken[page_zonenum(page)] += nr_pages; + list_move(&page->lru, dst); } /* From 725cac1c7e345c2e35a2de2db57233af279b851f Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:08:06 -0800 Subject: [PATCH 589/633] mm/workingset.c: avoid unnecessary max_nodes estimation in count_shadow_nodes() If list_lru_shrink_count is 0, we always return SHRINK_EMPTY regardless of the value of max_nodes. So we can return early if nodes == 0 to save some cpu cycles of approximating a reasonable limit for the nodes. Link: https://lkml.kernel.org/r/20210123073825.46709-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Andrew Morton Acked-by: Vlastimil Babka Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/workingset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/workingset.c b/mm/workingset.c index 10e96de945b3..7db8f3dad13c 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -461,6 +461,8 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, unsigned long pages; nodes = list_lru_shrink_count(&shadow_nodes, sc); + if (!nodes) + return SHRINK_EMPTY; /* * Approximate a reasonable limit for the nodes @@ -503,9 +505,6 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, max_nodes = pages >> (XA_CHUNK_SHIFT - 3); - if (!nodes) - return SHRINK_EMPTY; - if (nodes <= max_nodes) return 0; return nodes - max_nodes; From 42895ea73bcd37c4a79e4c9f681ab8b82243c7f7 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:09 -0800 Subject: [PATCH 590/633] mm/vmscan.c: use add_page_to_lru_list() Patch series "mm: lru related cleanups", v2. The cleanups are intended to reduce the verbosity in lru list operations and make them less error-prone. A typical example would be how the patches change __activate_page(): static void __activate_page(struct page *page, struct lruvec *lruvec) { if (!PageActive(page) && !PageUnevictable(page)) { - int lru = page_lru_base_type(page); int nr_pages = thp_nr_pages(page); - del_page_from_lru_list(page, lruvec, lru); + del_page_from_lru_list(page, lruvec); SetPageActive(page); - lru += LRU_ACTIVE; - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); trace_mm_lru_activate(page); There are a few more places like __activate_page() and they are unnecessarily repetitive in terms of figuring out which list a page should be added onto or deleted from. And with the duplicated code removed, they are easier to read, IMO. Patch 1 to 5 basically cover the above. Patch 6 and 7 make code more robust by improving bug reporting. Patch 8, 9 and 10 take care of some dangling helpers left in header files. This patch (of 10): There is add_page_to_lru_list(), and move_pages_to_lru() should reuse it, not duplicate it. Link: https://lkml.kernel.org/r/20210122220600.906146-1-yuzhao@google.com Link: https://lore.kernel.org/linux-mm/20201207220949.830352-2-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-2-yuzhao@google.com Signed-off-by: Yu Zhao Reviewed-by: Alex Shi Acked-by: Vlastimil Babka Reviewed-by: Miaohe Lin Cc: Hugh Dickins Cc: Michal Hocko Cc: Johannes Weiner Cc: Vladimir Davydov Cc: Roman Gushchin Cc: Matthew Wilcox Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/vmscan.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 04509994aed4..19875660e8f8 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1823,7 +1823,6 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, int nr_pages, nr_moved = 0; LIST_HEAD(pages_to_free); struct page *page; - enum lru_list lru; while (!list_empty(list)) { page = lru_to_page(list); @@ -1868,11 +1867,8 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, * inhibits memcg migration). */ VM_BUG_ON_PAGE(!lruvec_holds_page_lru_lock(page, lruvec), page); - lru = page_lru(page); + add_page_to_lru_list(page, lruvec, page_lru(page)); nr_pages = thp_nr_pages(page); - - update_lru_size(lruvec, lru, page_zonenum(page), nr_pages); - list_add(&page->lru, &lruvec->lists[lru]); nr_moved += nr_pages; if (PageActive(page)) workingset_age_nonresident(lruvec, nr_pages); From f90d8191ac864df33b1898bc7edc54eaa24e22bc Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:13 -0800 Subject: [PATCH 591/633] include/linux/mm_inline.h: shuffle lru list addition and deletion functions These functions will call page_lru() in the following patches. Move them below page_lru() to avoid the forward declaration. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-3-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-3-yuzhao@google.com Signed-off-by: Yu Zhao Acked-by: Vlastimil Babka Reviewed-by: Miaohe Lin Cc: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 8fc71e9d7bb0..2889741f450a 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -45,27 +45,6 @@ static __always_inline void update_lru_size(struct lruvec *lruvec, #endif } -static __always_inline void add_page_to_lru_list(struct page *page, - struct lruvec *lruvec, enum lru_list lru) -{ - update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); - list_add(&page->lru, &lruvec->lists[lru]); -} - -static __always_inline void add_page_to_lru_list_tail(struct page *page, - struct lruvec *lruvec, enum lru_list lru) -{ - update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); - list_add_tail(&page->lru, &lruvec->lists[lru]); -} - -static __always_inline void del_page_from_lru_list(struct page *page, - struct lruvec *lruvec, enum lru_list lru) -{ - list_del(&page->lru); - update_lru_size(lruvec, lru, page_zonenum(page), -thp_nr_pages(page)); -} - /** * page_lru_base_type - which LRU list type should a page be on? * @page: the page to test @@ -125,4 +104,25 @@ static __always_inline enum lru_list page_lru(struct page *page) } return lru; } + +static __always_inline void add_page_to_lru_list(struct page *page, + struct lruvec *lruvec, enum lru_list lru) +{ + update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); + list_add(&page->lru, &lruvec->lists[lru]); +} + +static __always_inline void add_page_to_lru_list_tail(struct page *page, + struct lruvec *lruvec, enum lru_list lru) +{ + update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); + list_add_tail(&page->lru, &lruvec->lists[lru]); +} + +static __always_inline void del_page_from_lru_list(struct page *page, + struct lruvec *lruvec, enum lru_list lru) +{ + list_del(&page->lru); + update_lru_size(lruvec, lru, page_zonenum(page), -thp_nr_pages(page)); +} #endif From 3a9c9788a3149d9745b7eb2eae811e57ef3b127c Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:17 -0800 Subject: [PATCH 592/633] mm: don't pass "enum lru_list" to lru list addition functions The "enum lru_list" parameter to add_page_to_lru_list() and add_page_to_lru_list_tail() is redundant in the sense that it can be extracted from the "struct page" parameter by page_lru(). A caveat is that we need to make sure PageActive() or PageUnevictable() is correctly set or cleared before calling these two functions. And they are indeed. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-4-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-4-yuzhao@google.com Signed-off-by: Yu Zhao Cc: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 8 ++++++-- mm/swap.c | 15 +++++++-------- mm/vmscan.c | 6 ++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 2889741f450a..130ba3201d3f 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -106,15 +106,19 @@ static __always_inline enum lru_list page_lru(struct page *page) } static __always_inline void add_page_to_lru_list(struct page *page, - struct lruvec *lruvec, enum lru_list lru) + struct lruvec *lruvec) { + enum lru_list lru = page_lru(page); + update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); list_add(&page->lru, &lruvec->lists[lru]); } static __always_inline void add_page_to_lru_list_tail(struct page *page, - struct lruvec *lruvec, enum lru_list lru) + struct lruvec *lruvec) { + enum lru_list lru = page_lru(page); + update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page)); list_add_tail(&page->lru, &lruvec->lists[lru]); } diff --git a/mm/swap.c b/mm/swap.c index 2cca7141470c..22fad34b9c18 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -231,7 +231,7 @@ static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec) if (!PageUnevictable(page)) { del_page_from_lru_list(page, lruvec, page_lru(page)); ClearPageActive(page); - add_page_to_lru_list_tail(page, lruvec, page_lru(page)); + add_page_to_lru_list_tail(page, lruvec); __count_vm_events(PGROTATED, thp_nr_pages(page)); } } @@ -313,8 +313,7 @@ static void __activate_page(struct page *page, struct lruvec *lruvec) del_page_from_lru_list(page, lruvec, lru); SetPageActive(page); - lru += LRU_ACTIVE; - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); trace_mm_lru_activate(page); __count_vm_events(PGACTIVATE, nr_pages); @@ -543,14 +542,14 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) * It can make readahead confusing. But race window * is _really_ small and it's non-critical problem. */ - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); SetPageReclaim(page); } else { /* * The page's writeback ends up during pagevec * We moves tha page into tail of inactive. */ - add_page_to_lru_list_tail(page, lruvec, lru); + add_page_to_lru_list_tail(page, lruvec); __count_vm_events(PGROTATED, nr_pages); } @@ -570,7 +569,7 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec) del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); ClearPageActive(page); ClearPageReferenced(page); - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); __count_vm_events(PGDEACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, @@ -595,7 +594,7 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec) * anonymous pages */ ClearPageSwapBacked(page); - add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); + add_page_to_lru_list(page, lruvec); __count_vm_events(PGLAZYFREE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, @@ -1005,7 +1004,7 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); } - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); trace_mm_lru_insertion(page, lru); } diff --git a/mm/vmscan.c b/mm/vmscan.c index 19875660e8f8..09e4f97488c9 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1867,7 +1867,7 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, * inhibits memcg migration). */ VM_BUG_ON_PAGE(!lruvec_holds_page_lru_lock(page, lruvec), page); - add_page_to_lru_list(page, lruvec, page_lru(page)); + add_page_to_lru_list(page, lruvec); nr_pages = thp_nr_pages(page); nr_moved += nr_pages; if (PageActive(page)) @@ -4282,12 +4282,10 @@ void check_move_unevictable_pages(struct pagevec *pvec) lruvec = relock_page_lruvec_irq(page, lruvec); if (page_evictable(page) && PageUnevictable(page)) { - enum lru_list lru = page_lru_base_type(page); - VM_BUG_ON_PAGE(PageActive(page), page); ClearPageUnevictable(page); del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE); - add_page_to_lru_list(page, lruvec, lru); + add_page_to_lru_list(page, lruvec); pgrescued += nr_pages; } SetPageLRU(page); From 861404536a3af3c39f1b10959a40def3d8efa2dd Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:21 -0800 Subject: [PATCH 593/633] mm/swap.c: don't pass "enum lru_list" to trace_mm_lru_insertion() The parameter is redundant in the sense that it can be extracted from the "struct page" parameter by page_lru() correctly. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-5-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-5-yuzhao@google.com Signed-off-by: Yu Zhao Reviewed-by: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/trace/events/pagemap.h | 11 ++++------- mm/swap.c | 5 +---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 8fd1babae761..e1735fe7c76a 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h @@ -27,24 +27,21 @@ TRACE_EVENT(mm_lru_insertion, - TP_PROTO( - struct page *page, - int lru - ), + TP_PROTO(struct page *page), - TP_ARGS(page, lru), + TP_ARGS(page), TP_STRUCT__entry( __field(struct page *, page ) __field(unsigned long, pfn ) - __field(int, lru ) + __field(enum lru_list, lru ) __field(unsigned long, flags ) ), TP_fast_assign( __entry->page = page; __entry->pfn = page_to_pfn(page); - __entry->lru = lru; + __entry->lru = page_lru(page); __entry->flags = trace_pagemap_flags(page); ), diff --git a/mm/swap.c b/mm/swap.c index 22fad34b9c18..a26e3a4fe17b 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -957,7 +957,6 @@ EXPORT_SYMBOL(__pagevec_release); static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) { - enum lru_list lru; int was_unevictable = TestClearPageUnevictable(page); int nr_pages = thp_nr_pages(page); @@ -993,11 +992,9 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) smp_mb__after_atomic(); if (page_evictable(page)) { - lru = page_lru(page); if (was_unevictable) __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); } else { - lru = LRU_UNEVICTABLE; ClearPageActive(page); SetPageUnevictable(page); if (!was_unevictable) @@ -1005,7 +1002,7 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) } add_page_to_lru_list(page, lruvec); - trace_mm_lru_insertion(page, lru); + trace_mm_lru_insertion(page); } /* From 46ae6b2cc2a47904a368d238425531ea91f3a2a5 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:25 -0800 Subject: [PATCH 594/633] mm/swap.c: don't pass "enum lru_list" to del_page_from_lru_list() The parameter is redundant in the sense that it can be potentially extracted from the "struct page" parameter by page_lru(). We need to make sure that existing PageActive() or PageUnevictable() remains until the function returns. A few places don't conform, and simple reordering fixes them. This patch may have left page_off_lru() seemingly odd, and we'll take care of it in the next patch. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-6-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-6-yuzhao@google.com Signed-off-by: Yu Zhao Cc: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 5 +++-- mm/compaction.c | 2 +- mm/mlock.c | 3 +-- mm/swap.c | 26 ++++++++++---------------- mm/vmscan.c | 4 ++-- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 130ba3201d3f..ffacc6273678 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -124,9 +124,10 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, } static __always_inline void del_page_from_lru_list(struct page *page, - struct lruvec *lruvec, enum lru_list lru) + struct lruvec *lruvec) { list_del(&page->lru); - update_lru_size(lruvec, lru, page_zonenum(page), -thp_nr_pages(page)); + update_lru_size(lruvec, page_lru(page), page_zonenum(page), + -thp_nr_pages(page)); } #endif diff --git a/mm/compaction.c b/mm/compaction.c index 8f52532675a9..3aef10c61d0e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1034,7 +1034,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, low_pfn += compound_nr(page) - 1; /* Successfully isolated */ - del_page_from_lru_list(page, lruvec, page_lru(page)); + del_page_from_lru_list(page, lruvec); mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_is_file_lru(page), thp_nr_pages(page)); diff --git a/mm/mlock.c b/mm/mlock.c index 55b3b3672977..73960bb3464d 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -278,8 +278,7 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone) */ if (TestClearPageLRU(page)) { lruvec = relock_page_lruvec_irq(page, lruvec); - del_page_from_lru_list(page, lruvec, - page_lru(page)); + del_page_from_lru_list(page, lruvec); continue; } else __munlock_isolation_failed(page); diff --git a/mm/swap.c b/mm/swap.c index a26e3a4fe17b..ff910f636df2 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -85,7 +85,8 @@ static void __page_cache_release(struct page *page) lruvec = lock_page_lruvec_irqsave(page, &flags); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); - del_page_from_lru_list(page, lruvec, page_off_lru(page)); + del_page_from_lru_list(page, lruvec); + page_off_lru(page); unlock_page_lruvec_irqrestore(lruvec, flags); } __ClearPageWaiters(page); @@ -229,7 +230,7 @@ static void pagevec_lru_move_fn(struct pagevec *pvec, static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec) { if (!PageUnevictable(page)) { - del_page_from_lru_list(page, lruvec, page_lru(page)); + del_page_from_lru_list(page, lruvec); ClearPageActive(page); add_page_to_lru_list_tail(page, lruvec); __count_vm_events(PGROTATED, thp_nr_pages(page)); @@ -308,10 +309,9 @@ void lru_note_cost_page(struct page *page) static void __activate_page(struct page *page, struct lruvec *lruvec) { if (!PageActive(page) && !PageUnevictable(page)) { - int lru = page_lru_base_type(page); int nr_pages = thp_nr_pages(page); - del_page_from_lru_list(page, lruvec, lru); + del_page_from_lru_list(page, lruvec); SetPageActive(page); add_page_to_lru_list(page, lruvec); trace_mm_lru_activate(page); @@ -518,8 +518,7 @@ void lru_cache_add_inactive_or_unevictable(struct page *page, */ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) { - int lru; - bool active; + bool active = PageActive(page); int nr_pages = thp_nr_pages(page); if (PageUnevictable(page)) @@ -529,10 +528,7 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) if (page_mapped(page)) return; - active = PageActive(page); - lru = page_lru_base_type(page); - - del_page_from_lru_list(page, lruvec, lru + active); + del_page_from_lru_list(page, lruvec); ClearPageActive(page); ClearPageReferenced(page); @@ -563,10 +559,9 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec) { if (PageActive(page) && !PageUnevictable(page)) { - int lru = page_lru_base_type(page); int nr_pages = thp_nr_pages(page); - del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); + del_page_from_lru_list(page, lruvec); ClearPageActive(page); ClearPageReferenced(page); add_page_to_lru_list(page, lruvec); @@ -581,11 +576,9 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec) { if (PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page) && !PageUnevictable(page)) { - bool active = PageActive(page); int nr_pages = thp_nr_pages(page); - del_page_from_lru_list(page, lruvec, - LRU_INACTIVE_ANON + active); + del_page_from_lru_list(page, lruvec); ClearPageActive(page); ClearPageReferenced(page); /* @@ -919,7 +912,8 @@ void release_pages(struct page **pages, int nr) VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); - del_page_from_lru_list(page, lruvec, page_off_lru(page)); + del_page_from_lru_list(page, lruvec); + page_off_lru(page); } __ClearPageWaiters(page); diff --git a/mm/vmscan.c b/mm/vmscan.c index 09e4f97488c9..7c65d47e6612 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1766,7 +1766,7 @@ int isolate_lru_page(struct page *page) get_page(page); lruvec = lock_page_lruvec_irq(page); - del_page_from_lru_list(page, lruvec, page_lru(page)); + del_page_from_lru_list(page, lruvec); unlock_page_lruvec_irq(lruvec); ret = 0; } @@ -4283,8 +4283,8 @@ void check_move_unevictable_pages(struct pagevec *pvec) lruvec = relock_page_lruvec_irq(page, lruvec); if (page_evictable(page) && PageUnevictable(page)) { VM_BUG_ON_PAGE(PageActive(page), page); + del_page_from_lru_list(page, lruvec); ClearPageUnevictable(page); - del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE); add_page_to_lru_list(page, lruvec); pgrescued += nr_pages; } From 875601796267214f286d3581fe74f2805d060fe8 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:28 -0800 Subject: [PATCH 595/633] mm: add __clear_page_lru_flags() to replace page_off_lru() Similar to page_off_lru(), the new function does non-atomic clearing of PageLRU() in addition to PageActive() and PageUnevictable(), on a page that has no references left. If PageActive() and PageUnevictable() are both set, refuse to clear either and leave them to bad_page(). This is a behavior change that is meant to help debug. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-7-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-7-yuzhao@google.com Signed-off-by: Yu Zhao Cc: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 28 ++++++++++------------------ mm/swap.c | 6 ++---- mm/vmscan.c | 3 +-- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index ffacc6273678..ef3fd79222e5 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -61,27 +61,19 @@ static inline enum lru_list page_lru_base_type(struct page *page) } /** - * page_off_lru - which LRU list was page on? clearing its lru flags. - * @page: the page to test - * - * Returns the LRU list a page was on, as an index into the array of LRU - * lists; and clears its Unevictable or Active flags, ready for freeing. + * __clear_page_lru_flags - clear page lru flags before releasing a page + * @page: the page that was on lru and now has a zero reference */ -static __always_inline enum lru_list page_off_lru(struct page *page) +static __always_inline void __clear_page_lru_flags(struct page *page) { - enum lru_list lru; + __ClearPageLRU(page); - if (PageUnevictable(page)) { - __ClearPageUnevictable(page); - lru = LRU_UNEVICTABLE; - } else { - lru = page_lru_base_type(page); - if (PageActive(page)) { - __ClearPageActive(page); - lru += LRU_ACTIVE; - } - } - return lru; + /* this shouldn't happen, so leave the flags to bad_page() */ + if (PageActive(page) && PageUnevictable(page)) + return; + + __ClearPageActive(page); + __ClearPageUnevictable(page); } /** diff --git a/mm/swap.c b/mm/swap.c index ff910f636df2..1455fa56a0ee 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -84,9 +84,8 @@ static void __page_cache_release(struct page *page) lruvec = lock_page_lruvec_irqsave(page, &flags); VM_BUG_ON_PAGE(!PageLRU(page), page); - __ClearPageLRU(page); del_page_from_lru_list(page, lruvec); - page_off_lru(page); + __clear_page_lru_flags(page); unlock_page_lruvec_irqrestore(lruvec, flags); } __ClearPageWaiters(page); @@ -911,9 +910,8 @@ void release_pages(struct page **pages, int nr) lock_batch = 0; VM_BUG_ON_PAGE(!PageLRU(page), page); - __ClearPageLRU(page); del_page_from_lru_list(page, lruvec); - page_off_lru(page); + __clear_page_lru_flags(page); } __ClearPageWaiters(page); diff --git a/mm/vmscan.c b/mm/vmscan.c index 7c65d47e6612..452dd3818aa3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1849,8 +1849,7 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, SetPageLRU(page); if (unlikely(put_page_testzero(page))) { - __ClearPageLRU(page); - __ClearPageActive(page); + __clear_page_lru_flags(page); if (unlikely(PageCompound(page))) { spin_unlock_irq(&lruvec->lru_lock); From bc7112719e1e80e4208eef3fc9bd8d2b6c263e7d Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:32 -0800 Subject: [PATCH 596/633] mm: VM_BUG_ON lru page flags Move scattered VM_BUG_ONs to two essential places that cover all lru list additions and deletions. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-8-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-8-yuzhao@google.com Signed-off-by: Yu Zhao Cc: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 4 ++++ mm/swap.c | 2 -- mm/vmscan.c | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index ef3fd79222e5..6d907a4dd6ad 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -66,6 +66,8 @@ static inline enum lru_list page_lru_base_type(struct page *page) */ static __always_inline void __clear_page_lru_flags(struct page *page) { + VM_BUG_ON_PAGE(!PageLRU(page), page); + __ClearPageLRU(page); /* this shouldn't happen, so leave the flags to bad_page() */ @@ -87,6 +89,8 @@ static __always_inline enum lru_list page_lru(struct page *page) { enum lru_list lru; + VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); + if (PageUnevictable(page)) lru = LRU_UNEVICTABLE; else { diff --git a/mm/swap.c b/mm/swap.c index 1455fa56a0ee..ab3258afcbeb 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -83,7 +83,6 @@ static void __page_cache_release(struct page *page) unsigned long flags; lruvec = lock_page_lruvec_irqsave(page, &flags); - VM_BUG_ON_PAGE(!PageLRU(page), page); del_page_from_lru_list(page, lruvec); __clear_page_lru_flags(page); unlock_page_lruvec_irqrestore(lruvec, flags); @@ -909,7 +908,6 @@ void release_pages(struct page **pages, int nr) if (prev_lruvec != lruvec) lock_batch = 0; - VM_BUG_ON_PAGE(!PageLRU(page), page); del_page_from_lru_list(page, lruvec); __clear_page_lru_flags(page); } diff --git a/mm/vmscan.c b/mm/vmscan.c index 452dd3818aa3..348a90096550 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4281,7 +4281,6 @@ void check_move_unevictable_pages(struct pagevec *pvec) lruvec = relock_page_lruvec_irq(page, lruvec); if (page_evictable(page) && PageUnevictable(page)) { - VM_BUG_ON_PAGE(PageActive(page), page); del_page_from_lru_list(page, lruvec); ClearPageUnevictable(page); add_page_to_lru_list(page, lruvec); From c1770e34f3e7640887d8129fc05d13fe17101301 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:36 -0800 Subject: [PATCH 597/633] include/linux/mm_inline.h: fold page_lru_base_type() into its sole caller We've removed all other references to this function. Link: https://lore.kernel.org/linux-mm/20201207220949.830352-9-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-9-yuzhao@google.com Signed-off-by: Yu Zhao Reviewed-by: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 6d907a4dd6ad..7183c7a03f09 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -45,21 +45,6 @@ static __always_inline void update_lru_size(struct lruvec *lruvec, #endif } -/** - * page_lru_base_type - which LRU list type should a page be on? - * @page: the page to test - * - * Used for LRU list index arithmetic. - * - * Returns the base LRU type - file or anon - @page should be on. - */ -static inline enum lru_list page_lru_base_type(struct page *page) -{ - if (page_is_file_lru(page)) - return LRU_INACTIVE_FILE; - return LRU_INACTIVE_ANON; -} - /** * __clear_page_lru_flags - clear page lru flags before releasing a page * @page: the page that was on lru and now has a zero reference @@ -92,12 +77,12 @@ static __always_inline enum lru_list page_lru(struct page *page) VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); if (PageUnevictable(page)) - lru = LRU_UNEVICTABLE; - else { - lru = page_lru_base_type(page); - if (PageActive(page)) - lru += LRU_ACTIVE; - } + return LRU_UNEVICTABLE; + + lru = page_is_file_lru(page) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON; + if (PageActive(page)) + lru += LRU_ACTIVE; + return lru; } From 289ccba18af436f2b65ec69b2be1b086ec9f24a4 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:40 -0800 Subject: [PATCH 598/633] include/linux/mm_inline.h: fold __update_lru_size() into its sole caller All other references to the function were removed after commit a892cb6b977f ("mm/vmscan.c: use update_lru_size() in update_lru_sizes()"). Link: https://lore.kernel.org/linux-mm/20201207220949.830352-10-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-10-yuzhao@google.com Signed-off-by: Yu Zhao Reviewed-by: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm_inline.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 7183c7a03f09..355ea1ee32bd 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -24,7 +24,7 @@ static inline int page_is_file_lru(struct page *page) return !PageSwapBacked(page); } -static __always_inline void __update_lru_size(struct lruvec *lruvec, +static __always_inline void update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, int nr_pages) { @@ -33,13 +33,6 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec, __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); __mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LRU_BASE + lru, nr_pages); -} - -static __always_inline void update_lru_size(struct lruvec *lruvec, - enum lru_list lru, enum zone_type zid, - int nr_pages) -{ - __update_lru_size(lruvec, lru, zid, nr_pages); #ifdef CONFIG_MEMCG mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); #endif From 2091339d59e7808e9b39a79f48e3d17ef7389b97 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Wed, 24 Feb 2021 12:08:44 -0800 Subject: [PATCH 599/633] mm/vmscan.c: make lruvec_lru_size() static All other references to the function were removed after commit b910718a948a ("mm: vmscan: detect file thrashing at the reclaim root"). Link: https://lore.kernel.org/linux-mm/20201207220949.830352-11-yuzhao@google.com/ Link: https://lkml.kernel.org/r/20210122220600.906146-11-yuzhao@google.com Signed-off-by: Yu Zhao Reviewed-by: Alex Shi Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Roman Gushchin Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mmzone.h | 2 -- mm/vmscan.c | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index fc99e9241846..9198b7ade85f 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -892,8 +892,6 @@ static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) #endif } -extern unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); - #ifdef CONFIG_HAVE_MEMORYLESS_NODES int local_memory_node(int node_id); #else diff --git a/mm/vmscan.c b/mm/vmscan.c index 348a90096550..fea6b43bc1f9 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -310,7 +310,8 @@ unsigned long zone_reclaimable_pages(struct zone *zone) * @lru: lru to use * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list) */ -unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx) +static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, + int zone_idx) { unsigned long size = 0; int zid; From aeddcee6c17bd8cf80675495d39c4daceaf5b506 Mon Sep 17 00:00:00 2001 From: Oscar Salvador Date: Wed, 24 Feb 2021 12:08:47 -0800 Subject: [PATCH 600/633] mm: workingset: clarify eviction order and distance calculation The premise of the refault distance is that it can be seen as a deficit of the inactive list space, so that if the inactive list would have had (R - E) more slots, the page would not have been evicted but promoted to the active list instead. However, the way the code is ordered right now set us to be off by one, so the real number of slots would be (R - E) + 1. I stumbled upon this when trying to understand the code and it puzzled me that the comments did not match what the code did. This it not an issue at all since evictions and refaults tend to happen in a number large enough that being off-by-one does not have any impact - and since the compiler and CPUs are free to rearrange the execution sequence anyway. But as Johannes says, it is better to re-arrange the code in the proper order since otherwise would be misleading to somebody who is actively reading and trying to understand the logic of the code - like it happened to me. Link: https://lkml.kernel.org/r/20210201060651.3781-1-osalvador@suse.de Signed-off-by: Oscar Salvador Acked-by: Johannes Weiner Acked-by: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/workingset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/workingset.c b/mm/workingset.c index 7db8f3dad13c..cd39902c1062 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -263,10 +263,10 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg) VM_BUG_ON_PAGE(!PageLocked(page), page); lruvec = mem_cgroup_lruvec(target_memcg, pgdat); - workingset_age_nonresident(lruvec, thp_nr_pages(page)); /* XXX: target_memcg can be NULL, go through lruvec */ memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); eviction = atomic_long_read(&lruvec->nonresident_age); + workingset_age_nonresident(lruvec, thp_nr_pages(page)); return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page)); } From d6995da311221a05c8aef3bda2629e5cb14c7302 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:08:51 -0800 Subject: [PATCH 601/633] hugetlb: use page.private for hugetlb specific page flags Patch series "create hugetlb flags to consolidate state", v3. While discussing a series of hugetlb fixes in [1], it became evident that the hugetlb specific page state information is stored in a somewhat haphazard manner. Code dealing with state information would be easier to read, understand and maintain if this information was stored in a consistent manner. This series uses page.private of the hugetlb head page for storing a set of hugetlb specific page flags. Routines are priovided for test, set and clear of the flags. [1] https://lore.kernel.org/r/20210106084739.63318-1-songmuchun@bytedance.com This patch (of 4): As hugetlbfs evolved, state information about hugetlb pages was added. One 'convenient' way of doing this was to use available fields in tail pages. Over time, it has become difficult to know the meaning or contents of fields simply by looking at a small bit of code. Sometimes, the naming is just confusing. For example: The PagePrivate flag indicates a huge page reservation was consumed and needs to be restored if an error is encountered and the page is freed before it is instantiated. The page.private field contains the pointer to a subpool if the page is associated with one. In an effort to make the code more readable, use page.private to contain hugetlb specific page flags. These flags will have test, set and clear functions similar to those used for 'normal' page flags. More importantly, an enum of flag values will be created with names that actually reflect their purpose. In this patch, - Create infrastructure for hugetlb specific page flag functions - Move subpool pointer to page[1].private to make way for flags Create routines with meaningful names to modify subpool field - Use new HPageRestoreReserve flag instead of PagePrivate Conversion of other state information will happen in subsequent patches. Link: https://lkml.kernel.org/r/20210122195231.324857-1-mike.kravetz@oracle.com Link: https://lkml.kernel.org/r/20210122195231.324857-2-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Acked-by: Michal Hocko Cc: Naoya Horiguchi Cc: Muchun Song Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Miaohe Lin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 12 ++------ include/linux/hugetlb.h | 68 +++++++++++++++++++++++++++++++++++++++++ mm/hugetlb.c | 48 +++++++++++++++-------------- 3 files changed, 96 insertions(+), 32 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index b7a72f577aab..907f6405e805 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -973,15 +973,9 @@ static int hugetlbfs_migrate_page(struct address_space *mapping, if (rc != MIGRATEPAGE_SUCCESS) return rc; - /* - * page_private is subpool pointer in hugetlb pages. Transfer to - * new page. PagePrivate is not associated with page_private for - * hugetlb pages and can not be set here as only page_huge_active - * pages can be migrated. - */ - if (page_private(page)) { - set_page_private(newpage, page_private(page)); - set_page_private(page, 0); + if (hugetlb_page_subpool(page)) { + hugetlb_set_page_subpool(newpage, hugetlb_page_subpool(page)); + hugetlb_set_page_subpool(page, NULL); } if (mode != MIGRATE_SYNC_NO_COPY) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ac3cb239a7ec..249c65e1d8ca 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -472,6 +472,60 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long flags); #endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */ +/* + * huegtlb page specific state flags. These flags are located in page.private + * of the hugetlb head page. Functions created via the below macros should be + * used to manipulate these flags. + * + * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at + * allocation time. Cleared when page is fully instantiated. Free + * routine checks flag to restore a reservation on error paths. + */ +enum hugetlb_page_flags { + HPG_restore_reserve = 0, + __NR_HPAGEFLAGS, +}; + +/* + * Macros to create test, set and clear function definitions for + * hugetlb specific page flags. + */ +#ifdef CONFIG_HUGETLB_PAGE +#define TESTHPAGEFLAG(uname, flname) \ +static inline int HPage##uname(struct page *page) \ + { return test_bit(HPG_##flname, &(page->private)); } + +#define SETHPAGEFLAG(uname, flname) \ +static inline void SetHPage##uname(struct page *page) \ + { set_bit(HPG_##flname, &(page->private)); } + +#define CLEARHPAGEFLAG(uname, flname) \ +static inline void ClearHPage##uname(struct page *page) \ + { clear_bit(HPG_##flname, &(page->private)); } +#else +#define TESTHPAGEFLAG(uname, flname) \ +static inline int HPage##uname(struct page *page) \ + { return 0; } + +#define SETHPAGEFLAG(uname, flname) \ +static inline void SetHPage##uname(struct page *page) \ + { } + +#define CLEARHPAGEFLAG(uname, flname) \ +static inline void ClearHPage##uname(struct page *page) \ + { } +#endif + +#define HPAGEFLAG(uname, flname) \ + TESTHPAGEFLAG(uname, flname) \ + SETHPAGEFLAG(uname, flname) \ + CLEARHPAGEFLAG(uname, flname) \ + +/* + * Create functions associated with hugetlb page flags + */ +HPAGEFLAG(RestoreReserve, restore_reserve) + #ifdef CONFIG_HUGETLB_PAGE #define HSTATE_NAME_LEN 32 @@ -531,6 +585,20 @@ extern unsigned int default_hstate_idx; #define default_hstate (hstates[default_hstate_idx]) +/* + * hugetlb page subpool pointer located in hpage[1].private + */ +static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) +{ + return (struct hugepage_subpool *)(hpage+1)->private; +} + +static inline void hugetlb_set_page_subpool(struct page *hpage, + struct hugepage_subpool *subpool) +{ + set_page_private(hpage+1, (unsigned long)subpool); +} + static inline struct hstate *hstate_file(struct file *f) { return hstate_inode(file_inode(f)); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8a1d1f85e21e..f5e85dabb7a3 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1143,7 +1143,7 @@ static struct page *dequeue_huge_page_vma(struct hstate *h, nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask); page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask); if (page && !avoid_reserve && vma_has_reserves(vma, chg)) { - SetPagePrivate(page); + SetHPageRestoreReserve(page); h->resv_huge_pages--; } @@ -1418,20 +1418,19 @@ static void __free_huge_page(struct page *page) */ struct hstate *h = page_hstate(page); int nid = page_to_nid(page); - struct hugepage_subpool *spool = - (struct hugepage_subpool *)page_private(page); + struct hugepage_subpool *spool = hugetlb_page_subpool(page); bool restore_reserve; VM_BUG_ON_PAGE(page_count(page), page); VM_BUG_ON_PAGE(page_mapcount(page), page); - set_page_private(page, 0); + hugetlb_set_page_subpool(page, NULL); page->mapping = NULL; - restore_reserve = PagePrivate(page); - ClearPagePrivate(page); + restore_reserve = HPageRestoreReserve(page); + ClearHPageRestoreReserve(page); /* - * If PagePrivate() was set on page, page allocation consumed a + * If HPageRestoreReserve was set on page, page allocation consumed a * reservation. If the page was associated with a subpool, there * would have been a page reserved in the subpool before allocation * via hugepage_subpool_get_pages(). Since we are 'restoring' the @@ -2263,24 +2262,24 @@ static long vma_add_reservation(struct hstate *h, * This routine is called to restore a reservation on error paths. In the * specific error paths, a huge page was allocated (via alloc_huge_page) * and is about to be freed. If a reservation for the page existed, - * alloc_huge_page would have consumed the reservation and set PagePrivate - * in the newly allocated page. When the page is freed via free_huge_page, - * the global reservation count will be incremented if PagePrivate is set. - * However, free_huge_page can not adjust the reserve map. Adjust the - * reserve map here to be consistent with global reserve count adjustments - * to be made by free_huge_page. + * alloc_huge_page would have consumed the reservation and set + * HPageRestoreReserve in the newly allocated page. When the page is freed + * via free_huge_page, the global reservation count will be incremented if + * HPageRestoreReserve is set. However, free_huge_page can not adjust the + * reserve map. Adjust the reserve map here to be consistent with global + * reserve count adjustments to be made by free_huge_page. */ static void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct page *page) { - if (unlikely(PagePrivate(page))) { + if (unlikely(HPageRestoreReserve(page))) { long rc = vma_needs_reservation(h, vma, address); if (unlikely(rc < 0)) { /* * Rare out of memory condition in reserve map - * manipulation. Clear PagePrivate so that + * manipulation. Clear HPageRestoreReserve so that * global reserve count will not be incremented * by free_huge_page. This will make it appear * as though the reservation for this page was @@ -2289,7 +2288,7 @@ static void restore_reserve_on_error(struct hstate *h, * is better than inconsistent global huge page * accounting of reserve counts. */ - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); } else if (rc) { rc = vma_add_reservation(h, vma, address); if (unlikely(rc < 0)) @@ -2297,7 +2296,7 @@ static void restore_reserve_on_error(struct hstate *h, * See above comment about rare out of * memory condition. */ - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); } else vma_end_reservation(h, vma, address); } @@ -2378,7 +2377,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, if (!page) goto out_uncharge_cgroup; if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { - SetPagePrivate(page); + SetHPageRestoreReserve(page); h->resv_huge_pages--; } spin_lock(&hugetlb_lock); @@ -2396,7 +2395,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, spin_unlock(&hugetlb_lock); - set_page_private(page, (unsigned long)spool); + hugetlb_set_page_subpool(page, spool); map_commit = vma_commit_reservation(h, vma, addr); if (unlikely(map_chg > map_commit)) { @@ -3170,6 +3169,9 @@ static int __init hugetlb_init(void) { int i; + BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE < + __NR_HPAGEFLAGS); + if (!hugepages_supported()) { if (hugetlb_max_hstate || default_hstate_max_huge_pages) pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); @@ -4207,7 +4209,7 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, spin_lock(ptl); ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) { - ClearPagePrivate(new_page); + ClearHPageRestoreReserve(new_page); /* Break COW */ huge_ptep_clear_flush(vma, haddr, ptep); @@ -4274,7 +4276,7 @@ int huge_add_to_page_cache(struct page *page, struct address_space *mapping, if (err) return err; - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); /* * set page dirty so that it will not be removed from cache/file @@ -4436,7 +4438,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, goto backout; if (anon_rmap) { - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); hugepage_add_new_anon_rmap(page, vma, haddr); } else page_dup_rmap(page, true); @@ -4750,7 +4752,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, if (vm_shared) { page_dup_rmap(page, true); } else { - ClearPagePrivate(page); + ClearHPageRestoreReserve(page); hugepage_add_new_anon_rmap(page, dst_vma, dst_addr); } From 8f251a3d5ce3bdea73bd045ed35db64f32e0d0d9 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:08:56 -0800 Subject: [PATCH 602/633] hugetlb: convert page_huge_active() HPageMigratable flag Use the new hugetlb page specific flag HPageMigratable to replace the page_huge_active interfaces. By it's name, page_huge_active implied that a huge page was on the active list. However, that is not really what code checking the flag wanted to know. It really wanted to determine if the huge page could be migrated. This happens when the page is actually added to the page cache and/or task page table. This is the reasoning behind the name change. The VM_BUG_ON_PAGE() calls in the *_huge_active() interfaces are not really necessary as we KNOW the page is a hugetlb page. Therefore, they are removed. The routine page_huge_active checked for PageHeadHuge before testing the active bit. This is unnecessary in the case where we hold a reference or lock and know it is a hugetlb head page. page_huge_active is also called without holding a reference or lock (scan_movable_pages), and can race with code freeing the page. The extra check in page_huge_active shortened the race window, but did not prevent the race. Offline code calling scan_movable_pages already deals with these races, so removing the check is acceptable. Add comment to racy code. [songmuchun@bytedance.com: remove set_page_huge_active() declaration from include/linux/hugetlb.h] Link: https://lkml.kernel.org/r/CAMZfGtUda+KoAZscU0718TN61cSFwp4zy=y2oZ=+6Z2TAZZwng@mail.gmail.com Link: https://lkml.kernel.org/r/20210122195231.324857-3-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Reviewed-by: Miaohe Lin Acked-by: Michal Hocko Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Naoya Horiguchi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- include/linux/hugetlb.h | 7 ++++-- include/linux/page-flags.h | 6 ----- mm/hugetlb.c | 45 ++++++++++---------------------------- mm/memory_hotplug.c | 9 +++++++- 5 files changed, 25 insertions(+), 44 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 907f6405e805..5a8ed6bd4f87 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -735,7 +735,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, mutex_unlock(&hugetlb_fault_mutex_table[hash]); - set_page_huge_active(page); + SetHPageMigratable(page); /* * unlock_page because locked by add_to_page_cache() * put_page() due to reference from alloc_huge_page() diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 249c65e1d8ca..77f2a032fe32 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -480,9 +480,13 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at * allocation time. Cleared when page is fully instantiated. Free * routine checks flag to restore a reservation on error paths. + * HPG_migratable - Set after a newly allocated page is added to the page + * cache and/or page tables. Indicates the page is a candidate for + * migration. */ enum hugetlb_page_flags { HPG_restore_reserve = 0, + HPG_migratable, __NR_HPAGEFLAGS, }; @@ -525,6 +529,7 @@ static inline void ClearHPage##uname(struct page *page) \ * Create functions associated with hugetlb page flags */ HPAGEFLAG(RestoreReserve, restore_reserve) +HPAGEFLAG(Migratable, migratable) #ifdef CONFIG_HUGETLB_PAGE @@ -838,8 +843,6 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, } #endif -void set_page_huge_active(struct page *page); - #else /* CONFIG_HUGETLB_PAGE */ struct hstate {}; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index ec5d0290e0ee..db914477057b 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -592,15 +592,9 @@ static inline void ClearPageCompound(struct page *page) #ifdef CONFIG_HUGETLB_PAGE int PageHuge(struct page *page); int PageHeadHuge(struct page *page); -bool page_huge_active(struct page *page); #else TESTPAGEFLAG_FALSE(Huge) TESTPAGEFLAG_FALSE(HeadHuge) - -static inline bool page_huge_active(struct page *page) -{ - return 0; -} #endif diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f5e85dabb7a3..727c09713627 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1364,30 +1364,6 @@ struct hstate *size_to_hstate(unsigned long size) return NULL; } -/* - * Test to determine whether the hugepage is "active/in-use" (i.e. being linked - * to hstate->hugepage_activelist.) - * - * This function can be called for tail pages, but never returns true for them. - */ -bool page_huge_active(struct page *page) -{ - return PageHeadHuge(page) && PagePrivate(&page[1]); -} - -/* never called for tail page */ -void set_page_huge_active(struct page *page) -{ - VM_BUG_ON_PAGE(!PageHeadHuge(page), page); - SetPagePrivate(&page[1]); -} - -static void clear_page_huge_active(struct page *page) -{ - VM_BUG_ON_PAGE(!PageHeadHuge(page), page); - ClearPagePrivate(&page[1]); -} - /* * Internal hugetlb specific page flag. Do not use outside of the hugetlb * code @@ -1449,7 +1425,7 @@ static void __free_huge_page(struct page *page) } spin_lock(&hugetlb_lock); - clear_page_huge_active(page); + ClearHPageMigratable(page); hugetlb_cgroup_uncharge_page(hstate_index(h), pages_per_huge_page(h), page); hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h), @@ -4218,7 +4194,7 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, make_huge_pte(vma, new_page, 1)); page_remove_rmap(old_page, true); hugepage_add_new_anon_rmap(new_page, vma, haddr); - set_page_huge_active(new_page); + SetHPageMigratable(new_page); /* Make the old page be freed below */ new_page = old_page; } @@ -4455,12 +4431,12 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, spin_unlock(ptl); /* - * Only make newly allocated pages active. Existing pages found - * in the pagecache could be !page_huge_active() if they have been - * isolated for migration. + * Only set HPageMigratable in newly allocated pages. Existing pages + * found in the pagecache may not have HPageMigratableset if they have + * been isolated for migration. */ if (new_page) - set_page_huge_active(page); + SetHPageMigratable(page); unlock_page(page); out: @@ -4771,7 +4747,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, update_mmu_cache(dst_vma, dst_addr, dst_pte); spin_unlock(ptl); - set_page_huge_active(page); + SetHPageMigratable(page); if (vm_shared) unlock_page(page); ret = 0; @@ -5610,12 +5586,13 @@ bool isolate_huge_page(struct page *page, struct list_head *list) bool ret = true; spin_lock(&hugetlb_lock); - if (!PageHeadHuge(page) || !page_huge_active(page) || + if (!PageHeadHuge(page) || + !HPageMigratable(page) || !get_page_unless_zero(page)) { ret = false; goto unlock; } - clear_page_huge_active(page); + ClearHPageMigratable(page); list_move_tail(&page->lru, list); unlock: spin_unlock(&hugetlb_lock); @@ -5625,7 +5602,7 @@ bool isolate_huge_page(struct page *page, struct list_head *list) void putback_active_hugepage(struct page *page) { spin_lock(&hugetlb_lock); - set_page_huge_active(page); + SetHPageMigratable(page); list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); spin_unlock(&hugetlb_lock); put_page(page); diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index ddcb1cd24c60..abe43c1ae920 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1260,7 +1260,14 @@ static int scan_movable_pages(unsigned long start, unsigned long end, if (!PageHuge(page)) continue; head = compound_head(page); - if (page_huge_active(head)) + /* + * This test is racy as we hold no reference or lock. The + * hugetlb page could have been free'ed and head is no longer + * a hugetlb page before the following check. In such unlikely + * cases false positives and negatives are possible. Calling + * code must deal with these scenarios. + */ + if (HPageMigratable(head)) goto found; skip = compound_nr(head) - (page - head); pfn += skip - 1; From 9157c31186c358c5750dea50ac5705d61d7fc917 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:00 -0800 Subject: [PATCH 603/633] hugetlb: convert PageHugeTemporary() to HPageTemporary flag Use new hugetlb specific HPageTemporary flag to replace the PageHugeTemporary() interfaces. PageHugeTemporary does contain a PageHuge() check. However, this interface is only used within hugetlb code where we know we are dealing with a hugetlb page. Therefore, the check can be eliminated. Link: https://lkml.kernel.org/r/20210122195231.324857-5-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Acked-by: Michal Hocko Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Naoya Horiguchi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/hugetlb.h | 6 ++++++ mm/hugetlb.c | 36 +++++++----------------------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 77f2a032fe32..e97498a21010 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -483,10 +483,15 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, * HPG_migratable - Set after a newly allocated page is added to the page * cache and/or page tables. Indicates the page is a candidate for * migration. + * HPG_temporary - - Set on a page that is temporarily allocated from the buddy + * allocator. Typically used for migration target pages when no pages + * are available in the pool. The hugetlb free page path will + * immediately free pages with this flag set to the buddy allocator. */ enum hugetlb_page_flags { HPG_restore_reserve = 0, HPG_migratable, + HPG_temporary, __NR_HPAGEFLAGS, }; @@ -530,6 +535,7 @@ static inline void ClearHPage##uname(struct page *page) \ */ HPAGEFLAG(RestoreReserve, restore_reserve) HPAGEFLAG(Migratable, migratable) +HPAGEFLAG(Temporary, temporary) #ifdef CONFIG_HUGETLB_PAGE diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 727c09713627..4d7d4535a313 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1364,28 +1364,6 @@ struct hstate *size_to_hstate(unsigned long size) return NULL; } -/* - * Internal hugetlb specific page flag. Do not use outside of the hugetlb - * code - */ -static inline bool PageHugeTemporary(struct page *page) -{ - if (!PageHuge(page)) - return false; - - return (unsigned long)page[2].mapping == -1U; -} - -static inline void SetPageHugeTemporary(struct page *page) -{ - page[2].mapping = (void *)-1U; -} - -static inline void ClearPageHugeTemporary(struct page *page) -{ - page[2].mapping = NULL; -} - static void __free_huge_page(struct page *page) { /* @@ -1433,9 +1411,9 @@ static void __free_huge_page(struct page *page) if (restore_reserve) h->resv_huge_pages++; - if (PageHugeTemporary(page)) { + if (HPageTemporary(page)) { list_del(&page->lru); - ClearPageHugeTemporary(page); + ClearHPageTemporary(page); update_and_free_page(h, page); } else if (h->surplus_huge_pages_node[nid]) { /* remove the page from active list */ @@ -1869,7 +1847,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, * codeflow */ if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { - SetPageHugeTemporary(page); + SetHPageTemporary(page); spin_unlock(&hugetlb_lock); put_page(page); return NULL; @@ -1900,7 +1878,7 @@ static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask, * We do not account these pages as surplus because they are only * temporary and will be released properly on the last reference */ - SetPageHugeTemporary(page); + SetHPageTemporary(page); return page; } @@ -5625,12 +5603,12 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) * here as well otherwise the global surplus count will not match * the per-node's. */ - if (PageHugeTemporary(newpage)) { + if (HPageTemporary(newpage)) { int old_nid = page_to_nid(oldpage); int new_nid = page_to_nid(newpage); - SetPageHugeTemporary(oldpage); - ClearPageHugeTemporary(newpage); + SetHPageTemporary(oldpage); + ClearHPageTemporary(newpage); spin_lock(&hugetlb_lock); if (h->surplus_huge_pages_node[old_nid]) { From 6c037149014027d50175da5be4ae4531374dcbe0 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:04 -0800 Subject: [PATCH 604/633] hugetlb: convert PageHugeFreed to HPageFreed flag Use new hugetlb specific HPageFreed flag to replace the PageHugeFreed interfaces. Link: https://lkml.kernel.org/r/20210122195231.324857-6-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Oscar Salvador Reviewed-by: Muchun Song Acked-by: Michal Hocko Cc: David Hildenbrand Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Michal Hocko Cc: Naoya Horiguchi Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 23 ++++------------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e97498a21010..eb2682fd97b6 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -487,11 +487,13 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, * allocator. Typically used for migration target pages when no pages * are available in the pool. The hugetlb free page path will * immediately free pages with this flag set to the buddy allocator. + * HPG_freed - Set when page is on the free lists. */ enum hugetlb_page_flags { HPG_restore_reserve = 0, HPG_migratable, HPG_temporary, + HPG_freed, __NR_HPAGEFLAGS, }; @@ -536,6 +538,7 @@ static inline void ClearHPage##uname(struct page *page) \ HPAGEFLAG(RestoreReserve, restore_reserve) HPAGEFLAG(Migratable, migratable) HPAGEFLAG(Temporary, temporary) +HPAGEFLAG(Freed, freed) #ifdef CONFIG_HUGETLB_PAGE diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4d7d4535a313..5377e1dad044 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -79,21 +79,6 @@ DEFINE_SPINLOCK(hugetlb_lock); static int num_fault_mutexes; struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; -static inline bool PageHugeFreed(struct page *head) -{ - return page_private(head + 4) == -1UL; -} - -static inline void SetPageHugeFreed(struct page *head) -{ - set_page_private(head + 4, -1UL); -} - -static inline void ClearPageHugeFreed(struct page *head) -{ - set_page_private(head + 4, 0); -} - /* Forward declaration */ static int hugetlb_acct_memory(struct hstate *h, long delta); @@ -1053,7 +1038,7 @@ static void enqueue_huge_page(struct hstate *h, struct page *page) list_move(&page->lru, &h->hugepage_freelists[nid]); h->free_huge_pages++; h->free_huge_pages_node[nid]++; - SetPageHugeFreed(page); + SetHPageFreed(page); } static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) @@ -1070,7 +1055,7 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid) list_move(&page->lru, &h->hugepage_activelist); set_page_refcounted(page); - ClearPageHugeFreed(page); + ClearHPageFreed(page); h->free_huge_pages--; h->free_huge_pages_node[nid]--; return page; @@ -1485,7 +1470,7 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) spin_lock(&hugetlb_lock); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++; - ClearPageHugeFreed(page); + ClearHPageFreed(page); spin_unlock(&hugetlb_lock); } @@ -1756,7 +1741,7 @@ int dissolve_free_huge_page(struct page *page) * We should make sure that the page is already on the free list * when it is dissolved. */ - if (unlikely(!PageHugeFreed(head))) { + if (unlikely(!HPageFreed(head))) { spin_unlock(&hugetlb_lock); cond_resched(); From d95c0337774b1dc74d271e7475a96fe8838332ea Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:08 -0800 Subject: [PATCH 605/633] include/linux/hugetlb.h: add synchronization information for new hugetlb specific flags Add comments, no functional change. Link: https://lkml.kernel.org/r/62a80585-2a73-10cc-4a2d-5721540d4ad2@oracle.com Signed-off-by: Mike Kravetz Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/hugetlb.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index eb2682fd97b6..3a541c6cf5c3 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -480,14 +480,24 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at * allocation time. Cleared when page is fully instantiated. Free * routine checks flag to restore a reservation on error paths. + * Synchronization: Examined or modified by code that knows it has + * the only reference to page. i.e. After allocation but before use + * or when the page is being freed. * HPG_migratable - Set after a newly allocated page is added to the page * cache and/or page tables. Indicates the page is a candidate for * migration. + * Synchronization: Initially set after new page allocation with no + * locking. When examined and modified during migration processing + * (isolate, migrate, putback) the hugetlb_lock is held. * HPG_temporary - - Set on a page that is temporarily allocated from the buddy * allocator. Typically used for migration target pages when no pages * are available in the pool. The hugetlb free page path will * immediately free pages with this flag set to the buddy allocator. + * Synchronization: Can be set after huge page allocation from buddy when + * code knows it has only reference. All other examinations and + * modifications require hugetlb_lock. * HPG_freed - Set when page is on the free lists. + * Synchronization: hugetlb_lock held for examination and modification. */ enum hugetlb_page_flags { HPG_restore_reserve = 0, From ff5461176213d5fd5cfb7e981f9add4d856e415a Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:11 -0800 Subject: [PATCH 606/633] hugetlb: fix uninitialized subpool pointer Gerald Schaefer reported a panic on s390 in hugepage_subpool_put_pages() with linux-next 5.12.0-20210222. Call trace: hugepage_subpool_put_pages.part.0+0x2c/0x138 __free_huge_page+0xce/0x310 alloc_pool_huge_page+0x102/0x120 set_max_huge_pages+0x13e/0x350 hugetlb_sysctl_handler_common+0xd8/0x110 hugetlb_sysctl_handler+0x48/0x58 proc_sys_call_handler+0x138/0x238 new_sync_write+0x10e/0x198 vfs_write.part.0+0x12c/0x238 ksys_write+0x68/0xf8 do_syscall+0x82/0xd0 __do_syscall+0xb4/0xc8 system_call+0x72/0x98 This is a result of the change which moved the hugetlb page subpool pointer from page->private to page[1]->private. When new pages are allocated from the buddy allocator, the private field of the head page will be cleared, but the private field of subpages is not modified. Therefore, old values may remain. Fix by initializing hugetlb page subpool pointer in prep_new_huge_page(). Link: https://lkml.kernel.org/r/20210223215544.313871-1-mike.kravetz@oracle.com Fixes: f1280272ae4d ("hugetlb: use page.private for hugetlb specific page flags") Signed-off-by: Mike Kravetz Reported-by: Gerald Schaefer Reviewed-by: Oscar Salvador Acked-by: Michal Hocko Cc: Gerald Schaefer Cc: Muchun Song Cc: Heiko Carstens Cc: Sven Schnelle Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/hugetlb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5377e1dad044..8e5f494291c6 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1465,6 +1465,7 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) { INIT_LIST_HEAD(&page->lru); set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); + hugetlb_set_page_subpool(page, NULL); set_hugetlb_cgroup(page, NULL); set_hugetlb_cgroup_rsvd(page, NULL); spin_lock(&hugetlb_lock); From 519983645a9f2ec339cabfa0c6ef7b09be985dd0 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Wed, 24 Feb 2021 12:09:15 -0800 Subject: [PATCH 607/633] mm/vmscan: restore zone_reclaim_mode ABI I went to go add a new RECLAIM_* mode for the zone_reclaim_mode sysctl. Like a good kernel developer, I also went to go update the documentation. I noticed that the bits in the documentation didn't match the bits in the #defines. The VM never explicitly checks the RECLAIM_ZONE bit. The bit is, however implicitly checked when checking 'node_reclaim_mode==0'. The RECLAIM_ZONE #define was removed in a cleanup. That, by itself is fine. But, when the bit was removed (bit 0) the _other_ bit locations also got changed. That's not OK because the bit values are documented to mean one specific thing. Users surely do not expect the meaning to change from kernel to kernel. The end result is that if someone had a script that did: sysctl vm.zone_reclaim_mode=1 it would have gone from enabling node reclaim for clean unmapped pages to writing out pages during node reclaim after the commit in question. That's not great. Put the bits back the way they were and add a comment so something like this is a bit harder to do again. Update the documentation to make it clear that the first bit is ignored. Link: https://lkml.kernel.org/r/20210219172555.FF0CDF23@viggo.jf.intel.com Signed-off-by: Dave Hansen Fixes: 648b5cf368e0 ("mm/vmscan: remove unused RECLAIM_OFF/RECLAIM_ZONE") Reviewed-by: Ben Widawsky Reviewed-by: Oscar Salvador Acked-by: David Rientjes Acked-by: Christoph Lameter Cc: Alex Shi Cc: Daniel Wagner Cc: "Tobin C. Harding" Cc: Christoph Lameter Cc: Andrew Morton Cc: Huang Ying Cc: Dan Williams Cc: Qian Cai Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/admin-guide/sysctl/vm.rst | 10 +++++----- mm/vmscan.c | 9 +++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index e35a3f2fb006..586cd4b86428 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -983,11 +983,11 @@ that benefit from having their data cached, zone_reclaim_mode should be left disabled as the caching effect is likely to be more important than data locality. -zone_reclaim may be enabled if it's known that the workload is partitioned -such that each partition fits within a NUMA node and that accessing remote -memory would cause a measurable performance reduction. The page allocator -will then reclaim easily reusable pages (those page cache pages that are -currently not used) before allocating off node pages. +Consider enabling one or more zone_reclaim mode bits if it's known that the +workload is partitioned such that each partition fits within a NUMA node +and that accessing remote memory would cause a measurable performance +reduction. The page allocator will take additional actions before +allocating off node pages. Allowing zone reclaim to write out pages stops processes that are writing large amounts of data from dirtying pages on other nodes. Zone diff --git a/mm/vmscan.c b/mm/vmscan.c index fea6b43bc1f9..562e87cbd7a1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4085,8 +4085,13 @@ module_init(kswapd_init) */ int node_reclaim_mode __read_mostly; -#define RECLAIM_WRITE (1<<0) /* Writeout pages during reclaim */ -#define RECLAIM_UNMAP (1<<1) /* Unmap pages during reclaim */ +/* + * These bit locations are exposed in the vm.zone_reclaim_mode sysctl + * ABI. New bits are OK, but existing bits can never change. + */ +#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */ +#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */ +#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */ /* * Priority for NODE_RECLAIM. This determines the fraction of pages From 70ad3196a68b0857b49811da7a94ad4f5a8e75bb Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:09:19 -0800 Subject: [PATCH 608/633] z3fold: remove unused attribute for release_z3fold_page Since commit dcf5aedb24f8 ("z3fold: stricter locking and more careful reclaim"), release_z3fold_page() is used again. So we can drop the unused attribute safely. Link: https://lkml.kernel.org/r/20210120084008.58432-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/z3fold.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/z3fold.c b/mm/z3fold.c index dacb0d70fa61..8e53cdf4d190 100644 --- a/mm/z3fold.c +++ b/mm/z3fold.c @@ -541,8 +541,7 @@ static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) spin_unlock(&pool->stale_lock); } -static void __attribute__((__unused__)) - release_z3fold_page(struct kref *ref) +static void release_z3fold_page(struct kref *ref) { struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, refcount); From c457cd96f18c7137287700c409d2ae16c6395256 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:09:22 -0800 Subject: [PATCH 609/633] z3fold: simplify the zhdr initialization code in init_z3fold_page() We can simplify the zhdr initialization by memset() the zhdr first instead of set struct member to zero one by one. This would also make code more compact and clear. Link: https://lkml.kernel.org/r/20210120085851.16159-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Vitaly Wool Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/z3fold.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mm/z3fold.c b/mm/z3fold.c index 8e53cdf4d190..c1ccf6bb0ffb 100644 --- a/mm/z3fold.c +++ b/mm/z3fold.c @@ -413,16 +413,10 @@ static struct z3fold_header *init_z3fold_page(struct page *page, bool headless, if (!slots) return NULL; + memset(zhdr, 0, sizeof(*zhdr)); spin_lock_init(&zhdr->page_lock); kref_init(&zhdr->refcount); - zhdr->first_chunks = 0; - zhdr->middle_chunks = 0; - zhdr->last_chunks = 0; - zhdr->first_num = 0; - zhdr->start_middle = 0; zhdr->cpu = -1; - zhdr->foreign_handles = 0; - zhdr->mapped_count = 0; zhdr->slots = slots; zhdr->pool = pool; INIT_LIST_HEAD(&zhdr->buddy); From d99fd5feb0ac1d56c36c760a8d922a46bd6c5521 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Wed, 24 Feb 2021 12:09:25 -0800 Subject: [PATCH 610/633] mm/compaction: remove rcu_read_lock during page compaction isolate_migratepages_block() used rcu_read_lock() with the intention of safeguarding against the mem_cgroup being destroyed concurrently; but its TestClearPageLRU already protects against that. Delete the unnecessary rcu_read_lock() and _unlock(). Hugh Dickins helped on commit log polishing, Thanks! Link: https://lkml.kernel.org/r/1608614453-10739-3-git-send-email-alex.shi@linux.alibaba.com Signed-off-by: Alex Shi Acked-by: Hugh Dickins Cc: Hugh Dickins Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/compaction.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 3aef10c61d0e..69ca3d6f9764 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -995,7 +995,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (!TestClearPageLRU(page)) goto isolate_fail_put; - rcu_read_lock(); lruvec = mem_cgroup_page_lruvec(page, pgdat); /* If we already hold the lock, we can skip some rechecking */ @@ -1005,7 +1004,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, compact_lock_irqsave(&lruvec->lru_lock, &flags, cc); locked = lruvec; - rcu_read_unlock(); lruvec_memcg_debug(lruvec, page); @@ -1026,8 +1024,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, SetPageLRU(page); goto isolate_fail_put; } - } else - rcu_read_unlock(); + } /* The whole page is taken off the LRU; skip the tail pages. */ if (PageCompound(page)) From e2d26aa5fb393e930eb03628e8add7bd600a8b97 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:09:29 -0800 Subject: [PATCH 611/633] mm/compaction: remove duplicated VM_BUG_ON_PAGE !PageLocked The VM_BUG_ON_PAGE(!PageLocked(page), page) is also done in PageMovable. Remove this explicitly one. Link: https://lkml.kernel.org/r/20210109081420.46030-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/compaction.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/compaction.c b/mm/compaction.c index 69ca3d6f9764..b5220f275525 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -137,7 +137,6 @@ EXPORT_SYMBOL(__SetPageMovable); void __ClearPageMovable(struct page *page) { - VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); /* * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE From 40d7e2032007f9b2ea9aad7c1399cff3bef0239c Mon Sep 17 00:00:00 2001 From: Charan Teja Reddy Date: Wed, 24 Feb 2021 12:09:32 -0800 Subject: [PATCH 612/633] mm/compaction: correct deferral logic for proactive compaction should_proactive_compact_node() returns true when sum of the weighted fragmentation score of all the zones in the node is greater than the wmark_high of compaction, which then triggers the proactive compaction that operates on the individual zones of the node. But proactive compaction runs on the zone only when its weighted fragmentation score is greater than wmark_low(=wmark_high - 10). This means that the sum of the weighted fragmentation scores of all the zones can exceed the wmark_high but individual weighted fragmentation zone scores can still be less than wmark_low which makes the unnecessary trigger of the proactive compaction only to return doing nothing. Issue with the return of proactive compaction with out even trying is its deferral. It is simply deferred for 1 << COMPACT_MAX_DEFER_SHIFT if the scores across the proactive compaction is same, thinking that compaction didn't make any progress but in reality it didn't even try. With the delay between successive retries for proactive compaction is 500msec, it can result into the deferral for ~30sec with out even trying the proactive compaction. Test scenario is that: compaction_proactiveness=50 thus the wmark_low = 50 and wmark_high = 60. System have 2 zones(Normal and Movable) with sizes 5GB and 6GB respectively. After opening some apps on the android, the weighted fragmentation scores of these zones are 47 and 49 respectively. Since the sum of these fragmentation scores are above the wmark_high which triggers the proactive compaction and there since the individual zones weighted fragmentation scores are below wmark_low, it returns without trying the proactive compaction. As a result the weighted fragmentation scores of the zones are still 47 and 49 which makes the existing logic to defer the compaction thinking that noprogress is made across the compaction. Fix this by checking just zone fragmentation score, not the weighted, in __compact_finished() and use the zones weighted fragmentation score in fragmentation_score_node(). In the test case above, If the weighted average of is above wmark_high, then individual score (not adjusted) of atleast one zone has to be above wmark_high. Thus it avoids the unnecessary trigger and deferrals of the proactive compaction. Link: https://lkml.kernel.org/r/1610989938-31374-1-git-send-email-charante@codeaurora.org Signed-off-by: Charan Teja Reddy Suggested-by: Vlastimil Babka Acked-by: Vlastimil Babka Reviewed-by: Khalid Aziz Acked-by: David Rientjes Cc: Michal Hocko Cc: Nitin Gupta Cc: Vinayak Menon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/compaction.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index b5220f275525..50b31e2ec6cb 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1922,20 +1922,28 @@ static bool kswapd_is_running(pg_data_t *pgdat) /* * A zone's fragmentation score is the external fragmentation wrt to the - * COMPACTION_HPAGE_ORDER scaled by the zone's size. It returns a value - * in the range [0, 100]. + * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100]. + */ +static unsigned int fragmentation_score_zone(struct zone *zone) +{ + return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER); +} + +/* + * A weighted zone's fragmentation score is the external fragmentation + * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It + * returns a value in the range [0, 100]. * * The scaling factor ensures that proactive compaction focuses on larger * zones like ZONE_NORMAL, rather than smaller, specialized zones like * ZONE_DMA32. For smaller zones, the score value remains close to zero, * and thus never exceeds the high threshold for proactive compaction. */ -static unsigned int fragmentation_score_zone(struct zone *zone) +static unsigned int fragmentation_score_zone_weighted(struct zone *zone) { unsigned long score; - score = zone->present_pages * - extfrag_for_order(zone, COMPACTION_HPAGE_ORDER); + score = zone->present_pages * fragmentation_score_zone(zone); return div64_ul(score, zone->zone_pgdat->node_present_pages + 1); } @@ -1955,7 +1963,7 @@ static unsigned int fragmentation_score_node(pg_data_t *pgdat) struct zone *zone; zone = &pgdat->node_zones[zoneid]; - score += fragmentation_score_zone(zone); + score += fragmentation_score_zone_weighted(zone); } return score; From 15d28d0d11609c7a4f217b3d85e26456d9beb134 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Wed, 24 Feb 2021 12:09:36 -0800 Subject: [PATCH 613/633] mm/compaction: fix misbehaviors of fast_find_migrateblock() In the fast_find_migrateblock(), it iterates ocer the freelist to find the proper pageblock. But there are some misbehaviors. First, if the page we found is equal to cc->migrate_pfn, it is considered that we didn't find a suitable pageblock. Secondly, if the loop was terminated because order is less than PAGE_ALLOC_COSTLY_ORDER, it could be considered that we found a suitable one. Thirdly, if the skip bit is set on the page block and we goto continue, it doesn't check nr_scanned. Fourthly, if the page block's skip bit is set, it checks that page block is the last of list, which is unnecessary. Link: https://lkml.kernel.org/r/20210128130411.6125-1-vvghjk1234@gmail.com Fixes: 70b44595eafe9 ("mm, compaction: use free lists to quickly locate a migration source") Signed-off-by: Wonhyuk Yang Acked-by: Vlastimil Babka Cc: Mel Gorman Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/compaction.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 50b31e2ec6cb..3d9be07633d9 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1698,6 +1698,7 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) unsigned long pfn = cc->migrate_pfn; unsigned long high_pfn; int order; + bool found_block = false; /* Skip hints are relied on to avoid repeats on the fast search */ if (cc->ignore_skip_hint) @@ -1740,7 +1741,7 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance); for (order = cc->order - 1; - order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit; + order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit; order--) { struct free_area *area = &cc->zone->free_area[order]; struct list_head *freelist; @@ -1755,7 +1756,11 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) list_for_each_entry(freepage, freelist, lru) { unsigned long free_pfn; - nr_scanned++; + if (nr_scanned++ >= limit) { + move_freelist_tail(freelist, freepage); + break; + } + free_pfn = page_to_pfn(freepage); if (free_pfn < high_pfn) { /* @@ -1764,12 +1769,8 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) * the list assumes an entry is deleted, not * reordered. */ - if (get_pageblock_skip(freepage)) { - if (list_is_last(freelist, &freepage->lru)) - break; - + if (get_pageblock_skip(freepage)) continue; - } /* Reorder to so a future search skips recent pages */ move_freelist_tail(freelist, freepage); @@ -1777,15 +1778,10 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) update_fast_start_pfn(cc, free_pfn); pfn = pageblock_start_pfn(free_pfn); cc->fast_search_fail = 0; + found_block = true; set_pageblock_skip(freepage); break; } - - if (nr_scanned >= limit) { - cc->fast_search_fail++; - move_freelist_tail(freelist, freepage); - break; - } } spin_unlock_irqrestore(&cc->zone->lock, flags); } @@ -1796,9 +1792,10 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) * If fast scanning failed then use a cached entry for a page block * that had free pages as the basis for starting a linear scan. */ - if (pfn == cc->migrate_pfn) + if (!found_block) { + cc->fast_search_fail++; pfn = reinit_migrate_pfn(cc); - + } return pfn; } From 6e2b7044c199229a3d20cefbd3184968238c4184 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 24 Feb 2021 12:09:39 -0800 Subject: [PATCH 614/633] mm, compaction: make fast_isolate_freepages() stay within zone Compaction always operates on pages from a single given zone when isolating both pages to migrate and freepages. Pageblock boundaries are intersected with zone boundaries to be safe in case zone starts or ends in the middle of pageblock. The use of pageblock_pfn_to_page() protects against non-contiguous pageblocks. The functions fast_isolate_freepages() and fast_isolate_around() don't currently protect the fast freepage isolation thoroughly enough against these corner cases, and can result in freepage isolation operate outside of zone boundaries: - in fast_isolate_freepages() if we get a pfn from the first pageblock of a zone that starts in the middle of that pageblock, 'highest' can be a pfn outside of the zone. If we fail to isolate anything in this function, we may then call fast_isolate_around() on a pfn outside of the zone and there effectively do a set_pageblock_skip(page_to_pfn(highest)) which may currently hit a VM_BUG_ON() in some configurations - fast_isolate_around() checks only the zone end boundary and not beginning, nor that the pageblock is contiguous (with pageblock_pfn_to_page()) so it's possible that we end up calling isolate_freepages_block() on a range of pfn's from two different zones and end up e.g. isolating freepages under the wrong zone's lock. This patch should fix the above issues. Link: https://lkml.kernel.org/r/20210217173300.6394-1-vbabka@suse.cz Fixes: 5a811889de10 ("mm, compaction: use free lists to quickly locate a migration target") Signed-off-by: Vlastimil Babka Acked-by: David Rientjes Acked-by: Mel Gorman Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: Michal Hocko Cc: Mike Rapoport Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/compaction.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 3d9be07633d9..e04f4476e68e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1284,7 +1284,7 @@ static void fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated) { unsigned long start_pfn, end_pfn; - struct page *page = pfn_to_page(pfn); + struct page *page; /* Do not search around if there are enough pages already */ if (cc->nr_freepages >= cc->nr_migratepages) @@ -1295,8 +1295,12 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long return; /* Pageblock boundaries */ - start_pfn = pageblock_start_pfn(pfn); - end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)) - 1; + start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn); + end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)); + + page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone); + if (!page) + return; /* Scan before */ if (start_pfn != pfn) { @@ -1398,7 +1402,8 @@ fast_isolate_freepages(struct compact_control *cc) pfn = page_to_pfn(freepage); if (pfn >= highest) - highest = pageblock_start_pfn(pfn); + highest = max(pageblock_start_pfn(pfn), + cc->zone->zone_start_pfn); if (pfn >= low_pfn) { cc->fast_search_fail = 0; @@ -1468,7 +1473,8 @@ fast_isolate_freepages(struct compact_control *cc) } else { if (cc->direct_compaction && pfn_valid(min_pfn)) { page = pageblock_pfn_to_page(min_pfn, - pageblock_end_pfn(min_pfn), + min(pageblock_end_pfn(min_pfn), + zone_end_pfn(cc->zone)), cc->zone); cc->free_pfn = min_pfn; } From bda420b985054a3badafef23807c4b4fa38a3dff Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Wed, 24 Feb 2021 12:09:43 -0800 Subject: [PATCH 615/633] numa balancing: migrate on fault among multiple bound nodes Now, NUMA balancing can only optimize the page placement among the NUMA nodes if the default memory policy is used. Because the memory policy specified explicitly should take precedence. But this seems too strict in some situations. For example, on a system with 4 NUMA nodes, if the memory of an application is bound to the node 0 and 1, NUMA balancing can potentially migrate the pages between the node 0 and 1 to reduce cross-node accessing without breaking the explicit memory binding policy. So in this patch, we add MPOL_F_NUMA_BALANCING mode flag to set_mempolicy() when mode is MPOL_BIND. With the flag specified, NUMA balancing will be enabled within the thread to optimize the page placement within the constrains of the specified memory binding policy. With the newly added flag, the NUMA balancing control mechanism becomes, - sysctl knob numa_balancing can enable/disable the NUMA balancing globally. - even if sysctl numa_balancing is enabled, the NUMA balancing will be disabled for the memory areas or applications with the explicit memory policy by default. - MPOL_F_NUMA_BALANCING can be used to enable the NUMA balancing for the applications when specifying the explicit memory policy (MPOL_BIND). Various page placement optimization based on the NUMA balancing can be done with these flags. As the first step, in this patch, if the memory of the application is bound to multiple nodes (MPOL_BIND), and in the hint page fault handler the accessing node are in the policy nodemask, the page will be tried to be migrated to the accessing node to reduce the cross-node accessing. If the newly added MPOL_F_NUMA_BALANCING flag is specified by an application on an old kernel version without its support, set_mempolicy() will return -1 and errno will be set to EINVAL. The application can use this behavior to run on both old and new kernel versions. And if the MPOL_F_NUMA_BALANCING flag is specified for the mode other than MPOL_BIND, set_mempolicy() will return -1 and errno will be set to EINVAL as before. Because we don't support optimization based on the NUMA balancing for these modes. In the previous version of the patch, we tried to reuse MPOL_MF_LAZY for mbind(). But that flag is tied to MPOL_MF_MOVE.*, so it seems not a good API/ABI for the purpose of the patch. And because it's not clear whether it's necessary to enable NUMA balancing for a specific memory area inside an application, so we only add the flag at the thread level (set_mempolicy()) instead of the memory area level (mbind()). We can do that when it become necessary. To test the patch, we run a test case as follows on a 4-node machine with 192 GB memory (48 GB per node). 1. Change pmbench memory accessing benchmark to call set_mempolicy() to bind its memory to node 1 and 3 and enable NUMA balancing. Some related code snippets are as follows, #include #include struct bitmask *bmp; int ret; bmp = numa_parse_nodestring("1,3"); ret = set_mempolicy(MPOL_BIND | MPOL_F_NUMA_BALANCING, bmp->maskp, bmp->size + 1); /* If MPOL_F_NUMA_BALANCING isn't supported, fall back to MPOL_BIND */ if (ret < 0 && errno == EINVAL) ret = set_mempolicy(MPOL_BIND, bmp->maskp, bmp->size + 1); if (ret < 0) { perror("Failed to call set_mempolicy"); exit(-1); } 2. Run a memory eater on node 3 to use 40 GB memory before running pmbench. 3. Run pmbench with 64 processes, the working-set size of each process is 640 MB, so the total working-set size is 64 * 640 MB = 40 GB. The CPU and the memory (as in step 1.) of all pmbench processes is bound to node 1 and 3. So, after CPU usage is balanced, some pmbench processes run on the CPUs of the node 3 will access the memory of the node 1. 4. After the pmbench processes run for 100 seconds, kill the memory eater. Now it's possible for some pmbench processes to migrate their pages from node 1 to node 3 to reduce cross-node accessing. Test results show that, with the patch, the pages can be migrated from node 1 to node 3 after killing the memory eater, and the pmbench score can increase about 17.5%. Link: https://lkml.kernel.org/r/20210120061235.148637-2-ying.huang@intel.com Signed-off-by: "Huang, Ying" Acked-by: Mel Gorman Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Rik van Riel Cc: Johannes Weiner Cc: "Matthew Wilcox (Oracle)" Cc: Dave Hansen Cc: Andi Kleen Cc: Michal Hocko Cc: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/uapi/linux/mempolicy.h | 4 +++- mm/mempolicy.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 3354774af61e..8948467b3992 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -28,12 +28,14 @@ enum { /* Flags for set_mempolicy */ #define MPOL_F_STATIC_NODES (1 << 15) #define MPOL_F_RELATIVE_NODES (1 << 14) +#define MPOL_F_NUMA_BALANCING (1 << 13) /* Optimize with NUMA balancing if possible */ /* * MPOL_MODE_FLAGS is the union of all possible optional mode flags passed to * either set_mempolicy() or mbind(). */ -#define MPOL_MODE_FLAGS (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES) +#define MPOL_MODE_FLAGS \ + (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES | MPOL_F_NUMA_BALANCING) /* Flags for get_mempolicy */ #define MPOL_F_NODE (1<<0) /* return next IL mode instead of node mask */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 2c3a86502053..6961238c7ef5 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -875,6 +875,16 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, goto out; } + if (flags & MPOL_F_NUMA_BALANCING) { + if (new && new->mode == MPOL_BIND) { + new->flags |= (MPOL_F_MOF | MPOL_F_MORON); + } else { + ret = -EINVAL; + mpol_put(new); + goto out; + } + } + ret = mpol_set_nodemask(new, nodes, scratch); if (ret) { mpol_put(new); @@ -2486,6 +2496,12 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long break; case MPOL_BIND: + /* Optimize placement among multiple nodes via NUMA balancing */ + if (pol->flags & MPOL_F_MORON) { + if (node_isset(thisnid, pol->v.nodes)) + break; + goto out; + } /* * allows binding to multiple nodes. From ce33135cdee6e2c2874e9d1198a6df0c5f356080 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:09:47 -0800 Subject: [PATCH 616/633] mm/mempolicy: use helper range_in_vma() in queue_pages_test_walk() The helper range_in_vma() is introduced via commit 017b1660df89 ("mm: migration: fix migration of huge PMD shared pages"). But we forgot to use it in queue_pages_test_walk(). Link: https://lkml.kernel.org/r/20210130091352.20220-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/mempolicy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 6961238c7ef5..ab51132547b8 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -677,7 +677,7 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, unsigned long flags = qp->flags; /* range check first */ - VM_BUG_ON_VMA((vma->vm_start > start) || (vma->vm_end < end), vma); + VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma); if (!qp->first) { qp->first = vma; From f8159c13905bba26f3e1782a521dacf7a66fc1ce Mon Sep 17 00:00:00 2001 From: Tang Yizhou Date: Wed, 24 Feb 2021 12:09:50 -0800 Subject: [PATCH 617/633] mm, oom: fix a comment in dump_task() If p is a kthread, it will be checked in oom_unkillable_task() so we can delete the corresponding comment. Link: https://lkml.kernel.org/r/20210125133006.7242-1-tangyizhou@huawei.com Signed-off-by: Tang Yizhou Acked-by: David Rientjes Acked-by: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/oom_kill.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/oom_kill.c b/mm/oom_kill.c index c9a33ffe38b7..9efaf430cfd3 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -395,9 +395,8 @@ static int dump_task(struct task_struct *p, void *arg) task = find_lock_task_mm(p); if (!task) { /* - * This is a kthread or all of p's threads have already - * detached their mm's. There's no need to report - * them; they can't be oom killed anyway. + * All of p's threads have already detached their mm's. There's + * no need to report them; they can't be oom killed anyway. */ return 0; } From 33b8f84a4ee78491a8f4f9e4c5520c9da4a10983 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:54 -0800 Subject: [PATCH 618/633] mm/hugetlb: change hugetlb_reserve_pages() to type bool While reviewing a bug in hugetlb_reserve_pages, it was noticed that all callers ignore the return value. Any failure is considered an ENOMEM error by the callers. Change the function to be of type bool. The function will return true if the reservation was successful, false otherwise. Callers currently assume a zero return code indicates success. Change the callers to look for true to indicate success. No functional change, only code cleanup. Link: https://lkml.kernel.org/r/20201221192542.15732-1-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Reviewed-by: Matthew Wilcox (Oracle) Cc: David Hildenbrand Cc: Dan Carpenter Cc: Michal Hocko Cc: Davidlohr Bueso Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 37 ++++++++++++++----------------------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 5a8ed6bd4f87..3eca85a4d940 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -171,7 +171,7 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) file_accessed(file); ret = -ENOMEM; - if (hugetlb_reserve_pages(inode, + if (!hugetlb_reserve_pages(inode, vma->vm_pgoff >> huge_page_order(h), len >> huge_page_shift(h), vma, vma->vm_flags)) @@ -1493,7 +1493,7 @@ struct file *hugetlb_file_setup(const char *name, size_t size, inode->i_size = size; clear_nlink(inode); - if (hugetlb_reserve_pages(inode, 0, + if (!hugetlb_reserve_pages(inode, 0, size >> huge_page_shift(hstate_inode(inode)), NULL, acctflag)) file = ERR_PTR(-ENOMEM); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3a541c6cf5c3..cccd1aab69dd 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -139,7 +139,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, struct page **pagep); -int hugetlb_reserve_pages(struct inode *inode, long from, long to, +bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, vm_flags_t vm_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8e5f494291c6..8fb42c6dd74b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5016,12 +5016,13 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, return pages << h->order; } -int hugetlb_reserve_pages(struct inode *inode, +/* Return true if reservation was successful, false otherwise. */ +bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, vm_flags_t vm_flags) { - long ret, chg, add = -1; + long chg, add = -1; struct hstate *h = hstate_inode(inode); struct hugepage_subpool *spool = subpool_inode(inode); struct resv_map *resv_map; @@ -5031,7 +5032,7 @@ int hugetlb_reserve_pages(struct inode *inode, /* This should never happen */ if (from > to) { VM_WARN(1, "%s called with a negative range\n", __func__); - return -EINVAL; + return false; } /* @@ -5040,7 +5041,7 @@ int hugetlb_reserve_pages(struct inode *inode, * without using reserves */ if (vm_flags & VM_NORESERVE) - return 0; + return true; /* * Shared mappings base their reservation on the number of pages that @@ -5062,7 +5063,7 @@ int hugetlb_reserve_pages(struct inode *inode, /* Private mapping. */ resv_map = resv_map_alloc(); if (!resv_map) - return -ENOMEM; + return false; chg = to - from; @@ -5070,18 +5071,12 @@ int hugetlb_reserve_pages(struct inode *inode, set_vma_resv_flags(vma, HPAGE_RESV_OWNER); } - if (chg < 0) { - ret = chg; + if (chg < 0) goto out_err; - } - ret = hugetlb_cgroup_charge_cgroup_rsvd( - hstate_index(h), chg * pages_per_huge_page(h), &h_cg); - - if (ret < 0) { - ret = -ENOMEM; + if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h), + chg * pages_per_huge_page(h), &h_cg) < 0) goto out_err; - } if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { /* For private mappings, the hugetlb_cgroup uncharge info hangs @@ -5096,19 +5091,15 @@ int hugetlb_reserve_pages(struct inode *inode, * reservations already in place (gbl_reserve). */ gbl_reserve = hugepage_subpool_get_pages(spool, chg); - if (gbl_reserve < 0) { - ret = -ENOSPC; + if (gbl_reserve < 0) goto out_uncharge_cgroup; - } /* * Check enough hugepages are available for the reservation. * Hand the pages back to the subpool if there are not */ - ret = hugetlb_acct_memory(h, gbl_reserve); - if (ret < 0) { + if (hugetlb_acct_memory(h, gbl_reserve) < 0) goto out_put_pages; - } /* * Account for the reservations made. Shared mappings record regions @@ -5126,7 +5117,6 @@ int hugetlb_reserve_pages(struct inode *inode, if (unlikely(add < 0)) { hugetlb_acct_memory(h, -gbl_reserve); - ret = add; goto out_put_pages; } else if (unlikely(chg > add)) { /* @@ -5147,7 +5137,8 @@ int hugetlb_reserve_pages(struct inode *inode, hugetlb_acct_memory(h, -rsv_adjust); } } - return 0; + return true; + out_put_pages: /* put back original number of pages, chg */ (void)hugepage_subpool_put_pages(spool, chg); @@ -5163,7 +5154,7 @@ int hugetlb_reserve_pages(struct inode *inode, region_abort(resv_map, from, to, regions_needed); if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) kref_put(&resv_map->refs, resv_map_release); - return ret; + return false; } long hugetlb_unreserve_pages(struct inode *inode, long start, long end, From a4fa34cdcd18296c097e2648fe894d28c5cf9709 Mon Sep 17 00:00:00 2001 From: Mike Kravetz Date: Wed, 24 Feb 2021 12:09:58 -0800 Subject: [PATCH 619/633] hugetlbfs: remove special hugetlbfs_set_page_dirty() Matthew Wilcox noticed that hugetlbfs_set_page_dirty always returns 0. Instead, it should return 1 or 0 depending on the previous state of the dirty bit. In addition, the call to compound_head is redundant as it is also performed in calling routine set_page_dirty. Replace the hugetlbfs specific routine hugetlbfs_set_page_dirty with __set_page_dirty_no_writeback as it addresses both of these issues. Link: https://lkml.kernel.org/r/20201221192542.15732-2-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz Suggested-by: Matthew Wilcox Cc: Dan Carpenter Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 3eca85a4d940..dc4aceffa2c7 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -952,17 +952,6 @@ static int hugetlbfs_symlink(struct user_namespace *mnt_userns, return error; } -/* - * mark the head page dirty - */ -static int hugetlbfs_set_page_dirty(struct page *page) -{ - struct page *head = compound_head(page); - - SetPageDirty(head); - return 0; -} - static int hugetlbfs_migrate_page(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) @@ -1150,7 +1139,7 @@ static void hugetlbfs_destroy_inode(struct inode *inode) static const struct address_space_operations hugetlbfs_aops = { .write_begin = hugetlbfs_write_begin, .write_end = hugetlbfs_write_end, - .set_page_dirty = hugetlbfs_set_page_dirty, + .set_page_dirty = __set_page_dirty_no_writeback, .migratepage = hugetlbfs_migrate_page, .error_remove_page = hugetlbfs_error_remove_page, }; From d0146756a0993d3a01407b38cd87d965ccda72c6 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:01 -0800 Subject: [PATCH 620/633] hugetlbfs: remove useless BUG_ON(!inode) in hugetlbfs_setattr() When we reach here with inode = NULL, we should have crashed as inode has already been dereferenced via hstate_inode. So this BUG_ON(!inode) does not take effect and should be removed. Link: https://lkml.kernel.org/r/20210118110700.52506-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index dc4aceffa2c7..71a00dbfa612 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -761,8 +761,6 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, unsigned int ia_valid = attr->ia_valid; struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); - BUG_ON(!inode); - error = setattr_prepare(&init_user_ns, dentry, attr); if (error) return error; From 3b2275a8d83a29e579b4f96f4c431d824e5f4a16 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:04 -0800 Subject: [PATCH 621/633] hugetlbfs: use helper macro default_hstate in init_hugetlbfs_fs Since commit e5ff215941d5 ("hugetlb: multiple hstates for multiple page sizes"), we can use macro default_hstate to get the struct hstate which we use by default. But init_hugetlbfs_fs() forgot to use it. Link: https://lkml.kernel.org/r/20210116091827.20982-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 71a00dbfa612..32c8ae4e1e48 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1543,7 +1543,7 @@ static int __init init_hugetlbfs_fs(void) goto out_free; /* default hstate mount is required */ - mnt = mount_one_hugetlbfs(&hstates[default_hstate_idx]); + mnt = mount_one_hugetlbfs(&default_hstate); if (IS_ERR(mnt)) { error = PTR_ERR(mnt); goto out_unreg; From c7e285e31f76453bc958006ebe5311a6cca909e3 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:08 -0800 Subject: [PATCH 622/633] hugetlbfs: correct obsolete function name in hugetlbfs_read_iter() Since commit 36e789144267 ("kill do_generic_mapping_read"), the function do_generic_mapping_read() is renamed to do_generic_file_read(). And then commit 47c27bc46946 ("fs: pass iocb to do_generic_file_read") renamed it to generic_file_buffered_read(). So replace do_generic_mapping_read() with generic_file_buffered_read() to keep comment uptodate. Link: https://lkml.kernel.org/r/20210118063210.47118-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 32c8ae4e1e48..974eee6736d2 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -310,7 +310,7 @@ hugetlbfs_read_actor(struct page *page, unsigned long offset, /* * Support for read() - Find the page attached to f_mapping and copy out the - * data. Its *very* similar to do_generic_mapping_read(), we can't use that + * data. Its *very* similar to generic_file_buffered_read(), we can't use that * since it has PAGE_SIZE assumptions. */ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) From 88ce3fef47f3f382985ecefe8f290b6ff05b4335 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:11 -0800 Subject: [PATCH 623/633] hugetlbfs: remove meaningless variable avoid_reserve The variable avoid_reserve is meaningless because we never changed its value and just passed it to alloc_huge_page(). So remove it to make code more clear that in hugetlbfs_fallocate, we never avoid reserve when alloc hugepage yet. Also add a comment offered by Mike Kravetz to explain this. Link: https://lkml.kernel.org/r/20210120071508.9078-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 974eee6736d2..7982adc3d98d 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -680,7 +680,6 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, */ struct page *page; unsigned long addr; - int avoid_reserve = 0; cond_resched(); @@ -716,8 +715,15 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, continue; } - /* Allocate page and add to page cache */ - page = alloc_huge_page(&pseudo_vma, addr, avoid_reserve); + /* + * Allocate page without setting the avoid_reserve argument. + * There certainly are no reserves associated with the + * pseudo_vma. However, there could be shared mappings with + * reserves for the file at the inode level. If we fallocate + * pages in these areas, we need to consume the reserves + * to keep reservation accounting consistent. + */ + page = alloc_huge_page(&pseudo_vma, addr, 0); hugetlb_drop_vma_policy(&pseudo_vma); if (IS_ERR(page)) { mutex_unlock(&hugetlb_fault_mutex_table[hash]); From a25fddced835ae53d18eb4bddabd719b4cebf624 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:14 -0800 Subject: [PATCH 624/633] hugetlbfs: make hugepage size conversion more readable The calculation 1U << (h->order + PAGE_SHIFT - 10) is actually equal to (PAGE_SHIFT << (h->order)) >> 10. So we can make it more readable by replace it with huge_page_size(h) >> 10. Link: https://lkml.kernel.org/r/20210122083141.24548-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 7982adc3d98d..46fad3d9265b 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1520,8 +1520,8 @@ static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h) put_fs_context(fc); } if (IS_ERR(mnt)) - pr_err("Cannot mount internal hugetlbfs for page size %uK", - 1U << (h->order + PAGE_SHIFT - 10)); + pr_err("Cannot mount internal hugetlbfs for page size %luK", + huge_page_size(h) >> 10); return mnt; } From 398c0da7364c907ccc662416585c19c5523cf678 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:18 -0800 Subject: [PATCH 625/633] hugetlbfs: correct some obsolete comments about inode i_mutex Since commit 9902af79c01a ("parallel lookups: actual switch to rwsem"), i_mutex of inode is converted to i_rwsem. So replace i_mutex with i_rwsem to make comments up to date. Link: https://lkml.kernel.org/r/20210127093111.36672-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 46fad3d9265b..f5758ba372ae 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -604,7 +604,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) inode_lock(inode); - /* protected by i_mutex */ + /* protected by i_rwsem */ if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { inode_unlock(inode); return -EPERM; @@ -777,7 +777,7 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, if (newsize & ~huge_page_mask(h)) return -EINVAL; - /* protected by i_mutex */ + /* protected by i_rwsem */ if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || (newsize > oldsize && (info->seals & F_SEAL_GROW))) return -EPERM; From 1935ebd3cf6c44038479bb2e7b4dd99bd492b3f2 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:21 -0800 Subject: [PATCH 626/633] hugetlbfs: fix some comment typos Fix typos reserv to reserve, minimim to minimum. No functional change intended. Link: https://lkml.kernel.org/r/20210130092351.28072-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Cc: Mike Kravetz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index f5758ba372ae..394da2ab08ad 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -442,15 +442,15 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end) * * truncation is indicated by end of range being LLONG_MAX * In this case, we first scan the range and release found pages. - * After releasing pages, hugetlb_unreserve_pages cleans up region/reserv + * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve * maps and global counts. Page faults can not race with truncation * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents * page faults in the truncated range by checking i_size. i_size is * modified while holding i_mmap_rwsem. * hole punch is indicated if end is not LLONG_MAX * In the hole punch case we scan the range and release found pages. - * Only when releasing a page is the associated region/reserv map - * deleted. The region/reserv map for ranges without associated + * Only when releasing a page is the associated region/reserve map + * deleted. The region/reserve map for ranges without associated * pages are not modified. Page faults can race with hole punch. * This is indicated if we find a mapped page. * Note: If the passed end of range value is beyond the end of file, but @@ -1343,7 +1343,7 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) /* * Allocate and initialize subpool if maximum or minimum size is - * specified. Any needed reservations (for minimim size) are taken + * specified. Any needed reservations (for minimum size) are taken * taken when the subpool is created. */ if (ctx->max_hpages != -1 || ctx->min_hpages != -1) { From e5d319dedafd21211fd19ea28a3f50da7368d6ff Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 24 Feb 2021 12:10:25 -0800 Subject: [PATCH 627/633] hugetlbfs: remove unneeded return value of hugetlb_vmtruncate() The function hugetlb_vmtruncate() is guaranteed to always success since commit 7aa91e104028 ("hugetlb: allow extending ftruncate on hugetlbfs"). So we should remove the unneeded return value which is always 0. Link: https://lkml.kernel.org/r/20210208084637.47789-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Mike Kravetz Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 394da2ab08ad..701c82c36138 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -567,7 +567,7 @@ static void hugetlbfs_evict_inode(struct inode *inode) clear_inode(inode); } -static int hugetlb_vmtruncate(struct inode *inode, loff_t offset) +static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) { pgoff_t pgoff; struct address_space *mapping = inode->i_mapping; @@ -582,7 +582,6 @@ static int hugetlb_vmtruncate(struct inode *inode, loff_t offset) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0); i_mmap_unlock_write(mapping); remove_inode_hugepages(inode, offset, LLONG_MAX); - return 0; } static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) @@ -781,9 +780,7 @@ static int hugetlbfs_setattr(struct user_namespace *mnt_userns, if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || (newsize > oldsize && (info->seals & F_SEAL_GROW))) return -EPERM; - error = hugetlb_vmtruncate(inode, newsize); - if (error) - return error; + hugetlb_vmtruncate(inode, newsize); } setattr_copy(&init_user_ns, inode, attr); From a553e3cd2053501b658feec2be9a3b662eb1b22b Mon Sep 17 00:00:00 2001 From: Chengyang Fan Date: Wed, 24 Feb 2021 12:10:28 -0800 Subject: [PATCH 628/633] mm/migrate: remove unneeded semicolons Remove superfluous semicolons after function definitions. Link: https://lkml.kernel.org/r/20210115110131.2359683-1-cy.fan@huawei.com Signed-off-by: Chengyang Fan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/migrate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 4594838a0f7c..3a389633b68f 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -89,7 +89,7 @@ extern int PageMovable(struct page *page); extern void __SetPageMovable(struct page *page, struct address_space *mapping); extern void __ClearPageMovable(struct page *page); #else -static inline int PageMovable(struct page *page) { return 0; }; +static inline int PageMovable(struct page *page) { return 0; } static inline void __SetPageMovable(struct page *page, struct address_space *mapping) { From d7cc16b4a3b84d61c0c58f6785f43a494efd0699 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 12 Feb 2020 11:16:51 +0100 Subject: [PATCH 629/633] nds32: Replace by The Andes platform code is not a clock provider, and just needs to call of_clk_init(). Hence it can include instead of . Signed-off-by: Geert Uytterhoeven Acked-by: Greentime Hu Reviewed-by: Stephen Boyd Signed-off-by: Greentime Hu --- arch/nds32/kernel/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/nds32/kernel/time.c b/arch/nds32/kernel/time.c index ac9d78ce3a81..574a3d0a8539 100644 --- a/arch/nds32/kernel/time.c +++ b/arch/nds32/kernel/time.c @@ -2,7 +2,7 @@ // Copyright (C) 2005-2017 Andes Technology Corporation #include -#include +#include void __init time_init(void) { From e99da8af9cbc8f68e27c28ddeb57a40ee1006081 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 17 Feb 2020 17:49:18 +0100 Subject: [PATCH 630/633] nds32: configs: Cleanup CONFIG_CROSS_COMPILE CONFIG_CROSS_COMPILE is gone since commit f1089c92da79 ("kbuild: remove CONFIG_CROSS_COMPILE support"). Signed-off-by: Krzysztof Kozlowski Acked-by: Greentime Hu Signed-off-by: Greentime Hu --- arch/nds32/configs/defconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/nds32/configs/defconfig b/arch/nds32/configs/defconfig index 40313a635075..f9a89cf00aa6 100644 --- a/arch/nds32/configs/defconfig +++ b/arch/nds32/configs/defconfig @@ -1,4 +1,3 @@ -CONFIG_CROSS_COMPILE="nds32le-linux-" CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_HIGH_RES_TIMERS=y From 9d63fecfcb2c7d379b6dd06892c534068a03a470 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Jul 2020 13:44:47 +0200 Subject: [PATCH 631/633] nds32: remove dump_instr dump_inst has a return before actually doing anything, so just drop the whole thing. Signed-off-by: Christoph Hellwig Acked-by: Nick Hu Acked-by: Greentime Hu Signed-off-by: Greentime Hu --- arch/nds32/kernel/traps.c | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c index 6a9772ba7392..b66f889bc6df 100644 --- a/arch/nds32/kernel/traps.c +++ b/arch/nds32/kernel/traps.c @@ -62,40 +62,6 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) EXPORT_SYMBOL(dump_mem); -static void dump_instr(struct pt_regs *regs) -{ - unsigned long addr = instruction_pointer(regs); - mm_segment_t fs; - char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; - int i; - - return; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - - pr_emerg("Code: "); - for (i = -4; i < 1; i++) { - unsigned int val, bad; - - bad = __get_user(val, &((u32 *) addr)[i]); - - if (!bad) { - p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); - } else { - p += sprintf(p, "bad PC value"); - break; - } - } - pr_emerg("Code: %s\n", str); - - set_fs(fs); -} - #define LOOP_TIMES (100) static void __dump(struct task_struct *tsk, unsigned long *base_reg, const char *loglvl) @@ -179,7 +145,6 @@ void die(const char *str, struct pt_regs *regs, int err) if (!user_mode(regs) || in_interrupt()) { dump_mem("Stack: ", regs->sp, (regs->sp + PAGE_SIZE) & PAGE_MASK); - dump_instr(regs); dump_stack(); } From fa2f478a348efa483abd4159c9f5478a3867bcc0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Jul 2020 13:44:48 +0200 Subject: [PATCH 632/633] nds32: use get_kernel_nofault in dump_mem Use the proper get_kernel_nofault helper to access an unsafe kernel pointer without faulting instead of playing with set_fs and get_user. Signed-off-by: Christoph Hellwig Acked-by: Nick Hu Acked-by: Greentime Hu Signed-off-by: Greentime Hu --- arch/nds32/kernel/traps.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c index b66f889bc6df..ee0d9ae192a5 100644 --- a/arch/nds32/kernel/traps.c +++ b/arch/nds32/kernel/traps.c @@ -25,17 +25,8 @@ extern void show_pte(struct mm_struct *mm, unsigned long addr); void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) { unsigned long first; - mm_segment_t fs; int i; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - pr_emerg("%s(0x%08lx to 0x%08lx)\n", lvl, bottom, top); for (first = bottom & ~31; first < top; first += 32) { @@ -48,7 +39,9 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { if (p >= bottom && p < top) { unsigned long val; - if (__get_user(val, (unsigned long *)p) == 0) + + if (get_kernel_nofault(val, + (unsigned long *)p) == 0) sprintf(str + i * 9, " %08lx", val); else sprintf(str + i * 9, " ????????"); @@ -56,8 +49,6 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) } pr_emerg("%s%04lx:%s\n", lvl, first & 0xffff, str); } - - set_fs(fs); } EXPORT_SYMBOL(dump_mem); From 40e0dd851e7b7afe219820fb270b09016e41d4fc Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Thu, 27 Aug 2020 15:24:34 +0200 Subject: [PATCH 633/633] nds32: Fix bogus reference to Andestech(nds32) never had . Signed-off-by: Geert Uytterhoeven Acked-by: Greentime Hu Signed-off-by: Greentime Hu --- arch/nds32/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c index c356e484dcab..af82e996f412 100644 --- a/arch/nds32/kernel/setup.c +++ b/arch/nds32/kernel/setup.c @@ -52,7 +52,7 @@ EXPORT_SYMBOL(elf_hwcap); /* * The following string table, must sync with HWCAP_xx bitmask, - * which is defined in + * which is defined above */ static const char *hwcap_str[] = { "mfusr_pc",