diff options
Diffstat (limited to 'drivers/pci')
121 files changed, 8871 insertions, 2428 deletions
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index 0c473d75e625..43e615aa12ff 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -110,7 +110,7 @@ config PCI_PF_STUB config XEN_PCIDEV_FRONTEND tristate "Xen PCI Frontend" - depends on X86 && XEN + depends on XEN_PV select PCI_XEN select XEN_XENBUS_FRONTEND default y diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 6d7d64939f82..c967ad6e2626 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -376,7 +376,7 @@ int pci_enable_pasid(struct pci_dev *pdev, int features) if (WARN_ON(pdev->pasid_enabled)) return -EBUSY; - if (!pdev->eetlp_prefix_path) + if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp) return -EINVAL; if (!pasid) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 2f2c8a1729f9..7fc5135ffbbf 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -37,6 +37,15 @@ config PCI_FTPCI100 depends on OF default ARCH_GEMINI +config PCI_IXP4XX + bool "Intel IXP4xx PCI controller" + depends on ARM && OF + depends on ARCH_IXP4XX || COMPILE_TEST + default ARCH_IXP4XX + help + Say Y here if you want support for the PCI host controller found + in the Intel IXP4xx XScale-based network processor SoC. + config PCI_TEGRA bool "NVIDIA Tegra PCIe controller" depends on ARCH_TEGRA || COMPILE_TEST @@ -245,7 +254,7 @@ config PCIE_MEDIATEK_GEN3 MediaTek SoCs. config VMD - depends on PCI_MSI && X86_64 && SRCU + depends on PCI_MSI && X86_64 && SRCU && !UML tristate "Intel Volume Management Device Driver" help Adds support for the Intel Volume Management Device (VMD). VMD is a @@ -261,7 +270,8 @@ config VMD config PCIE_BRCMSTB tristate "Broadcom Brcmstb PCIe host controller" - depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || COMPILE_TEST + depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || \ + BMIPS_GENERIC || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN default ARCH_BRCMSTB @@ -303,6 +313,32 @@ config PCIE_HISI_ERR Say Y here if you want error handling support for the PCIe controller's errors on HiSilicon HIP SoCs +config PCIE_APPLE_MSI_DOORBELL_ADDR + hex + default 0xfffff000 + depends on PCIE_APPLE + +config PCIE_APPLE + tristate "Apple PCIe controller" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF + depends on PCI_MSI_IRQ_DOMAIN + select PCI_HOST_COMMON + help + Say Y here if you want to enable PCIe controller support on Apple + system-on-chips, like the Apple M1. This is required for the USB + type-A ports, Ethernet, Wi-Fi, and Bluetooth. + + If unsure, say Y if you have an Apple Silicon system. + +config PCIE_MT7621 + bool "MediaTek MT7621 PCIe Controller" + depends on SOC_MT7621 || (MIPS && COMPILE_TEST) + select PHY_MT7621_PCI + default SOC_MT7621 + help + This selects a driver for the MediaTek MT7621 PCIe Controller. + source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" source "drivers/pci/controller/cadence/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 63e3880a3e87..37c8663de7fe 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_PCIE_CADENCE) += cadence/ obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o +obj-$(CONFIG_PCI_IXP4XX) += pci-ixp4xx.o obj-$(CONFIG_PCI_HYPERV) += pci-hyperv.o obj-$(CONFIG_PCI_HYPERV_INTERFACE) += pci-hyperv-intf.o obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o @@ -36,6 +37,9 @@ obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o +obj-$(CONFIG_PCIE_APPLE) += pcie-apple.o +obj-$(CONFIG_PCIE_MT7621) += pcie-mt7621.o + # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ obj-y += mobiveil/ diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 35e61048e133..918e11082e6a 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -27,6 +27,7 @@ #define STATUS_REG_SYS_2 0x508 #define STATUS_CLR_REG_SYS_2 0x708 #define LINK_DOWN BIT(1) +#define J7200_LINK_DOWN BIT(10) #define J721E_PCIE_USER_CMD_STATUS 0x4 #define LINK_TRAINING_ENABLE BIT(0) @@ -57,6 +58,7 @@ struct j721e_pcie { struct cdns_pcie *cdns_pcie; void __iomem *user_cfg_base; void __iomem *intd_cfg_base; + u32 linkdown_irq_regfield; }; enum j721e_pcie_mode { @@ -66,7 +68,10 @@ enum j721e_pcie_mode { struct j721e_pcie_data { enum j721e_pcie_mode mode; - bool quirk_retrain_flag; + unsigned int quirk_retrain_flag:1; + unsigned int quirk_detect_quiet_flag:1; + u32 linkdown_irq_regfield; + unsigned int byte_access_allowed:1; }; static inline u32 j721e_pcie_user_readl(struct j721e_pcie *pcie, u32 offset) @@ -98,12 +103,12 @@ static irqreturn_t j721e_pcie_link_irq_handler(int irq, void *priv) u32 reg; reg = j721e_pcie_intd_readl(pcie, STATUS_REG_SYS_2); - if (!(reg & LINK_DOWN)) + if (!(reg & pcie->linkdown_irq_regfield)) return IRQ_NONE; dev_err(dev, "LINK DOWN!\n"); - j721e_pcie_intd_writel(pcie, STATUS_CLR_REG_SYS_2, LINK_DOWN); + j721e_pcie_intd_writel(pcie, STATUS_CLR_REG_SYS_2, pcie->linkdown_irq_regfield); return IRQ_HANDLED; } @@ -112,7 +117,7 @@ static void j721e_pcie_config_link_irq(struct j721e_pcie *pcie) u32 reg; reg = j721e_pcie_intd_readl(pcie, ENABLE_REG_SYS_2); - reg |= LINK_DOWN; + reg |= pcie->linkdown_irq_regfield; j721e_pcie_intd_writel(pcie, ENABLE_REG_SYS_2, reg); } @@ -284,10 +289,36 @@ static struct pci_ops cdns_ti_pcie_host_ops = { static const struct j721e_pcie_data j721e_pcie_rc_data = { .mode = PCI_MODE_RC, .quirk_retrain_flag = true, + .byte_access_allowed = false, + .linkdown_irq_regfield = LINK_DOWN, }; static const struct j721e_pcie_data j721e_pcie_ep_data = { .mode = PCI_MODE_EP, + .linkdown_irq_regfield = LINK_DOWN, +}; + +static const struct j721e_pcie_data j7200_pcie_rc_data = { + .mode = PCI_MODE_RC, + .quirk_detect_quiet_flag = true, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, +}; + +static const struct j721e_pcie_data j7200_pcie_ep_data = { + .mode = PCI_MODE_EP, + .quirk_detect_quiet_flag = true, +}; + +static const struct j721e_pcie_data am64_pcie_rc_data = { + .mode = PCI_MODE_RC, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, +}; + +static const struct j721e_pcie_data am64_pcie_ep_data = { + .mode = PCI_MODE_EP, + .linkdown_irq_regfield = J7200_LINK_DOWN, }; static const struct of_device_id of_j721e_pcie_match[] = { @@ -299,6 +330,22 @@ static const struct of_device_id of_j721e_pcie_match[] = { .compatible = "ti,j721e-pcie-ep", .data = &j721e_pcie_ep_data, }, + { + .compatible = "ti,j7200-pcie-host", + .data = &j7200_pcie_rc_data, + }, + { + .compatible = "ti,j7200-pcie-ep", + .data = &j7200_pcie_ep_data, + }, + { + .compatible = "ti,am64-pcie-host", + .data = &am64_pcie_rc_data, + }, + { + .compatible = "ti,am64-pcie-ep", + .data = &am64_pcie_ep_data, + }, {}, }; @@ -332,6 +379,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->mode = mode; + pcie->linkdown_irq_regfield = data->linkdown_irq_regfield; base = devm_platform_ioremap_resource_byname(pdev, "intd_cfg"); if (IS_ERR(base)) @@ -391,9 +439,11 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - bridge->ops = &cdns_ti_pcie_host_ops; + if (!data->byte_access_allowed) + bridge->ops = &cdns_ti_pcie_host_ops; rc = pci_host_bridge_priv(bridge); rc->quirk_retrain_flag = data->quirk_retrain_flag; + rc->quirk_detect_quiet_flag = data->quirk_detect_quiet_flag; cdns_pcie = &rc->pcie; cdns_pcie->dev = dev; @@ -424,7 +474,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) ret = clk_prepare_enable(clk); if (ret) { dev_err(dev, "failed to enable pcie_refclk\n"); - goto err_get_sync; + goto err_pcie_setup; } pcie->refclk = clk; @@ -459,6 +509,7 @@ static int j721e_pcie_probe(struct platform_device *pdev) ret = -ENOMEM; goto err_get_sync; } + ep->quirk_detect_quiet_flag = data->quirk_detect_quiet_flag; cdns_pcie = &ep->pcie; cdns_pcie->dev = dev; diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c index 897cdde02bd8..88e05b9c2e5b 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -16,11 +16,37 @@ #define CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE 0x1 #define CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY 0x3 -static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, +static u8 cdns_pcie_get_fn_from_vfn(struct cdns_pcie *pcie, u8 fn, u8 vfn) +{ + u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET; + u32 first_vf_offset, stride; + + if (vfn == 0) + return fn; + + first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_OFFSET); + stride = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_STRIDE); + fn = fn + first_vf_offset + ((vfn - 1) * stride); + + return fn; +} + +static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_header *hdr) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET; struct cdns_pcie *pcie = &ep->pcie; + u32 reg; + + if (vfn > 1) { + dev_err(&epc->dev, "Only Virtual Function #1 has deviceID\n"); + return -EINVAL; + } else if (vfn == 1) { + reg = cap + PCI_SRIOV_VF_DID; + cdns_pcie_ep_fn_writew(pcie, fn, reg, hdr->deviceid); + return 0; + } cdns_pcie_ep_fn_writew(pcie, fn, PCI_DEVICE_ID, hdr->deviceid); cdns_pcie_ep_fn_writeb(pcie, fn, PCI_REVISION_ID, hdr->revid); @@ -47,7 +73,7 @@ static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, return 0; } -static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, +static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_bar *epf_bar) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); @@ -92,32 +118,36 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, addr0 = lower_32_bits(bar_phys); addr1 = upper_32_bits(bar_phys); + + if (vfn == 1) + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn); + else + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn); + b = (bar < BAR_4) ? bar : bar - BAR_4; + + if (vfn == 0 || vfn == 1) { + cfg = cdns_pcie_readl(pcie, reg); + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); + cdns_pcie_writel(pcie, reg, cfg); + } + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), addr0); cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), addr1); - if (bar < BAR_4) { - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn); - b = bar; - } else { - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn); - b = bar - BAR_4; - } - - cfg = cdns_pcie_readl(pcie, reg); - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); - cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); - cdns_pcie_writel(pcie, reg, cfg); - + if (vfn > 0) + epf = &epf->epf[vfn - 1]; epf->epf_bar[bar] = epf_bar; return 0; } -static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, +static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_bar *epf_bar) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); @@ -126,29 +156,32 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, enum pci_barno bar = epf_bar->barno; u32 reg, cfg, b, ctrl; - if (bar < BAR_4) { - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn); - b = bar; - } else { - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn); - b = bar - BAR_4; + if (vfn == 1) + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn); + else + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn); + b = (bar < BAR_4) ? bar : bar - BAR_4; + + if (vfn == 0 || vfn == 1) { + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; + cfg = cdns_pcie_readl(pcie, reg); + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); + cdns_pcie_writel(pcie, reg, cfg); } - ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; - cfg = cdns_pcie_readl(pcie, reg); - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); - cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); - cdns_pcie_writel(pcie, reg, cfg); - + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), 0); cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), 0); + if (vfn > 0) + epf = &epf->epf[vfn - 1]; epf->epf_bar[bar] = NULL; } -static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, phys_addr_t addr, - u64 pci_addr, size_t size) +static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr, u64 pci_addr, size_t size) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; @@ -161,6 +194,7 @@ static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, phys_addr_t addr, return -EINVAL; } + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); cdns_pcie_set_outbound_region(pcie, 0, fn, r, false, addr, pci_addr, size); set_bit(r, &ep->ob_region_map); @@ -169,7 +203,7 @@ static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, phys_addr_t addr, return 0; } -static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, +static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); @@ -189,13 +223,15 @@ static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, clear_bit(r, &ep->ob_region_map); } -static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 mmc) +static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 mmc) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET; u16 flags; + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + /* * Set the Multiple Message Capable bitfield into the Message Control * register. @@ -209,13 +245,15 @@ static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 mmc) return 0; } -static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn) +static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET; u16 flags, mme; + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + /* Validate that the MSI feature is actually enabled. */ flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); if (!(flags & PCI_MSI_FLAGS_ENABLE)) @@ -230,13 +268,15 @@ static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn) return mme; } -static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) +static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET; u32 val, reg; + func_no = cdns_pcie_get_fn_from_vfn(pcie, func_no, vfunc_no); + reg = cap + PCI_MSIX_FLAGS; val = cdns_pcie_ep_fn_readw(pcie, func_no, reg); if (!(val & PCI_MSIX_FLAGS_ENABLE)) @@ -247,14 +287,17 @@ static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) return val; } -static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u16 interrupts, - enum pci_barno bir, u32 offset) +static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn, + u16 interrupts, enum pci_barno bir, + u32 offset) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); struct cdns_pcie *pcie = &ep->pcie; u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET; u32 val, reg; + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + reg = cap + PCI_MSIX_FLAGS; val = cdns_pcie_ep_fn_readw(pcie, fn, reg); val &= ~PCI_MSIX_FLAGS_QSIZE; @@ -274,8 +317,8 @@ static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u16 interrupts, return 0; } -static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, - u8 intx, bool is_asserted) +static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, u8 intx, + bool is_asserted) { struct cdns_pcie *pcie = &ep->pcie; unsigned long flags; @@ -317,7 +360,8 @@ static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, writel(0, ep->irq_cpu_addr + offset); } -static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 intx) +static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, + u8 intx) { u16 cmd; @@ -334,7 +378,7 @@ static int cdns_pcie_ep_send_legacy_irq(struct cdns_pcie_ep *ep, u8 fn, u8 intx) return 0; } -static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, +static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, u8 interrupt_num) { struct cdns_pcie *pcie = &ep->pcie; @@ -343,6 +387,8 @@ static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, u8 msi_count; u64 pci_addr, pci_addr_mask = 0xff; + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + /* Check whether the MSI feature has been enabled by the PCI host. */ flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); if (!(flags & PCI_MSI_FLAGS_ENABLE)) @@ -382,7 +428,7 @@ static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, return 0; } -static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, +static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr, u8 interrupt_num, u32 entry_size, u32 *msi_data, u32 *msi_addr_offset) @@ -396,6 +442,8 @@ static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, int ret; int i; + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + /* Check whether the MSI feature has been enabled by the PCI host. */ flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); if (!(flags & PCI_MSI_FLAGS_ENABLE)) @@ -419,7 +467,7 @@ static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, pci_addr &= GENMASK_ULL(63, 2); for (i = 0; i < interrupt_num; i++) { - ret = cdns_pcie_ep_map_addr(epc, fn, addr, + ret = cdns_pcie_ep_map_addr(epc, fn, vfn, addr, pci_addr & ~pci_addr_mask, entry_size); if (ret) @@ -433,7 +481,7 @@ static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, return 0; } -static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, +static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, u16 interrupt_num) { u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET; @@ -446,6 +494,12 @@ static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u16 flags; u8 bir; + epf = &ep->epf[fn]; + if (vfn > 0) + epf = &epf->epf[vfn - 1]; + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + /* Check whether the MSI-X feature has been enabled by the PCI host. */ flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSIX_FLAGS); if (!(flags & PCI_MSIX_FLAGS_ENABLE)) @@ -456,7 +510,6 @@ static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, bir = tbl_offset & PCI_MSIX_TABLE_BIR; tbl_offset &= PCI_MSIX_TABLE_OFFSET; - epf = &ep->epf[fn]; msix_tbl = epf->epf_bar[bir]->addr + tbl_offset; msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; msg_data = msix_tbl[(interrupt_num - 1)].msg_data; @@ -478,21 +531,27 @@ static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, return 0; } -static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, +static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, enum pci_epc_irq_type type, u16 interrupt_num) { struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + struct device *dev = pcie->dev; switch (type) { case PCI_EPC_IRQ_LEGACY: - return cdns_pcie_ep_send_legacy_irq(ep, fn, 0); + if (vfn > 0) { + dev_err(dev, "Cannot raise legacy interrupts for VF\n"); + return -EINVAL; + } + return cdns_pcie_ep_send_legacy_irq(ep, fn, vfn, 0); case PCI_EPC_IRQ_MSI: - return cdns_pcie_ep_send_msi_irq(ep, fn, interrupt_num); + return cdns_pcie_ep_send_msi_irq(ep, fn, vfn, interrupt_num); case PCI_EPC_IRQ_MSIX: - return cdns_pcie_ep_send_msix_irq(ep, fn, interrupt_num); + return cdns_pcie_ep_send_msix_irq(ep, fn, vfn, interrupt_num); default: break; @@ -523,6 +582,13 @@ static int cdns_pcie_ep_start(struct pci_epc *epc) return 0; } +static const struct pci_epc_features cdns_pcie_epc_vf_features = { + .linkup_notifier = false, + .msi_capable = true, + .msix_capable = true, + .align = 65536, +}; + static const struct pci_epc_features cdns_pcie_epc_features = { .linkup_notifier = false, .msi_capable = true, @@ -531,9 +597,12 @@ static const struct pci_epc_features cdns_pcie_epc_features = { }; static const struct pci_epc_features* -cdns_pcie_ep_get_features(struct pci_epc *epc, u8 func_no) +cdns_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { - return &cdns_pcie_epc_features; + if (!vfunc_no) + return &cdns_pcie_epc_features; + + return &cdns_pcie_epc_vf_features; } static const struct pci_epc_ops cdns_pcie_epc_ops = { @@ -559,9 +628,11 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) struct platform_device *pdev = to_platform_device(dev); struct device_node *np = dev->of_node; struct cdns_pcie *pcie = &ep->pcie; + struct cdns_pcie_epf *epf; struct resource *res; struct pci_epc *epc; int ret; + int i; pcie->is_rc = false; @@ -606,6 +677,25 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) if (!ep->epf) return -ENOMEM; + epc->max_vfs = devm_kcalloc(dev, epc->max_functions, + sizeof(*epc->max_vfs), GFP_KERNEL); + if (!epc->max_vfs) + return -ENOMEM; + + ret = of_property_read_u8_array(np, "max-virtual-functions", + epc->max_vfs, epc->max_functions); + if (ret == 0) { + for (i = 0; i < epc->max_functions; i++) { + epf = &ep->epf[i]; + if (epc->max_vfs[i] == 0) + continue; + epf->epf = devm_kcalloc(dev, epc->max_vfs[i], + sizeof(*ep->epf), GFP_KERNEL); + if (!epf->epf) + return -ENOMEM; + } + } + ret = pci_epc_mem_init(epc, pcie->mem_res->start, resource_size(pcie->mem_res), PAGE_SIZE); if (ret < 0) { @@ -623,6 +713,10 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) ep->irq_pci_addr = CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE; /* Reserve region 0 for IRQs */ set_bit(0, &ep->ob_region_map); + + if (ep->quirk_detect_quiet_flag) + cdns_pcie_detect_quiet_min_delay_set(&ep->pcie); + spin_lock_init(&ep->lock); return 0; diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index ae1c55503513..fb96d37a135c 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -498,6 +498,9 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) return PTR_ERR(rc->cfg_base); rc->cfg_res = res; + if (rc->quirk_detect_quiet_flag) + cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); + ret = cdns_pcie_start_link(pcie); if (ret) { dev_err(dev, "Failed to start link\n"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-plat.c b/drivers/pci/controller/cadence/pcie-cadence-plat.c index 5fee0f89ab59..a224afadbcc0 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-plat.c +++ b/drivers/pci/controller/cadence/pcie-cadence-plat.c @@ -127,6 +127,8 @@ static int cdns_plat_pcie_probe(struct platform_device *pdev) goto err_init; } + return 0; + err_init: err_get_sync: pm_runtime_put_sync(dev); diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index 3c3646502d05..52767f26048f 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -7,6 +7,22 @@ #include "pcie-cadence.h" +void cdns_pcie_detect_quiet_min_delay_set(struct cdns_pcie *pcie) +{ + u32 delay = 0x3; + u32 ltssm_control_cap; + + /* + * Set the LTSSM Detect Quiet state min. delay to 2ms. + */ + ltssm_control_cap = cdns_pcie_readl(pcie, CDNS_PCIE_LTSSM_CONTROL_CAP); + ltssm_control_cap = ((ltssm_control_cap & + ~CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK) | + CDNS_PCIE_DETECT_QUIET_MIN_DELAY(delay)); + + cdns_pcie_writel(pcie, CDNS_PCIE_LTSSM_CONTROL_CAP, ltssm_control_cap); +} + void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, u32 r, bool is_io, u64 cpu_addr, u64 pci_addr, size_t size) diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 254d2570f8c9..262421e5d917 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -8,6 +8,7 @@ #include <linux/kernel.h> #include <linux/pci.h> +#include <linux/pci-epf.h> #include <linux/phy/phy.h> /* Parameters for the waiting for link up routine */ @@ -46,10 +47,18 @@ #define CDNS_PCIE_LM_EP_ID_BUS_SHIFT 8 /* Endpoint Function f BAR b Configuration Registers */ +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_4) ? CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn)) #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) \ (CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008) #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \ (CDNS_PCIE_LM_BASE + 0x0244 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_4) ? CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn)) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) \ + (CDNS_PCIE_LM_BASE + 0x0280 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn) \ + (CDNS_PCIE_LM_BASE + 0x0284 + (fn) * 0x0008) #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \ (GENMASK(4, 0) << ((b) * 8)) #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \ @@ -114,6 +123,7 @@ #define CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET 0x90 #define CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET 0xb0 +#define CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET 0x200 /* * Root Port Registers (PCI configuration space for the root port function) @@ -189,6 +199,14 @@ /* AXI link down register */ #define CDNS_PCIE_AT_LINKDOWN (CDNS_PCIE_AT_BASE + 0x0824) +/* LTSSM Capabilities register */ +#define CDNS_PCIE_LTSSM_CONTROL_CAP (CDNS_PCIE_LM_BASE + 0x0054) +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK GENMASK(2, 1) +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY_SHIFT 1 +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY(delay) \ + (((delay) << CDNS_PCIE_DETECT_QUIET_MIN_DELAY_SHIFT) & \ + CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK) + enum cdns_pcie_rp_bar { RP_BAR_UNDEFINED = -1, RP_BAR0, @@ -263,9 +281,12 @@ struct cdns_pcie_ops { * struct cdns_pcie - private data for Cadence PCIe controller drivers * @reg_base: IO mapped register base * @mem_res: start/end offsets in the physical system memory to map PCI accesses + * @dev: PCIe controller * @is_rc: tell whether the PCIe controller mode is Root Complex or Endpoint. - * @bus: In Root Complex mode, the bus number - * @ops: Platform specific ops to control various inputs from Cadence PCIe + * @phy_count: number of supported PHY devices + * @phy: list of pointers to specific PHY control blocks + * @link: list of pointers to corresponding device link representations + * @ops: Platform-specific ops to control various inputs from Cadence PCIe * wrapper */ struct cdns_pcie { @@ -292,6 +313,7 @@ struct cdns_pcie { * @avail_ib_bar: Satus of RP_BAR0, RP_BAR1 and RP_NO_BAR if it's free or * available * @quirk_retrain_flag: Retrain link as quirk for PCIe Gen2 + * @quirk_detect_quiet_flag: LTSSM Detect Quiet min delay set as quirk */ struct cdns_pcie_rc { struct cdns_pcie pcie; @@ -300,14 +322,17 @@ struct cdns_pcie_rc { u32 vendor_id; u32 device_id; bool avail_ib_bar[CDNS_PCIE_RP_MAX_IB]; - bool quirk_retrain_flag; + unsigned int quirk_retrain_flag:1; + unsigned int quirk_detect_quiet_flag:1; }; /** * struct cdns_pcie_epf - Structure to hold info about endpoint function + * @epf: Info about virtual functions attached to the physical function * @epf_bar: reference to the pci_epf_bar for the six Base Address Registers */ struct cdns_pcie_epf { + struct cdns_pcie_epf *epf; struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; }; @@ -331,6 +356,7 @@ struct cdns_pcie_epf { * registers fields (RMW) accessible by both remote RC and EP to * minimize time between read and write * @epf: Structure to hold info about endpoint function + * @quirk_detect_quiet_flag: LTSSM Detect Quiet min delay set as quirk */ struct cdns_pcie_ep { struct cdns_pcie pcie; @@ -345,6 +371,7 @@ struct cdns_pcie_ep { /* protect writing to PCI_STATUS while raising legacy interrupts */ spinlock_t lock; struct cdns_pcie_epf *epf; + unsigned int quirk_detect_quiet_flag:1; }; @@ -505,6 +532,9 @@ static inline int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) return 0; } #endif + +void cdns_pcie_detect_quiet_min_delay_set(struct cdns_pcie *pcie); + void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, u32 r, bool is_io, u64 cpu_addr, u64 pci_addr, size_t size); diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig index 423d35872ce4..62ce3abf0f19 100644 --- a/drivers/pci/controller/dwc/Kconfig +++ b/drivers/pci/controller/dwc/Kconfig @@ -8,22 +8,20 @@ config PCIE_DW config PCIE_DW_HOST bool - depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW config PCIE_DW_EP bool - depends on PCI_ENDPOINT select PCIE_DW config PCI_DRA7XX - bool + tristate config PCI_DRA7XX_HOST - bool "TI DRA7xx PCIe controller Host Mode" + tristate "TI DRA7xx PCIe controller Host Mode" depends on SOC_DRA7XX || COMPILE_TEST - depends on PCI_MSI_IRQ_DOMAIN depends on OF && HAS_IOMEM && TI_PIPE3 + depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST select PCI_DRA7XX default y if SOC_DRA7XX @@ -36,10 +34,10 @@ config PCI_DRA7XX_HOST This uses the DesignWare core. config PCI_DRA7XX_EP - bool "TI DRA7xx PCIe controller Endpoint Mode" + tristate "TI DRA7xx PCIe controller Endpoint Mode" depends on SOC_DRA7XX || COMPILE_TEST - depends on PCI_ENDPOINT depends on OF && HAS_IOMEM && TI_PIPE3 + depends on PCI_ENDPOINT select PCIE_DW_EP select PCI_DRA7XX help @@ -55,7 +53,7 @@ config PCIE_DW_PLAT config PCIE_DW_PLAT_HOST bool "Platform bus based DesignWare PCIe Controller - Host mode" - depends on PCI && PCI_MSI_IRQ_DOMAIN + depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST select PCIE_DW_PLAT help @@ -138,8 +136,8 @@ config PCI_LAYERSCAPE bool "Freescale Layerscape PCIe controller - Host mode" depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST) depends on PCI_MSI_IRQ_DOMAIN - select MFD_SYSCON select PCIE_DW_HOST + select MFD_SYSCON help Say Y here if you want to enable PCIe controller support on Layerscape SoCs to work in Host mode. @@ -180,6 +178,16 @@ config PCIE_QCOM PCIe controller uses the DesignWare core plus Qualcomm-specific hardware wrappers. +config PCIE_QCOM_EP + tristate "Qualcomm PCIe controller - Endpoint mode" + depends on OF && (ARCH_QCOM || COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + help + Say Y here to enable support for the PCIe controllers on Qualcomm SoCs + to work in endpoint mode. The PCIe controller uses the DesignWare core + plus Qualcomm-specific hardware wrappers. + config PCIE_ARMADA_8K bool "Marvell Armada-8K PCIe controller" depends on ARCH_MVEBU || COMPILE_TEST @@ -214,6 +222,17 @@ config PCIE_ARTPEC6_EP Enables support for the PCIe controller in the ARTPEC-6 SoC to work in endpoint mode. This uses the DesignWare core. +config PCIE_ROCKCHIP_DW_HOST + bool "Rockchip DesignWare PCIe controller" + select PCIE_DW + select PCIE_DW_HOST + depends on PCI_MSI_IRQ_DOMAIN + depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on OF + help + Enables support for the DesignWare PCIe controller in the + Rockchip SoC except RK3399. + config PCIE_INTEL_GW bool "Intel Gateway PCIe host controller support" depends on OF && (X86 || COMPILE_TEST) @@ -225,9 +244,37 @@ config PCIE_INTEL_GW The PCIe controller uses the DesignWare core plus Intel-specific hardware wrappers. +config PCIE_KEEMBAY + bool + +config PCIE_KEEMBAY_HOST + bool "Intel Keem Bay PCIe controller - Host mode" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on PCI && PCI_MSI_IRQ_DOMAIN + select PCIE_DW_HOST + select PCIE_KEEMBAY + help + Say 'Y' here to enable support for the PCIe controller in Keem Bay + to work in host mode. + The PCIe controller is based on DesignWare Hardware and uses + DesignWare core functions. + +config PCIE_KEEMBAY_EP + bool "Intel Keem Bay PCIe controller - Endpoint mode" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on PCI && PCI_MSI_IRQ_DOMAIN + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_KEEMBAY + help + Say 'Y' here to enable support for the PCIe controller in Keem Bay + to work in endpoint mode. + The PCIe controller is based on DesignWare Hardware and uses + DesignWare core functions. + config PCIE_KIRIN depends on OF && (ARM64 || COMPILE_TEST) - bool "HiSilicon Kirin series SoCs PCIe controllers" + tristate "HiSilicon Kirin series SoCs PCIe controllers" depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST help @@ -244,8 +291,8 @@ config PCIE_HISI_STB config PCI_MESON tristate "MESON PCIe controller" - depends on PCI_MSI_IRQ_DOMAIN default m if ARCH_MESON + depends on PCI_MSI_IRQ_DOMAIN select PCIE_DW_HOST help Say Y here if you want to enable PCI controller support on Amlogic @@ -286,6 +333,15 @@ config PCIE_TEGRA194_EP in order to enable device-specific features PCIE_TEGRA194_EP must be selected. This uses the DesignWare core. +config PCIE_VISCONTI_HOST + bool "Toshiba Visconti PCIe controllers" + depends on ARCH_VISCONTI || COMPILE_TEST + depends on PCI_MSI_IRQ_DOMAIN + select PCIE_DW_HOST + help + Say Y here if you want PCIe controller support on Toshiba Visconti SoC. + This driver supports TMPV7708 SoC. + config PCIE_UNIPHIER bool "Socionext UniPhier PCIe host controllers" depends on ARCH_UNIPHIER || COMPILE_TEST diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile index 9e6ce0dc2f53..8ba7b67f5e50 100644 --- a/drivers/pci/controller/dwc/Makefile +++ b/drivers/pci/controller/dwc/Makefile @@ -12,15 +12,19 @@ obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone.o obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o obj-$(CONFIG_PCI_LAYERSCAPE_EP) += pci-layerscape-ep.o obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o +obj-$(CONFIG_PCIE_QCOM_EP) += pcie-qcom-ep.o obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o +obj-$(CONFIG_PCIE_ROCKCHIP_DW_HOST) += pcie-dw-rockchip.o obj-$(CONFIG_PCIE_INTEL_GW) += pcie-intel-gw.o +obj-$(CONFIG_PCIE_KEEMBAY) += pcie-keembay.o obj-$(CONFIG_PCIE_KIRIN) += pcie-kirin.o obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o obj-$(CONFIG_PCI_MESON) += pci-meson.o obj-$(CONFIG_PCIE_TEGRA194) += pcie-tegra194.o obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o obj-$(CONFIG_PCIE_UNIPHIER_EP) += pcie-uniphier-ep.o +obj-$(CONFIG_PCIE_VISCONTI_HOST) += pcie-visconti.o # The following drivers are for devices that use the generic ACPI # pci_root.c driver but don't support standard ECAM config access. diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c index 047cfbdc1330..a4221f6f3629 100644 --- a/drivers/pci/controller/dwc/pci-dra7xx.c +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -7,6 +7,7 @@ * Authors: Kishon Vijay Abraham I <kishon@ti.com> */ +#include <linux/clk.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/err.h> @@ -14,7 +15,7 @@ #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> -#include <linux/init.h> +#include <linux/module.h> #include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_pci.h> @@ -90,6 +91,7 @@ struct dra7xx_pcie { int phy_count; /* DT phy-names count */ struct phy **phy; struct irq_domain *irq_domain; + struct clk *clk; enum dw_pcie_device_mode mode; }; @@ -204,7 +206,7 @@ static int dra7xx_pcie_handle_msi(struct pcie_port *pp, int index) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); unsigned long val; - int pos, irq; + int pos; val = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + (index * MSI_REG_CTRL_BLOCK_SIZE)); @@ -213,9 +215,8 @@ static int dra7xx_pcie_handle_msi(struct pcie_port *pp, int index) pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, 0); while (pos != MAX_MSI_IRQS_PER_CTRL) { - irq = irq_find_mapping(pp->irq_domain, - (index * MAX_MSI_IRQS_PER_CTRL) + pos); - generic_handle_irq(irq); + generic_handle_domain_irq(pp->irq_domain, + (index * MAX_MSI_IRQS_PER_CTRL) + pos); pos++; pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, pos); } @@ -257,7 +258,7 @@ static void dra7xx_pcie_msi_irq_handler(struct irq_desc *desc) struct dw_pcie *pci; struct pcie_port *pp; unsigned long reg; - u32 virq, bit; + u32 bit; chained_irq_enter(chip, desc); @@ -276,11 +277,8 @@ static void dra7xx_pcie_msi_irq_handler(struct irq_desc *desc) case INTB: case INTC: case INTD: - for_each_set_bit(bit, ®, PCI_NUM_INTX) { - virq = irq_find_mapping(dra7xx->irq_domain, bit); - if (virq) - generic_handle_irq(virq); - } + for_each_set_bit(bit, ®, PCI_NUM_INTX) + generic_handle_domain_irq(dra7xx->irq_domain, bit); break; } @@ -611,6 +609,7 @@ static const struct of_device_id of_dra7xx_pcie_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, of_dra7xx_pcie_match); /* * dra7xx_pcie_unaligned_memaccess: workaround for AM572x/AM571x Errata i870 @@ -744,6 +743,15 @@ static int dra7xx_pcie_probe(struct platform_device *pdev) if (!link) return -ENOMEM; + dra7xx->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(dra7xx->clk)) + return dev_err_probe(dev, PTR_ERR(dra7xx->clk), + "clock request failed"); + + ret = clk_prepare_enable(dra7xx->clk); + if (ret) + return ret; + for (i = 0; i < phy_count; i++) { snprintf(name, sizeof(name), "pcie-phy%d", i); phy[i] = devm_phy_get(dev, name); @@ -929,6 +937,8 @@ static void dra7xx_pcie_shutdown(struct platform_device *pdev) pm_runtime_disable(dev); dra7xx_pcie_disable_phy(dra7xx); + + clk_disable_unprepare(dra7xx->clk); } static const struct dev_pm_ops dra7xx_pcie_pm_ops = { @@ -947,4 +957,8 @@ static struct platform_driver dra7xx_pcie_driver = { }, .shutdown = dra7xx_pcie_shutdown, }; -builtin_platform_driver(dra7xx_pcie_driver); +module_platform_driver(dra7xx_pcie_driver); + +MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); +MODULE_DESCRIPTION("PCIe controller driver for TI DRA7xx SoCs"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c index c24dab383654..722dacdd5a17 100644 --- a/drivers/pci/controller/dwc/pci-exynos.c +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -19,6 +19,7 @@ #include <linux/platform_device.h> #include <linux/phy/phy.h> #include <linux/regulator/consumer.h> +#include <linux/module.h> #include "pcie-designware.h" diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 0cf1333c0440..26f49f797b0f 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -37,6 +37,7 @@ #define IMX8MQ_GPR_PCIE_REF_USE_PAD BIT(9) #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN BIT(10) #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE BIT(11) +#define IMX8MQ_GPR_PCIE_VREG_BYPASS BIT(12) #define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE GENMASK(11, 8) #define IMX8MQ_PCIE2_BASE_ADDR 0x33c00000 @@ -80,6 +81,7 @@ struct imx6_pcie { u32 tx_swing_full; u32 tx_swing_low; struct regulator *vpcie; + struct regulator *vph; void __iomem *phy_base; /* power domain for pcie */ @@ -621,6 +623,17 @@ static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie) imx6_pcie_grp_offset(imx6_pcie), IMX8MQ_GPR_PCIE_REF_USE_PAD, IMX8MQ_GPR_PCIE_REF_USE_PAD); + /* + * Regarding the datasheet, the PCIE_VPH is suggested + * to be 1.8V. If the PCIE_VPH is supplied by 3.3V, the + * VREG_BYPASS should be cleared to zero. + */ + if (imx6_pcie->vph && + regulator_get_voltage(imx6_pcie->vph) > 3000000) + regmap_update_bits(imx6_pcie->iomuxc_gpr, + imx6_pcie_grp_offset(imx6_pcie), + IMX8MQ_GPR_PCIE_VREG_BYPASS, + 0); break; case IMX7D: regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, @@ -1002,10 +1015,8 @@ static int imx6_pcie_probe(struct platform_device *pdev) return ret; } imx6_pcie->phy_base = devm_ioremap_resource(dev, &res); - if (IS_ERR(imx6_pcie->phy_base)) { - dev_err(dev, "Unable to map PCIe PHY\n"); + if (IS_ERR(imx6_pcie->phy_base)) return PTR_ERR(imx6_pcie->phy_base); - } } dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -1121,7 +1132,7 @@ static int imx6_pcie_probe(struct platform_device *pdev) /* Limit link speed */ pci->link_gen = 1; - ret = of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen); + of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen); imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); if (IS_ERR(imx6_pcie->vpcie)) { @@ -1130,6 +1141,13 @@ static int imx6_pcie_probe(struct platform_device *pdev) imx6_pcie->vpcie = NULL; } + imx6_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph"); + if (IS_ERR(imx6_pcie->vph)) { + if (PTR_ERR(imx6_pcie->vph) != -ENODEV) + return PTR_ERR(imx6_pcie->vph); + imx6_pcie->vph = NULL; + } + platform_set_drvdata(pdev, imx6_pcie); ret = imx6_pcie_attach_pd(dev); @@ -1175,6 +1193,7 @@ static const struct imx6_pcie_drvdata drvdata[] = { .variant = IMX6QP, .flags = IMX6_PCIE_FLAG_IMX6_PHY | IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE, + .dbi_length = 0x200, }, [IMX7D] = { .variant = IMX7D, diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c index bde3b2824e89..865258d8c53c 100644 --- a/drivers/pci/controller/dwc/pci-keystone.c +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -259,14 +259,12 @@ static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie, struct dw_pcie *pci = ks_pcie->pci; struct device *dev = pci->dev; u32 pending; - int virq; pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS(offset)); if (BIT(0) & pending) { - virq = irq_linear_revmap(ks_pcie->legacy_irq_domain, offset); - dev_dbg(dev, ": irq: irq_offset %d, virq %d\n", offset, virq); - generic_handle_irq(virq); + dev_dbg(dev, ": irq: irq_offset %d", offset); + generic_handle_domain_irq(ks_pcie->legacy_irq_domain, offset); } /* EOI the INTx interrupt */ @@ -579,7 +577,7 @@ static void ks_pcie_msi_irq_handler(struct irq_desc *desc) struct pcie_port *pp = &pci->pp; struct device *dev = pci->dev; struct irq_chip *chip = irq_desc_get_chip(desc); - u32 vector, virq, reg, pos; + u32 vector, reg, pos; dev_dbg(dev, "%s, irq %d\n", __func__, irq); @@ -600,10 +598,8 @@ static void ks_pcie_msi_irq_handler(struct irq_desc *desc) continue; vector = offset + (pos << 3); - virq = irq_linear_revmap(pp->irq_domain, vector); - dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n", pos, vector, - virq); - generic_handle_irq(virq); + dev_dbg(dev, "irq: bit %d, vector %d\n", pos, vector); + generic_handle_domain_irq(pp->irq_domain, vector); } chained_irq_exit(chip, desc); diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c index 597c282f586c..c91fc1954432 100644 --- a/drivers/pci/controller/dwc/pcie-artpec6.c +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -384,6 +384,7 @@ static int artpec6_pcie_probe(struct platform_device *pdev) const struct artpec_pcie_of_data *data; enum artpec_pcie_variants variant; enum dw_pcie_device_mode mode; + u32 val; match = of_match_device(artpec6_pcie_of_match, dev); if (!match) @@ -432,9 +433,7 @@ static int artpec6_pcie_probe(struct platform_device *pdev) if (ret < 0) return ret; break; - case DW_PCIE_EP_TYPE: { - u32 val; - + case DW_PCIE_EP_TYPE: if (!IS_ENABLED(CONFIG_PCIE_ARTPEC6_EP)) return -ENODEV; @@ -445,8 +444,6 @@ static int artpec6_pcie_probe(struct platform_device *pdev) pci->ep.ops = &pcie_ep_ops; return dw_pcie_ep_init(&pci->ep); - break; - } default: dev_err(dev, "INVALID device type %d\n", artpec6_pcie->mode); } diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 8d028a88b375..0eda8236c125 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -83,6 +83,7 @@ void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) for (func_no = 0; func_no < funcs; func_no++) __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); } +EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie_ep *ep, u8 func_no, u8 cap_ptr, u8 cap) @@ -125,7 +126,7 @@ static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) return __dw_pcie_ep_find_next_cap(ep, func_no, next_cap_ptr, cap); } -static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, +static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *hdr) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); @@ -202,7 +203,7 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no, return 0; } -static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, +static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); @@ -217,7 +218,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, ep->epf_bar[bar] = NULL; } -static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, +static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { int ret; @@ -276,7 +277,7 @@ static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, return -EINVAL; } -static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, +static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t addr) { int ret; @@ -292,9 +293,8 @@ static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, clear_bit(atu_index, ep->ob_window_map); } -static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, - phys_addr_t addr, - u64 pci_addr, size_t size) +static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + phys_addr_t addr, u64 pci_addr, size_t size) { int ret; struct dw_pcie_ep *ep = epc_get_drvdata(epc); @@ -309,7 +309,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, return 0; } -static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no) +static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -333,7 +333,8 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no) return val; } -static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) +static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u8 interrupts) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -358,7 +359,7 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) return 0; } -static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) +static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -382,8 +383,8 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) return val; } -static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, - enum pci_barno bir, u32 offset) +static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u16 interrupts, enum pci_barno bir, u32 offset) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -418,7 +419,7 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, return 0; } -static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, +static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, enum pci_epc_irq_type type, u16 interrupt_num) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); @@ -450,7 +451,7 @@ static int dw_pcie_ep_start(struct pci_epc *epc) } static const struct pci_epc_features* -dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no) +dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { struct dw_pcie_ep *ep = epc_get_drvdata(epc); @@ -485,6 +486,7 @@ int dw_pcie_ep_raise_legacy_irq(struct dw_pcie_ep *ep, u8 func_no) return -EINVAL; } +EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_legacy_irq); int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num) @@ -525,17 +527,18 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, aligned_offset = msg_addr_lower & (epc->mem->window.page_size - 1); msg_addr = ((u64)msg_addr_upper) << 32 | (msg_addr_lower & ~aligned_offset); - ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr, + ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, epc->mem->window.page_size); if (ret) return ret; writel(msg_data | (interrupt_num - 1), ep->msi_mem + aligned_offset); - dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); return 0; } +EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num) @@ -593,14 +596,14 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, } aligned_offset = msg_addr & (epc->mem->window.page_size - 1); - ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr, + ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, epc->mem->window.page_size); if (ret) return ret; writel(msg_data, ep->msi_mem + aligned_offset); - dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); return 0; } diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index a608ae1fad57..f4755f3a03be 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -55,7 +55,7 @@ static struct msi_domain_info dw_pcie_msi_domain_info = { /* MSI int handler */ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp) { - int i, pos, irq; + int i, pos; unsigned long val; u32 status, num_ctrls; irqreturn_t ret = IRQ_NONE; @@ -74,10 +74,9 @@ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp) pos = 0; while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, pos)) != MAX_MSI_IRQS_PER_CTRL) { - irq = irq_find_mapping(pp->irq_domain, - (i * MAX_MSI_IRQS_PER_CTRL) + - pos); - generic_handle_irq(irq); + generic_handle_domain_irq(pp->irq_domain, + (i * MAX_MSI_IRQS_PER_CTRL) + + pos); pos++; } } @@ -336,6 +335,16 @@ int dw_pcie_host_init(struct pcie_port *pp) if (pci->link_gen < 1) pci->link_gen = of_pci_get_max_link_speed(np); + /* Set default bus ops */ + bridge->ops = &dw_pcie_ops; + bridge->child_ops = &dw_child_pcie_ops; + + if (pp->ops->host_init) { + ret = pp->ops->host_init(pp); + if (ret) + return ret; + } + if (pci_msi_enabled()) { pp->has_msi_ctrl = !(pp->ops->msi_host_init || of_property_read_bool(np, "msi-parent") || @@ -389,15 +398,6 @@ int dw_pcie_host_init(struct pcie_port *pp) } } - /* Set default bus ops */ - bridge->ops = &dw_pcie_ops; - bridge->child_ops = &dw_child_pcie_ops; - - if (pp->ops->host_init) { - ret = pp->ops->host_init(pp); - if (ret) - goto err_free_msi; - } dw_pcie_iatu_detect(pci); dw_pcie_setup_rc(pp); diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c index 9b397c807261..8851eb161a0e 100644 --- a/drivers/pci/controller/dwc/pcie-designware-plat.c +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -164,7 +164,6 @@ static int dw_plat_pcie_probe(struct platform_device *pdev) pci->ep.ops = &pcie_ep_ops; return dw_pcie_ep_init(&pci->ep); - break; default: dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->mode); } diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index a945f0c0e73d..850b4533f4ef 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -538,6 +538,7 @@ int dw_pcie_link_up(struct dw_pcie *pci) return ((val & PCIE_PORT_DEBUG1_LINK_UP) && (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING))); } +EXPORT_SYMBOL_GPL(dw_pcie_link_up); void dw_pcie_upconfig_setup(struct dw_pcie *pci) { diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c new file mode 100644 index 000000000000..c9b341e55cbb --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Rockchip SoCs. + * + * Copyright (C) 2021 Rockchip Electronics Co., Ltd. + * http://www.rock-chips.com + * + * Author: Simon Xue <xxm@rock-chips.com> + */ + +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include "pcie-designware.h" + +/* + * The upper 16 bits of PCIE_CLIENT_CONFIG are a write + * mask for the lower 16 bits. + */ +#define HIWORD_UPDATE(mask, val) (((mask) << 16) | (val)) +#define HIWORD_UPDATE_BIT(val) HIWORD_UPDATE(val, val) + +#define to_rockchip_pcie(x) dev_get_drvdata((x)->dev) + +#define PCIE_CLIENT_RC_MODE HIWORD_UPDATE_BIT(0x40) +#define PCIE_CLIENT_ENABLE_LTSSM HIWORD_UPDATE_BIT(0xc) +#define PCIE_SMLH_LINKUP BIT(16) +#define PCIE_RDLH_LINKUP BIT(17) +#define PCIE_LINKUP (PCIE_SMLH_LINKUP | PCIE_RDLH_LINKUP) +#define PCIE_L0S_ENTRY 0x11 +#define PCIE_CLIENT_GENERAL_CONTROL 0x0 +#define PCIE_CLIENT_GENERAL_DEBUG 0x104 +#define PCIE_CLIENT_HOT_RESET_CTRL 0x180 +#define PCIE_CLIENT_LTSSM_STATUS 0x300 +#define PCIE_LTSSM_ENABLE_ENHANCE BIT(4) +#define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) + +struct rockchip_pcie { + struct dw_pcie pci; + void __iomem *apb_base; + struct phy *phy; + struct clk_bulk_data *clks; + unsigned int clk_cnt; + struct reset_control *rst; + struct gpio_desc *rst_gpio; + struct regulator *vpcie3v3; +}; + +static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, + u32 reg) +{ + return readl_relaxed(rockchip->apb_base + reg); +} + +static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, + u32 val, u32 reg) +{ + writel_relaxed(val, rockchip->apb_base + reg); +} + +static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) +{ + rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, + PCIE_CLIENT_GENERAL_CONTROL); +} + +static int rockchip_pcie_link_up(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); + + if ((val & PCIE_LINKUP) == PCIE_LINKUP && + (val & PCIE_LTSSM_STATUS_MASK) == PCIE_L0S_ENTRY) + return 1; + + return 0; +} + +static int rockchip_pcie_start_link(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + /* Reset device */ + gpiod_set_value_cansleep(rockchip->rst_gpio, 0); + + rockchip_pcie_enable_ltssm(rockchip); + + /* + * PCIe requires the refclk to be stable for 100µs prior to releasing + * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI + * Express Card Electromechanical Specification, 1.1. However, we don't + * know if the refclk is coming from RC's PHY or external OSC. If it's + * from RC, so enabling LTSSM is the just right place to release #PERST. + * We need more extra time as before, rather than setting just + * 100us as we don't know how long should the device need to reset. + */ + msleep(100); + gpiod_set_value_cansleep(rockchip->rst_gpio, 1); + + return 0; +} + +static int rockchip_pcie_host_init(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = HIWORD_UPDATE_BIT(PCIE_LTSSM_ENABLE_ENHANCE); + + /* LTSSM enable control mode */ + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); + + rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_RC_MODE, + PCIE_CLIENT_GENERAL_CONTROL); + + return 0; +} + +static const struct dw_pcie_host_ops rockchip_pcie_host_ops = { + .host_init = rockchip_pcie_host_init, +}; + +static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + int ret; + + ret = devm_clk_bulk_get_all(dev, &rockchip->clks); + if (ret < 0) + return ret; + + rockchip->clk_cnt = ret; + + return clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks); +} + +static int rockchip_pcie_resource_get(struct platform_device *pdev, + struct rockchip_pcie *rockchip) +{ + rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(rockchip->apb_base)) + return PTR_ERR(rockchip->apb_base); + + rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", + GPIOD_OUT_HIGH); + if (IS_ERR(rockchip->rst_gpio)) + return PTR_ERR(rockchip->rst_gpio); + + return 0; +} + +static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + int ret; + + rockchip->phy = devm_phy_get(dev, "pcie-phy"); + if (IS_ERR(rockchip->phy)) + return dev_err_probe(dev, PTR_ERR(rockchip->phy), + "missing PHY\n"); + + ret = phy_init(rockchip->phy); + if (ret < 0) + return ret; + + ret = phy_power_on(rockchip->phy); + if (ret) + phy_exit(rockchip->phy); + + return ret; +} + +static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip) +{ + phy_exit(rockchip->phy); + phy_power_off(rockchip->phy); +} + +static int rockchip_pcie_reset_control_release(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + + rockchip->rst = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(rockchip->rst)) + return dev_err_probe(dev, PTR_ERR(rockchip->rst), + "failed to get reset lines\n"); + + return reset_control_deassert(rockchip->rst); +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = rockchip_pcie_link_up, + .start_link = rockchip_pcie_start_link, +}; + +static int rockchip_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_pcie *rockchip; + struct pcie_port *pp; + int ret; + + rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL); + if (!rockchip) + return -ENOMEM; + + platform_set_drvdata(pdev, rockchip); + + rockchip->pci.dev = dev; + rockchip->pci.ops = &dw_pcie_ops; + + pp = &rockchip->pci.pp; + pp->ops = &rockchip_pcie_host_ops; + + ret = rockchip_pcie_resource_get(pdev, rockchip); + if (ret) + return ret; + + /* DON'T MOVE ME: must be enable before PHY init */ + rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); + if (IS_ERR(rockchip->vpcie3v3)) { + if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) + return dev_err_probe(dev, PTR_ERR(rockchip->vpcie3v3), + "failed to get vpcie3v3 regulator\n"); + rockchip->vpcie3v3 = NULL; + } else { + ret = regulator_enable(rockchip->vpcie3v3); + if (ret) { + dev_err(dev, "failed to enable vpcie3v3 regulator\n"); + return ret; + } + } + + ret = rockchip_pcie_phy_init(rockchip); + if (ret) + goto disable_regulator; + + ret = rockchip_pcie_reset_control_release(rockchip); + if (ret) + goto deinit_phy; + + ret = rockchip_pcie_clk_init(rockchip); + if (ret) + goto deinit_phy; + + ret = dw_pcie_host_init(pp); + if (!ret) + return 0; + + clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks); +deinit_phy: + rockchip_pcie_phy_deinit(rockchip); +disable_regulator: + if (rockchip->vpcie3v3) + regulator_disable(rockchip->vpcie3v3); + + return ret; +} + +static const struct of_device_id rockchip_pcie_of_match[] = { + { .compatible = "rockchip,rk3568-pcie", }, + {}, +}; + +static struct platform_driver rockchip_pcie_driver = { + .driver = { + .name = "rockchip-dw-pcie", + .of_match_table = rockchip_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = rockchip_pcie_probe, +}; +builtin_platform_driver(rockchip_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index f89a7d24ba28..d15cf35fa7f2 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -39,6 +39,10 @@ #define PCIE_APP_IRN_PM_TO_ACK BIT(9) #define PCIE_APP_IRN_LINK_AUTO_BW_STAT BIT(11) #define PCIE_APP_IRN_BW_MGT BIT(12) +#define PCIE_APP_IRN_INTA BIT(13) +#define PCIE_APP_IRN_INTB BIT(14) +#define PCIE_APP_IRN_INTC BIT(15) +#define PCIE_APP_IRN_INTD BIT(16) #define PCIE_APP_IRN_MSG_LTR BIT(18) #define PCIE_APP_IRN_SYS_ERR_RC BIT(29) #define PCIE_APP_INTX_OFST 12 @@ -48,10 +52,8 @@ PCIE_APP_IRN_RX_VDM_MSG | PCIE_APP_IRN_SYS_ERR_RC | \ PCIE_APP_IRN_PM_TO_ACK | PCIE_APP_IRN_MSG_LTR | \ PCIE_APP_IRN_BW_MGT | PCIE_APP_IRN_LINK_AUTO_BW_STAT | \ - (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTA) | \ - (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTB) | \ - (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTC) | \ - (PCIE_APP_INTX_OFST + PCI_INTERRUPT_INTD)) + PCIE_APP_IRN_INTA | PCIE_APP_IRN_INTB | \ + PCIE_APP_IRN_INTC | PCIE_APP_IRN_INTD) #define BUS_IATU_OFFSET SZ_256M #define RESET_INTERVAL_MS 100 diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c new file mode 100644 index 000000000000..1ac29a6eef22 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe controller driver for Intel Keem Bay + * Copyright (C) 2020 Intel Corporation + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/gpio/consumer.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/kernel.h> +#include <linux/mod_devicetable.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/property.h> + +#include "pcie-designware.h" + +/* PCIE_REGS_APB_SLV Registers */ +#define PCIE_REGS_PCIE_CFG 0x0004 +#define PCIE_DEVICE_TYPE BIT(8) +#define PCIE_RSTN BIT(0) +#define PCIE_REGS_PCIE_APP_CNTRL 0x0008 +#define APP_LTSSM_ENABLE BIT(0) +#define PCIE_REGS_INTERRUPT_ENABLE 0x0028 +#define MSI_CTRL_INT_EN BIT(8) +#define EDMA_INT_EN GENMASK(7, 0) +#define PCIE_REGS_INTERRUPT_STATUS 0x002c +#define MSI_CTRL_INT BIT(8) +#define PCIE_REGS_PCIE_SII_PM_STATE 0x00b0 +#define SMLH_LINK_UP BIT(19) +#define RDLH_LINK_UP BIT(8) +#define PCIE_REGS_PCIE_SII_LINK_UP (SMLH_LINK_UP | RDLH_LINK_UP) +#define PCIE_REGS_PCIE_PHY_CNTL 0x0164 +#define PHY0_SRAM_BYPASS BIT(8) +#define PCIE_REGS_PCIE_PHY_STAT 0x0168 +#define PHY0_MPLLA_STATE BIT(1) +#define PCIE_REGS_LJPLL_STA 0x016c +#define LJPLL_LOCK BIT(0) +#define PCIE_REGS_LJPLL_CNTRL_0 0x0170 +#define LJPLL_EN BIT(29) +#define LJPLL_FOUT_EN GENMASK(24, 21) +#define PCIE_REGS_LJPLL_CNTRL_2 0x0178 +#define LJPLL_REF_DIV GENMASK(17, 12) +#define LJPLL_FB_DIV GENMASK(11, 0) +#define PCIE_REGS_LJPLL_CNTRL_3 0x017c +#define LJPLL_POST_DIV3A GENMASK(24, 22) +#define LJPLL_POST_DIV2A GENMASK(18, 16) + +#define PERST_DELAY_US 1000 +#define AUX_CLK_RATE_HZ 24000000 + +struct keembay_pcie { + struct dw_pcie pci; + void __iomem *apb_base; + enum dw_pcie_device_mode mode; + + struct clk *clk_master; + struct clk *clk_aux; + struct gpio_desc *reset; +}; + +struct keembay_pcie_of_data { + enum dw_pcie_device_mode mode; +}; + +static void keembay_ep_reset_assert(struct keembay_pcie *pcie) +{ + gpiod_set_value_cansleep(pcie->reset, 1); + usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); +} + +static void keembay_ep_reset_deassert(struct keembay_pcie *pcie) +{ + /* + * Ensure that PERST# is asserted for a minimum of 100ms. + * + * For more details, refer to PCI Express Card Electromechanical + * Specification Revision 1.1, Table-2.4. + */ + msleep(100); + + gpiod_set_value_cansleep(pcie->reset, 0); + usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); +} + +static void keembay_pcie_ltssm_set(struct keembay_pcie *pcie, bool enable) +{ + u32 val; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_APP_CNTRL); + if (enable) + val |= APP_LTSSM_ENABLE; + else + val &= ~APP_LTSSM_ENABLE; + writel(val, pcie->apb_base + PCIE_REGS_PCIE_APP_CNTRL); +} + +static int keembay_pcie_link_up(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_SII_PM_STATE); + + return (val & PCIE_REGS_PCIE_SII_LINK_UP) == PCIE_REGS_PCIE_SII_LINK_UP; +} + +static int keembay_pcie_start_link(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + int ret; + + if (pcie->mode == DW_PCIE_EP_TYPE) + return 0; + + keembay_pcie_ltssm_set(pcie, false); + + ret = readl_poll_timeout(pcie->apb_base + PCIE_REGS_PCIE_PHY_STAT, + val, val & PHY0_MPLLA_STATE, 20, + 500 * USEC_PER_MSEC); + if (ret) { + dev_err(pci->dev, "MPLLA is not locked\n"); + return ret; + } + + keembay_pcie_ltssm_set(pcie, true); + + return 0; +} + +static void keembay_pcie_stop_link(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + + keembay_pcie_ltssm_set(pcie, false); +} + +static const struct dw_pcie_ops keembay_pcie_ops = { + .link_up = keembay_pcie_link_up, + .start_link = keembay_pcie_start_link, + .stop_link = keembay_pcie_stop_link, +}; + +static inline struct clk *keembay_pcie_probe_clock(struct device *dev, + const char *id, u64 rate) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get(dev, id); + if (IS_ERR(clk)) + return clk; + + if (rate) { + ret = clk_set_rate(clk, rate); + if (ret) + return ERR_PTR(ret); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ERR_PTR(ret); + + ret = devm_add_action_or_reset(dev, + (void(*)(void *))clk_disable_unprepare, + clk); + if (ret) + return ERR_PTR(ret); + + return clk; +} + +static int keembay_pcie_probe_clocks(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pci->dev; + + pcie->clk_master = keembay_pcie_probe_clock(dev, "master", 0); + if (IS_ERR(pcie->clk_master)) + return dev_err_probe(dev, PTR_ERR(pcie->clk_master), + "Failed to enable master clock"); + + pcie->clk_aux = keembay_pcie_probe_clock(dev, "aux", AUX_CLK_RATE_HZ); + if (IS_ERR(pcie->clk_aux)) + return dev_err_probe(dev, PTR_ERR(pcie->clk_aux), + "Failed to enable auxiliary clock"); + + return 0; +} + +/* + * Initialize the internal PCIe PLL in Host mode. + * See the following sections in Keem Bay data book, + * (1) 6.4.6.1 PCIe Subsystem Example Initialization, + * (2) 6.8 PCIe Low Jitter PLL for Ref Clk Generation. + */ +static int keembay_pcie_pll_init(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val; + int ret; + + val = FIELD_PREP(LJPLL_REF_DIV, 0) | FIELD_PREP(LJPLL_FB_DIV, 0x32); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_2); + + val = FIELD_PREP(LJPLL_POST_DIV3A, 0x2) | + FIELD_PREP(LJPLL_POST_DIV2A, 0x2); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_3); + + val = FIELD_PREP(LJPLL_EN, 0x1) | FIELD_PREP(LJPLL_FOUT_EN, 0xc); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_0); + + ret = readl_poll_timeout(pcie->apb_base + PCIE_REGS_LJPLL_STA, + val, val & LJPLL_LOCK, 20, + 500 * USEC_PER_MSEC); + if (ret) + dev_err(pci->dev, "Low jitter PLL is not locked\n"); + + return ret; +} + +static void keembay_pcie_msi_irq_handler(struct irq_desc *desc) +{ + struct keembay_pcie *pcie = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + u32 val, mask, status; + struct pcie_port *pp; + + /* + * Keem Bay PCIe Controller provides an additional IP logic on top of + * standard DWC IP to clear MSI IRQ by writing '1' to the respective + * bit of the status register. + * + * So, a chained irq handler is defined to handle this additional + * IP logic. + */ + + chained_irq_enter(chip, desc); + + pp = &pcie->pci.pp; + val = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_STATUS); + mask = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + + status = val & mask; + + if (status & MSI_CTRL_INT) { + dw_handle_msi_irq(pp); + writel(status, pcie->apb_base + PCIE_REGS_INTERRUPT_STATUS); + } + + chained_irq_exit(chip, desc); +} + +static int keembay_pcie_setup_msi_irq(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + int irq; + + irq = platform_get_irq_byname(pdev, "pcie"); + if (irq < 0) + return irq; + + irq_set_chained_handler_and_data(irq, keembay_pcie_msi_irq_handler, + pcie); + + return 0; +} + +static void keembay_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + + writel(EDMA_INT_EN, pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); +} + +static int keembay_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + enum pci_epc_irq_type type, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_EPC_IRQ_LEGACY: + /* Legacy interrupts are not supported in Keem Bay */ + dev_err(pci->dev, "Legacy IRQ is not supported\n"); + return -EINVAL; + case PCI_EPC_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_EPC_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "Unknown IRQ type %d\n", type); + return -EINVAL; + } +} + +static const struct pci_epc_features keembay_pcie_epc_features = { + .linkup_notifier = false, + .msi_capable = true, + .msix_capable = true, + .reserved_bar = BIT(BAR_1) | BIT(BAR_3) | BIT(BAR_5), + .bar_fixed_64bit = BIT(BAR_0) | BIT(BAR_2) | BIT(BAR_4), + .align = SZ_16K, +}; + +static const struct pci_epc_features * +keembay_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &keembay_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops keembay_pcie_ep_ops = { + .ep_init = keembay_pcie_ep_init, + .raise_irq = keembay_pcie_ep_raise_irq, + .get_features = keembay_pcie_get_features, +}; + +static const struct dw_pcie_host_ops keembay_pcie_host_ops = { +}; + +static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct pcie_port *pp = &pci->pp; + struct device *dev = &pdev->dev; + u32 val; + int ret; + + pp->ops = &keembay_pcie_host_ops; + pp->msi_irq = -ENODEV; + + ret = keembay_pcie_setup_msi_irq(pcie); + if (ret) + return ret; + + pcie->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(pcie->reset)) + return PTR_ERR(pcie->reset); + + ret = keembay_pcie_probe_clocks(pcie); + if (ret) + return ret; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_PHY_CNTL); + val |= PHY0_SRAM_BYPASS; + writel(val, pcie->apb_base + PCIE_REGS_PCIE_PHY_CNTL); + + writel(PCIE_DEVICE_TYPE, pcie->apb_base + PCIE_REGS_PCIE_CFG); + + ret = keembay_pcie_pll_init(pcie); + if (ret) + return ret; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_CFG); + writel(val | PCIE_RSTN, pcie->apb_base + PCIE_REGS_PCIE_CFG); + keembay_ep_reset_deassert(pcie); + + ret = dw_pcie_host_init(pp); + if (ret) { + keembay_ep_reset_assert(pcie); + dev_err(dev, "Failed to initialize host: %d\n", ret); + return ret; + } + + val = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + if (IS_ENABLED(CONFIG_PCI_MSI)) + val |= MSI_CTRL_INT_EN; + writel(val, pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + + return 0; +} + +static int keembay_pcie_probe(struct platform_device *pdev) +{ + const struct keembay_pcie_of_data *data; + struct device *dev = &pdev->dev; + struct keembay_pcie *pcie; + struct dw_pcie *pci; + enum dw_pcie_device_mode mode; + + data = device_get_match_data(dev); + if (!data) + return -ENODEV; + + mode = (enum dw_pcie_device_mode)data->mode; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = dev; + pci->ops = &keembay_pcie_ops; + + pcie->mode = mode; + + pcie->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(pcie->apb_base)) + return PTR_ERR(pcie->apb_base); + + platform_set_drvdata(pdev, pcie); + + switch (pcie->mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_KEEMBAY_HOST)) + return -ENODEV; + + return keembay_pcie_add_pcie_port(pcie, pdev); + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_KEEMBAY_EP)) + return -ENODEV; + + pci->ep.ops = &keembay_pcie_ep_ops; + return dw_pcie_ep_init(&pci->ep); + default: + dev_err(dev, "Invalid device type %d\n", pcie->mode); + return -ENODEV; + } +} + +static const struct keembay_pcie_of_data keembay_pcie_rc_of_data = { + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct keembay_pcie_of_data keembay_pcie_ep_of_data = { + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id keembay_pcie_of_match[] = { + { + .compatible = "intel,keembay-pcie", + .data = &keembay_pcie_rc_of_data, + }, + { + .compatible = "intel,keembay-pcie-ep", + .data = &keembay_pcie_ep_of_data, + }, + {} +}; + +static struct platform_driver keembay_pcie_driver = { + .driver = { + .name = "keembay-pcie", + .of_match_table = keembay_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = keembay_pcie_probe, +}; +builtin_platform_driver(keembay_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index 026fd1e42a55..095afbccf9c1 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -8,16 +8,18 @@ * Author: Xiaowei Song <songxiaowei@huawei.com> */ -#include <linux/compiler.h> #include <linux/clk.h> +#include <linux/compiler.h> #include <linux/delay.h> #include <linux/err.h> #include <linux/gpio.h> #include <linux/interrupt.h> #include <linux/mfd/syscon.h> #include <linux/of_address.h> +#include <linux/of_device.h> #include <linux/of_gpio.h> #include <linux/of_pci.h> +#include <linux/phy/phy.h> #include <linux/pci.h> #include <linux/pci_regs.h> #include <linux/platform_device.h> @@ -28,26 +30,16 @@ #define to_kirin_pcie(x) dev_get_drvdata((x)->dev) -#define REF_CLK_FREQ 100000000 - /* PCIe ELBI registers */ #define SOC_PCIECTRL_CTRL0_ADDR 0x000 #define SOC_PCIECTRL_CTRL1_ADDR 0x004 -#define SOC_PCIEPHY_CTRL2_ADDR 0x008 -#define SOC_PCIEPHY_CTRL3_ADDR 0x00c #define PCIE_ELBI_SLV_DBI_ENABLE (0x1 << 21) /* info located in APB */ #define PCIE_APP_LTSSM_ENABLE 0x01c -#define PCIE_APB_PHY_CTRL0 0x0 -#define PCIE_APB_PHY_CTRL1 0x4 #define PCIE_APB_PHY_STATUS0 0x400 #define PCIE_LINKUP_ENABLE (0x8020) #define PCIE_LTSSM_ENABLE_BIT (0x1 << 11) -#define PIPE_CLK_STABLE (0x1 << 19) -#define PHY_REF_PAD_BIT (0x1 << 8) -#define PHY_PWR_DOWN_BIT (0x1 << 22) -#define PHY_RST_ACK_BIT (0x1 << 16) /* info located in sysctrl */ #define SCTRL_PCIE_CMOS_OFFSET 0x60 @@ -60,17 +52,70 @@ #define PCIE_DEBOUNCE_PARAM 0xF0F400 #define PCIE_OE_BYPASS (0x3 << 28) +/* + * Max number of connected PCI slots at an external PCI bridge + * + * This is used on HiKey 970, which has a PEX 8606 bridge with 4 connected + * lanes (lane 0 upstream, and the other three lanes, one connected to an + * in-board Ethernet adapter and the other two connected to M.2 and mini + * PCI slots. + * + * Each slot has a different clock source and uses a separate PERST# pin. + */ +#define MAX_PCI_SLOTS 3 + +enum pcie_kirin_phy_type { + PCIE_KIRIN_INTERNAL_PHY, + PCIE_KIRIN_EXTERNAL_PHY +}; + +struct kirin_pcie { + enum pcie_kirin_phy_type type; + + struct dw_pcie *pci; + struct regmap *apb; + struct phy *phy; + void *phy_priv; /* only for PCIE_KIRIN_INTERNAL_PHY */ + + /* DWC PERST# */ + int gpio_id_dwc_perst; + + /* Per-slot PERST# */ + int num_slots; + int gpio_id_reset[MAX_PCI_SLOTS]; + const char *reset_names[MAX_PCI_SLOTS]; + + /* Per-slot clkreq */ + int n_gpio_clkreq; + int gpio_id_clkreq[MAX_PCI_SLOTS]; + const char *clkreq_names[MAX_PCI_SLOTS]; +}; + +/* + * Kirin 960 PHY. Can't be split into a PHY driver without changing the + * DT schema. + */ + +#define REF_CLK_FREQ 100000000 + +/* PHY info located in APB */ +#define PCIE_APB_PHY_CTRL0 0x0 +#define PCIE_APB_PHY_CTRL1 0x4 +#define PCIE_APB_PHY_STATUS0 0x400 +#define PIPE_CLK_STABLE BIT(19) +#define PHY_REF_PAD_BIT BIT(8) +#define PHY_PWR_DOWN_BIT BIT(22) +#define PHY_RST_ACK_BIT BIT(16) + /* peri_crg ctrl */ #define CRGCTRL_PCIE_ASSERT_OFFSET 0x88 #define CRGCTRL_PCIE_ASSERT_BIT 0x8c000000 /* Time for delay */ -#define REF_2_PERST_MIN 20000 +#define REF_2_PERST_MIN 21000 #define REF_2_PERST_MAX 25000 #define PERST_2_ACCESS_MIN 10000 #define PERST_2_ACCESS_MAX 12000 -#define LINK_WAIT_MIN 900 -#define LINK_WAIT_MAX 1000 #define PIPE_CLK_WAIT_MIN 550 #define PIPE_CLK_WAIT_MAX 600 #define TIME_CMOS_MIN 100 @@ -78,118 +123,101 @@ #define TIME_PHY_PD_MIN 10 #define TIME_PHY_PD_MAX 11 -struct kirin_pcie { - struct dw_pcie *pci; - void __iomem *apb_base; - void __iomem *phy_base; +struct hi3660_pcie_phy { + struct device *dev; + void __iomem *base; struct regmap *crgctrl; struct regmap *sysctrl; struct clk *apb_sys_clk; struct clk *apb_phy_clk; struct clk *phy_ref_clk; - struct clk *pcie_aclk; - struct clk *pcie_aux_clk; - int gpio_id_reset; + struct clk *aclk; + struct clk *aux_clk; }; -/* Registers in PCIeCTRL */ -static inline void kirin_apb_ctrl_writel(struct kirin_pcie *kirin_pcie, - u32 val, u32 reg) -{ - writel(val, kirin_pcie->apb_base + reg); -} - -static inline u32 kirin_apb_ctrl_readl(struct kirin_pcie *kirin_pcie, u32 reg) -{ - return readl(kirin_pcie->apb_base + reg); -} - /* Registers in PCIePHY */ -static inline void kirin_apb_phy_writel(struct kirin_pcie *kirin_pcie, +static inline void kirin_apb_phy_writel(struct hi3660_pcie_phy *hi3660_pcie_phy, u32 val, u32 reg) { - writel(val, kirin_pcie->phy_base + reg); + writel(val, hi3660_pcie_phy->base + reg); } -static inline u32 kirin_apb_phy_readl(struct kirin_pcie *kirin_pcie, u32 reg) +static inline u32 kirin_apb_phy_readl(struct hi3660_pcie_phy *hi3660_pcie_phy, + u32 reg) { - return readl(kirin_pcie->phy_base + reg); + return readl(hi3660_pcie_phy->base + reg); } -static long kirin_pcie_get_clk(struct kirin_pcie *kirin_pcie, - struct platform_device *pdev) +static int hi3660_pcie_phy_get_clk(struct hi3660_pcie_phy *phy) { - struct device *dev = &pdev->dev; + struct device *dev = phy->dev; - kirin_pcie->phy_ref_clk = devm_clk_get(dev, "pcie_phy_ref"); - if (IS_ERR(kirin_pcie->phy_ref_clk)) - return PTR_ERR(kirin_pcie->phy_ref_clk); + phy->phy_ref_clk = devm_clk_get(dev, "pcie_phy_ref"); + if (IS_ERR(phy->phy_ref_clk)) + return PTR_ERR(phy->phy_ref_clk); - kirin_pcie->pcie_aux_clk = devm_clk_get(dev, "pcie_aux"); - if (IS_ERR(kirin_pcie->pcie_aux_clk)) - return PTR_ERR(kirin_pcie->pcie_aux_clk); + phy->aux_clk = devm_clk_get(dev, "pcie_aux"); + if (IS_ERR(phy->aux_clk)) + return PTR_ERR(phy->aux_clk); - kirin_pcie->apb_phy_clk = devm_clk_get(dev, "pcie_apb_phy"); - if (IS_ERR(kirin_pcie->apb_phy_clk)) - return PTR_ERR(kirin_pcie->apb_phy_clk); + phy->apb_phy_clk = devm_clk_get(dev, "pcie_apb_phy"); + if (IS_ERR(phy->apb_phy_clk)) + return PTR_ERR(phy->apb_phy_clk); - kirin_pcie->apb_sys_clk = devm_clk_get(dev, "pcie_apb_sys"); - if (IS_ERR(kirin_pcie->apb_sys_clk)) - return PTR_ERR(kirin_pcie->apb_sys_clk); + phy->apb_sys_clk = devm_clk_get(dev, "pcie_apb_sys"); + if (IS_ERR(phy->apb_sys_clk)) + return PTR_ERR(phy->apb_sys_clk); - kirin_pcie->pcie_aclk = devm_clk_get(dev, "pcie_aclk"); - if (IS_ERR(kirin_pcie->pcie_aclk)) - return PTR_ERR(kirin_pcie->pcie_aclk); + phy->aclk = devm_clk_get(dev, "pcie_aclk"); + if (IS_ERR(phy->aclk)) + return PTR_ERR(phy->aclk); return 0; } -static long kirin_pcie_get_resource(struct kirin_pcie *kirin_pcie, - struct platform_device *pdev) +static int hi3660_pcie_phy_get_resource(struct hi3660_pcie_phy *phy) { - kirin_pcie->apb_base = - devm_platform_ioremap_resource_byname(pdev, "apb"); - if (IS_ERR(kirin_pcie->apb_base)) - return PTR_ERR(kirin_pcie->apb_base); - - kirin_pcie->phy_base = - devm_platform_ioremap_resource_byname(pdev, "phy"); - if (IS_ERR(kirin_pcie->phy_base)) - return PTR_ERR(kirin_pcie->phy_base); - - kirin_pcie->crgctrl = - syscon_regmap_lookup_by_compatible("hisilicon,hi3660-crgctrl"); - if (IS_ERR(kirin_pcie->crgctrl)) - return PTR_ERR(kirin_pcie->crgctrl); - - kirin_pcie->sysctrl = - syscon_regmap_lookup_by_compatible("hisilicon,hi3660-sctrl"); - if (IS_ERR(kirin_pcie->sysctrl)) - return PTR_ERR(kirin_pcie->sysctrl); + struct device *dev = phy->dev; + struct platform_device *pdev; + + /* registers */ + pdev = container_of(dev, struct platform_device, dev); + + phy->base = devm_platform_ioremap_resource_byname(pdev, "phy"); + if (IS_ERR(phy->base)) + return PTR_ERR(phy->base); + + phy->crgctrl = syscon_regmap_lookup_by_compatible("hisilicon,hi3660-crgctrl"); + if (IS_ERR(phy->crgctrl)) + return PTR_ERR(phy->crgctrl); + + phy->sysctrl = syscon_regmap_lookup_by_compatible("hisilicon,hi3660-sctrl"); + if (IS_ERR(phy->sysctrl)) + return PTR_ERR(phy->sysctrl); return 0; } -static int kirin_pcie_phy_init(struct kirin_pcie *kirin_pcie) +static int hi3660_pcie_phy_start(struct hi3660_pcie_phy *phy) { - struct device *dev = kirin_pcie->pci->dev; + struct device *dev = phy->dev; u32 reg_val; - reg_val = kirin_apb_phy_readl(kirin_pcie, PCIE_APB_PHY_CTRL1); + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL1); reg_val &= ~PHY_REF_PAD_BIT; - kirin_apb_phy_writel(kirin_pcie, reg_val, PCIE_APB_PHY_CTRL1); + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL1); - reg_val = kirin_apb_phy_readl(kirin_pcie, PCIE_APB_PHY_CTRL0); + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL0); reg_val &= ~PHY_PWR_DOWN_BIT; - kirin_apb_phy_writel(kirin_pcie, reg_val, PCIE_APB_PHY_CTRL0); + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL0); usleep_range(TIME_PHY_PD_MIN, TIME_PHY_PD_MAX); - reg_val = kirin_apb_phy_readl(kirin_pcie, PCIE_APB_PHY_CTRL1); + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL1); reg_val &= ~PHY_RST_ACK_BIT; - kirin_apb_phy_writel(kirin_pcie, reg_val, PCIE_APB_PHY_CTRL1); + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL1); usleep_range(PIPE_CLK_WAIT_MIN, PIPE_CLK_WAIT_MAX); - reg_val = kirin_apb_phy_readl(kirin_pcie, PCIE_APB_PHY_STATUS0); + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_STATUS0); if (reg_val & PIPE_CLK_STABLE) { dev_err(dev, "PIPE clk is not stable\n"); return -EINVAL; @@ -198,102 +226,274 @@ static int kirin_pcie_phy_init(struct kirin_pcie *kirin_pcie) return 0; } -static void kirin_pcie_oe_enable(struct kirin_pcie *kirin_pcie) +static void hi3660_pcie_phy_oe_enable(struct hi3660_pcie_phy *phy) { u32 val; - regmap_read(kirin_pcie->sysctrl, SCTRL_PCIE_OE_OFFSET, &val); + regmap_read(phy->sysctrl, SCTRL_PCIE_OE_OFFSET, &val); val |= PCIE_DEBOUNCE_PARAM; val &= ~PCIE_OE_BYPASS; - regmap_write(kirin_pcie->sysctrl, SCTRL_PCIE_OE_OFFSET, val); + regmap_write(phy->sysctrl, SCTRL_PCIE_OE_OFFSET, val); } -static int kirin_pcie_clk_ctrl(struct kirin_pcie *kirin_pcie, bool enable) +static int hi3660_pcie_phy_clk_ctrl(struct hi3660_pcie_phy *phy, bool enable) { int ret = 0; if (!enable) goto close_clk; - ret = clk_set_rate(kirin_pcie->phy_ref_clk, REF_CLK_FREQ); + ret = clk_set_rate(phy->phy_ref_clk, REF_CLK_FREQ); if (ret) return ret; - ret = clk_prepare_enable(kirin_pcie->phy_ref_clk); + ret = clk_prepare_enable(phy->phy_ref_clk); if (ret) return ret; - ret = clk_prepare_enable(kirin_pcie->apb_sys_clk); + ret = clk_prepare_enable(phy->apb_sys_clk); if (ret) goto apb_sys_fail; - ret = clk_prepare_enable(kirin_pcie->apb_phy_clk); + ret = clk_prepare_enable(phy->apb_phy_clk); if (ret) goto apb_phy_fail; - ret = clk_prepare_enable(kirin_pcie->pcie_aclk); + ret = clk_prepare_enable(phy->aclk); if (ret) goto aclk_fail; - ret = clk_prepare_enable(kirin_pcie->pcie_aux_clk); + ret = clk_prepare_enable(phy->aux_clk); if (ret) goto aux_clk_fail; return 0; close_clk: - clk_disable_unprepare(kirin_pcie->pcie_aux_clk); + clk_disable_unprepare(phy->aux_clk); aux_clk_fail: - clk_disable_unprepare(kirin_pcie->pcie_aclk); + clk_disable_unprepare(phy->aclk); aclk_fail: - clk_disable_unprepare(kirin_pcie->apb_phy_clk); + clk_disable_unprepare(phy->apb_phy_clk); apb_phy_fail: - clk_disable_unprepare(kirin_pcie->apb_sys_clk); + clk_disable_unprepare(phy->apb_sys_clk); apb_sys_fail: - clk_disable_unprepare(kirin_pcie->phy_ref_clk); + clk_disable_unprepare(phy->phy_ref_clk); return ret; } -static int kirin_pcie_power_on(struct kirin_pcie *kirin_pcie) +static int hi3660_pcie_phy_power_on(struct kirin_pcie *pcie) { + struct hi3660_pcie_phy *phy = pcie->phy_priv; int ret; /* Power supply for Host */ - regmap_write(kirin_pcie->sysctrl, + regmap_write(phy->sysctrl, SCTRL_PCIE_CMOS_OFFSET, SCTRL_PCIE_CMOS_BIT); usleep_range(TIME_CMOS_MIN, TIME_CMOS_MAX); - kirin_pcie_oe_enable(kirin_pcie); - ret = kirin_pcie_clk_ctrl(kirin_pcie, true); + hi3660_pcie_phy_oe_enable(phy); + + ret = hi3660_pcie_phy_clk_ctrl(phy, true); if (ret) return ret; /* ISO disable, PCIeCtrl, PHY assert and clk gate clear */ - regmap_write(kirin_pcie->sysctrl, + regmap_write(phy->sysctrl, SCTRL_PCIE_ISO_OFFSET, SCTRL_PCIE_ISO_BIT); - regmap_write(kirin_pcie->crgctrl, + regmap_write(phy->crgctrl, CRGCTRL_PCIE_ASSERT_OFFSET, CRGCTRL_PCIE_ASSERT_BIT); - regmap_write(kirin_pcie->sysctrl, + regmap_write(phy->sysctrl, SCTRL_PCIE_HPCLK_OFFSET, SCTRL_PCIE_HPCLK_BIT); - ret = kirin_pcie_phy_init(kirin_pcie); + ret = hi3660_pcie_phy_start(phy); if (ret) - goto close_clk; + goto disable_clks; - /* perst assert Endpoint */ - if (!gpio_request(kirin_pcie->gpio_id_reset, "pcie_perst")) { - usleep_range(REF_2_PERST_MIN, REF_2_PERST_MAX); - ret = gpio_direction_output(kirin_pcie->gpio_id_reset, 1); - if (ret) - goto close_clk; - usleep_range(PERST_2_ACCESS_MIN, PERST_2_ACCESS_MAX); + return 0; + +disable_clks: + hi3660_pcie_phy_clk_ctrl(phy, false); + return ret; +} + +static int hi3660_pcie_phy_init(struct platform_device *pdev, + struct kirin_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct hi3660_pcie_phy *phy; + int ret; + phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + pcie->phy_priv = phy; + phy->dev = dev; + + /* registers */ + pdev = container_of(dev, struct platform_device, dev); + + ret = hi3660_pcie_phy_get_clk(phy); + if (ret) + return ret; + + return hi3660_pcie_phy_get_resource(phy); +} + +static int hi3660_pcie_phy_power_off(struct kirin_pcie *pcie) +{ + struct hi3660_pcie_phy *phy = pcie->phy_priv; + + /* Drop power supply for Host */ + regmap_write(phy->sysctrl, SCTRL_PCIE_CMOS_OFFSET, 0x00); + + hi3660_pcie_phy_clk_ctrl(phy, false); + + return 0; +} + +/* + * The non-PHY part starts here + */ + +static const struct regmap_config pcie_kirin_regmap_conf = { + .name = "kirin_pcie_apb", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static int kirin_pcie_get_gpio_enable(struct kirin_pcie *pcie, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + char name[32]; + int ret, i; + + /* This is an optional property */ + ret = of_gpio_named_count(np, "hisilicon,clken-gpios"); + if (ret < 0) return 0; + + if (ret > MAX_PCI_SLOTS) { + dev_err(dev, "Too many GPIO clock requests!\n"); + return -EINVAL; } -close_clk: - kirin_pcie_clk_ctrl(kirin_pcie, false); + pcie->n_gpio_clkreq = ret; + + for (i = 0; i < pcie->n_gpio_clkreq; i++) { + pcie->gpio_id_clkreq[i] = of_get_named_gpio(dev->of_node, + "hisilicon,clken-gpios", i); + if (pcie->gpio_id_clkreq[i] < 0) + return pcie->gpio_id_clkreq[i]; + + sprintf(name, "pcie_clkreq_%d", i); + pcie->clkreq_names[i] = devm_kstrdup_const(dev, name, + GFP_KERNEL); + if (!pcie->clkreq_names[i]) + return -ENOMEM; + } + + return 0; +} + +static int kirin_pcie_parse_port(struct kirin_pcie *pcie, + struct platform_device *pdev, + struct device_node *node) +{ + struct device *dev = &pdev->dev; + struct device_node *parent, *child; + int ret, slot, i; + char name[32]; + + for_each_available_child_of_node(node, parent) { + for_each_available_child_of_node(parent, child) { + i = pcie->num_slots; + + pcie->gpio_id_reset[i] = of_get_named_gpio(child, + "reset-gpios", 0); + if (pcie->gpio_id_reset[i] < 0) + continue; + + pcie->num_slots++; + if (pcie->num_slots > MAX_PCI_SLOTS) { + dev_err(dev, "Too many PCI slots!\n"); + ret = -EINVAL; + goto put_node; + } + + ret = of_pci_get_devfn(child); + if (ret < 0) { + dev_err(dev, "failed to parse devfn: %d\n", ret); + goto put_node; + } + + slot = PCI_SLOT(ret); + + sprintf(name, "pcie_perst_%d", slot); + pcie->reset_names[i] = devm_kstrdup_const(dev, name, + GFP_KERNEL); + if (!pcie->reset_names[i]) { + ret = -ENOMEM; + goto put_node; + } + } + } + + return 0; + +put_node: + of_node_put(child); + of_node_put(parent); + return ret; +} + +static long kirin_pcie_get_resource(struct kirin_pcie *kirin_pcie, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *child, *node = dev->of_node; + void __iomem *apb_base; + int ret; + + apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(apb_base)) + return PTR_ERR(apb_base); + + kirin_pcie->apb = devm_regmap_init_mmio(dev, apb_base, + &pcie_kirin_regmap_conf); + if (IS_ERR(kirin_pcie->apb)) + return PTR_ERR(kirin_pcie->apb); + + /* pcie internal PERST# gpio */ + kirin_pcie->gpio_id_dwc_perst = of_get_named_gpio(dev->of_node, + "reset-gpios", 0); + if (kirin_pcie->gpio_id_dwc_perst == -EPROBE_DEFER) { + return -EPROBE_DEFER; + } else if (!gpio_is_valid(kirin_pcie->gpio_id_dwc_perst)) { + dev_err(dev, "unable to get a valid gpio pin\n"); + return -ENODEV; + } + + ret = kirin_pcie_get_gpio_enable(kirin_pcie, pdev); + if (ret) + return ret; + + /* Parse OF children */ + for_each_available_child_of_node(node, child) { + ret = kirin_pcie_parse_port(kirin_pcie, pdev, child); + if (ret) + goto put_node; + } + + return 0; + +put_node: + of_node_put(child); return ret; } @@ -302,13 +502,13 @@ static void kirin_pcie_sideband_dbi_w_mode(struct kirin_pcie *kirin_pcie, { u32 val; - val = kirin_apb_ctrl_readl(kirin_pcie, SOC_PCIECTRL_CTRL0_ADDR); + regmap_read(kirin_pcie->apb, SOC_PCIECTRL_CTRL0_ADDR, &val); if (on) val = val | PCIE_ELBI_SLV_DBI_ENABLE; else val = val & ~PCIE_ELBI_SLV_DBI_ENABLE; - kirin_apb_ctrl_writel(kirin_pcie, val, SOC_PCIECTRL_CTRL0_ADDR); + regmap_write(kirin_pcie->apb, SOC_PCIECTRL_CTRL0_ADDR, val); } static void kirin_pcie_sideband_dbi_r_mode(struct kirin_pcie *kirin_pcie, @@ -316,13 +516,13 @@ static void kirin_pcie_sideband_dbi_r_mode(struct kirin_pcie *kirin_pcie, { u32 val; - val = kirin_apb_ctrl_readl(kirin_pcie, SOC_PCIECTRL_CTRL1_ADDR); + regmap_read(kirin_pcie->apb, SOC_PCIECTRL_CTRL1_ADDR, &val); if (on) val = val | PCIE_ELBI_SLV_DBI_ENABLE; else val = val & ~PCIE_ELBI_SLV_DBI_ENABLE; - kirin_apb_ctrl_writel(kirin_pcie, val, SOC_PCIECTRL_CTRL1_ADDR); + regmap_write(kirin_pcie->apb, SOC_PCIECTRL_CTRL1_ADDR, val); } static int kirin_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, @@ -351,9 +551,32 @@ static int kirin_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, return PCIBIOS_SUCCESSFUL; } +static int kirin_pcie_add_bus(struct pci_bus *bus) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + int i, ret; + + if (!kirin_pcie->num_slots) + return 0; + + /* Send PERST# to each slot */ + for (i = 0; i < kirin_pcie->num_slots; i++) { + ret = gpio_direction_output(kirin_pcie->gpio_id_reset[i], 1); + if (ret) { + dev_err(pci->dev, "PERST# %s error: %d\n", + kirin_pcie->reset_names[i], ret); + } + } + usleep_range(PERST_2_ACCESS_MIN, PERST_2_ACCESS_MAX); + + return 0; +} + static struct pci_ops kirin_pci_ops = { .read = kirin_pcie_rd_own_conf, .write = kirin_pcie_wr_own_conf, + .add_bus = kirin_pcie_add_bus, }; static u32 kirin_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, @@ -382,8 +605,9 @@ static void kirin_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, static int kirin_pcie_link_up(struct dw_pcie *pci) { struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); - u32 val = kirin_apb_ctrl_readl(kirin_pcie, PCIE_APB_PHY_STATUS0); + u32 val; + regmap_read(kirin_pcie->apb, PCIE_APB_PHY_STATUS0, &val); if ((val & PCIE_LINKUP_ENABLE) == PCIE_LINKUP_ENABLE) return 1; @@ -395,8 +619,8 @@ static int kirin_pcie_start_link(struct dw_pcie *pci) struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); /* assert LTSSM enable */ - kirin_apb_ctrl_writel(kirin_pcie, PCIE_LTSSM_ENABLE_BIT, - PCIE_APP_LTSSM_ENABLE); + regmap_write(kirin_pcie->apb, PCIE_APP_LTSSM_ENABLE, + PCIE_LTSSM_ENABLE_BIT); return 0; } @@ -408,6 +632,44 @@ static int kirin_pcie_host_init(struct pcie_port *pp) return 0; } +static int kirin_pcie_gpio_request(struct kirin_pcie *kirin_pcie, + struct device *dev) +{ + int ret, i; + + for (i = 0; i < kirin_pcie->num_slots; i++) { + if (!gpio_is_valid(kirin_pcie->gpio_id_reset[i])) { + dev_err(dev, "unable to get a valid %s gpio\n", + kirin_pcie->reset_names[i]); + return -ENODEV; + } + + ret = devm_gpio_request(dev, kirin_pcie->gpio_id_reset[i], + kirin_pcie->reset_names[i]); + if (ret) + return ret; + } + + for (i = 0; i < kirin_pcie->n_gpio_clkreq; i++) { + if (!gpio_is_valid(kirin_pcie->gpio_id_clkreq[i])) { + dev_err(dev, "unable to get a valid %s gpio\n", + kirin_pcie->clkreq_names[i]); + return -ENODEV; + } + + ret = devm_gpio_request(dev, kirin_pcie->gpio_id_clkreq[i], + kirin_pcie->clkreq_names[i]); + if (ret) + return ret; + + ret = gpio_direction_output(kirin_pcie->gpio_id_clkreq[i], 0); + if (ret) + return ret; + } + + return 0; +} + static const struct dw_pcie_ops kirin_dw_pcie_ops = { .read_dbi = kirin_pcie_read_dbi, .write_dbi = kirin_pcie_write_dbi, @@ -419,8 +681,99 @@ static const struct dw_pcie_host_ops kirin_pcie_host_ops = { .host_init = kirin_pcie_host_init, }; +static int kirin_pcie_power_off(struct kirin_pcie *kirin_pcie) +{ + int i; + + if (kirin_pcie->type == PCIE_KIRIN_INTERNAL_PHY) + return hi3660_pcie_phy_power_off(kirin_pcie); + + for (i = 0; i < kirin_pcie->n_gpio_clkreq; i++) + gpio_direction_output(kirin_pcie->gpio_id_clkreq[i], 1); + + phy_power_off(kirin_pcie->phy); + phy_exit(kirin_pcie->phy); + + return 0; +} + +static int kirin_pcie_power_on(struct platform_device *pdev, + struct kirin_pcie *kirin_pcie) +{ + struct device *dev = &pdev->dev; + int ret; + + if (kirin_pcie->type == PCIE_KIRIN_INTERNAL_PHY) { + ret = hi3660_pcie_phy_init(pdev, kirin_pcie); + if (ret) + return ret; + + ret = hi3660_pcie_phy_power_on(kirin_pcie); + if (ret) + return ret; + } else { + kirin_pcie->phy = devm_of_phy_get(dev, dev->of_node, NULL); + if (IS_ERR(kirin_pcie->phy)) + return PTR_ERR(kirin_pcie->phy); + + ret = kirin_pcie_gpio_request(kirin_pcie, dev); + if (ret) + return ret; + + ret = phy_init(kirin_pcie->phy); + if (ret) + goto err; + + ret = phy_power_on(kirin_pcie->phy); + if (ret) + goto err; + } + + /* perst assert Endpoint */ + usleep_range(REF_2_PERST_MIN, REF_2_PERST_MAX); + + if (!gpio_request(kirin_pcie->gpio_id_dwc_perst, "pcie_perst_bridge")) { + ret = gpio_direction_output(kirin_pcie->gpio_id_dwc_perst, 1); + if (ret) + goto err; + } + + usleep_range(PERST_2_ACCESS_MIN, PERST_2_ACCESS_MAX); + + return 0; +err: + kirin_pcie_power_off(kirin_pcie); + + return ret; +} + +static int __exit kirin_pcie_remove(struct platform_device *pdev) +{ + struct kirin_pcie *kirin_pcie = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&kirin_pcie->pci->pp); + + kirin_pcie_power_off(kirin_pcie); + + return 0; +} + +static const struct of_device_id kirin_pcie_match[] = { + { + .compatible = "hisilicon,kirin960-pcie", + .data = (void *)PCIE_KIRIN_INTERNAL_PHY + }, + { + .compatible = "hisilicon,kirin970-pcie", + .data = (void *)PCIE_KIRIN_EXTERNAL_PHY + }, + {}, +}; + static int kirin_pcie_probe(struct platform_device *pdev) { + enum pcie_kirin_phy_type phy_type; + const struct of_device_id *of_id; struct device *dev = &pdev->dev; struct kirin_pcie *kirin_pcie; struct dw_pcie *pci; @@ -431,6 +784,14 @@ static int kirin_pcie_probe(struct platform_device *pdev) return -EINVAL; } + of_id = of_match_device(kirin_pcie_match, dev); + if (!of_id) { + dev_err(dev, "OF data missing\n"); + return -EINVAL; + } + + phy_type = (long)of_id->data; + kirin_pcie = devm_kzalloc(dev, sizeof(struct kirin_pcie), GFP_KERNEL); if (!kirin_pcie) return -ENOMEM; @@ -443,44 +804,33 @@ static int kirin_pcie_probe(struct platform_device *pdev) pci->ops = &kirin_dw_pcie_ops; pci->pp.ops = &kirin_pcie_host_ops; kirin_pcie->pci = pci; - - ret = kirin_pcie_get_clk(kirin_pcie, pdev); - if (ret) - return ret; + kirin_pcie->type = phy_type; ret = kirin_pcie_get_resource(kirin_pcie, pdev); if (ret) return ret; - kirin_pcie->gpio_id_reset = of_get_named_gpio(dev->of_node, - "reset-gpios", 0); - if (kirin_pcie->gpio_id_reset == -EPROBE_DEFER) { - return -EPROBE_DEFER; - } else if (!gpio_is_valid(kirin_pcie->gpio_id_reset)) { - dev_err(dev, "unable to get a valid gpio pin\n"); - return -ENODEV; - } + platform_set_drvdata(pdev, kirin_pcie); - ret = kirin_pcie_power_on(kirin_pcie); + ret = kirin_pcie_power_on(pdev, kirin_pcie); if (ret) return ret; - platform_set_drvdata(pdev, kirin_pcie); - return dw_pcie_host_init(&pci->pp); } -static const struct of_device_id kirin_pcie_match[] = { - { .compatible = "hisilicon,kirin960-pcie" }, - {}, -}; - static struct platform_driver kirin_pcie_driver = { .probe = kirin_pcie_probe, + .remove = __exit_p(kirin_pcie_remove), .driver = { .name = "kirin-pcie", - .of_match_table = kirin_pcie_match, - .suppress_bind_attrs = true, + .of_match_table = kirin_pcie_match, + .suppress_bind_attrs = true, }, }; -builtin_platform_driver(kirin_pcie_driver); +module_platform_driver(kirin_pcie_driver); + +MODULE_DEVICE_TABLE(of, kirin_pcie_match); +MODULE_DESCRIPTION("PCIe host controller driver for Kirin Phone SoCs"); +MODULE_AUTHOR("Xiaowei Song <songxiaowei@huawei.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c new file mode 100644 index 000000000000..cfe66bf04c1d --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm PCIe Endpoint controller driver + * + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Author: Siddartha Mohanadoss <smohanad@codeaurora.org + * + * Copyright (c) 2021, Linaro Ltd. + * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/mfd/syscon.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/module.h> + +#include "pcie-designware.h" + +/* PARF registers */ +#define PARF_SYS_CTRL 0x00 +#define PARF_DB_CTRL 0x10 +#define PARF_PM_CTRL 0x20 +#define PARF_MHI_BASE_ADDR_LOWER 0x178 +#define PARF_MHI_BASE_ADDR_UPPER 0x17c +#define PARF_DEBUG_INT_EN 0x190 +#define PARF_AXI_MSTR_RD_HALT_NO_WRITES 0x1a4 +#define PARF_AXI_MSTR_WR_ADDR_HALT 0x1a8 +#define PARF_Q2A_FLUSH 0x1ac +#define PARF_LTSSM 0x1b0 +#define PARF_CFG_BITS 0x210 +#define PARF_INT_ALL_STATUS 0x224 +#define PARF_INT_ALL_CLEAR 0x228 +#define PARF_INT_ALL_MASK 0x22c +#define PARF_SLV_ADDR_MSB_CTRL 0x2c0 +#define PARF_DBI_BASE_ADDR 0x350 +#define PARF_DBI_BASE_ADDR_HI 0x354 +#define PARF_SLV_ADDR_SPACE_SIZE 0x358 +#define PARF_SLV_ADDR_SPACE_SIZE_HI 0x35c +#define PARF_ATU_BASE_ADDR 0x634 +#define PARF_ATU_BASE_ADDR_HI 0x638 +#define PARF_SRIS_MODE 0x644 +#define PARF_DEVICE_TYPE 0x1000 +#define PARF_BDF_TO_SID_CFG 0x2c00 + +/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ +#define PARF_INT_ALL_LINK_DOWN BIT(1) +#define PARF_INT_ALL_BME BIT(2) +#define PARF_INT_ALL_PM_TURNOFF BIT(3) +#define PARF_INT_ALL_DEBUG BIT(4) +#define PARF_INT_ALL_LTR BIT(5) +#define PARF_INT_ALL_MHI_Q6 BIT(6) +#define PARF_INT_ALL_MHI_A7 BIT(7) +#define PARF_INT_ALL_DSTATE_CHANGE BIT(8) +#define PARF_INT_ALL_L1SUB_TIMEOUT BIT(9) +#define PARF_INT_ALL_MMIO_WRITE BIT(10) +#define PARF_INT_ALL_CFG_WRITE BIT(11) +#define PARF_INT_ALL_BRIDGE_FLUSH_N BIT(12) +#define PARF_INT_ALL_LINK_UP BIT(13) +#define PARF_INT_ALL_AER_LEGACY BIT(14) +#define PARF_INT_ALL_PLS_ERR BIT(15) +#define PARF_INT_ALL_PME_LEGACY BIT(16) +#define PARF_INT_ALL_PLS_PME BIT(17) + +/* PARF_BDF_TO_SID_CFG register fields */ +#define PARF_BDF_TO_SID_BYPASS BIT(0) + +/* PARF_DEBUG_INT_EN register fields */ +#define PARF_DEBUG_INT_PM_DSTATE_CHANGE BIT(1) +#define PARF_DEBUG_INT_CFG_BUS_MASTER_EN BIT(2) +#define PARF_DEBUG_INT_RADM_PM_TURNOFF BIT(3) + +/* PARF_DEVICE_TYPE register fields */ +#define PARF_DEVICE_TYPE_EP 0x0 + +/* PARF_PM_CTRL register fields */ +#define PARF_PM_CTRL_REQ_EXIT_L1 BIT(1) +#define PARF_PM_CTRL_READY_ENTR_L23 BIT(2) +#define PARF_PM_CTRL_REQ_NOT_ENTR_L1 BIT(5) + +/* PARF_AXI_MSTR_RD_HALT_NO_WRITES register fields */ +#define PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN BIT(0) + +/* PARF_AXI_MSTR_WR_ADDR_HALT register fields */ +#define PARF_AXI_MSTR_WR_ADDR_HALT_EN BIT(31) + +/* PARF_Q2A_FLUSH register fields */ +#define PARF_Q2A_FLUSH_EN BIT(16) + +/* PARF_SYS_CTRL register fields */ +#define PARF_SYS_CTRL_AUX_PWR_DET BIT(4) +#define PARF_SYS_CTRL_CORE_CLK_CGC_DIS BIT(6) +#define PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE BIT(11) + +/* PARF_DB_CTRL register fields */ +#define PARF_DB_CTRL_INSR_DBNCR_BLOCK BIT(0) +#define PARF_DB_CTRL_RMVL_DBNCR_BLOCK BIT(1) +#define PARF_DB_CTRL_DBI_WKP_BLOCK BIT(4) +#define PARF_DB_CTRL_SLV_WKP_BLOCK BIT(5) +#define PARF_DB_CTRL_MST_WKP_BLOCK BIT(6) + +/* PARF_CFG_BITS register fields */ +#define PARF_CFG_BITS_REQ_EXIT_L1SS_MSI_LTR_EN BIT(1) + +/* ELBI registers */ +#define ELBI_SYS_STTS 0x08 + +/* DBI registers */ +#define DBI_CON_STATUS 0x44 + +/* DBI register fields */ +#define DBI_CON_STATUS_POWER_STATE_MASK GENMASK(1, 0) + +#define XMLH_LINK_UP 0x400 +#define CORE_RESET_TIME_US_MIN 1000 +#define CORE_RESET_TIME_US_MAX 1005 +#define WAKE_DELAY_US 2000 /* 2 ms */ + +#define to_pcie_ep(x) dev_get_drvdata((x)->dev) + +enum qcom_pcie_ep_link_status { + QCOM_PCIE_EP_LINK_DISABLED, + QCOM_PCIE_EP_LINK_ENABLED, + QCOM_PCIE_EP_LINK_UP, + QCOM_PCIE_EP_LINK_DOWN, +}; + +static struct clk_bulk_data qcom_pcie_ep_clks[] = { + { .id = "cfg" }, + { .id = "aux" }, + { .id = "bus_master" }, + { .id = "bus_slave" }, + { .id = "ref" }, + { .id = "sleep" }, + { .id = "slave_q2a" }, +}; + +struct qcom_pcie_ep { + struct dw_pcie pci; + + void __iomem *parf; + void __iomem *elbi; + struct regmap *perst_map; + struct resource *mmio_res; + + struct reset_control *core_reset; + struct gpio_desc *reset; + struct gpio_desc *wake; + struct phy *phy; + + u32 perst_en; + u32 perst_sep_en; + + enum qcom_pcie_ep_link_status link_status; + int global_irq; + int perst_irq; +}; + +static int qcom_pcie_ep_core_reset(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + int ret; + + ret = reset_control_assert(pcie_ep->core_reset); + if (ret) { + dev_err(dev, "Cannot assert core reset\n"); + return ret; + } + + usleep_range(CORE_RESET_TIME_US_MIN, CORE_RESET_TIME_US_MAX); + + ret = reset_control_deassert(pcie_ep->core_reset); + if (ret) { + dev_err(dev, "Cannot de-assert core reset\n"); + return ret; + } + + usleep_range(CORE_RESET_TIME_US_MIN, CORE_RESET_TIME_US_MAX); + + return 0; +} + +/* + * Delatch PERST_EN and PERST_SEPARATION_ENABLE with TCSR to avoid + * device reset during host reboot and hibernation. The driver is + * expected to handle this situation. + */ +static void qcom_pcie_ep_configure_tcsr(struct qcom_pcie_ep *pcie_ep) +{ + regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0); + regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0); +} + +static int qcom_pcie_dw_link_up(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + u32 reg; + + reg = readl_relaxed(pcie_ep->elbi + ELBI_SYS_STTS); + + return reg & XMLH_LINK_UP; +} + +static int qcom_pcie_dw_start_link(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + + enable_irq(pcie_ep->perst_irq); + + return 0; +} + +static void qcom_pcie_dw_stop_link(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + + disable_irq(pcie_ep->perst_irq); +} + +static int qcom_pcie_perst_deassert(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + struct device *dev = pci->dev; + u32 val, offset; + int ret; + + ret = clk_bulk_prepare_enable(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + if (ret) + return ret; + + ret = qcom_pcie_ep_core_reset(pcie_ep); + if (ret) + goto err_disable_clk; + + ret = phy_init(pcie_ep->phy); + if (ret) + goto err_disable_clk; + + ret = phy_power_on(pcie_ep->phy); + if (ret) + goto err_phy_exit; + + /* Assert WAKE# to RC to indicate device is ready */ + gpiod_set_value_cansleep(pcie_ep->wake, 1); + usleep_range(WAKE_DELAY_US, WAKE_DELAY_US + 500); + gpiod_set_value_cansleep(pcie_ep->wake, 0); + + qcom_pcie_ep_configure_tcsr(pcie_ep); + + /* Disable BDF to SID mapping */ + val = readl_relaxed(pcie_ep->parf + PARF_BDF_TO_SID_CFG); + val |= PARF_BDF_TO_SID_BYPASS; + writel_relaxed(val, pcie_ep->parf + PARF_BDF_TO_SID_CFG); + + /* Enable debug IRQ */ + val = readl_relaxed(pcie_ep->parf + PARF_DEBUG_INT_EN); + val |= PARF_DEBUG_INT_RADM_PM_TURNOFF | + PARF_DEBUG_INT_CFG_BUS_MASTER_EN | + PARF_DEBUG_INT_PM_DSTATE_CHANGE; + writel_relaxed(val, pcie_ep->parf + PARF_DEBUG_INT_EN); + + /* Configure PCIe to endpoint mode */ + writel_relaxed(PARF_DEVICE_TYPE_EP, pcie_ep->parf + PARF_DEVICE_TYPE); + + /* Allow entering L1 state */ + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val &= ~PARF_PM_CTRL_REQ_NOT_ENTR_L1; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + + /* Read halts write */ + val = readl_relaxed(pcie_ep->parf + PARF_AXI_MSTR_RD_HALT_NO_WRITES); + val &= ~PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN; + writel_relaxed(val, pcie_ep->parf + PARF_AXI_MSTR_RD_HALT_NO_WRITES); + + /* Write after write halt */ + val = readl_relaxed(pcie_ep->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + val |= PARF_AXI_MSTR_WR_ADDR_HALT_EN; + writel_relaxed(val, pcie_ep->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + + /* Q2A flush disable */ + val = readl_relaxed(pcie_ep->parf + PARF_Q2A_FLUSH); + val &= ~PARF_Q2A_FLUSH_EN; + writel_relaxed(val, pcie_ep->parf + PARF_Q2A_FLUSH); + + /* Disable DBI Wakeup, core clock CGC and enable AUX power */ + val = readl_relaxed(pcie_ep->parf + PARF_SYS_CTRL); + val |= PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE | + PARF_SYS_CTRL_CORE_CLK_CGC_DIS | + PARF_SYS_CTRL_AUX_PWR_DET; + writel_relaxed(val, pcie_ep->parf + PARF_SYS_CTRL); + + /* Disable the debouncers */ + val = readl_relaxed(pcie_ep->parf + PARF_DB_CTRL); + val |= PARF_DB_CTRL_INSR_DBNCR_BLOCK | PARF_DB_CTRL_RMVL_DBNCR_BLOCK | + PARF_DB_CTRL_DBI_WKP_BLOCK | PARF_DB_CTRL_SLV_WKP_BLOCK | + PARF_DB_CTRL_MST_WKP_BLOCK; + writel_relaxed(val, pcie_ep->parf + PARF_DB_CTRL); + + /* Request to exit from L1SS for MSI and LTR MSG */ + val = readl_relaxed(pcie_ep->parf + PARF_CFG_BITS); + val |= PARF_CFG_BITS_REQ_EXIT_L1SS_MSI_LTR_EN; + writel_relaxed(val, pcie_ep->parf + PARF_CFG_BITS); + + dw_pcie_dbi_ro_wr_en(pci); + + /* Set the L0s Exit Latency to 2us-4us = 0x6 */ + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_L0SEL; + val |= FIELD_PREP(PCI_EXP_LNKCAP_L0SEL, 0x6); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val); + + /* Set the L1 Exit Latency to be 32us-64 us = 0x6 */ + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_L1EL; + val |= FIELD_PREP(PCI_EXP_LNKCAP_L1EL, 0x6); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val); + + dw_pcie_dbi_ro_wr_dis(pci); + + writel_relaxed(0, pcie_ep->parf + PARF_INT_ALL_MASK); + val = PARF_INT_ALL_LINK_DOWN | PARF_INT_ALL_BME | + PARF_INT_ALL_PM_TURNOFF | PARF_INT_ALL_DSTATE_CHANGE | + PARF_INT_ALL_LINK_UP; + writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_MASK); + + ret = dw_pcie_ep_init_complete(&pcie_ep->pci.ep); + if (ret) { + dev_err(dev, "Failed to complete initialization: %d\n", ret); + goto err_phy_power_off; + } + + /* + * The physical address of the MMIO region which is exposed as the BAR + * should be written to MHI BASE registers. + */ + writel_relaxed(pcie_ep->mmio_res->start, + pcie_ep->parf + PARF_MHI_BASE_ADDR_LOWER); + writel_relaxed(0, pcie_ep->parf + PARF_MHI_BASE_ADDR_UPPER); + + dw_pcie_ep_init_notify(&pcie_ep->pci.ep); + + /* Enable LTSSM */ + val = readl_relaxed(pcie_ep->parf + PARF_LTSSM); + val |= BIT(8); + writel_relaxed(val, pcie_ep->parf + PARF_LTSSM); + + return 0; + +err_phy_power_off: + phy_power_off(pcie_ep->phy); +err_phy_exit: + phy_exit(pcie_ep->phy); +err_disable_clk: + clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + + return ret; +} + +static void qcom_pcie_perst_assert(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + struct device *dev = pci->dev; + + if (pcie_ep->link_status == QCOM_PCIE_EP_LINK_DISABLED) { + dev_dbg(dev, "Link is already disabled\n"); + return; + } + + phy_power_off(pcie_ep->phy); + phy_exit(pcie_ep->phy); + clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; +} + +/* Common DWC controller ops */ +static const struct dw_pcie_ops pci_ops = { + .link_up = qcom_pcie_dw_link_up, + .start_link = qcom_pcie_dw_start_link, + .stop_link = qcom_pcie_dw_stop_link, +}; + +static int qcom_pcie_ep_get_io_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci = &pcie_ep->pci; + struct device_node *syscon; + struct resource *res; + int ret; + + pcie_ep->parf = devm_platform_ioremap_resource_byname(pdev, "parf"); + if (IS_ERR(pcie_ep->parf)) + return PTR_ERR(pcie_ep->parf); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + pci->dbi_base2 = pci->dbi_base; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); + pcie_ep->elbi = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pcie_ep->elbi)) + return PTR_ERR(pcie_ep->elbi); + + pcie_ep->mmio_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "mmio"); + + syscon = of_parse_phandle(dev->of_node, "qcom,perst-regs", 0); + if (!syscon) { + dev_err(dev, "Failed to parse qcom,perst-regs\n"); + return -EINVAL; + } + + pcie_ep->perst_map = syscon_node_to_regmap(syscon); + of_node_put(syscon); + if (IS_ERR(pcie_ep->perst_map)) + return PTR_ERR(pcie_ep->perst_map); + + ret = of_property_read_u32_index(dev->of_node, "qcom,perst-regs", + 1, &pcie_ep->perst_en); + if (ret < 0) { + dev_err(dev, "No Perst Enable offset in syscon\n"); + return ret; + } + + ret = of_property_read_u32_index(dev->of_node, "qcom,perst-regs", + 2, &pcie_ep->perst_sep_en); + if (ret < 0) { + dev_err(dev, "No Perst Separation Enable offset in syscon\n"); + return ret; + } + + return 0; +} + +static int qcom_pcie_ep_get_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = &pdev->dev; + int ret; + + ret = qcom_pcie_ep_get_io_resources(pdev, pcie_ep); + if (ret) { + dev_err(&pdev->dev, "Failed to get io resources %d\n", ret); + return ret; + } + + ret = devm_clk_bulk_get(dev, ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + if (ret) + return ret; + + pcie_ep->core_reset = devm_reset_control_get_exclusive(dev, "core"); + if (IS_ERR(pcie_ep->core_reset)) + return PTR_ERR(pcie_ep->core_reset); + + pcie_ep->reset = devm_gpiod_get(dev, "reset", GPIOD_IN); + if (IS_ERR(pcie_ep->reset)) + return PTR_ERR(pcie_ep->reset); + + pcie_ep->wake = devm_gpiod_get_optional(dev, "wake", GPIOD_OUT_LOW); + if (IS_ERR(pcie_ep->wake)) + return PTR_ERR(pcie_ep->wake); + + pcie_ep->phy = devm_phy_optional_get(&pdev->dev, "pciephy"); + if (IS_ERR(pcie_ep->phy)) + ret = PTR_ERR(pcie_ep->phy); + + return ret; +} + +/* TODO: Notify clients about PCIe state change */ +static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) +{ + struct qcom_pcie_ep *pcie_ep = data; + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS); + u32 mask = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_MASK); + u32 dstate, val; + + writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR); + status &= mask; + + if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { + dev_dbg(dev, "Received Linkdown event\n"); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_DOWN; + } else if (FIELD_GET(PARF_INT_ALL_BME, status)) { + dev_dbg(dev, "Received BME event. Link is enabled!\n"); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_ENABLED; + } else if (FIELD_GET(PARF_INT_ALL_PM_TURNOFF, status)) { + dev_dbg(dev, "Received PM Turn-off event! Entering L23\n"); + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val |= PARF_PM_CTRL_READY_ENTR_L23; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + } else if (FIELD_GET(PARF_INT_ALL_DSTATE_CHANGE, status)) { + dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) & + DBI_CON_STATUS_POWER_STATE_MASK; + dev_dbg(dev, "Received D%d state event\n", dstate); + if (dstate == 3) { + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val |= PARF_PM_CTRL_REQ_EXIT_L1; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + } + } else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { + dev_dbg(dev, "Received Linkup event. Enumeration complete!\n"); + dw_pcie_ep_linkup(&pci->ep); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_UP; + } else { + dev_dbg(dev, "Received unknown event: %d\n", status); + } + + return IRQ_HANDLED; +} + +static irqreturn_t qcom_pcie_ep_perst_irq_thread(int irq, void *data) +{ + struct qcom_pcie_ep *pcie_ep = data; + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + u32 perst; + + perst = gpiod_get_value(pcie_ep->reset); + if (perst) { + dev_dbg(dev, "PERST asserted by host. Shutting down the PCIe link!\n"); + qcom_pcie_perst_assert(pci); + } else { + dev_dbg(dev, "PERST de-asserted by host. Starting link training!\n"); + qcom_pcie_perst_deassert(pci); + } + + irq_set_irq_type(gpiod_to_irq(pcie_ep->reset), + (perst ? IRQF_TRIGGER_HIGH : IRQF_TRIGGER_LOW)); + + return IRQ_HANDLED; +} + +static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + int irq, ret; + + irq = platform_get_irq_byname(pdev, "global"); + if (irq < 0) { + dev_err(&pdev->dev, "Failed to get Global IRQ\n"); + return irq; + } + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + qcom_pcie_ep_global_irq_thread, + IRQF_ONESHOT, + "global_irq", pcie_ep); + if (ret) { + dev_err(&pdev->dev, "Failed to request Global IRQ\n"); + return ret; + } + + pcie_ep->perst_irq = gpiod_to_irq(pcie_ep->reset); + irq_set_status_flags(pcie_ep->perst_irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->perst_irq, NULL, + qcom_pcie_ep_perst_irq_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + "perst_irq", pcie_ep); + if (ret) { + dev_err(&pdev->dev, "Failed to request PERST IRQ\n"); + disable_irq(irq); + return ret; + } + + return 0; +} + +static int qcom_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + enum pci_epc_irq_type type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_EPC_IRQ_LEGACY: + return dw_pcie_ep_raise_legacy_irq(ep, func_no); + case PCI_EPC_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "Unknown IRQ type\n"); + return -EINVAL; + } +} + +static const struct pci_epc_features qcom_pcie_epc_features = { + .linkup_notifier = true, + .core_init_notifier = true, + .msi_capable = true, + .msix_capable = false, +}; + +static const struct pci_epc_features * +qcom_pcie_epc_get_features(struct dw_pcie_ep *pci_ep) +{ + return &qcom_pcie_epc_features; +} + +static void qcom_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar; + + for (bar = BAR_0; bar <= BAR_5; bar++) + dw_pcie_ep_reset_bar(pci, bar); +} + +static struct dw_pcie_ep_ops pci_ep_ops = { + .ep_init = qcom_pcie_ep_init, + .raise_irq = qcom_pcie_ep_raise_irq, + .get_features = qcom_pcie_epc_get_features, +}; + +static int qcom_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qcom_pcie_ep *pcie_ep; + int ret; + + pcie_ep = devm_kzalloc(dev, sizeof(*pcie_ep), GFP_KERNEL); + if (!pcie_ep) + return -ENOMEM; + + pcie_ep->pci.dev = dev; + pcie_ep->pci.ops = &pci_ops; + pcie_ep->pci.ep.ops = &pci_ep_ops; + platform_set_drvdata(pdev, pcie_ep); + + ret = qcom_pcie_ep_get_resources(pdev, pcie_ep); + if (ret) + return ret; + + ret = clk_bulk_prepare_enable(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + if (ret) + return ret; + + ret = qcom_pcie_ep_core_reset(pcie_ep); + if (ret) + goto err_disable_clk; + + ret = phy_init(pcie_ep->phy); + if (ret) + goto err_disable_clk; + + /* PHY needs to be powered on for dw_pcie_ep_init() */ + ret = phy_power_on(pcie_ep->phy); + if (ret) + goto err_phy_exit; + + ret = dw_pcie_ep_init(&pcie_ep->pci.ep); + if (ret) { + dev_err(dev, "Failed to initialize endpoint: %d\n", ret); + goto err_phy_power_off; + } + + ret = qcom_pcie_ep_enable_irq_resources(pdev, pcie_ep); + if (ret) + goto err_phy_power_off; + + return 0; + +err_phy_power_off: + phy_power_off(pcie_ep->phy); +err_phy_exit: + phy_exit(pcie_ep->phy); +err_disable_clk: + clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + + return ret; +} + +static int qcom_pcie_ep_remove(struct platform_device *pdev) +{ + struct qcom_pcie_ep *pcie_ep = platform_get_drvdata(pdev); + + if (pcie_ep->link_status == QCOM_PCIE_EP_LINK_DISABLED) + return 0; + + phy_power_off(pcie_ep->phy); + phy_exit(pcie_ep->phy); + clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), + qcom_pcie_ep_clks); + + return 0; +} + +static const struct of_device_id qcom_pcie_ep_match[] = { + { .compatible = "qcom,sdx55-pcie-ep", }, + { } +}; + +static struct platform_driver qcom_pcie_ep_driver = { + .probe = qcom_pcie_ep_probe, + .remove = qcom_pcie_ep_remove, + .driver = { + .name = "qcom-pcie-ep", + .of_match_table = qcom_pcie_ep_match, + }, +}; +builtin_platform_driver(qcom_pcie_ep_driver); + +MODULE_AUTHOR("Siddartha Mohanadoss <smohanad@codeaurora.org>"); +MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm PCIe Endpoint controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 8a7a300163e5..1c3d1116bb60 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -166,6 +166,9 @@ struct qcom_pcie_resources_2_7_0 { struct regulator_bulk_data supplies[2]; struct reset_control *pci_reset; struct clk *pipe_clk; + struct clk *pipe_clk_src; + struct clk *phy_pipe_clk; + struct clk *ref_clk_src; }; union qcom_pcie_resources { @@ -189,6 +192,11 @@ struct qcom_pcie_ops { int (*config_sid)(struct qcom_pcie *pcie); }; +struct qcom_pcie_cfg { + const struct qcom_pcie_ops *ops; + unsigned int pipe_clk_need_muxing:1; +}; + struct qcom_pcie { struct dw_pcie *pci; void __iomem *parf; /* DT parf */ @@ -197,6 +205,7 @@ struct qcom_pcie { struct phy *phy; struct gpio_desc *reset; const struct qcom_pcie_ops *ops; + unsigned int pipe_clk_need_muxing:1; }; #define to_qcom_pcie(x) dev_get_drvdata((x)->dev) @@ -1167,6 +1176,20 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) if (ret < 0) return ret; + if (pcie->pipe_clk_need_muxing) { + res->pipe_clk_src = devm_clk_get(dev, "pipe_mux"); + if (IS_ERR(res->pipe_clk_src)) + return PTR_ERR(res->pipe_clk_src); + + res->phy_pipe_clk = devm_clk_get(dev, "phy_pipe"); + if (IS_ERR(res->phy_pipe_clk)) + return PTR_ERR(res->phy_pipe_clk); + + res->ref_clk_src = devm_clk_get(dev, "ref"); + if (IS_ERR(res->ref_clk_src)) + return PTR_ERR(res->ref_clk_src); + } + res->pipe_clk = devm_clk_get(dev, "pipe"); return PTR_ERR_OR_ZERO(res->pipe_clk); } @@ -1185,6 +1208,10 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) return ret; } + /* Set TCXO as clock source for pcie_pipe_clk_src */ + if (pcie->pipe_clk_need_muxing) + clk_set_parent(res->pipe_clk_src, res->ref_clk_src); + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); if (ret < 0) goto err_disable_regulators; @@ -1256,6 +1283,10 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) { struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + /* Set pipe clock as clock source for pcie_pipe_clk_src */ + if (pcie->pipe_clk_need_muxing) + clk_set_parent(res->pipe_clk_src, res->phy_pipe_clk); + return clk_prepare_enable(res->pipe_clk); } @@ -1456,6 +1487,39 @@ static const struct qcom_pcie_ops ops_1_9_0 = { .config_sid = qcom_pcie_config_sid_sm8250, }; +static const struct qcom_pcie_cfg apq8084_cfg = { + .ops = &ops_1_0_0, +}; + +static const struct qcom_pcie_cfg ipq8064_cfg = { + .ops = &ops_2_1_0, +}; + +static const struct qcom_pcie_cfg msm8996_cfg = { + .ops = &ops_2_3_2, +}; + +static const struct qcom_pcie_cfg ipq8074_cfg = { + .ops = &ops_2_3_3, +}; + +static const struct qcom_pcie_cfg ipq4019_cfg = { + .ops = &ops_2_4_0, +}; + +static const struct qcom_pcie_cfg sdm845_cfg = { + .ops = &ops_2_7_0, +}; + +static const struct qcom_pcie_cfg sm8250_cfg = { + .ops = &ops_1_9_0, +}; + +static const struct qcom_pcie_cfg sc7280_cfg = { + .ops = &ops_1_9_0, + .pipe_clk_need_muxing = true, +}; + static const struct dw_pcie_ops dw_pcie_ops = { .link_up = qcom_pcie_link_up, .start_link = qcom_pcie_start_link, @@ -1467,6 +1531,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct pcie_port *pp; struct dw_pcie *pci; struct qcom_pcie *pcie; + const struct qcom_pcie_cfg *pcie_cfg; int ret; pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); @@ -1488,7 +1553,14 @@ static int qcom_pcie_probe(struct platform_device *pdev) pcie->pci = pci; - pcie->ops = of_device_get_match_data(dev); + pcie_cfg = of_device_get_match_data(dev); + if (!pcie_cfg || !pcie_cfg->ops) { + dev_err(dev, "Invalid platform data\n"); + return -EINVAL; + } + + pcie->ops = pcie_cfg->ops; + pcie->pipe_clk_need_muxing = pcie_cfg->pipe_clk_need_muxing; pcie->reset = devm_gpiod_get_optional(dev, "perst", GPIOD_OUT_HIGH); if (IS_ERR(pcie->reset)) { @@ -1545,16 +1617,18 @@ err_pm_runtime_put: } static const struct of_device_id qcom_pcie_match[] = { - { .compatible = "qcom,pcie-apq8084", .data = &ops_1_0_0 }, - { .compatible = "qcom,pcie-ipq8064", .data = &ops_2_1_0 }, - { .compatible = "qcom,pcie-ipq8064-v2", .data = &ops_2_1_0 }, - { .compatible = "qcom,pcie-apq8064", .data = &ops_2_1_0 }, - { .compatible = "qcom,pcie-msm8996", .data = &ops_2_3_2 }, - { .compatible = "qcom,pcie-ipq8074", .data = &ops_2_3_3 }, - { .compatible = "qcom,pcie-ipq4019", .data = &ops_2_4_0 }, - { .compatible = "qcom,pcie-qcs404", .data = &ops_2_4_0 }, - { .compatible = "qcom,pcie-sdm845", .data = &ops_2_7_0 }, - { .compatible = "qcom,pcie-sm8250", .data = &ops_1_9_0 }, + { .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg }, + { .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg }, + { .compatible = "qcom,pcie-ipq8064-v2", .data = &ipq8064_cfg }, + { .compatible = "qcom,pcie-apq8064", .data = &ipq8064_cfg }, + { .compatible = "qcom,pcie-msm8996", .data = &msm8996_cfg }, + { .compatible = "qcom,pcie-ipq8074", .data = &ipq8074_cfg }, + { .compatible = "qcom,pcie-ipq4019", .data = &ipq4019_cfg }, + { .compatible = "qcom,pcie-qcs404", .data = &ipq4019_cfg }, + { .compatible = "qcom,pcie-sdm845", .data = &sdm845_cfg }, + { .compatible = "qcom,pcie-sm8250", .data = &sm8250_cfg }, + { .compatible = "qcom,pcie-sc8180x", .data = &sm8250_cfg }, + { .compatible = "qcom,pcie-sc7280", .data = &sc7280_cfg }, { } }; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 504669e3afe0..904976913081 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -497,19 +497,19 @@ static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) struct tegra_pcie_dw *pcie = arg; struct dw_pcie_ep *ep = &pcie->pci.ep; int spurious = 1; - u32 val, tmp; + u32 status_l0, status_l1, link_status; - val = appl_readl(pcie, APPL_INTR_STATUS_L0); - if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); - appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); + status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); - if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) + if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) pex_ep_event_hot_rst_done(pcie); - if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { - tmp = appl_readl(pcie, APPL_LINK_STATUS); - if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { + if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { + link_status = appl_readl(pcie, APPL_LINK_STATUS); + if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) { dev_dbg(pcie->dev, "Link is up with Host\n"); dw_pcie_ep_linkup(ep); } @@ -518,11 +518,11 @@ static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) spurious = 0; } - if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); - appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); + if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15); - if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) + if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) return IRQ_WAKE_THREAD; spurious = 0; @@ -530,8 +530,8 @@ static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) if (spurious) { dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", - val); - appl_writel(pcie, val, APPL_INTR_STATUS_L0); + status_l0); + appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0); } return IRQ_HANDLED; @@ -1493,6 +1493,16 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) return; } + /* + * PCIe controller exits from L2 only if reset is applied, so + * controller doesn't handle interrupts. But in cases where + * L2 entry fails, PERST# is asserted which can trigger surprise + * link down AER. However this function call happens in + * suspend_noirq(), so AER interrupt will not be processed. + * Disable all interrupts to avoid such a scenario. + */ + appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0); + if (tegra_pcie_try_link_l2(pcie)) { dev_info(pcie->dev, "Link didn't transition to L2 state\n"); /* @@ -1763,7 +1773,7 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); val |= MSIX_ADDR_MATCH_LOW_OFF_EN; dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); - val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); + val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); ret = dw_pcie_ep_init_complete(ep); @@ -1826,7 +1836,7 @@ static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) if (unlikely(irq > 31)) return -EINVAL; - appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); + appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1); return 0; } @@ -1935,13 +1945,6 @@ static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, return ret; } - name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", - pcie->cid); - if (!name) { - dev_err(dev, "Failed to create PCIe EP work thread string\n"); - return -ENOMEM; - } - pm_runtime_enable(dev); ret = dw_pcie_ep_init(ep); @@ -2214,6 +2217,8 @@ static int tegra_pcie_dw_resume_noirq(struct device *dev) goto fail_host_init; } + dw_pcie_setup_rc(&pcie->pci.pp); + ret = tegra_pcie_dw_start_link(&pcie->pci); if (ret < 0) goto fail_host_init; @@ -2234,6 +2239,11 @@ static int tegra_pcie_dw_resume_early(struct device *dev) struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); u32 val; + if (pcie->mode == DW_PCIE_EP_TYPE) { + dev_err(dev, "Suspend is not supported in EP mode"); + return -ENOTSUPP; + } + if (!pcie->link_state) return 0; diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c index 7e8bad326770..d05be942956e 100644 --- a/drivers/pci/controller/dwc/pcie-uniphier.c +++ b/drivers/pci/controller/dwc/pcie-uniphier.c @@ -168,30 +168,21 @@ static void uniphier_pcie_irq_enable(struct uniphier_pcie_priv *priv) writel(PCL_RCV_INTX_ALL_ENABLE, priv->base + PCL_RCV_INTX); } -static void uniphier_pcie_irq_ack(struct irq_data *d) -{ - struct pcie_port *pp = irq_data_get_irq_chip_data(d); - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci); - u32 val; - - val = readl(priv->base + PCL_RCV_INTX); - val &= ~PCL_RCV_INTX_ALL_STATUS; - val |= BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_STATUS_SHIFT); - writel(val, priv->base + PCL_RCV_INTX); -} - static void uniphier_pcie_irq_mask(struct irq_data *d) { struct pcie_port *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci); + unsigned long flags; u32 val; + raw_spin_lock_irqsave(&pp->lock, flags); + val = readl(priv->base + PCL_RCV_INTX); - val &= ~PCL_RCV_INTX_ALL_MASK; val |= BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT); writel(val, priv->base + PCL_RCV_INTX); + + raw_spin_unlock_irqrestore(&pp->lock, flags); } static void uniphier_pcie_irq_unmask(struct irq_data *d) @@ -199,17 +190,20 @@ static void uniphier_pcie_irq_unmask(struct irq_data *d) struct pcie_port *pp = irq_data_get_irq_chip_data(d); struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci); + unsigned long flags; u32 val; + raw_spin_lock_irqsave(&pp->lock, flags); + val = readl(priv->base + PCL_RCV_INTX); - val &= ~PCL_RCV_INTX_ALL_MASK; val &= ~BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT); writel(val, priv->base + PCL_RCV_INTX); + + raw_spin_unlock_irqrestore(&pp->lock, flags); } static struct irq_chip uniphier_pcie_irq_chip = { .name = "PCI", - .irq_ack = uniphier_pcie_irq_ack, .irq_mask = uniphier_pcie_irq_mask, .irq_unmask = uniphier_pcie_irq_unmask, }; @@ -235,7 +229,7 @@ static void uniphier_pcie_irq_handler(struct irq_desc *desc) struct uniphier_pcie_priv *priv = to_uniphier_pcie(pci); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned long reg; - u32 val, bit, virq; + u32 val, bit; /* INT for debug */ val = readl(priv->base + PCL_RCV_INT); @@ -257,10 +251,8 @@ static void uniphier_pcie_irq_handler(struct irq_desc *desc) val = readl(priv->base + PCL_RCV_INTX); reg = FIELD_GET(PCL_RCV_INTX_ALL_STATUS, val); - for_each_set_bit(bit, ®, PCI_NUM_INTX) { - virq = irq_linear_revmap(priv->legacy_irq_domain, bit); - generic_handle_irq(virq); - } + for_each_set_bit(bit, ®, PCI_NUM_INTX) + generic_handle_domain_irq(priv->legacy_irq_domain, bit); chained_irq_exit(chip, desc); } diff --git a/drivers/pci/controller/dwc/pcie-visconti.c b/drivers/pci/controller/dwc/pcie-visconti.c new file mode 100644 index 000000000000..50f80f07e4db --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-visconti.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DWC PCIe RC driver for Toshiba Visconti ARM SoC + * + * Copyright (C) 2021 Toshiba Electronic Device & Storage Corporation + * Copyright (C) 2021 TOSHIBA CORPORATION + * + * Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/types.h> + +#include "pcie-designware.h" +#include "../../pci.h" + +struct visconti_pcie { + struct dw_pcie pci; + void __iomem *ulreg_base; + void __iomem *smu_base; + void __iomem *mpu_base; + struct clk *refclk; + struct clk *coreclk; + struct clk *auxclk; +}; + +#define PCIE_UL_REG_S_PCIE_MODE 0x00F4 +#define PCIE_UL_REG_S_PCIE_MODE_EP 0x00 +#define PCIE_UL_REG_S_PCIE_MODE_RC 0x04 + +#define PCIE_UL_REG_S_PERSTN_CTRL 0x00F8 +#define PCIE_UL_IOM_PCIE_PERSTN_I_EN BIT(3) +#define PCIE_UL_DIRECT_PERSTN_EN BIT(2) +#define PCIE_UL_PERSTN_OUT BIT(1) +#define PCIE_UL_DIRECT_PERSTN BIT(0) +#define PCIE_UL_REG_S_PERSTN_CTRL_INIT (PCIE_UL_IOM_PCIE_PERSTN_I_EN | \ + PCIE_UL_DIRECT_PERSTN_EN | \ + PCIE_UL_DIRECT_PERSTN) + +#define PCIE_UL_REG_S_PHY_INIT_02 0x0104 +#define PCIE_UL_PHY0_SRAM_EXT_LD_DONE BIT(0) + +#define PCIE_UL_REG_S_PHY_INIT_03 0x0108 +#define PCIE_UL_PHY0_SRAM_INIT_DONE BIT(0) + +#define PCIE_UL_REG_S_INT_EVENT_MASK1 0x0138 +#define PCIE_UL_CFG_PME_INT BIT(0) +#define PCIE_UL_CFG_LINK_EQ_REQ_INT BIT(1) +#define PCIE_UL_EDMA_INT0 BIT(2) +#define PCIE_UL_EDMA_INT1 BIT(3) +#define PCIE_UL_EDMA_INT2 BIT(4) +#define PCIE_UL_EDMA_INT3 BIT(5) +#define PCIE_UL_S_INT_EVENT_MASK1_ALL (PCIE_UL_CFG_PME_INT | \ + PCIE_UL_CFG_LINK_EQ_REQ_INT | \ + PCIE_UL_EDMA_INT0 | \ + PCIE_UL_EDMA_INT1 | \ + PCIE_UL_EDMA_INT2 | \ + PCIE_UL_EDMA_INT3) + +#define PCIE_UL_REG_S_SB_MON 0x0198 +#define PCIE_UL_REG_S_SIG_MON 0x019C +#define PCIE_UL_CORE_RST_N_MON BIT(0) + +#define PCIE_UL_REG_V_SII_DBG_00 0x0844 +#define PCIE_UL_REG_V_SII_GEN_CTRL_01 0x0860 +#define PCIE_UL_APP_LTSSM_ENABLE BIT(0) + +#define PCIE_UL_REG_V_PHY_ST_00 0x0864 +#define PCIE_UL_SMLH_LINK_UP BIT(0) + +#define PCIE_UL_REG_V_PHY_ST_02 0x0868 +#define PCIE_UL_S_DETECT_ACT 0x01 +#define PCIE_UL_S_L0 0x11 + +#define PISMU_CKON_PCIE 0x0038 +#define PISMU_CKON_PCIE_AUX_CLK BIT(1) +#define PISMU_CKON_PCIE_MSTR_ACLK BIT(0) + +#define PISMU_RSOFF_PCIE 0x0538 +#define PISMU_RSOFF_PCIE_ULREG_RST_N BIT(1) +#define PISMU_RSOFF_PCIE_PWR_UP_RST_N BIT(0) + +#define PCIE_MPU_REG_MP_EN 0x0 +#define MPU_MP_EN_DISABLE BIT(0) + +/* Access registers in PCIe ulreg */ +static void visconti_ulreg_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->ulreg_base + reg); +} + +static u32 visconti_ulreg_readl(struct visconti_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->ulreg_base + reg); +} + +/* Access registers in PCIe smu */ +static void visconti_smu_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->smu_base + reg); +} + +/* Access registers in PCIe mpu */ +static void visconti_mpu_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->mpu_base + reg); +} + +static u32 visconti_mpu_readl(struct visconti_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->mpu_base + reg); +} + +static int visconti_pcie_link_up(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr = pcie->ulreg_base; + u32 val = readl_relaxed(addr + PCIE_UL_REG_V_PHY_ST_02); + + return !!(val & PCIE_UL_S_L0); +} + +static int visconti_pcie_start_link(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr = pcie->ulreg_base; + u32 val; + int ret; + + visconti_ulreg_writel(pcie, PCIE_UL_APP_LTSSM_ENABLE, + PCIE_UL_REG_V_SII_GEN_CTRL_01); + + ret = readl_relaxed_poll_timeout(addr + PCIE_UL_REG_V_PHY_ST_02, + val, (val & PCIE_UL_S_L0), + 90000, 100000); + if (ret) + return ret; + + visconti_ulreg_writel(pcie, PCIE_UL_S_INT_EVENT_MASK1_ALL, + PCIE_UL_REG_S_INT_EVENT_MASK1); + + if (dw_pcie_link_up(pci)) { + val = visconti_mpu_readl(pcie, PCIE_MPU_REG_MP_EN); + visconti_mpu_writel(pcie, val & ~MPU_MP_EN_DISABLE, + PCIE_MPU_REG_MP_EN); + } + + return 0; +} + +static void visconti_pcie_stop_link(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + + val = visconti_ulreg_readl(pcie, PCIE_UL_REG_V_SII_GEN_CTRL_01); + val &= ~PCIE_UL_APP_LTSSM_ENABLE; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_V_SII_GEN_CTRL_01); + + val = visconti_mpu_readl(pcie, PCIE_MPU_REG_MP_EN); + visconti_mpu_writel(pcie, val | MPU_MP_EN_DISABLE, PCIE_MPU_REG_MP_EN); +} + +/* + * In this SoC specification, the CPU bus outputs the offset value from + * 0x40000000 to the PCIe bus, so 0x40000000 is subtracted from the CPU + * bus address. This 0x40000000 is also based on io_base from DT. + */ +static u64 visconti_pcie_cpu_addr_fixup(struct dw_pcie *pci, u64 cpu_addr) +{ + struct pcie_port *pp = &pci->pp; + + return cpu_addr & ~pp->io_base; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .cpu_addr_fixup = visconti_pcie_cpu_addr_fixup, + .link_up = visconti_pcie_link_up, + .start_link = visconti_pcie_start_link, + .stop_link = visconti_pcie_stop_link, +}; + +static int visconti_pcie_host_init(struct pcie_port *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr; + int err; + u32 val; + + visconti_smu_writel(pcie, + PISMU_CKON_PCIE_AUX_CLK | PISMU_CKON_PCIE_MSTR_ACLK, + PISMU_CKON_PCIE); + ndelay(250); + + visconti_smu_writel(pcie, PISMU_RSOFF_PCIE_ULREG_RST_N, + PISMU_RSOFF_PCIE); + visconti_ulreg_writel(pcie, PCIE_UL_REG_S_PCIE_MODE_RC, + PCIE_UL_REG_S_PCIE_MODE); + + val = PCIE_UL_REG_S_PERSTN_CTRL_INIT; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_S_PERSTN_CTRL); + udelay(100); + + val |= PCIE_UL_PERSTN_OUT; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_S_PERSTN_CTRL); + udelay(100); + + visconti_smu_writel(pcie, PISMU_RSOFF_PCIE_PWR_UP_RST_N, + PISMU_RSOFF_PCIE); + + addr = pcie->ulreg_base + PCIE_UL_REG_S_PHY_INIT_03; + err = readl_relaxed_poll_timeout(addr, val, + (val & PCIE_UL_PHY0_SRAM_INIT_DONE), + 100, 1000); + if (err) + return err; + + visconti_ulreg_writel(pcie, PCIE_UL_PHY0_SRAM_EXT_LD_DONE, + PCIE_UL_REG_S_PHY_INIT_02); + + addr = pcie->ulreg_base + PCIE_UL_REG_S_SIG_MON; + return readl_relaxed_poll_timeout(addr, val, + (val & PCIE_UL_CORE_RST_N_MON), 100, + 1000); +} + +static const struct dw_pcie_host_ops visconti_pcie_host_ops = { + .host_init = visconti_pcie_host_init, +}; + +static int visconti_get_resources(struct platform_device *pdev, + struct visconti_pcie *pcie) +{ + struct device *dev = &pdev->dev; + + pcie->ulreg_base = devm_platform_ioremap_resource_byname(pdev, "ulreg"); + if (IS_ERR(pcie->ulreg_base)) + return PTR_ERR(pcie->ulreg_base); + + pcie->smu_base = devm_platform_ioremap_resource_byname(pdev, "smu"); + if (IS_ERR(pcie->smu_base)) + return PTR_ERR(pcie->smu_base); + + pcie->mpu_base = devm_platform_ioremap_resource_byname(pdev, "mpu"); + if (IS_ERR(pcie->mpu_base)) + return PTR_ERR(pcie->mpu_base); + + pcie->refclk = devm_clk_get(dev, "ref"); + if (IS_ERR(pcie->refclk)) + return dev_err_probe(dev, PTR_ERR(pcie->refclk), + "Failed to get ref clock\n"); + + pcie->coreclk = devm_clk_get(dev, "core"); + if (IS_ERR(pcie->coreclk)) + return dev_err_probe(dev, PTR_ERR(pcie->coreclk), + "Failed to get core clock\n"); + + pcie->auxclk = devm_clk_get(dev, "aux"); + if (IS_ERR(pcie->auxclk)) + return dev_err_probe(dev, PTR_ERR(pcie->auxclk), + "Failed to get aux clock\n"); + + return 0; +} + +static int visconti_add_pcie_port(struct visconti_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct pcie_port *pp = &pci->pp; + + pp->irq = platform_get_irq_byname(pdev, "intr"); + if (pp->irq < 0) + return pp->irq; + + pp->ops = &visconti_pcie_host_ops; + + return dw_pcie_host_init(pp); +} + +static int visconti_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct visconti_pcie *pcie; + struct dw_pcie *pci; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + ret = visconti_get_resources(pdev, pcie); + if (ret) + return ret; + + platform_set_drvdata(pdev, pcie); + + return visconti_add_pcie_port(pcie, pdev); +} + +static const struct of_device_id visconti_pcie_match[] = { + { .compatible = "toshiba,visconti-pcie" }, + {}, +}; + +static struct platform_driver visconti_pcie_driver = { + .probe = visconti_pcie_probe, + .driver = { + .name = "visconti-pcie", + .of_match_table = visconti_pcie_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(visconti_pcie_driver); diff --git a/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c b/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c index ee0156921ebc..306950272fd6 100644 --- a/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c +++ b/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c @@ -42,17 +42,6 @@ struct ls_pcie_g4 { int irq; }; -static inline u32 ls_pcie_g4_lut_readl(struct ls_pcie_g4 *pcie, u32 off) -{ - return ioread32(pcie->pci.csr_axi_slave_base + PCIE_LUT_OFF + off); -} - -static inline void ls_pcie_g4_lut_writel(struct ls_pcie_g4 *pcie, - u32 off, u32 val) -{ - iowrite32(val, pcie->pci.csr_axi_slave_base + PCIE_LUT_OFF + off); -} - static inline u32 ls_pcie_g4_pf_readl(struct ls_pcie_g4 *pcie, u32 off) { return ioread32(pcie->pci.csr_axi_slave_base + PCIE_PF_OFF + off); diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c index c637de3a389b..f3547aa60140 100644 --- a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c @@ -92,7 +92,7 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) u32 msi_data, msi_addr_lo, msi_addr_hi; u32 intr_status, msi_status; unsigned long shifted_status; - u32 bit, virq, val, mask; + u32 bit, val, mask; /* * The core provides a single interrupt for both INTx/MSI messages. @@ -114,11 +114,10 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) shifted_status >>= PAB_INTX_START; do { for_each_set_bit(bit, &shifted_status, PCI_NUM_INTX) { - virq = irq_find_mapping(rp->intx_domain, - bit + 1); - if (virq) - generic_handle_irq(virq); - else + int ret; + ret = generic_handle_domain_irq(rp->intx_domain, + bit + 1); + if (ret) dev_err_ratelimited(dev, "unexpected IRQ, INT%d\n", bit); @@ -155,9 +154,7 @@ static void mobiveil_pcie_isr(struct irq_desc *desc) dev_dbg(dev, "MSI registers, data: %08x, addr: %08x:%08x\n", msi_data, msi_addr_hi, msi_addr_lo); - virq = irq_find_mapping(msi->dev_domain, msi_data); - if (virq) - generic_handle_irq(virq); + generic_handle_domain_irq(msi->dev_domain, msi_data); msi_status = readl_relaxed(pcie->apb_csr_base + MSI_STATUS_OFFSET); diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index e3f5e7ab7606..c3b725afa11f 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -31,9 +31,6 @@ /* PCIe core registers */ #define PCIE_CORE_DEV_ID_REG 0x0 #define PCIE_CORE_CMD_STATUS_REG 0x4 -#define PCIE_CORE_CMD_IO_ACCESS_EN BIT(0) -#define PCIE_CORE_CMD_MEM_ACCESS_EN BIT(1) -#define PCIE_CORE_CMD_MEM_IO_REQ_EN BIT(2) #define PCIE_CORE_DEV_REV_REG 0x8 #define PCIE_CORE_PCIEXP_CAP 0xc0 #define PCIE_CORE_ERR_CAPCTL_REG 0x118 @@ -57,7 +54,8 @@ #define PIO_COMPLETION_STATUS_UR 1 #define PIO_COMPLETION_STATUS_CRS 2 #define PIO_COMPLETION_STATUS_CA 4 -#define PIO_NON_POSTED_REQ BIT(0) +#define PIO_NON_POSTED_REQ BIT(10) +#define PIO_ERR_STATUS BIT(11) #define PIO_ADDR_LS (PIO_BASE_ADDR + 0x8) #define PIO_ADDR_MS (PIO_BASE_ADDR + 0xc) #define PIO_WR_DATA (PIO_BASE_ADDR + 0x10) @@ -98,6 +96,7 @@ #define PCIE_CORE_CTRL2_MSI_ENABLE BIT(10) #define PCIE_CORE_REF_CLK_REG (CONTROL_BASE_ADDR + 0x14) #define PCIE_CORE_REF_CLK_TX_ENABLE BIT(1) +#define PCIE_CORE_REF_CLK_RX_ENABLE BIT(2) #define PCIE_MSG_LOG_REG (CONTROL_BASE_ADDR + 0x30) #define PCIE_ISR0_REG (CONTROL_BASE_ADDR + 0x40) #define PCIE_MSG_PM_PME_MASK BIT(7) @@ -105,27 +104,111 @@ #define PCIE_ISR0_MSI_INT_PENDING BIT(24) #define PCIE_ISR0_INTX_ASSERT(val) BIT(16 + (val)) #define PCIE_ISR0_INTX_DEASSERT(val) BIT(20 + (val)) -#define PCIE_ISR0_ALL_MASK GENMASK(26, 0) +#define PCIE_ISR0_ALL_MASK GENMASK(31, 0) #define PCIE_ISR1_REG (CONTROL_BASE_ADDR + 0x48) #define PCIE_ISR1_MASK_REG (CONTROL_BASE_ADDR + 0x4C) #define PCIE_ISR1_POWER_STATE_CHANGE BIT(4) #define PCIE_ISR1_FLUSH BIT(5) #define PCIE_ISR1_INTX_ASSERT(val) BIT(8 + (val)) -#define PCIE_ISR1_ALL_MASK GENMASK(11, 4) +#define PCIE_ISR1_ALL_MASK GENMASK(31, 0) #define PCIE_MSI_ADDR_LOW_REG (CONTROL_BASE_ADDR + 0x50) #define PCIE_MSI_ADDR_HIGH_REG (CONTROL_BASE_ADDR + 0x54) #define PCIE_MSI_STATUS_REG (CONTROL_BASE_ADDR + 0x58) #define PCIE_MSI_MASK_REG (CONTROL_BASE_ADDR + 0x5C) #define PCIE_MSI_PAYLOAD_REG (CONTROL_BASE_ADDR + 0x9C) +#define PCIE_MSI_DATA_MASK GENMASK(15, 0) + +/* PCIe window configuration */ +#define OB_WIN_BASE_ADDR 0x4c00 +#define OB_WIN_BLOCK_SIZE 0x20 +#define OB_WIN_COUNT 8 +#define OB_WIN_REG_ADDR(win, offset) (OB_WIN_BASE_ADDR + \ + OB_WIN_BLOCK_SIZE * (win) + \ + (offset)) +#define OB_WIN_MATCH_LS(win) OB_WIN_REG_ADDR(win, 0x00) +#define OB_WIN_ENABLE BIT(0) +#define OB_WIN_MATCH_MS(win) OB_WIN_REG_ADDR(win, 0x04) +#define OB_WIN_REMAP_LS(win) OB_WIN_REG_ADDR(win, 0x08) +#define OB_WIN_REMAP_MS(win) OB_WIN_REG_ADDR(win, 0x0c) +#define OB_WIN_MASK_LS(win) OB_WIN_REG_ADDR(win, 0x10) +#define OB_WIN_MASK_MS(win) OB_WIN_REG_ADDR(win, 0x14) +#define OB_WIN_ACTIONS(win) OB_WIN_REG_ADDR(win, 0x18) +#define OB_WIN_DEFAULT_ACTIONS (OB_WIN_ACTIONS(OB_WIN_COUNT-1) + 0x4) +#define OB_WIN_FUNC_NUM_MASK GENMASK(31, 24) +#define OB_WIN_FUNC_NUM_SHIFT 24 +#define OB_WIN_FUNC_NUM_ENABLE BIT(23) +#define OB_WIN_BUS_NUM_BITS_MASK GENMASK(22, 20) +#define OB_WIN_BUS_NUM_BITS_SHIFT 20 +#define OB_WIN_MSG_CODE_ENABLE BIT(22) +#define OB_WIN_MSG_CODE_MASK GENMASK(21, 14) +#define OB_WIN_MSG_CODE_SHIFT 14 +#define OB_WIN_MSG_PAYLOAD_LEN BIT(12) +#define OB_WIN_ATTR_ENABLE BIT(11) +#define OB_WIN_ATTR_TC_MASK GENMASK(10, 8) +#define OB_WIN_ATTR_TC_SHIFT 8 +#define OB_WIN_ATTR_RELAXED BIT(7) +#define OB_WIN_ATTR_NOSNOOP BIT(6) +#define OB_WIN_ATTR_POISON BIT(5) +#define OB_WIN_ATTR_IDO BIT(4) +#define OB_WIN_TYPE_MASK GENMASK(3, 0) +#define OB_WIN_TYPE_SHIFT 0 +#define OB_WIN_TYPE_MEM 0x0 +#define OB_WIN_TYPE_IO 0x4 +#define OB_WIN_TYPE_CONFIG_TYPE0 0x8 +#define OB_WIN_TYPE_CONFIG_TYPE1 0x9 +#define OB_WIN_TYPE_MSG 0xc /* LMI registers base address and register offsets */ #define LMI_BASE_ADDR 0x6000 #define CFG_REG (LMI_BASE_ADDR + 0x0) #define LTSSM_SHIFT 24 #define LTSSM_MASK 0x3f -#define LTSSM_L0 0x10 #define RC_BAR_CONFIG 0x300 +/* LTSSM values in CFG_REG */ +enum { + LTSSM_DETECT_QUIET = 0x0, + LTSSM_DETECT_ACTIVE = 0x1, + LTSSM_POLLING_ACTIVE = 0x2, + LTSSM_POLLING_COMPLIANCE = 0x3, + LTSSM_POLLING_CONFIGURATION = 0x4, + LTSSM_CONFIG_LINKWIDTH_START = 0x5, + LTSSM_CONFIG_LINKWIDTH_ACCEPT = 0x6, + LTSSM_CONFIG_LANENUM_ACCEPT = 0x7, + LTSSM_CONFIG_LANENUM_WAIT = 0x8, + LTSSM_CONFIG_COMPLETE = 0x9, + LTSSM_CONFIG_IDLE = 0xa, + LTSSM_RECOVERY_RCVR_LOCK = 0xb, + LTSSM_RECOVERY_SPEED = 0xc, + LTSSM_RECOVERY_RCVR_CFG = 0xd, + LTSSM_RECOVERY_IDLE = 0xe, + LTSSM_L0 = 0x10, + LTSSM_RX_L0S_ENTRY = 0x11, + LTSSM_RX_L0S_IDLE = 0x12, + LTSSM_RX_L0S_FTS = 0x13, + LTSSM_TX_L0S_ENTRY = 0x14, + LTSSM_TX_L0S_IDLE = 0x15, + LTSSM_TX_L0S_FTS = 0x16, + LTSSM_L1_ENTRY = 0x17, + LTSSM_L1_IDLE = 0x18, + LTSSM_L2_IDLE = 0x19, + LTSSM_L2_TRANSMIT_WAKE = 0x1a, + LTSSM_DISABLED = 0x20, + LTSSM_LOOPBACK_ENTRY_MASTER = 0x21, + LTSSM_LOOPBACK_ACTIVE_MASTER = 0x22, + LTSSM_LOOPBACK_EXIT_MASTER = 0x23, + LTSSM_LOOPBACK_ENTRY_SLAVE = 0x24, + LTSSM_LOOPBACK_ACTIVE_SLAVE = 0x25, + LTSSM_LOOPBACK_EXIT_SLAVE = 0x26, + LTSSM_HOT_RESET = 0x27, + LTSSM_RECOVERY_EQUALIZATION_PHASE0 = 0x28, + LTSSM_RECOVERY_EQUALIZATION_PHASE1 = 0x29, + LTSSM_RECOVERY_EQUALIZATION_PHASE2 = 0x2a, + LTSSM_RECOVERY_EQUALIZATION_PHASE3 = 0x2b, +}; + +#define VENDOR_ID_REG (LMI_BASE_ADDR + 0x44) + /* PCIe core controller registers */ #define CTRL_CORE_BASE_ADDR 0x18000 #define CTRL_CONFIG_REG (CTRL_CORE_BASE_ADDR + 0x0) @@ -156,7 +239,7 @@ #define PCIE_IRQ_MSI_INT2_DET BIT(21) #define PCIE_IRQ_RC_DBELL_DET BIT(22) #define PCIE_IRQ_EP_STATUS BIT(23) -#define PCIE_IRQ_ALL_MASK 0xfff0fb +#define PCIE_IRQ_ALL_MASK GENMASK(31, 0) #define PCIE_IRQ_ENABLE_INTS_MASK PCIE_IRQ_CORE_INT /* Transaction types */ @@ -165,7 +248,7 @@ #define PCIE_CONFIG_WR_TYPE0 0xa #define PCIE_CONFIG_WR_TYPE1 0xb -#define PIO_RETRY_CNT 500 +#define PIO_RETRY_CNT 750000 /* 1.5 s */ #define PIO_RETRY_DELAY 2 /* 2 us*/ #define LINK_WAIT_MAX_RETRIES 10 @@ -176,11 +259,21 @@ #define MSI_IRQ_NUM 32 +#define CFG_RD_CRS_VAL 0xffff0001 + struct advk_pcie { struct platform_device *pdev; void __iomem *base; + struct { + phys_addr_t match; + phys_addr_t remap; + phys_addr_t mask; + u32 actions; + } wins[OB_WIN_COUNT]; + u8 wins_count; struct irq_domain *irq_domain; struct irq_chip irq_chip; + raw_spinlock_t irq_lock; struct irq_domain *msi_domain; struct irq_domain *msi_inner_domain; struct irq_chip msi_bottom_irq_chip; @@ -205,18 +298,49 @@ static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg) return readl(pcie->base + reg); } -static inline u16 advk_read16(struct advk_pcie *pcie, u64 reg) +static u8 advk_pcie_ltssm_state(struct advk_pcie *pcie) { - return advk_readl(pcie, (reg & ~0x3)) >> ((reg & 0x3) * 8); + u32 val; + u8 ltssm_state; + + val = advk_readl(pcie, CFG_REG); + ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK; + return ltssm_state; } -static int advk_pcie_link_up(struct advk_pcie *pcie) +static inline bool advk_pcie_link_up(struct advk_pcie *pcie) { - u32 val, ltssm_state; + /* check if LTSSM is in normal operation - some L* state */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ltssm_state >= LTSSM_L0 && ltssm_state < LTSSM_DISABLED; +} - val = advk_readl(pcie, CFG_REG); - ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK; - return ltssm_state >= LTSSM_L0; +static inline bool advk_pcie_link_active(struct advk_pcie *pcie) +{ + /* + * According to PCIe Base specification 3.0, Table 4-14: Link + * Status Mapped to the LTSSM, and 4.2.6.3.6 Configuration.Idle + * is Link Up mapped to LTSSM Configuration.Idle, Recovery, L0, + * L0s, L1 and L2 states. And according to 3.2.1. Data Link + * Control and Management State Machine Rules is DL Up status + * reported in DL Active state. + */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ltssm_state >= LTSSM_CONFIG_IDLE && ltssm_state < LTSSM_DISABLED; +} + +static inline bool advk_pcie_link_training(struct advk_pcie *pcie) +{ + /* + * According to PCIe Base specification 3.0, Table 4-14: Link + * Status Mapped to the LTSSM is Link Training mapped to LTSSM + * Configuration and Recovery states. + */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ((ltssm_state >= LTSSM_CONFIG_LINKWIDTH_START && + ltssm_state < LTSSM_L0) || + (ltssm_state >= LTSSM_RECOVERY_EQUALIZATION_PHASE0 && + ltssm_state <= LTSSM_RECOVERY_EQUALIZATION_PHASE3)); } static int advk_pcie_wait_for_link(struct advk_pcie *pcie) @@ -239,7 +363,7 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie) size_t retries; for (retries = 0; retries < RETRAIN_WAIT_MAX_RETRIES; ++retries) { - if (!advk_pcie_link_up(pcie)) + if (advk_pcie_link_training(pcie)) break; udelay(RETRAIN_WAIT_USLEEP_US); } @@ -247,23 +371,9 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie) static void advk_pcie_issue_perst(struct advk_pcie *pcie) { - u32 reg; - if (!pcie->reset_gpio) return; - /* - * As required by PCI Express spec (PCI Express Base Specification, REV. - * 4.0 PCI Express, February 19 2014, 6.6.1 Conventional Reset) a delay - * for at least 100ms after de-asserting PERST# signal is needed before - * link training is enabled. So ensure that link training is disabled - * prior de-asserting PERST# signal to fulfill that PCI Express spec - * requirement. - */ - reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); - reg &= ~LINK_TRAINING_EN; - advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); - /* 10ms delay is needed for some cards */ dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n"); gpiod_set_value_cansleep(pcie->reset_gpio, 1); @@ -271,54 +381,47 @@ static void advk_pcie_issue_perst(struct advk_pcie *pcie) gpiod_set_value_cansleep(pcie->reset_gpio, 0); } -static int advk_pcie_train_at_gen(struct advk_pcie *pcie, int gen) +static void advk_pcie_train_link(struct advk_pcie *pcie) { - int ret, neg_gen; + struct device *dev = &pcie->pdev->dev; u32 reg; + int ret; - /* Setup link speed */ + /* + * Setup PCIe rev / gen compliance based on device tree property + * 'max-link-speed' which also forces maximal link speed. + */ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); reg &= ~PCIE_GEN_SEL_MSK; - if (gen == 3) + if (pcie->link_gen == 3) reg |= SPEED_GEN_3; - else if (gen == 2) + else if (pcie->link_gen == 2) reg |= SPEED_GEN_2; else reg |= SPEED_GEN_1; advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); /* - * Enable link training. This is not needed in every call to this - * function, just once suffices, but it does not break anything either. + * Set maximal link speed value also into PCIe Link Control 2 register. + * Armada 3700 Functional Specification says that default value is based + * on SPEED_GEN but tests showed that default value is always 8.0 GT/s. */ + reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2); + reg &= ~PCI_EXP_LNKCTL2_TLS; + if (pcie->link_gen == 3) + reg |= PCI_EXP_LNKCTL2_TLS_8_0GT; + else if (pcie->link_gen == 2) + reg |= PCI_EXP_LNKCTL2_TLS_5_0GT; + else + reg |= PCI_EXP_LNKCTL2_TLS_2_5GT; + advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2); + + /* Enable link training after selecting PCIe generation */ reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); reg |= LINK_TRAINING_EN; advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); /* - * Start link training immediately after enabling it. - * This solves problems for some buggy cards. - */ - reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL); - reg |= PCI_EXP_LNKCTL_RL; - advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL); - - ret = advk_pcie_wait_for_link(pcie); - if (ret) - return ret; - - reg = advk_read16(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKSTA); - neg_gen = reg & PCI_EXP_LNKSTA_CLS; - - return neg_gen; -} - -static void advk_pcie_train_link(struct advk_pcie *pcie) -{ - struct device *dev = &pcie->pdev->dev; - int neg_gen = -1, gen; - - /* * Reset PCIe card via PERST# signal. Some cards are not detected * during link training when they are in some non-initial state. */ @@ -328,50 +431,63 @@ static void advk_pcie_train_link(struct advk_pcie *pcie) * PERST# signal could have been asserted by pinctrl subsystem before * probe() callback has been called or issued explicitly by reset gpio * function advk_pcie_issue_perst(), making the endpoint going into - * fundamental reset. As required by PCI Express spec a delay for at - * least 100ms after such a reset before link training is needed. - */ - msleep(PCI_PM_D3COLD_WAIT); - - /* - * Try link training at link gen specified by device tree property - * 'max-link-speed'. If this fails, iteratively train at lower gen. - */ - for (gen = pcie->link_gen; gen > 0; --gen) { - neg_gen = advk_pcie_train_at_gen(pcie, gen); - if (neg_gen > 0) - break; - } - - if (neg_gen < 0) - goto err; - - /* - * After successful training if negotiated gen is lower than requested, - * train again on negotiated gen. This solves some stability issues for - * some buggy gen1 cards. + * fundamental reset. As required by PCI Express spec (PCI Express + * Base Specification, REV. 4.0 PCI Express, February 19 2014, 6.6.1 + * Conventional Reset) a delay for at least 100ms after such a reset + * before sending a Configuration Request to the device is needed. + * So wait until PCIe link is up. Function advk_pcie_wait_for_link() + * waits for link at least 900ms. */ - if (neg_gen < gen) { - gen = neg_gen; - neg_gen = advk_pcie_train_at_gen(pcie, gen); - } + ret = advk_pcie_wait_for_link(pcie); + if (ret < 0) + dev_err(dev, "link never came up\n"); + else + dev_info(dev, "link up\n"); +} - if (neg_gen == gen) { - dev_info(dev, "link up at gen %i\n", gen); - return; - } +/* + * Set PCIe address window register which could be used for memory + * mapping. + */ +static void advk_pcie_set_ob_win(struct advk_pcie *pcie, u8 win_num, + phys_addr_t match, phys_addr_t remap, + phys_addr_t mask, u32 actions) +{ + advk_writel(pcie, OB_WIN_ENABLE | + lower_32_bits(match), OB_WIN_MATCH_LS(win_num)); + advk_writel(pcie, upper_32_bits(match), OB_WIN_MATCH_MS(win_num)); + advk_writel(pcie, lower_32_bits(remap), OB_WIN_REMAP_LS(win_num)); + advk_writel(pcie, upper_32_bits(remap), OB_WIN_REMAP_MS(win_num)); + advk_writel(pcie, lower_32_bits(mask), OB_WIN_MASK_LS(win_num)); + advk_writel(pcie, upper_32_bits(mask), OB_WIN_MASK_MS(win_num)); + advk_writel(pcie, actions, OB_WIN_ACTIONS(win_num)); +} -err: - dev_err(dev, "link never came up\n"); +static void advk_pcie_disable_ob_win(struct advk_pcie *pcie, u8 win_num) +{ + advk_writel(pcie, 0, OB_WIN_MATCH_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_MATCH_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_REMAP_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_REMAP_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_MASK_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_MASK_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_ACTIONS(win_num)); } static void advk_pcie_setup_hw(struct advk_pcie *pcie) { u32 reg; + int i; - /* Enable TX */ + /* + * Configure PCIe Reference clock. Direction is from the PCIe + * controller to the endpoint card, so enable transmitting of + * Reference clock differential signal off-chip and disable + * receiving off-chip differential signal. + */ reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG); reg |= PCIE_CORE_REF_CLK_TX_ENABLE; + reg &= ~PCIE_CORE_REF_CLK_RX_ENABLE; advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG); /* Set to Direct mode */ @@ -385,6 +501,41 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) reg |= (IS_RC_MSK << IS_RC_SHIFT); advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + /* + * Replace incorrect PCI vendor id value 0x1b4b by correct value 0x11ab. + * VENDOR_ID_REG contains vendor id in low 16 bits and subsystem vendor + * id in high 16 bits. Updating this register changes readback value of + * read-only vendor id bits in PCIE_CORE_DEV_ID_REG register. Workaround + * for erratum 4.1: "The value of device and vendor ID is incorrect". + */ + reg = (PCI_VENDOR_ID_MARVELL << 16) | PCI_VENDOR_ID_MARVELL; + advk_writel(pcie, reg, VENDOR_ID_REG); + + /* + * Change Class Code of PCI Bridge device to PCI Bridge (0x600400), + * because the default value is Mass storage controller (0x010400). + * + * Note that this Aardvark PCI Bridge does not have compliant Type 1 + * Configuration Space and it even cannot be accessed via Aardvark's + * PCI config space access method. Something like config space is + * available in internal Aardvark registers starting at offset 0x0 + * and is reported as Type 0. In range 0x10 - 0x34 it has totally + * different registers. + * + * Therefore driver uses emulation of PCI Bridge which emulates + * access to configuration space via internal Aardvark registers or + * emulated configuration buffer. + */ + reg = advk_readl(pcie, PCIE_CORE_DEV_REV_REG); + reg &= ~0xffffff00; + reg |= (PCI_CLASS_BRIDGE_PCI << 8) << 8; + advk_writel(pcie, reg, PCIE_CORE_DEV_REV_REG); + + /* Disable Root Bridge I/O space, memory space and bus mastering */ + reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); + reg &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG); + /* Set Advanced Error Capabilities and Control PF0 register */ reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX | PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN | @@ -396,8 +547,9 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL); reg &= ~PCI_EXP_DEVCTL_RELAX_EN; reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN; + reg &= ~PCI_EXP_DEVCTL_PAYLOAD; reg &= ~PCI_EXP_DEVCTL_READRQ; - reg |= PCI_EXP_DEVCTL_PAYLOAD; /* Set max payload size */ + reg |= PCI_EXP_DEVCTL_PAYLOAD_512B; reg |= PCI_EXP_DEVCTL_READRQ_512B; advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL); @@ -436,67 +588,159 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie) reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK); advk_writel(pcie, reg, HOST_CTRL_INT_MASK_REG); + /* + * Enable AXI address window location generation: + * When it is enabled, the default outbound window + * configurations (Default User Field: 0xD0074CFC) + * are used to transparent address translation for + * the outbound transactions. Thus, PCIe address + * windows are not required for transparent memory + * access when default outbound window configuration + * is set for memory access. + */ reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG); reg |= PCIE_CORE_CTRL2_OB_WIN_ENABLE; advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG); - /* Bypass the address window mapping for PIO */ + /* + * Set memory access in Default User Field so it + * is not required to configure PCIe address for + * transparent memory access. + */ + advk_writel(pcie, OB_WIN_TYPE_MEM, OB_WIN_DEFAULT_ACTIONS); + + /* + * Bypass the address window mapping for PIO: + * Since PIO access already contains all required + * info over AXI interface by PIO registers, the + * address window is not required. + */ reg = advk_readl(pcie, PIO_CTRL); reg |= PIO_CTRL_ADDR_WIN_DISABLE; advk_writel(pcie, reg, PIO_CTRL); - advk_pcie_train_link(pcie); - /* - * FIXME: The following register update is suspicious. This register is - * applicable only when the PCI controller is configured for Endpoint - * mode, not as a Root Complex. But apparently when this code is - * removed, some cards stop working. This should be investigated and - * a comment explaining this should be put here. + * Configure PCIe address windows for non-memory or + * non-transparent access as by default PCIe uses + * transparent memory access. */ - reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); - reg |= PCIE_CORE_CMD_MEM_ACCESS_EN | - PCIE_CORE_CMD_IO_ACCESS_EN | - PCIE_CORE_CMD_MEM_IO_REQ_EN; - advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG); + for (i = 0; i < pcie->wins_count; i++) + advk_pcie_set_ob_win(pcie, i, + pcie->wins[i].match, pcie->wins[i].remap, + pcie->wins[i].mask, pcie->wins[i].actions); + + /* Disable remaining PCIe outbound windows */ + for (i = pcie->wins_count; i < OB_WIN_COUNT; i++) + advk_pcie_disable_ob_win(pcie, i); + + advk_pcie_train_link(pcie); } -static void advk_pcie_check_pio_status(struct advk_pcie *pcie) +static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_crs, u32 *val) { struct device *dev = &pcie->pdev->dev; u32 reg; unsigned int status; char *strcomp_status, *str_posted; + int ret; reg = advk_readl(pcie, PIO_STAT); status = (reg & PIO_COMPLETION_STATUS_MASK) >> PIO_COMPLETION_STATUS_SHIFT; - if (!status) - return; - + /* + * According to HW spec, the PIO status check sequence as below: + * 1) even if COMPLETION_STATUS(bit9:7) indicates successful, + * it still needs to check Error Status(bit11), only when this bit + * indicates no error happen, the operation is successful. + * 2) value Unsupported Request(1) of COMPLETION_STATUS(bit9:7) only + * means a PIO write error, and for PIO read it is successful with + * a read value of 0xFFFFFFFF. + * 3) value Completion Retry Status(CRS) of COMPLETION_STATUS(bit9:7) + * only means a PIO write error, and for PIO read it is successful + * with a read value of 0xFFFF0001. + * 4) value Completer Abort (CA) of COMPLETION_STATUS(bit9:7) means + * error for both PIO read and PIO write operation. + * 5) other errors are indicated as 'unknown'. + */ switch (status) { + case PIO_COMPLETION_STATUS_OK: + if (reg & PIO_ERR_STATUS) { + strcomp_status = "COMP_ERR"; + ret = -EFAULT; + break; + } + /* Get the read result */ + if (val) + *val = advk_readl(pcie, PIO_RD_DATA); + /* No error */ + strcomp_status = NULL; + ret = 0; + break; case PIO_COMPLETION_STATUS_UR: strcomp_status = "UR"; + ret = -EOPNOTSUPP; break; case PIO_COMPLETION_STATUS_CRS: + if (allow_crs && val) { + /* PCIe r4.0, sec 2.3.2, says: + * If CRS Software Visibility is enabled: + * For a Configuration Read Request that includes both + * bytes of the Vendor ID field of a device Function's + * Configuration Space Header, the Root Complex must + * complete the Request to the host by returning a + * read-data value of 0001h for the Vendor ID field and + * all '1's for any additional bytes included in the + * request. + * + * So CRS in this case is not an error status. + */ + *val = CFG_RD_CRS_VAL; + strcomp_status = NULL; + ret = 0; + break; + } + /* PCIe r4.0, sec 2.3.2, says: + * If CRS Software Visibility is not enabled, the Root Complex + * must re-issue the Configuration Request as a new Request. + * If CRS Software Visibility is enabled: For a Configuration + * Write Request or for any other Configuration Read Request, + * the Root Complex must re-issue the Configuration Request as + * a new Request. + * A Root Complex implementation may choose to limit the number + * of Configuration Request/CRS Completion Status loops before + * determining that something is wrong with the target of the + * Request and taking appropriate action, e.g., complete the + * Request to the host as a failed transaction. + * + * So return -EAGAIN and caller (pci-aardvark.c driver) will + * re-issue request again up to the PIO_RETRY_CNT retries. + */ strcomp_status = "CRS"; + ret = -EAGAIN; break; case PIO_COMPLETION_STATUS_CA: strcomp_status = "CA"; + ret = -ECANCELED; break; default: strcomp_status = "Unknown"; + ret = -EINVAL; break; } + if (!strcomp_status) + return ret; + if (reg & PIO_NON_POSTED_REQ) str_posted = "Non-posted"; else str_posted = "Posted"; - dev_err(dev, "%s PIO Response Status: %s, %#x @ %#x\n", + dev_dbg(dev, "%s PIO Response Status: %s, %#x @ %#x\n", str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS)); + + return ret; } static int advk_pcie_wait_pio(struct advk_pcie *pcie) @@ -504,13 +748,13 @@ static int advk_pcie_wait_pio(struct advk_pcie *pcie) struct device *dev = &pcie->pdev->dev; int i; - for (i = 0; i < PIO_RETRY_CNT; i++) { + for (i = 1; i <= PIO_RETRY_CNT; i++) { u32 start, isr; start = advk_readl(pcie, PIO_START); isr = advk_readl(pcie, PIO_ISR); if (!start && isr) - return 0; + return i; udelay(PIO_RETRY_DELAY); } @@ -518,6 +762,64 @@ static int advk_pcie_wait_pio(struct advk_pcie *pcie) return -ETIMEDOUT; } +static pci_bridge_emul_read_status_t +advk_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case PCI_COMMAND: + *value = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); + return PCI_BRIDGE_EMUL_HANDLED; + + case PCI_INTERRUPT_LINE: { + /* + * From the whole 32bit register we support reading from HW only + * one bit: PCI_BRIDGE_CTL_BUS_RESET. + * Other bits are retrieved only from emulated config buffer. + */ + __le32 *cfgspace = (__le32 *)&bridge->conf; + u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]); + if (advk_readl(pcie, PCIE_CORE_CTRL1_REG) & HOT_RESET_GEN) + val |= PCI_BRIDGE_CTL_BUS_RESET << 16; + else + val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16); + *value = val; + return PCI_BRIDGE_EMUL_HANDLED; + } + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } +} + +static void +advk_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case PCI_COMMAND: + advk_writel(pcie, new, PCIE_CORE_CMD_STATUS_REG); + break; + + case PCI_INTERRUPT_LINE: + if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) { + u32 val = advk_readl(pcie, PCIE_CORE_CTRL1_REG); + if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16)) + val |= HOT_RESET_GEN; + else + val &= ~HOT_RESET_GEN; + advk_writel(pcie, val, PCIE_CORE_CTRL1_REG); + } + break; + + default: + break; + } +} static pci_bridge_emul_read_status_t advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, @@ -534,6 +836,8 @@ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, case PCI_EXP_RTCTL: { u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG); *value = (val & PCIE_MSG_PM_PME_MASK) ? 0 : PCI_EXP_RTCTL_PMEIE; + *value |= le16_to_cpu(bridge->pcie_conf.rootctl) & PCI_EXP_RTCTL_CRSSVE; + *value |= PCI_EXP_RTCAP_CRSVIS << 16; return PCI_BRIDGE_EMUL_HANDLED; } @@ -544,12 +848,26 @@ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, return PCI_BRIDGE_EMUL_HANDLED; } + case PCI_EXP_LNKCAP: { + u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg); + /* + * PCI_EXP_LNKCAP_DLLLARC bit is hardwired in aardvark HW to 0. + * But support for PCI_EXP_LNKSTA_DLLLA is emulated via ltssm + * state so explicitly enable PCI_EXP_LNKCAP_DLLLARC flag. + */ + val |= PCI_EXP_LNKCAP_DLLLARC; + *value = val; + return PCI_BRIDGE_EMUL_HANDLED; + } + case PCI_EXP_LNKCTL: { /* u32 contains both PCI_EXP_LNKCTL and PCI_EXP_LNKSTA */ u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg) & ~(PCI_EXP_LNKSTA_LT << 16); - if (!advk_pcie_link_up(pcie)) + if (advk_pcie_link_training(pcie)) val |= (PCI_EXP_LNKSTA_LT << 16); + if (advk_pcie_link_active(pcie)) + val |= (PCI_EXP_LNKSTA_DLLLA << 16); *value = val; return PCI_BRIDGE_EMUL_HANDLED; } @@ -557,7 +875,6 @@ advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, case PCI_CAP_LIST_ID: case PCI_EXP_DEVCAP: case PCI_EXP_DEVCTL: - case PCI_EXP_LNKCAP: *value = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg); return PCI_BRIDGE_EMUL_HANDLED; default: @@ -604,6 +921,8 @@ advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge, } static struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = { + .read_base = advk_pci_bridge_emul_base_conf_read, + .write_base = advk_pci_bridge_emul_base_conf_write, .read_pcie = advk_pci_bridge_emul_pcie_conf_read, .write_pcie = advk_pci_bridge_emul_pcie_conf_write, }; @@ -634,6 +953,9 @@ static int advk_sw_pci_bridge_init(struct advk_pcie *pcie) /* Support interrupt A for MSI feature */ bridge->conf.intpin = PCIE_CORE_INT_A_ASSERT_ENABLE; + /* Indicates supports for Completion Retry Status */ + bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS); + bridge->has_pcie = true; bridge->data = pcie; bridge->ops = &advk_pci_bridge_emul_ops; @@ -690,6 +1012,8 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, int size, u32 *val) { struct advk_pcie *pcie = bus->sysdata; + int retry_count; + bool allow_crs; u32 reg; int ret; @@ -702,10 +1026,17 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, return pci_bridge_emul_conf_read(&pcie->bridge, where, size, val); - if (advk_pcie_pio_is_running(pcie)) { - *val = 0xffffffff; - return PCIBIOS_SET_FAILED; - } + /* + * Completion Retry Status is possible to return only when reading all + * 4 bytes from PCI_VENDOR_ID and PCI_DEVICE_ID registers at once and + * CRSSVE flag on Root Bridge is enabled. + */ + allow_crs = (where == PCI_VENDOR_ID) && (size == 4) && + (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & + PCI_EXP_RTCTL_CRSSVE); + + if (advk_pcie_pio_is_running(pcie)) + goto try_crs; /* Program the control register */ reg = advk_readl(pcie, PIO_CTRL); @@ -724,26 +1055,45 @@ static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, /* Program the data strobe */ advk_writel(pcie, 0xf, PIO_WR_DATA_STRB); - /* Clear PIO DONE ISR and start the transfer */ - advk_writel(pcie, 1, PIO_ISR); - advk_writel(pcie, 1, PIO_START); + retry_count = 0; + do { + /* Clear PIO DONE ISR and start the transfer */ + advk_writel(pcie, 1, PIO_ISR); + advk_writel(pcie, 1, PIO_START); - ret = advk_pcie_wait_pio(pcie); - if (ret < 0) { - *val = 0xffffffff; - return PCIBIOS_SET_FAILED; - } + ret = advk_pcie_wait_pio(pcie); + if (ret < 0) + goto try_crs; + + retry_count += ret; - advk_pcie_check_pio_status(pcie); + /* Check PIO status and get the read result */ + ret = advk_pcie_check_pio_status(pcie, allow_crs, val); + } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT); + + if (ret < 0) + goto fail; - /* Get the read result */ - *val = advk_readl(pcie, PIO_RD_DATA); if (size == 1) *val = (*val >> (8 * (where & 3))) & 0xff; else if (size == 2) *val = (*val >> (8 * (where & 3))) & 0xffff; return PCIBIOS_SUCCESSFUL; + +try_crs: + /* + * If it is possible, return Completion Retry Status so that caller + * tries to issue the request again instead of failing. + */ + if (allow_crs) { + *val = CFG_RD_CRS_VAL; + return PCIBIOS_SUCCESSFUL; + } + +fail: + *val = 0xffffffff; + return PCIBIOS_SET_FAILED; } static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn, @@ -752,6 +1102,7 @@ static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn, struct advk_pcie *pcie = bus->sysdata; u32 reg; u32 data_strobe = 0x0; + int retry_count; int offset; int ret; @@ -793,17 +1144,22 @@ static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn, /* Program the data strobe */ advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB); - /* Clear PIO DONE ISR and start the transfer */ - advk_writel(pcie, 1, PIO_ISR); - advk_writel(pcie, 1, PIO_START); + retry_count = 0; + do { + /* Clear PIO DONE ISR and start the transfer */ + advk_writel(pcie, 1, PIO_ISR); + advk_writel(pcie, 1, PIO_START); - ret = advk_pcie_wait_pio(pcie); - if (ret < 0) - return PCIBIOS_SET_FAILED; + ret = advk_pcie_wait_pio(pcie); + if (ret < 0) + return PCIBIOS_SET_FAILED; - advk_pcie_check_pio_status(pcie); + retry_count += ret; - return PCIBIOS_SUCCESSFUL; + ret = advk_pcie_check_pio_status(pcie, false, NULL); + } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT); + + return ret < 0 ? PCIBIOS_SET_FAILED : PCIBIOS_SUCCESSFUL; } static struct pci_ops advk_pcie_ops = { @@ -852,7 +1208,7 @@ static int advk_msi_irq_domain_alloc(struct irq_domain *domain, domain->host_data, handle_simple_irq, NULL, NULL); - return hwirq; + return 0; } static void advk_msi_irq_domain_free(struct irq_domain *domain, @@ -875,22 +1231,28 @@ static void advk_pcie_irq_mask(struct irq_data *d) { struct advk_pcie *pcie = d->domain->host_data; irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; u32 mask; + raw_spin_lock_irqsave(&pcie->irq_lock, flags); mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); mask |= PCIE_ISR1_INTX_ASSERT(hwirq); advk_writel(pcie, mask, PCIE_ISR1_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); } static void advk_pcie_irq_unmask(struct irq_data *d) { struct advk_pcie *pcie = d->domain->host_data; irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; u32 mask; + raw_spin_lock_irqsave(&pcie->irq_lock, flags); mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); mask &= ~PCIE_ISR1_INTX_ASSERT(hwirq); advk_writel(pcie, mask, PCIE_ISR1_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); } static int advk_pcie_irq_map(struct irq_domain *h, @@ -974,6 +1336,8 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie) struct irq_chip *irq_chip; int ret = 0; + raw_spin_lock_init(&pcie->irq_lock); + pcie_intc_node = of_get_next_child(node, NULL); if (!pcie_intc_node) { dev_err(dev, "No PCIe Intc node found\n"); @@ -1025,8 +1389,12 @@ static void advk_pcie_handle_msi(struct advk_pcie *pcie) if (!(BIT(msi_idx) & msi_status)) continue; + /* + * msi_idx contains bits [4:0] of the msi_data and msi_data + * contains 16bit MSI interrupt number + */ advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG); - msi_data = advk_readl(pcie, PCIE_MSI_PAYLOAD_REG) & 0xFF; + msi_data = advk_readl(pcie, PCIE_MSI_PAYLOAD_REG) & PCIE_MSI_DATA_MASK; generic_handle_irq(msi_data); } @@ -1038,7 +1406,7 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie) { u32 isr0_val, isr0_mask, isr0_status; u32 isr1_val, isr1_mask, isr1_status; - int i, virq; + int i; isr0_val = advk_readl(pcie, PCIE_ISR0_REG); isr0_mask = advk_readl(pcie, PCIE_ISR0_MASK_REG); @@ -1048,12 +1416,6 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie) isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK); - if (!isr0_status && !isr1_status) { - advk_writel(pcie, isr0_val, PCIE_ISR0_REG); - advk_writel(pcie, isr1_val, PCIE_ISR1_REG); - return; - } - /* Process MSI interrupts */ if (isr0_status & PCIE_ISR0_MSI_INT_PENDING) advk_pcie_handle_msi(pcie); @@ -1066,8 +1428,7 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie) advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i), PCIE_ISR1_REG); - virq = irq_find_mapping(pcie->irq_domain, i); - generic_handle_irq(virq); + generic_handle_domain_irq(pcie->irq_domain, i); } } @@ -1151,6 +1512,7 @@ static int advk_pcie_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct advk_pcie *pcie; struct pci_host_bridge *bridge; + struct resource_entry *entry; int ret, irq; bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct advk_pcie)); @@ -1161,6 +1523,80 @@ static int advk_pcie_probe(struct platform_device *pdev) pcie->pdev = pdev; platform_set_drvdata(pdev, pcie); + resource_list_for_each_entry(entry, &bridge->windows) { + resource_size_t start = entry->res->start; + resource_size_t size = resource_size(entry->res); + unsigned long type = resource_type(entry->res); + u64 win_size; + + /* + * Aardvark hardware allows to configure also PCIe window + * for config type 0 and type 1 mapping, but driver uses + * only PIO for issuing configuration transfers which does + * not use PCIe window configuration. + */ + if (type != IORESOURCE_MEM && type != IORESOURCE_MEM_64 && + type != IORESOURCE_IO) + continue; + + /* + * Skip transparent memory resources. Default outbound access + * configuration is set to transparent memory access so it + * does not need window configuration. + */ + if ((type == IORESOURCE_MEM || type == IORESOURCE_MEM_64) && + entry->offset == 0) + continue; + + /* + * The n-th PCIe window is configured by tuple (match, remap, mask) + * and an access to address A uses this window if A matches the + * match with given mask. + * So every PCIe window size must be a power of two and every start + * address must be aligned to window size. Minimal size is 64 KiB + * because lower 16 bits of mask must be zero. Remapped address + * may have set only bits from the mask. + */ + while (pcie->wins_count < OB_WIN_COUNT && size > 0) { + /* Calculate the largest aligned window size */ + win_size = (1ULL << (fls64(size)-1)) | + (start ? (1ULL << __ffs64(start)) : 0); + win_size = 1ULL << __ffs64(win_size); + if (win_size < 0x10000) + break; + + dev_dbg(dev, + "Configuring PCIe window %d: [0x%llx-0x%llx] as %lu\n", + pcie->wins_count, (unsigned long long)start, + (unsigned long long)start + win_size, type); + + if (type == IORESOURCE_IO) { + pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_IO; + pcie->wins[pcie->wins_count].match = pci_pio_to_address(start); + } else { + pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_MEM; + pcie->wins[pcie->wins_count].match = start; + } + pcie->wins[pcie->wins_count].remap = start - entry->offset; + pcie->wins[pcie->wins_count].mask = ~(win_size - 1); + + if (pcie->wins[pcie->wins_count].remap & (win_size - 1)) + break; + + start += win_size; + size -= win_size; + pcie->wins_count++; + } + + if (size > 0) { + dev_err(&pcie->pdev->dev, + "Invalid PCIe region [0x%llx-0x%llx]\n", + (unsigned long long)entry->res->start, + (unsigned long long)entry->res->end + 1); + return -EINVAL; + } + } + pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); @@ -1241,6 +1677,7 @@ static int advk_pcie_remove(struct platform_device *pdev) { struct advk_pcie *pcie = platform_get_drvdata(pdev); struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + int i; pci_lock_rescan_remove(); pci_stop_root_bus(bridge->bus); @@ -1250,6 +1687,10 @@ static int advk_pcie_remove(struct platform_device *pdev) advk_pcie_remove_msi_irq_domain(pcie); advk_pcie_remove_irq_domain(pcie); + /* Disable outbound address windows mapping */ + for (i = 0; i < OB_WIN_COUNT; i++) + advk_pcie_disable_ob_win(pcie, i); + return 0; } diff --git a/drivers/pci/controller/pci-ftpci100.c b/drivers/pci/controller/pci-ftpci100.c index da3cd216da00..88980a44461d 100644 --- a/drivers/pci/controller/pci-ftpci100.c +++ b/drivers/pci/controller/pci-ftpci100.c @@ -34,12 +34,12 @@ * Special configuration registers directly in the first few words * in I/O space. */ -#define PCI_IOSIZE 0x00 -#define PCI_PROT 0x04 /* AHB protection */ -#define PCI_CTRL 0x08 /* PCI control signal */ -#define PCI_SOFTRST 0x10 /* Soft reset counter and response error enable */ -#define PCI_CONFIG 0x28 /* PCI configuration command register */ -#define PCI_DATA 0x2C +#define FTPCI_IOSIZE 0x00 +#define FTPCI_PROT 0x04 /* AHB protection */ +#define FTPCI_CTRL 0x08 /* PCI control signal */ +#define FTPCI_SOFTRST 0x10 /* Soft reset counter and response error enable */ +#define FTPCI_CONFIG 0x28 /* PCI configuration command register */ +#define FTPCI_DATA 0x2C #define FARADAY_PCI_STATUS_CMD 0x04 /* Status and command */ #define FARADAY_PCI_PMC 0x40 /* Power management control */ @@ -195,9 +195,9 @@ static int faraday_raw_pci_read_config(struct faraday_pci *p, int bus_number, PCI_CONF_FUNCTION(PCI_FUNC(fn)) | PCI_CONF_WHERE(config) | PCI_CONF_ENABLE, - p->base + PCI_CONFIG); + p->base + FTPCI_CONFIG); - *value = readl(p->base + PCI_DATA); + *value = readl(p->base + FTPCI_DATA); if (size == 1) *value = (*value >> (8 * (config & 3))) & 0xFF; @@ -230,17 +230,17 @@ static int faraday_raw_pci_write_config(struct faraday_pci *p, int bus_number, PCI_CONF_FUNCTION(PCI_FUNC(fn)) | PCI_CONF_WHERE(config) | PCI_CONF_ENABLE, - p->base + PCI_CONFIG); + p->base + FTPCI_CONFIG); switch (size) { case 4: - writel(value, p->base + PCI_DATA); + writel(value, p->base + FTPCI_DATA); break; case 2: - writew(value, p->base + PCI_DATA + (config & 3)); + writew(value, p->base + FTPCI_DATA + (config & 3)); break; case 1: - writeb(value, p->base + PCI_DATA + (config & 3)); + writeb(value, p->base + FTPCI_DATA + (config & 3)); break; default: ret = PCIBIOS_BAD_REGISTER_NUMBER; @@ -314,7 +314,7 @@ static void faraday_pci_irq_handler(struct irq_desc *desc) for (i = 0; i < 4; i++) { if ((irq_stat & BIT(i)) == 0) continue; - generic_handle_irq(irq_find_mapping(p->irqdomain, i)); + generic_handle_domain_irq(p->irqdomain, i); } chained_irq_exit(irqchip, desc); @@ -469,7 +469,7 @@ static int faraday_pci_probe(struct platform_device *pdev) if (!faraday_res_to_memcfg(io->start - win->offset, resource_size(io), &val)) { /* setup I/O space size */ - writel(val, p->base + PCI_IOSIZE); + writel(val, p->base + FTPCI_IOSIZE); } else { dev_err(dev, "illegal IO mem size\n"); return -EINVAL; @@ -477,11 +477,11 @@ static int faraday_pci_probe(struct platform_device *pdev) } /* Setup hostbridge */ - val = readl(p->base + PCI_CTRL); + val = readl(p->base + FTPCI_CTRL); val |= PCI_COMMAND_IO; val |= PCI_COMMAND_MEMORY; val |= PCI_COMMAND_MASTER; - writel(val, p->base + PCI_CTRL); + writel(val, p->base + FTPCI_CTRL); /* Mask and clear all interrupts */ faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2 + 2, 2, 0xF000); if (variant->cascaded_irq) { diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 6511648271b2..6733cb14e775 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -40,6 +40,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/pci-ecam.h> #include <linux/delay.h> #include <linux/semaphore.h> #include <linux/irqdomain.h> @@ -64,6 +65,7 @@ enum pci_protocol_version_t { PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1), /* Win10 */ PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2), /* RS1 */ PCI_PROTOCOL_VERSION_1_3 = PCI_MAKE_VERSION(1, 3), /* Vibranium */ + PCI_PROTOCOL_VERSION_1_4 = PCI_MAKE_VERSION(1, 4), /* WS2022 */ }; #define CPU_AFFINITY_ALL -1ULL @@ -73,6 +75,7 @@ enum pci_protocol_version_t { * first. */ static enum pci_protocol_version_t pci_protocol_versions[] = { + PCI_PROTOCOL_VERSION_1_4, PCI_PROTOCOL_VERSION_1_3, PCI_PROTOCOL_VERSION_1_2, PCI_PROTOCOL_VERSION_1_1, @@ -122,6 +125,8 @@ enum pci_message_type { PCI_CREATE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x17, PCI_DELETE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x18, /* unused */ PCI_BUS_RELATIONS2 = PCI_MESSAGE_BASE + 0x19, + PCI_RESOURCES_ASSIGNED3 = PCI_MESSAGE_BASE + 0x1A, + PCI_CREATE_INTERRUPT_MESSAGE3 = PCI_MESSAGE_BASE + 0x1B, PCI_MESSAGE_MAXIMUM }; @@ -235,6 +240,21 @@ struct hv_msi_desc2 { u16 processor_array[32]; } __packed; +/* + * struct hv_msi_desc3 - 1.3 version of hv_msi_desc + * Everything is the same as in 'hv_msi_desc2' except that the size of the + * 'vector' field is larger to support bigger vector values. For ex: LPI + * vectors on ARM. + */ +struct hv_msi_desc3 { + u32 vector; + u8 delivery_mode; + u8 reserved; + u16 vector_count; + u16 processor_count; + u16 processor_array[32]; +} __packed; + /** * struct tran_int_desc * @reserved: unused, padding @@ -383,6 +403,12 @@ struct pci_create_interrupt2 { struct hv_msi_desc2 int_desc; } __packed; +struct pci_create_interrupt3 { + struct pci_message message_type; + union win_slot_encoding wslot; + struct hv_msi_desc3 int_desc; +} __packed; + struct pci_delete_interrupt { struct pci_message message_type; union win_slot_encoding wslot; @@ -444,16 +470,20 @@ enum hv_pcibus_state { hv_pcibus_probed, hv_pcibus_installed, hv_pcibus_removing, - hv_pcibus_removed, hv_pcibus_maximum }; struct hv_pcibus_device { +#ifdef CONFIG_X86 struct pci_sysdata sysdata; +#elif defined(CONFIG_ARM64) + struct pci_config_window sysdata; +#endif + struct pci_host_bridge *bridge; + struct fwnode_handle *fwnode; /* Protocol version negotiated with the host */ enum pci_protocol_version_t protocol_version; enum hv_pcibus_state state; - refcount_t remove_lock; struct hv_device *hdev; resource_size_t low_mmio_space; resource_size_t high_mmio_space; @@ -461,14 +491,11 @@ struct hv_pcibus_device { struct resource *low_mmio_res; struct resource *high_mmio_res; struct completion *survey_event; - struct completion remove_event; struct pci_bus *pci_bus; spinlock_t config_lock; /* Avoid two threads writing index page */ spinlock_t device_list_lock; /* Protect lists below */ void __iomem *cfg_addr; - struct list_head resources_for_children; - struct list_head children; struct list_head dr_list; @@ -593,9 +620,6 @@ static void put_pcichild(struct hv_pci_dev *hpdev) kfree(hpdev); } -static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus); -static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus); - /* * There is no good way to get notified from vmbus_onoffer_rescind(), * so let's use polling here, since this is not a hot path. @@ -1334,6 +1358,15 @@ static u32 hv_compose_msi_req_v1( return sizeof(*int_pkt); } +/* + * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten + * by subsequent retarget in hv_irq_unmask(). + */ +static int hv_compose_msi_req_get_cpu(struct cpumask *affinity) +{ + return cpumask_first_and(affinity, cpu_online_mask); +} + static u32 hv_compose_msi_req_v2( struct pci_create_interrupt2 *int_pkt, struct cpumask *affinity, u32 slot, u8 vector) @@ -1345,12 +1378,27 @@ static u32 hv_compose_msi_req_v2( int_pkt->int_desc.vector = vector; int_pkt->int_desc.vector_count = 1; int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED; + cpu = hv_compose_msi_req_get_cpu(affinity); + int_pkt->int_desc.processor_array[0] = + hv_cpu_number_to_vp_number(cpu); + int_pkt->int_desc.processor_count = 1; - /* - * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten - * by subsequent retarget in hv_irq_unmask(). - */ - cpu = cpumask_first_and(affinity, cpu_online_mask); + return sizeof(*int_pkt); +} + +static u32 hv_compose_msi_req_v3( + struct pci_create_interrupt3 *int_pkt, struct cpumask *affinity, + u32 slot, u32 vector) +{ + int cpu; + + int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE3; + int_pkt->wslot.slot = slot; + int_pkt->int_desc.vector = vector; + int_pkt->int_desc.reserved = 0; + int_pkt->int_desc.vector_count = 1; + int_pkt->int_desc.delivery_mode = APIC_DELIVERY_MODE_FIXED; + cpu = hv_compose_msi_req_get_cpu(affinity); int_pkt->int_desc.processor_array[0] = hv_cpu_number_to_vp_number(cpu); int_pkt->int_desc.processor_count = 1; @@ -1385,6 +1433,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) union { struct pci_create_interrupt v1; struct pci_create_interrupt2 v2; + struct pci_create_interrupt3 v3; } int_pkts; } __packed ctxt; @@ -1432,6 +1481,13 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) cfg->vector); break; + case PCI_PROTOCOL_VERSION_1_4: + size = hv_compose_msi_req_v3(&ctxt.int_pkts.v3, + dest, + hpdev->desc.win_slot.slot, + cfg->vector); + break; + default: /* As we only negotiate protocol versions known to this driver, * this path should never hit. However, this is it not a hot @@ -1572,7 +1628,7 @@ static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus) hbus->msi_info.handler = handle_edge_irq; hbus->msi_info.handler_name = "edge"; hbus->msi_info.data = hbus; - hbus->irq_domain = pci_msi_create_irq_domain(hbus->sysdata.fwnode, + hbus->irq_domain = pci_msi_create_irq_domain(hbus->fwnode, &hbus->msi_info, x86_vector_domain); if (!hbus->irq_domain) { @@ -1581,6 +1637,8 @@ static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus) return -ENODEV; } + dev_set_msi_domain(&hbus->bridge->dev, hbus->irq_domain); + return 0; } @@ -1803,7 +1861,7 @@ static void hv_pci_assign_slots(struct hv_pcibus_device *hbus) slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot)); snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser); - hpdev->pci_slot = pci_create_slot(hbus->pci_bus, slot_nr, + hpdev->pci_slot = pci_create_slot(hbus->bridge->bus, slot_nr, name, NULL); if (IS_ERR(hpdev->pci_slot)) { pr_warn("pci_create slot %s failed\n", name); @@ -1833,7 +1891,7 @@ static void hv_pci_remove_slots(struct hv_pcibus_device *hbus) static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) { struct pci_dev *dev; - struct pci_bus *bus = hbus->pci_bus; + struct pci_bus *bus = hbus->bridge->bus; struct hv_pci_dev *hv_dev; list_for_each_entry(dev, &bus->devices, bus_list) { @@ -1856,21 +1914,22 @@ static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) */ static int create_root_hv_pci_bus(struct hv_pcibus_device *hbus) { - /* Register the device */ - hbus->pci_bus = pci_create_root_bus(&hbus->hdev->device, - 0, /* bus number is always zero */ - &hv_pcifront_ops, - &hbus->sysdata, - &hbus->resources_for_children); - if (!hbus->pci_bus) - return -ENODEV; + int error; + struct pci_host_bridge *bridge = hbus->bridge; + + bridge->dev.parent = &hbus->hdev->device; + bridge->sysdata = &hbus->sysdata; + bridge->ops = &hv_pcifront_ops; + + error = pci_scan_root_bus_bridge(bridge); + if (error) + return error; pci_lock_rescan_remove(); - pci_scan_child_bus(hbus->pci_bus); hv_pci_assign_numa_node(hbus); - pci_bus_assign_resources(hbus->pci_bus); + pci_bus_assign_resources(bridge->bus); hv_pci_assign_slots(hbus); - pci_bus_add_devices(hbus->pci_bus); + pci_bus_add_devices(bridge->bus); pci_unlock_rescan_remove(); hbus->state = hv_pcibus_installed; return 0; @@ -2064,10 +2123,8 @@ static void pci_devices_present_work(struct work_struct *work) } spin_unlock_irqrestore(&hbus->device_list_lock, flags); - if (!dr) { - put_hvpcibus(hbus); + if (!dr) return; - } /* First, mark all existing children as reported missing. */ spin_lock_irqsave(&hbus->device_list_lock, flags); @@ -2135,7 +2192,7 @@ static void pci_devices_present_work(struct work_struct *work) * because there may have been changes. */ pci_lock_rescan_remove(); - pci_scan_child_bus(hbus->pci_bus); + pci_scan_child_bus(hbus->bridge->bus); hv_pci_assign_numa_node(hbus); hv_pci_assign_slots(hbus); pci_unlock_rescan_remove(); @@ -2150,7 +2207,6 @@ static void pci_devices_present_work(struct work_struct *work) break; } - put_hvpcibus(hbus); kfree(dr); } @@ -2191,12 +2247,10 @@ static int hv_pci_start_relations_work(struct hv_pcibus_device *hbus, list_add_tail(&dr->list_entry, &hbus->dr_list); spin_unlock_irqrestore(&hbus->device_list_lock, flags); - if (pending_dr) { + if (pending_dr) kfree(dr_wrk); - } else { - get_hvpcibus(hbus); + else queue_work(hbus->wq, &dr_wrk->wrk); - } return 0; } @@ -2306,11 +2360,11 @@ static void hv_eject_device_work(struct work_struct *work) /* * Ejection can come before or after the PCI bus has been set up, so * attempt to find it and tear down the bus state, if it exists. This - * must be done without constructs like pci_domain_nr(hbus->pci_bus) - * because hbus->pci_bus may not exist yet. + * must be done without constructs like pci_domain_nr(hbus->bridge->bus) + * because hbus->bridge->bus may not exist yet. */ wslot = wslot_to_devfn(hpdev->desc.win_slot.slot); - pdev = pci_get_domain_bus_and_slot(hbus->sysdata.domain, 0, wslot); + pdev = pci_get_domain_bus_and_slot(hbus->bridge->domain_nr, 0, wslot); if (pdev) { pci_lock_rescan_remove(); pci_stop_and_remove_bus_device(pdev); @@ -2339,8 +2393,6 @@ static void hv_eject_device_work(struct work_struct *work) put_pcichild(hpdev); put_pcichild(hpdev); /* hpdev has been freed. Do not use it any more. */ - - put_hvpcibus(hbus); } /** @@ -2364,7 +2416,6 @@ static void hv_pci_eject_device(struct hv_pci_dev *hpdev) hpdev->state = hv_pcichild_ejecting; get_pcichild(hpdev); INIT_WORK(&hpdev->wrk, hv_eject_device_work); - get_hvpcibus(hbus); queue_work(hbus->wq, &hpdev->wrk); } @@ -2676,8 +2727,7 @@ static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus) /* Modify this resource to become a bridge window. */ hbus->low_mmio_res->flags |= IORESOURCE_WINDOW; hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY; - pci_add_resource(&hbus->resources_for_children, - hbus->low_mmio_res); + pci_add_resource(&hbus->bridge->windows, hbus->low_mmio_res); } if (hbus->high_mmio_space) { @@ -2696,8 +2746,7 @@ static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus) /* Modify this resource to become a bridge window. */ hbus->high_mmio_res->flags |= IORESOURCE_WINDOW; hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY; - pci_add_resource(&hbus->resources_for_children, - hbus->high_mmio_res); + pci_add_resource(&hbus->bridge->windows, hbus->high_mmio_res); } return 0; @@ -2964,17 +3013,6 @@ static int hv_send_resources_released(struct hv_device *hdev) return 0; } -static void get_hvpcibus(struct hv_pcibus_device *hbus) -{ - refcount_inc(&hbus->remove_lock); -} - -static void put_hvpcibus(struct hv_pcibus_device *hbus) -{ - if (refcount_dec_and_test(&hbus->remove_lock)) - complete(&hbus->remove_event); -} - #define HVPCI_DOM_MAP_SIZE (64 * 1024) static DECLARE_BITMAP(hvpci_dom_map, HVPCI_DOM_MAP_SIZE); @@ -3027,6 +3065,7 @@ static void hv_put_dom_num(u16 dom) static int hv_pci_probe(struct hv_device *hdev, const struct hv_vmbus_device_id *dev_id) { + struct pci_host_bridge *bridge; struct hv_pcibus_device *hbus; u16 dom_req, dom; char *name; @@ -3039,6 +3078,10 @@ static int hv_pci_probe(struct hv_device *hdev, */ BUILD_BUG_ON(sizeof(*hbus) > HV_HYP_PAGE_SIZE); + bridge = devm_pci_alloc_host_bridge(&hdev->device, 0); + if (!bridge) + return -ENOMEM; + /* * With the recent 59bb47985c1d ("mm, sl[aou]b: guarantee natural * alignment for kmalloc(power-of-two)"), kzalloc() is able to allocate @@ -3060,6 +3103,8 @@ static int hv_pci_probe(struct hv_device *hdev, hbus = kzalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL); if (!hbus) return -ENOMEM; + + hbus->bridge = bridge; hbus->state = hv_pcibus_init; hbus->wslot_res_allocated = -1; @@ -3081,29 +3126,29 @@ static int hv_pci_probe(struct hv_device *hdev, if (dom == HVPCI_DOM_INVALID) { dev_err(&hdev->device, - "Unable to use dom# 0x%hx or other numbers", dom_req); + "Unable to use dom# 0x%x or other numbers", dom_req); ret = -EINVAL; goto free_bus; } if (dom != dom_req) dev_info(&hdev->device, - "PCI dom# 0x%hx has collision, using 0x%hx", + "PCI dom# 0x%x has collision, using 0x%x", dom_req, dom); + hbus->bridge->domain_nr = dom; +#ifdef CONFIG_X86 hbus->sysdata.domain = dom; +#endif hbus->hdev = hdev; - refcount_set(&hbus->remove_lock, 1); INIT_LIST_HEAD(&hbus->children); INIT_LIST_HEAD(&hbus->dr_list); - INIT_LIST_HEAD(&hbus->resources_for_children); spin_lock_init(&hbus->config_lock); spin_lock_init(&hbus->device_list_lock); spin_lock_init(&hbus->retarget_msi_interrupt_lock); - init_completion(&hbus->remove_event); hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0, - hbus->sysdata.domain); + hbus->bridge->domain_nr); if (!hbus->wq) { ret = -ENOMEM; goto free_dom; @@ -3140,9 +3185,9 @@ static int hv_pci_probe(struct hv_device *hdev, goto unmap; } - hbus->sysdata.fwnode = irq_domain_alloc_named_fwnode(name); + hbus->fwnode = irq_domain_alloc_named_fwnode(name); kfree(name); - if (!hbus->sysdata.fwnode) { + if (!hbus->fwnode) { ret = -ENOMEM; goto unmap; } @@ -3220,7 +3265,7 @@ exit_d0: free_irq_domain: irq_domain_remove(hbus->irq_domain); free_fwnode: - irq_domain_free_fwnode(hbus->sysdata.fwnode); + irq_domain_free_fwnode(hbus->fwnode); unmap: iounmap(hbus->cfg_addr); free_config: @@ -3230,7 +3275,7 @@ close: destroy_wq: destroy_workqueue(hbus->wq); free_dom: - hv_put_dom_num(hbus->sysdata.domain); + hv_put_dom_num(hbus->bridge->domain_nr); free_bus: kfree(hbus); return ret; @@ -3243,8 +3288,9 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) struct pci_packet teardown_packet; u8 buffer[sizeof(struct pci_message)]; } pkt; - struct hv_dr_state *dr; struct hv_pci_compl comp_pkt; + struct hv_pci_dev *hpdev, *tmp; + unsigned long flags; int ret; /* @@ -3255,10 +3301,24 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) return 0; if (!keep_devs) { - /* Delete any children which might still exist. */ - dr = kzalloc(sizeof(*dr), GFP_KERNEL); - if (dr && hv_pci_start_relations_work(hbus, dr)) - kfree(dr); + struct list_head removed; + + /* Move all present children to the list on stack */ + INIT_LIST_HEAD(&removed); + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) + list_move_tail(&hpdev->list_entry, &removed); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + /* Remove all children in the list */ + list_for_each_entry_safe(hpdev, tmp, &removed, list_entry) { + list_del(&hpdev->list_entry); + if (hpdev->pci_slot) + pci_destroy_slot(hpdev->pci_slot); + /* For the two refs got in new_pcichild_device() */ + put_pcichild(hpdev); + put_pcichild(hpdev); + } } ret = hv_send_resources_released(hdev); @@ -3301,13 +3361,23 @@ static int hv_pci_remove(struct hv_device *hdev) hbus = hv_get_drvdata(hdev); if (hbus->state == hv_pcibus_installed) { + tasklet_disable(&hdev->channel->callback_event); + hbus->state = hv_pcibus_removing; + tasklet_enable(&hdev->channel->callback_event); + destroy_workqueue(hbus->wq); + hbus->wq = NULL; + /* + * At this point, no work is running or can be scheduled + * on hbus-wq. We can't race with hv_pci_devices_present() + * or hv_pci_eject_device(), it's safe to proceed. + */ + /* Remove the bus from PCI's point of view. */ pci_lock_rescan_remove(); - pci_stop_root_bus(hbus->pci_bus); + pci_stop_root_bus(hbus->bridge->bus); hv_pci_remove_slots(hbus); - pci_remove_root_bus(hbus->pci_bus); + pci_remove_root_bus(hbus->bridge->bus); pci_unlock_rescan_remove(); - hbus->state = hv_pcibus_removed; } ret = hv_pci_bus_exit(hdev, false); @@ -3316,15 +3386,11 @@ static int hv_pci_remove(struct hv_device *hdev) iounmap(hbus->cfg_addr); hv_free_config_window(hbus); - pci_free_resource_list(&hbus->resources_for_children); hv_pci_free_bridge_windows(hbus); irq_domain_remove(hbus->irq_domain); - irq_domain_free_fwnode(hbus->sysdata.fwnode); - put_hvpcibus(hbus); - wait_for_completion(&hbus->remove_event); - destroy_workqueue(hbus->wq); + irq_domain_free_fwnode(hbus->fwnode); - hv_put_dom_num(hbus->sysdata.domain); + hv_put_dom_num(hbus->bridge->domain_nr); kfree(hbus); return ret; @@ -3402,7 +3468,7 @@ static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg) */ static void hv_pci_restore_msi_state(struct hv_pcibus_device *hbus) { - pci_walk_bus(hbus->pci_bus, hv_pci_restore_msi_msg, NULL); + pci_walk_bus(hbus->bridge->bus, hv_pci_restore_msi_msg, NULL); } static int hv_pci_resume(struct hv_device *hdev) @@ -3476,6 +3542,9 @@ static void __exit exit_hv_pci_drv(void) static int __init init_hv_pci_drv(void) { + if (!hv_is_hyperv_initialized()) + return -ENODEV; + /* Set the invalid domain number's bit, so it will not be used */ set_bit(HVPCI_DOM_INVALID, hvpci_dom_map); diff --git a/drivers/pci/controller/pci-ixp4xx.c b/drivers/pci/controller/pci-ixp4xx.c new file mode 100644 index 000000000000..654ac4a82beb --- /dev/null +++ b/drivers/pci/controller/pci-ixp4xx.c @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Intel IXP4xx PCI host controller + * + * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org> + * + * Based on the IXP4xx arch/arm/mach-ixp4xx/common-pci.c driver + * Copyright (C) 2002 Intel Corporation + * Copyright (C) 2003 Greg Ungerer <gerg@linux-m68k.org> + * Copyright (C) 2003-2004 MontaVista Software, Inc. + * Copyright (C) 2005 Deepak Saxena <dsaxena@plexity.net> + * Copyright (C) 2005 Alessandro Zummo <a.zummo@towertech.it> + * + * TODO: + * - Test IO-space access + * - DMA support + */ + +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/bits.h> + +/* Register offsets */ +#define IXP4XX_PCI_NP_AD 0x00 +#define IXP4XX_PCI_NP_CBE 0x04 +#define IXP4XX_PCI_NP_WDATA 0x08 +#define IXP4XX_PCI_NP_RDATA 0x0c +#define IXP4XX_PCI_CRP_AD_CBE 0x10 +#define IXP4XX_PCI_CRP_WDATA 0x14 +#define IXP4XX_PCI_CRP_RDATA 0x18 +#define IXP4XX_PCI_CSR 0x1c +#define IXP4XX_PCI_ISR 0x20 +#define IXP4XX_PCI_INTEN 0x24 +#define IXP4XX_PCI_DMACTRL 0x28 +#define IXP4XX_PCI_AHBMEMBASE 0x2c +#define IXP4XX_PCI_AHBIOBASE 0x30 +#define IXP4XX_PCI_PCIMEMBASE 0x34 +#define IXP4XX_PCI_AHBDOORBELL 0x38 +#define IXP4XX_PCI_PCIDOORBELL 0x3c +#define IXP4XX_PCI_ATPDMA0_AHBADDR 0x40 +#define IXP4XX_PCI_ATPDMA0_PCIADDR 0x44 +#define IXP4XX_PCI_ATPDMA0_LENADDR 0x48 +#define IXP4XX_PCI_ATPDMA1_AHBADDR 0x4c +#define IXP4XX_PCI_ATPDMA1_PCIADDR 0x50 +#define IXP4XX_PCI_ATPDMA1_LENADDR 0x54 + +/* CSR bit definitions */ +#define IXP4XX_PCI_CSR_HOST BIT(0) +#define IXP4XX_PCI_CSR_ARBEN BIT(1) +#define IXP4XX_PCI_CSR_ADS BIT(2) +#define IXP4XX_PCI_CSR_PDS BIT(3) +#define IXP4XX_PCI_CSR_ABE BIT(4) +#define IXP4XX_PCI_CSR_DBT BIT(5) +#define IXP4XX_PCI_CSR_ASE BIT(8) +#define IXP4XX_PCI_CSR_IC BIT(15) +#define IXP4XX_PCI_CSR_PRST BIT(16) + +/* ISR (Interrupt status) Register bit definitions */ +#define IXP4XX_PCI_ISR_PSE BIT(0) +#define IXP4XX_PCI_ISR_PFE BIT(1) +#define IXP4XX_PCI_ISR_PPE BIT(2) +#define IXP4XX_PCI_ISR_AHBE BIT(3) +#define IXP4XX_PCI_ISR_APDC BIT(4) +#define IXP4XX_PCI_ISR_PADC BIT(5) +#define IXP4XX_PCI_ISR_ADB BIT(6) +#define IXP4XX_PCI_ISR_PDB BIT(7) + +/* INTEN (Interrupt Enable) Register bit definitions */ +#define IXP4XX_PCI_INTEN_PSE BIT(0) +#define IXP4XX_PCI_INTEN_PFE BIT(1) +#define IXP4XX_PCI_INTEN_PPE BIT(2) +#define IXP4XX_PCI_INTEN_AHBE BIT(3) +#define IXP4XX_PCI_INTEN_APDC BIT(4) +#define IXP4XX_PCI_INTEN_PADC BIT(5) +#define IXP4XX_PCI_INTEN_ADB BIT(6) +#define IXP4XX_PCI_INTEN_PDB BIT(7) + +/* Shift value for byte enable on NP cmd/byte enable register */ +#define IXP4XX_PCI_NP_CBE_BESL 4 + +/* PCI commands supported by NP access unit */ +#define NP_CMD_IOREAD 0x2 +#define NP_CMD_IOWRITE 0x3 +#define NP_CMD_CONFIGREAD 0xa +#define NP_CMD_CONFIGWRITE 0xb +#define NP_CMD_MEMREAD 0x6 +#define NP_CMD_MEMWRITE 0x7 + +/* Constants for CRP access into local config space */ +#define CRP_AD_CBE_BESL 20 +#define CRP_AD_CBE_WRITE 0x00010000 + +/* Special PCI configuration space registers for this controller */ +#define IXP4XX_PCI_RTOTTO 0x40 + +struct ixp4xx_pci { + struct device *dev; + void __iomem *base; + bool errata_hammer; + bool host_mode; +}; + +/* + * The IXP4xx has a peculiar address bus that will change the + * byte order on SoC peripherals depending on whether the device + * operates in big-endian or little-endian mode. That means that + * readl() and writel() that always use little-endian access + * will not work for SoC peripherals such as the PCI controller + * when used in big-endian mode. The accesses to the individual + * PCI devices on the other hand, are always little-endian and + * can use readl() and writel(). + * + * For local AHB bus access we need to use __raw_[readl|writel]() + * to make sure that we access the SoC devices in the CPU native + * endianness. + */ +static inline u32 ixp4xx_readl(struct ixp4xx_pci *p, u32 reg) +{ + return __raw_readl(p->base + reg); +} + +static inline void ixp4xx_writel(struct ixp4xx_pci *p, u32 reg, u32 val) +{ + __raw_writel(val, p->base + reg); +} + +static int ixp4xx_pci_check_master_abort(struct ixp4xx_pci *p) +{ + u32 isr = ixp4xx_readl(p, IXP4XX_PCI_ISR); + + if (isr & IXP4XX_PCI_ISR_PFE) { + /* Make sure the master abort bit is reset */ + ixp4xx_writel(p, IXP4XX_PCI_ISR, IXP4XX_PCI_ISR_PFE); + dev_dbg(p->dev, "master abort detected\n"); + return -EINVAL; + } + + return 0; +} + +static int ixp4xx_pci_read_indirect(struct ixp4xx_pci *p, u32 addr, u32 cmd, u32 *data) +{ + ixp4xx_writel(p, IXP4XX_PCI_NP_AD, addr); + + if (p->errata_hammer) { + int i; + + /* + * PCI workaround - only works if NP PCI space reads have + * no side effects. Hammer the register and read twice 8 + * times. last one will be good. + */ + for (i = 0; i < 8; i++) { + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + } + } else { + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + } + + return ixp4xx_pci_check_master_abort(p); +} + +static int ixp4xx_pci_write_indirect(struct ixp4xx_pci *p, u32 addr, u32 cmd, u32 data) +{ + ixp4xx_writel(p, IXP4XX_PCI_NP_AD, addr); + + /* Set up the write */ + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + + /* Execute the write by writing to NP_WDATA */ + ixp4xx_writel(p, IXP4XX_PCI_NP_WDATA, data); + + return ixp4xx_pci_check_master_abort(p); +} + +static u32 ixp4xx_config_addr(u8 bus_num, u16 devfn, int where) +{ + /* Root bus is always 0 in this hardware */ + if (bus_num == 0) { + /* type 0 */ + return BIT(32-PCI_SLOT(devfn)) | ((PCI_FUNC(devfn)) << 8) | + (where & ~3); + } else { + /* type 1 */ + return (bus_num << 16) | ((PCI_SLOT(devfn)) << 11) | + ((PCI_FUNC(devfn)) << 8) | (where & ~3) | 1; + } +} + +/* + * CRP functions are "Controller Configuration Port" accesses + * initiated from within this driver itself to read/write PCI + * control information in the config space. + */ +static u32 ixp4xx_crp_byte_lane_enable_bits(u32 n, int size) +{ + if (size == 1) + return (0xf & ~BIT(n)) << CRP_AD_CBE_BESL; + if (size == 2) + return (0xf & ~(BIT(n) | BIT(n+1))) << CRP_AD_CBE_BESL; + if (size == 4) + return 0; + return 0xffffffff; +} + +static int ixp4xx_crp_read_config(struct ixp4xx_pci *p, int where, int size, + u32 *value) +{ + u32 n, cmd, val; + + n = where % 4; + cmd = where & ~3; + + dev_dbg(p->dev, "%s from %d size %d cmd %08x\n", + __func__, where, size, cmd); + + ixp4xx_writel(p, IXP4XX_PCI_CRP_AD_CBE, cmd); + val = ixp4xx_readl(p, IXP4XX_PCI_CRP_RDATA); + + val >>= (8*n); + switch (size) { + case 1: + val &= U8_MAX; + dev_dbg(p->dev, "%s read byte %02x\n", __func__, val); + break; + case 2: + val &= U16_MAX; + dev_dbg(p->dev, "%s read word %04x\n", __func__, val); + break; + case 4: + val &= U32_MAX; + dev_dbg(p->dev, "%s read long %08x\n", __func__, val); + break; + default: + /* Should not happen */ + dev_err(p->dev, "%s illegal size\n", __func__); + return PCIBIOS_DEVICE_NOT_FOUND; + } + *value = val; + + return PCIBIOS_SUCCESSFUL; +} + +static int ixp4xx_crp_write_config(struct ixp4xx_pci *p, int where, int size, + u32 value) +{ + u32 n, cmd, val; + + n = where % 4; + cmd = ixp4xx_crp_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + cmd |= where & ~3; + cmd |= CRP_AD_CBE_WRITE; + + val = value << (8*n); + + dev_dbg(p->dev, "%s to %d size %d cmd %08x val %08x\n", + __func__, where, size, cmd, val); + + ixp4xx_writel(p, IXP4XX_PCI_CRP_AD_CBE, cmd); + ixp4xx_writel(p, IXP4XX_PCI_CRP_WDATA, val); + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Then follows the functions that read and write from the common PCI + * configuration space. + */ +static u32 ixp4xx_byte_lane_enable_bits(u32 n, int size) +{ + if (size == 1) + return (0xf & ~BIT(n)) << 4; + if (size == 2) + return (0xf & ~(BIT(n) | BIT(n+1))) << 4; + if (size == 4) + return 0; + return 0xffffffff; +} + +static int ixp4xx_pci_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + struct ixp4xx_pci *p = bus->sysdata; + u32 n, addr, val, cmd; + u8 bus_num = bus->number; + int ret; + + *value = 0xffffffff; + n = where % 4; + cmd = ixp4xx_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + + addr = ixp4xx_config_addr(bus_num, devfn, where); + cmd |= NP_CMD_CONFIGREAD; + dev_dbg(p->dev, "read_config from %d size %d dev %d:%d:%d address: %08x cmd: %08x\n", + where, size, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn), addr, cmd); + + ret = ixp4xx_pci_read_indirect(p, addr, cmd, &val); + if (ret) + return PCIBIOS_DEVICE_NOT_FOUND; + + val >>= (8*n); + switch (size) { + case 1: + val &= U8_MAX; + dev_dbg(p->dev, "%s read byte %02x\n", __func__, val); + break; + case 2: + val &= U16_MAX; + dev_dbg(p->dev, "%s read word %04x\n", __func__, val); + break; + case 4: + val &= U32_MAX; + dev_dbg(p->dev, "%s read long %08x\n", __func__, val); + break; + default: + /* Should not happen */ + dev_err(p->dev, "%s illegal size\n", __func__); + return PCIBIOS_DEVICE_NOT_FOUND; + } + *value = val; + + return PCIBIOS_SUCCESSFUL; +} + +static int ixp4xx_pci_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + struct ixp4xx_pci *p = bus->sysdata; + u32 n, addr, val, cmd; + u8 bus_num = bus->number; + int ret; + + n = where % 4; + cmd = ixp4xx_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + + addr = ixp4xx_config_addr(bus_num, devfn, where); + cmd |= NP_CMD_CONFIGWRITE; + val = value << (8*n); + + dev_dbg(p->dev, "write_config_byte %#x to %d size %d dev %d:%d:%d addr: %08x cmd %08x\n", + value, where, size, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn), addr, cmd); + + ret = ixp4xx_pci_write_indirect(p, addr, cmd, val); + if (ret) + return PCIBIOS_DEVICE_NOT_FOUND; + + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops ixp4xx_pci_ops = { + .read = ixp4xx_pci_read_config, + .write = ixp4xx_pci_write_config, +}; + +static u32 ixp4xx_pci_addr_to_64mconf(phys_addr_t addr) +{ + u8 base; + + base = ((addr & 0xff000000) >> 24); + return (base << 24) | ((base + 1) << 16) + | ((base + 2) << 8) | (base + 3); +} + +static int ixp4xx_pci_parse_map_ranges(struct ixp4xx_pci *p) +{ + struct device *dev = p->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(p); + struct resource_entry *win; + struct resource *res; + phys_addr_t addr; + + win = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); + if (win) { + u32 pcimembase; + + res = win->res; + addr = res->start - win->offset; + + if (res->flags & IORESOURCE_PREFETCH) + res->name = "IXP4xx PCI PRE-MEM"; + else + res->name = "IXP4xx PCI NON-PRE-MEM"; + + dev_dbg(dev, "%s window %pR, bus addr %pa\n", + res->name, res, &addr); + if (resource_size(res) != SZ_64M) { + dev_err(dev, "memory range is not 64MB\n"); + return -EINVAL; + } + + pcimembase = ixp4xx_pci_addr_to_64mconf(addr); + /* Commit configuration */ + ixp4xx_writel(p, IXP4XX_PCI_PCIMEMBASE, pcimembase); + } else { + dev_err(dev, "no AHB memory mapping defined\n"); + } + + win = resource_list_first_type(&bridge->windows, IORESOURCE_IO); + if (win) { + res = win->res; + + addr = pci_pio_to_address(res->start); + if (addr & 0xff) { + dev_err(dev, "IO mem at uneven address: %pa\n", &addr); + return -EINVAL; + } + + res->name = "IXP4xx PCI IO MEM"; + /* + * Setup I/O space location for PCI->AHB access, the + * upper 24 bits of the address goes into the lower + * 24 bits of this register. + */ + ixp4xx_writel(p, IXP4XX_PCI_AHBIOBASE, (addr >> 8)); + } else { + dev_info(dev, "no IO space AHB memory mapping defined\n"); + } + + return 0; +} + +static int ixp4xx_pci_parse_map_dma_ranges(struct ixp4xx_pci *p) +{ + struct device *dev = p->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(p); + struct resource_entry *win; + struct resource *res; + phys_addr_t addr; + u32 ahbmembase; + + win = resource_list_first_type(&bridge->dma_ranges, IORESOURCE_MEM); + if (win) { + res = win->res; + addr = res->start - win->offset; + + if (resource_size(res) != SZ_64M) { + dev_err(dev, "DMA memory range is not 64MB\n"); + return -EINVAL; + } + + dev_dbg(dev, "DMA MEM BASE: %pa\n", &addr); + /* + * 4 PCI-to-AHB windows of 16 MB each, write the 8 high bits + * into each byte of the PCI_AHBMEMBASE register. + */ + ahbmembase = ixp4xx_pci_addr_to_64mconf(addr); + /* Commit AHB membase */ + ixp4xx_writel(p, IXP4XX_PCI_AHBMEMBASE, ahbmembase); + } else { + dev_err(dev, "no DMA memory range defined\n"); + } + + return 0; +} + +/* Only used to get context for abort handling */ +static struct ixp4xx_pci *ixp4xx_pci_abort_singleton; + +static int ixp4xx_pci_abort_handler(unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + struct ixp4xx_pci *p = ixp4xx_pci_abort_singleton; + u32 isr, status; + int ret; + + isr = ixp4xx_readl(p, IXP4XX_PCI_ISR); + ret = ixp4xx_crp_read_config(p, PCI_STATUS, 2, &status); + if (ret) { + dev_err(p->dev, "unable to read abort status\n"); + return -EINVAL; + } + + dev_err(p->dev, + "PCI: abort_handler addr = %#lx, isr = %#x, status = %#x\n", + addr, isr, status); + + /* Make sure the Master Abort bit is reset */ + ixp4xx_writel(p, IXP4XX_PCI_ISR, IXP4XX_PCI_ISR_PFE); + status |= PCI_STATUS_REC_MASTER_ABORT; + ret = ixp4xx_crp_write_config(p, PCI_STATUS, 2, status); + if (ret) + dev_err(p->dev, "unable to clear abort status bit\n"); + + /* + * If it was an imprecise abort, then we need to correct the + * return address to be _after_ the instruction. + */ + if (fsr & (1 << 10)) { + dev_err(p->dev, "imprecise abort\n"); + regs->ARM_pc += 4; + } + + return 0; +} + +static int __init ixp4xx_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct ixp4xx_pci *p; + struct pci_host_bridge *host; + int ret; + u32 val; + phys_addr_t addr; + u32 basereg[4] = { + PCI_BASE_ADDRESS_0, + PCI_BASE_ADDRESS_1, + PCI_BASE_ADDRESS_2, + PCI_BASE_ADDRESS_3, + }; + int i; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*p)); + if (!host) + return -ENOMEM; + + host->ops = &ixp4xx_pci_ops; + p = pci_host_bridge_priv(host); + host->sysdata = p; + p->dev = dev; + dev_set_drvdata(dev, p); + + /* + * Set up quirk for erratic behaviour in the 42x variant + * when accessing config space. + */ + if (of_device_is_compatible(np, "intel,ixp42x-pci")) { + p->errata_hammer = true; + dev_info(dev, "activate hammering errata\n"); + } + + p->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(p->base)) + return PTR_ERR(p->base); + + val = ixp4xx_readl(p, IXP4XX_PCI_CSR); + p->host_mode = !!(val & IXP4XX_PCI_CSR_HOST); + dev_info(dev, "controller is in %s mode\n", + p->host_mode ? "host" : "option"); + + /* Hook in our fault handler for PCI errors */ + ixp4xx_pci_abort_singleton = p; + hook_fault_code(16+6, ixp4xx_pci_abort_handler, SIGBUS, 0, + "imprecise external abort"); + + ret = ixp4xx_pci_parse_map_ranges(p); + if (ret) + return ret; + + ret = ixp4xx_pci_parse_map_dma_ranges(p); + if (ret) + return ret; + + /* This is only configured in host mode */ + if (p->host_mode) { + addr = __pa(PAGE_OFFSET); + /* This is a noop (0x00) but explains what is going on */ + addr |= PCI_BASE_ADDRESS_SPACE_MEMORY; + + for (i = 0; i < 4; i++) { + /* Write this directly into the config space */ + ret = ixp4xx_crp_write_config(p, basereg[i], 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_%d\n", i); + else + dev_info(dev, "set PCI_BASE_ADDR_%d to %pa\n", i, &addr); + addr += SZ_16M; + } + + /* + * Enable CSR window at 64 MiB to allow PCI masters to continue + * prefetching past the 64 MiB boundary, if all AHB to PCI + * windows are consecutive. + */ + ret = ixp4xx_crp_write_config(p, PCI_BASE_ADDRESS_4, 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_4\n"); + else + dev_info(dev, "set PCI_BASE_ADDR_4 to %pa\n", &addr); + + /* + * Put the IO memory window at the very end of physical memory + * at 0xfffffc00. This is when the system is trying to access IO + * memory over AHB. + */ + addr = 0xfffffc00; + addr |= PCI_BASE_ADDRESS_SPACE_IO; + ret = ixp4xx_crp_write_config(p, PCI_BASE_ADDRESS_5, 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_5\n"); + else + dev_info(dev, "set PCI_BASE_ADDR_5 to %pa\n", &addr); + + /* + * Retry timeout to 0x80 + * Transfer ready timeout to 0xff + */ + ret = ixp4xx_crp_write_config(p, IXP4XX_PCI_RTOTTO, 4, + 0x000080ff); + if (ret) + dev_err(dev, "failed to set up TRDY limit\n"); + else + dev_info(dev, "set TRDY limit to 0x80ff\n"); + } + + /* Clear interrupts */ + val = IXP4XX_PCI_ISR_PSE | IXP4XX_PCI_ISR_PFE | IXP4XX_PCI_ISR_PPE | IXP4XX_PCI_ISR_AHBE; + ixp4xx_writel(p, IXP4XX_PCI_ISR, val); + + /* + * Set Initialize Complete in PCI Control Register: allow IXP4XX to + * generate PCI configuration cycles. Specify that the AHB bus is + * operating in big-endian mode. Set up byte lane swapping between + * little-endian PCI and the big-endian AHB bus. + */ + val = IXP4XX_PCI_CSR_IC | IXP4XX_PCI_CSR_ABE; + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + val |= (IXP4XX_PCI_CSR_PDS | IXP4XX_PCI_CSR_ADS); + ixp4xx_writel(p, IXP4XX_PCI_CSR, val); + + ret = ixp4xx_crp_write_config(p, PCI_COMMAND, 2, PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + if (ret) + dev_err(dev, "unable to initialize master and command memory\n"); + else + dev_info(dev, "initialized as master\n"); + + pci_host_probe(host); + + return 0; +} + +static const struct of_device_id ixp4xx_pci_of_match[] = { + { + .compatible = "intel,ixp42x-pci", + }, + { + .compatible = "intel,ixp43x-pci", + }, + {}, +}; + +/* + * This driver needs to be a builtin module with suppressed bind + * attributes since the probe() is initializing a hard exception + * handler and this can only be done from __init-tagged code + * sections. This module cannot be removed and inserted at all. + */ +static struct platform_driver ixp4xx_pci_driver = { + .driver = { + .name = "ixp4xx-pci", + .suppress_bind_attrs = true, + .of_match_table = ixp4xx_pci_of_match, + }, +}; +builtin_platform_driver_probe(ixp4xx_pci_driver, ixp4xx_pci_probe); diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index 8069bd9232d4..cb0aa65d6934 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -372,11 +372,6 @@ struct tegra_pcie_port { struct gpio_desc *reset_gpio; }; -struct tegra_pcie_bus { - struct list_head list; - unsigned int nr; -}; - static inline void afi_writel(struct tegra_pcie *pcie, u32 value, unsigned long offset) { @@ -764,7 +759,7 @@ static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) static irqreturn_t tegra_pcie_isr(int irq, void *arg) { - const char *err_msg[] = { + static const char * const err_msg[] = { "Unknown", "AXI slave error", "AXI decode error", @@ -1553,12 +1548,10 @@ static void tegra_pcie_msi_irq(struct irq_desc *desc) while (reg) { unsigned int offset = find_first_bit(®, 32); unsigned int index = i * 32 + offset; - unsigned int irq; + int ret; - irq = irq_find_mapping(msi->domain->parent, index); - if (irq) { - generic_handle_irq(irq); - } else { + ret = generic_handle_domain_irq(msi->domain->parent, index); + if (ret) { /* * that's weird who triggered this? * just clear it @@ -2193,13 +2186,15 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) rp->np = port; rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs); - if (IS_ERR(rp->base)) - return PTR_ERR(rp->base); + if (IS_ERR(rp->base)) { + err = PTR_ERR(rp->base); + goto err_node_put; + } label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index); if (!label) { - dev_err(dev, "failed to create reset GPIO label\n"); - return -ENOMEM; + err = -ENOMEM; + goto err_node_put; } /* @@ -2217,7 +2212,8 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) } else { dev_err(dev, "failed to get reset GPIO: %ld\n", PTR_ERR(rp->reset_gpio)); - return PTR_ERR(rp->reset_gpio); + err = PTR_ERR(rp->reset_gpio); + goto err_node_put; } } @@ -2539,6 +2535,7 @@ static const struct of_device_id tegra_pcie_of_match[] = { { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie }, { }, }; +MODULE_DEVICE_TABLE(of, tegra_pcie_of_match); static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos) { @@ -2547,7 +2544,7 @@ static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos) if (list_empty(&pcie->ports)) return NULL; - seq_printf(s, "Index Status\n"); + seq_puts(s, "Index Status\n"); return seq_list_start(&pcie->ports, *pos); } @@ -2584,16 +2581,16 @@ static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v) seq_printf(s, "%2u ", port->index); if (up) - seq_printf(s, "up"); + seq_puts(s, "up"); if (active) { if (up) - seq_printf(s, ", "); + seq_puts(s, ", "); - seq_printf(s, "active"); + seq_puts(s, "active"); } - seq_printf(s, "\n"); + seq_puts(s, "\n"); return 0; } diff --git a/drivers/pci/controller/pci-thunder-ecam.c b/drivers/pci/controller/pci-thunder-ecam.c index ffd84656544f..e9d5ca245f5e 100644 --- a/drivers/pci/controller/pci-thunder-ecam.c +++ b/drivers/pci/controller/pci-thunder-ecam.c @@ -17,7 +17,7 @@ static void set_val(u32 v, int where, int size, u32 *val) { int shift = (where & 3) * 8; - pr_debug("set_val %04x: %08x\n", (unsigned)(where & ~3), v); + pr_debug("set_val %04x: %08x\n", (unsigned int)(where & ~3), v); v >>= shift; if (size == 1) v &= 0xff; @@ -187,7 +187,7 @@ static int thunder_ecam_config_read(struct pci_bus *bus, unsigned int devfn, pr_debug("%04x:%04x - Fix pass#: %08x, where: %03x, devfn: %03x\n", vendor_device & 0xffff, vendor_device >> 16, class_rev, - (unsigned) where, devfn); + (unsigned int)where, devfn); /* Check for non type-00 header */ if (cfg_type == 0) { diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c index 1c34c897a7e2..c50ff279903c 100644 --- a/drivers/pci/controller/pci-xgene-msi.c +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -291,8 +291,7 @@ static void xgene_msi_isr(struct irq_desc *desc) struct irq_chip *chip = irq_desc_get_chip(desc); struct xgene_msi_group *msi_groups; struct xgene_msi *xgene_msi; - unsigned int virq; - int msir_index, msir_val, hw_irq; + int msir_index, msir_val, hw_irq, ret; u32 intr_index, grp_select, msi_grp; chained_irq_enter(chip, desc); @@ -303,7 +302,7 @@ static void xgene_msi_isr(struct irq_desc *desc) /* * MSIINTn (n is 0..F) indicates if there is a pending MSI interrupt - * If bit x of this register is set (x is 0..7), one or more interupts + * If bit x of this register is set (x is 0..7), one or more interrupts * corresponding to MSInIRx is set. */ grp_select = xgene_msi_int_read(xgene_msi, msi_grp); @@ -330,10 +329,8 @@ static void xgene_msi_isr(struct irq_desc *desc) * CPU0 */ hw_irq = hwirq_to_canonical_hwirq(hw_irq); - virq = irq_find_mapping(xgene_msi->inner_domain, hw_irq); - WARN_ON(!virq); - if (virq != 0) - generic_handle_irq(virq); + ret = generic_handle_domain_irq(xgene_msi->inner_domain, hw_irq); + WARN_ON_ONCE(ret); msir_val &= ~(1 << intr_index); } grp_select &= ~(1 << msir_index); @@ -451,7 +448,6 @@ static int xgene_msi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); xgene_msi->msi_regs = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(xgene_msi->msi_regs)) { - dev_err(&pdev->dev, "no reg space\n"); rc = PTR_ERR(xgene_msi->msi_regs); goto error; } diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index 7f503dd4ff81..56d0d50338c8 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0+ -/** +/* * APM X-Gene PCIe Driver * * Copyright (c) 2014 Applied Micro Circuits Corporation. @@ -48,7 +48,6 @@ #define EN_COHERENCY 0xF0000000 #define EN_REG 0x00000001 #define OB_LO_IO 0x00000002 -#define XGENE_PCIE_VENDORID 0x10E8 #define XGENE_PCIE_DEVICEID 0xE004 #define SZ_1T (SZ_1G*1024ULL) #define PIPE_PHY_RATE_RD(src) ((0xc000 & (u32)(src)) >> 0xe) @@ -485,7 +484,7 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie_port *port, { void __iomem *cfg_base = port->cfg_base; struct device *dev = port->dev; - void *bar_addr; + void __iomem *bar_addr; u32 pim_reg; u64 cpu_addr = entry->res->start; u64 pci_addr = cpu_addr - entry->offset; @@ -560,7 +559,7 @@ static int xgene_pcie_setup(struct xgene_pcie_port *port) xgene_pcie_clear_config(port); /* setup the vendor and device IDs correctly */ - val = (XGENE_PCIE_DEVICEID << 16) | XGENE_PCIE_VENDORID; + val = (XGENE_PCIE_DEVICEID << 16) | PCI_VENDOR_ID_AMCC; xgene_pcie_writel(port, BRIDGE_CFG_0, val); ret = xgene_pcie_map_ranges(port); diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c index 98aa1dccc6e6..7b1d3ebc34ec 100644 --- a/drivers/pci/controller/pcie-altera-msi.c +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -55,7 +55,7 @@ static void altera_msi_isr(struct irq_desc *desc) struct altera_msi *msi; unsigned long status; u32 bit; - u32 virq; + int ret; chained_irq_enter(chip, desc); msi = irq_desc_get_handler_data(desc); @@ -65,11 +65,9 @@ static void altera_msi_isr(struct irq_desc *desc) /* Dummy read from vector to clear the interrupt */ readl_relaxed(msi->vector_base + (bit * sizeof(u32))); - virq = irq_find_mapping(msi->inner_domain, bit); - if (virq) - generic_handle_irq(virq); - else - dev_err(&msi->pdev->dev, "unexpected MSI\n"); + ret = generic_handle_domain_irq(msi->inner_domain, bit); + if (ret) + dev_err_ratelimited(&msi->pdev->dev, "unexpected MSI\n"); } } diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index 523bd928b380..2513e9363236 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -646,7 +646,7 @@ static void altera_pcie_isr(struct irq_desc *desc) struct device *dev; unsigned long status; u32 bit; - u32 virq; + int ret; chained_irq_enter(chip, desc); pcie = irq_desc_get_handler_data(desc); @@ -658,11 +658,9 @@ static void altera_pcie_isr(struct irq_desc *desc) /* clear interrupts */ cra_writel(pcie, 1 << bit, P2A_INT_STATUS); - virq = irq_find_mapping(pcie->irq_domain, bit); - if (virq) - generic_handle_irq(virq); - else - dev_err(dev, "unexpected IRQ, INT%d\n", bit); + ret = generic_handle_domain_irq(pcie->irq_domain, bit); + if (ret) + dev_err_ratelimited(dev, "unexpected IRQ, INT%d\n", bit); } } diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c new file mode 100644 index 000000000000..b090924b41fe --- /dev/null +++ b/drivers/pci/controller/pcie-apple.c @@ -0,0 +1,834 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host bridge driver for Apple system-on-chips. + * + * The HW is ECAM compliant, so once the controller is initialized, + * the driver mostly deals MSI mapping and handling of per-port + * interrupts (INTx, management and error signals). + * + * Initialization requires enabling power and clocks, along with a + * number of register pokes. + * + * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io> + * Copyright (C) 2021 Google LLC + * Copyright (C) 2021 Corellium LLC + * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org> + * + * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io> + * Author: Marc Zyngier <maz@kernel.org> + */ + +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/notifier.h> +#include <linux/of_irq.h> +#include <linux/pci-ecam.h> + +#define CORE_RC_PHYIF_CTL 0x00024 +#define CORE_RC_PHYIF_CTL_RUN BIT(0) +#define CORE_RC_PHYIF_STAT 0x00028 +#define CORE_RC_PHYIF_STAT_REFCLK BIT(4) +#define CORE_RC_CTL 0x00050 +#define CORE_RC_CTL_RUN BIT(0) +#define CORE_RC_STAT 0x00058 +#define CORE_RC_STAT_READY BIT(0) +#define CORE_FABRIC_STAT 0x04000 +#define CORE_FABRIC_STAT_MASK 0x001F001F +#define CORE_LANE_CFG(port) (0x84000 + 0x4000 * (port)) +#define CORE_LANE_CFG_REFCLK0REQ BIT(0) +#define CORE_LANE_CFG_REFCLK1 BIT(1) +#define CORE_LANE_CFG_REFCLK0ACK BIT(2) +#define CORE_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) +#define CORE_LANE_CTL(port) (0x84004 + 0x4000 * (port)) +#define CORE_LANE_CTL_CFGACC BIT(15) + +#define PORT_LTSSMCTL 0x00080 +#define PORT_LTSSMCTL_START BIT(0) +#define PORT_INTSTAT 0x00100 +#define PORT_INT_TUNNEL_ERR 31 +#define PORT_INT_CPL_TIMEOUT 23 +#define PORT_INT_RID2SID_MAPERR 22 +#define PORT_INT_CPL_ABORT 21 +#define PORT_INT_MSI_BAD_DATA 19 +#define PORT_INT_MSI_ERR 18 +#define PORT_INT_REQADDR_GT32 17 +#define PORT_INT_AF_TIMEOUT 15 +#define PORT_INT_LINK_DOWN 14 +#define PORT_INT_LINK_UP 12 +#define PORT_INT_LINK_BWMGMT 11 +#define PORT_INT_AER_MASK (15 << 4) +#define PORT_INT_PORT_ERR 4 +#define PORT_INT_INTx(i) i +#define PORT_INT_INTx_MASK 15 +#define PORT_INTMSK 0x00104 +#define PORT_INTMSKSET 0x00108 +#define PORT_INTMSKCLR 0x0010c +#define PORT_MSICFG 0x00124 +#define PORT_MSICFG_EN BIT(0) +#define PORT_MSICFG_L2MSINUM_SHIFT 4 +#define PORT_MSIBASE 0x00128 +#define PORT_MSIBASE_1_SHIFT 16 +#define PORT_MSIADDR 0x00168 +#define PORT_LINKSTS 0x00208 +#define PORT_LINKSTS_UP BIT(0) +#define PORT_LINKSTS_BUSY BIT(2) +#define PORT_LINKCMDSTS 0x00210 +#define PORT_OUTS_NPREQS 0x00284 +#define PORT_OUTS_NPREQS_REQ BIT(24) +#define PORT_OUTS_NPREQS_CPL BIT(16) +#define PORT_RXWR_FIFO 0x00288 +#define PORT_RXWR_FIFO_HDR GENMASK(15, 10) +#define PORT_RXWR_FIFO_DATA GENMASK(9, 0) +#define PORT_RXRD_FIFO 0x0028C +#define PORT_RXRD_FIFO_REQ GENMASK(6, 0) +#define PORT_OUTS_CPLS 0x00290 +#define PORT_OUTS_CPLS_SHRD GENMASK(14, 8) +#define PORT_OUTS_CPLS_WAIT GENMASK(6, 0) +#define PORT_APPCLK 0x00800 +#define PORT_APPCLK_EN BIT(0) +#define PORT_APPCLK_CGDIS BIT(8) +#define PORT_STATUS 0x00804 +#define PORT_STATUS_READY BIT(0) +#define PORT_REFCLK 0x00810 +#define PORT_REFCLK_EN BIT(0) +#define PORT_REFCLK_CGDIS BIT(8) +#define PORT_PERST 0x00814 +#define PORT_PERST_OFF BIT(0) +#define PORT_RID2SID(i16) (0x00828 + 4 * (i16)) +#define PORT_RID2SID_VALID BIT(31) +#define PORT_RID2SID_SID_SHIFT 16 +#define PORT_RID2SID_BUS_SHIFT 8 +#define PORT_RID2SID_DEV_SHIFT 3 +#define PORT_RID2SID_FUNC_SHIFT 0 +#define PORT_OUTS_PREQS_HDR 0x00980 +#define PORT_OUTS_PREQS_HDR_MASK GENMASK(9, 0) +#define PORT_OUTS_PREQS_DATA 0x00984 +#define PORT_OUTS_PREQS_DATA_MASK GENMASK(15, 0) +#define PORT_TUNCTRL 0x00988 +#define PORT_TUNCTRL_PERST_ON BIT(0) +#define PORT_TUNCTRL_PERST_ACK_REQ BIT(1) +#define PORT_TUNSTAT 0x0098c +#define PORT_TUNSTAT_PERST_ON BIT(0) +#define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) +#define PORT_PREFMEM_ENABLE 0x00994 + +#define MAX_RID2SID 64 + +/* + * The doorbell address is set to 0xfffff000, which by convention + * matches what MacOS does, and it is possible to use any other + * address (in the bottom 4GB, as the base register is only 32bit). + * However, it has to be excluded from the IOVA range, and the DART + * driver has to know about it. + */ +#define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR + +struct apple_pcie { + struct mutex lock; + struct device *dev; + void __iomem *base; + struct irq_domain *domain; + unsigned long *bitmap; + struct list_head ports; + struct completion event; + struct irq_fwspec fwspec; + u32 nvecs; +}; + +struct apple_pcie_port { + struct apple_pcie *pcie; + struct device_node *np; + void __iomem *base; + struct irq_domain *domain; + struct list_head entry; + DECLARE_BITMAP(sid_map, MAX_RID2SID); + int sid_map_sz; + int idx; +}; + +static void rmw_set(u32 set, void __iomem *addr) +{ + writel_relaxed(readl_relaxed(addr) | set, addr); +} + +static void rmw_clear(u32 clr, void __iomem *addr) +{ + writel_relaxed(readl_relaxed(addr) & ~clr, addr); +} + +static void apple_msi_top_irq_mask(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void apple_msi_top_irq_unmask(struct irq_data *d) +{ + pci_msi_unmask_irq(d); + irq_chip_unmask_parent(d); +} + +static struct irq_chip apple_msi_top_chip = { + .name = "PCIe MSI", + .irq_mask = apple_msi_top_irq_mask, + .irq_unmask = apple_msi_top_irq_unmask, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, +}; + +static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + msg->address_hi = upper_32_bits(DOORBELL_ADDR); + msg->address_lo = lower_32_bits(DOORBELL_ADDR); + msg->data = data->hwirq; +} + +static struct irq_chip apple_msi_bottom_chip = { + .name = "MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = apple_msi_compose_msg, +}; + +static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct apple_pcie *pcie = domain->host_data; + struct irq_fwspec fwspec = pcie->fwspec; + unsigned int i; + int ret, hwirq; + + mutex_lock(&pcie->lock); + + hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, + order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); + + if (hwirq < 0) + return -ENOSPC; + + fwspec.param[1] += hwirq; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &apple_msi_bottom_chip, + domain->host_data); + } + + return 0; +} + +static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct apple_pcie *pcie = domain->host_data; + + mutex_lock(&pcie->lock); + + bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); +} + +static const struct irq_domain_ops apple_msi_domain_ops = { + .alloc = apple_msi_domain_alloc, + .free = apple_msi_domain_free, +}; + +static struct msi_domain_info apple_msi_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), + .chip = &apple_msi_top_chip, +}; + +static void apple_port_irq_mask(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKSET); +} + +static void apple_port_irq_unmask(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKCLR); +} + +static bool hwirq_is_intx(unsigned int hwirq) +{ + return BIT(hwirq) & PORT_INT_INTx_MASK; +} + +static void apple_port_irq_ack(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + if (!hwirq_is_intx(data->hwirq)) + writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT); +} + +static int apple_port_irq_set_type(struct irq_data *data, unsigned int type) +{ + /* + * It doesn't seem that there is any way to configure the + * trigger, so assume INTx have to be level (as per the spec), + * and the rest is edge (which looks likely). + */ + if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK)) + return -EINVAL; + + irqd_set_trigger_type(data, type); + return 0; +} + +static struct irq_chip apple_port_irqchip = { + .name = "PCIe", + .irq_ack = apple_port_irq_ack, + .irq_mask = apple_port_irq_mask, + .irq_unmask = apple_port_irq_unmask, + .irq_set_type = apple_port_irq_set_type, +}; + +static int apple_port_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct apple_pcie_port *port = domain->host_data; + struct irq_fwspec *fwspec = args; + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_flow_handler_t flow = handle_edge_irq; + unsigned int type = IRQ_TYPE_EDGE_RISING; + + if (hwirq_is_intx(fwspec->param[0] + i)) { + flow = handle_level_irq; + type = IRQ_TYPE_LEVEL_HIGH; + } + + irq_domain_set_info(domain, virq + i, fwspec->param[0] + i, + &apple_port_irqchip, port, flow, + NULL, NULL); + + irq_set_irq_type(virq + i, type); + } + + return 0; +} + +static void apple_port_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops apple_port_irq_domain_ops = { + .translate = irq_domain_translate_onecell, + .alloc = apple_port_irq_domain_alloc, + .free = apple_port_irq_domain_free, +}; + +static void apple_port_irq_handler(struct irq_desc *desc) +{ + struct apple_pcie_port *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long stat; + int i; + + chained_irq_enter(chip, desc); + + stat = readl_relaxed(port->base + PORT_INTSTAT); + + for_each_set_bit(i, &stat, 32) + generic_handle_domain_irq(port->domain, i); + + chained_irq_exit(chip, desc); +} + +static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) +{ + struct fwnode_handle *fwnode = &port->np->fwnode; + unsigned int irq; + + /* FIXME: consider moving each interrupt under each port */ + irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), + port->idx); + if (!irq) + return -ENXIO; + + port->domain = irq_domain_create_linear(fwnode, 32, + &apple_port_irq_domain_ops, + port); + if (!port->domain) + return -ENOMEM; + + /* Disable all interrupts */ + writel_relaxed(~0, port->base + PORT_INTMSKSET); + writel_relaxed(~0, port->base + PORT_INTSTAT); + + irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); + + /* Configure MSI base address */ + BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); + writel_relaxed(lower_32_bits(DOORBELL_ADDR), port->base + PORT_MSIADDR); + + /* Enable MSIs, shared between all ports */ + writel_relaxed(0, port->base + PORT_MSIBASE); + writel_relaxed((ilog2(port->pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) | + PORT_MSICFG_EN, port->base + PORT_MSICFG); + + return 0; +} + +static irqreturn_t apple_pcie_port_irq(int irq, void *data) +{ + struct apple_pcie_port *port = data; + unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq; + + switch (hwirq) { + case PORT_INT_LINK_UP: + dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n", + port->np); + complete_all(&port->pcie->event); + break; + case PORT_INT_LINK_DOWN: + dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", + port->np); + break; + default: + return IRQ_NONE; + } + + return IRQ_HANDLED; +} + +static int apple_pcie_port_register_irqs(struct apple_pcie_port *port) +{ + static struct { + unsigned int hwirq; + const char *name; + } port_irqs[] = { + { PORT_INT_LINK_UP, "Link up", }, + { PORT_INT_LINK_DOWN, "Link down", }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(port_irqs); i++) { + struct irq_fwspec fwspec = { + .fwnode = &port->np->fwnode, + .param_count = 1, + .param = { + [0] = port_irqs[i].hwirq, + }, + }; + unsigned int irq; + int ret; + + irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE, + &fwspec); + if (WARN_ON(!irq)) + continue; + + ret = request_irq(irq, apple_pcie_port_irq, 0, + port_irqs[i].name, port); + WARN_ON(ret); + } + + return 0; +} + +static int apple_pcie_setup_refclk(struct apple_pcie *pcie, + struct apple_pcie_port *port) +{ + u32 stat; + int res; + + res = readl_relaxed_poll_timeout(pcie->base + CORE_RC_PHYIF_STAT, stat, + stat & CORE_RC_PHYIF_STAT_REFCLK, + 100, 50000); + if (res < 0) + return res; + + rmw_set(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); + rmw_set(CORE_LANE_CFG_REFCLK0REQ, pcie->base + CORE_LANE_CFG(port->idx)); + + res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), + stat, stat & CORE_LANE_CFG_REFCLK0ACK, + 100, 50000); + if (res < 0) + return res; + + rmw_set(CORE_LANE_CFG_REFCLK1, pcie->base + CORE_LANE_CFG(port->idx)); + res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), + stat, stat & CORE_LANE_CFG_REFCLK1, + 100, 50000); + + if (res < 0) + return res; + + rmw_clear(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); + + rmw_set(CORE_LANE_CFG_REFCLKEN, pcie->base + CORE_LANE_CFG(port->idx)); + rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK); + + return 0; +} + +static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, + int idx, u32 val) +{ + writel_relaxed(val, port->base + PORT_RID2SID(idx)); + /* Read back to ensure completion of the write */ + return readl_relaxed(port->base + PORT_RID2SID(idx)); +} + +static int apple_pcie_setup_port(struct apple_pcie *pcie, + struct device_node *np) +{ + struct platform_device *platform = to_platform_device(pcie->dev); + struct apple_pcie_port *port; + struct gpio_desc *reset; + u32 stat, idx; + int ret, i; + + reset = gpiod_get_from_of_node(np, "reset-gpios", 0, + GPIOD_OUT_LOW, "PERST#"); + if (IS_ERR(reset)) + return PTR_ERR(reset); + + port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + ret = of_property_read_u32_index(np, "reg", 0, &idx); + if (ret) + return ret; + + /* Use the first reg entry to work out the port index */ + port->idx = idx >> 11; + port->pcie = pcie; + port->np = np; + + port->base = devm_platform_ioremap_resource(platform, port->idx + 2); + if (IS_ERR(port->base)) + return PTR_ERR(port->base); + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* Assert PERST# before setting up the clock */ + gpiod_set_value(reset, 1); + + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + + /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 200); + + /* Deassert PERST# */ + rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); + gpiod_set_value(reset, 0); + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); + return ret; + } + + ret = apple_pcie_port_setup_irq(port); + if (ret) + return ret; + + /* Reset all RID/SID mappings, and check for RAZ/WI registers */ + for (i = 0; i < MAX_RID2SID; i++) { + if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) + break; + apple_pcie_rid2sid_write(port, i, 0); + } + + dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i); + + port->sid_map_sz = i; + + list_add_tail(&port->entry, &pcie->ports); + init_completion(&pcie->event); + + ret = apple_pcie_port_register_irqs(port); + WARN_ON(ret); + + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + + if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + + return 0; +} + +static int apple_msi_init(struct apple_pcie *pcie) +{ + struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); + struct of_phandle_args args = {}; + struct irq_domain *parent; + int ret; + + ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges", + "#interrupt-cells", 0, &args); + if (ret) + return ret; + + ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges", + args.args_count + 1, &pcie->nvecs); + if (ret) + return ret; + + of_phandle_args_to_fwspec(args.np, args.args, args.args_count, + &pcie->fwspec); + + pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL); + if (!pcie->bitmap) + return -ENOMEM; + + parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED); + if (!parent) { + dev_err(pcie->dev, "failed to find parent domain\n"); + return -ENXIO; + } + + parent = irq_domain_create_hierarchy(parent, 0, pcie->nvecs, fwnode, + &apple_msi_domain_ops, pcie); + if (!parent) { + dev_err(pcie->dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); + + pcie->domain = pci_msi_create_irq_domain(fwnode, &apple_msi_info, + parent); + if (!pcie->domain) { + dev_err(pcie->dev, "failed to create MSI domain\n"); + irq_domain_remove(parent); + return -ENOMEM; + } + + return 0; +} + +static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev) +{ + struct pci_config_window *cfg = pdev->sysdata; + struct apple_pcie *pcie = cfg->priv; + struct pci_dev *port_pdev; + struct apple_pcie_port *port; + + /* Find the root port this device is on */ + port_pdev = pcie_find_root_port(pdev); + + /* If finding the port itself, nothing to do */ + if (WARN_ON(!port_pdev) || pdev == port_pdev) + return NULL; + + list_for_each_entry(port, &pcie->ports, entry) { + if (port->idx == PCI_SLOT(port_pdev->devfn)) + return port; + } + + return NULL; +} + +static int apple_pcie_add_device(struct apple_pcie_port *port, + struct pci_dev *pdev) +{ + u32 sid, rid = PCI_DEVID(pdev->bus->number, pdev->devfn); + int idx, err; + + dev_dbg(&pdev->dev, "added to bus %s, index %d\n", + pci_name(pdev->bus->self), port->idx); + + err = of_map_id(port->pcie->dev->of_node, rid, "iommu-map", + "iommu-map-mask", NULL, &sid); + if (err) + return err; + + mutex_lock(&port->pcie->lock); + + idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); + if (idx >= 0) { + apple_pcie_rid2sid_write(port, idx, + PORT_RID2SID_VALID | + (sid << PORT_RID2SID_SID_SHIFT) | rid); + + dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n", + rid, sid, idx); + } + + mutex_unlock(&port->pcie->lock); + + return idx >= 0 ? 0 : -ENOSPC; +} + +static void apple_pcie_release_device(struct apple_pcie_port *port, + struct pci_dev *pdev) +{ + u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn); + int idx; + + mutex_lock(&port->pcie->lock); + + for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { + u32 val; + + val = readl_relaxed(port->base + PORT_RID2SID(idx)); + if ((val & 0xffff) == rid) { + apple_pcie_rid2sid_write(port, idx, 0); + bitmap_release_region(port->sid_map, idx, 0); + dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx); + break; + } + } + + mutex_unlock(&port->pcie->lock); +} + +static int apple_pcie_bus_notifier(struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct device *dev = data; + struct pci_dev *pdev = to_pci_dev(dev); + struct apple_pcie_port *port; + int err; + + /* + * This is a bit ugly. We assume that if we get notified for + * any PCI device, we must be in charge of it, and that there + * is no other PCI controller in the whole system. It probably + * holds for now, but who knows for how long? + */ + port = apple_pcie_get_port(pdev); + if (!port) + return NOTIFY_DONE; + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + err = apple_pcie_add_device(port, pdev); + if (err) + return notifier_from_errno(err); + break; + case BUS_NOTIFY_DEL_DEVICE: + apple_pcie_release_device(port, pdev); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block apple_pcie_nb = { + .notifier_call = apple_pcie_bus_notifier, +}; + +static int apple_pcie_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct platform_device *platform = to_platform_device(dev); + struct device_node *of_port; + struct apple_pcie *pcie; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pcie->dev = dev; + + mutex_init(&pcie->lock); + + pcie->base = devm_platform_ioremap_resource(platform, 1); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + cfg->priv = pcie; + INIT_LIST_HEAD(&pcie->ports); + + for_each_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_setup_port(pcie, of_port); + if (ret) { + dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret); + of_node_put(of_port); + return ret; + } + } + + return apple_msi_init(pcie); +} + +static int apple_pcie_probe(struct platform_device *pdev) +{ + int ret; + + ret = bus_register_notifier(&pci_bus_type, &apple_pcie_nb); + if (ret) + return ret; + + ret = pci_host_common_probe(pdev); + if (ret) + bus_unregister_notifier(&pci_bus_type, &apple_pcie_nb); + + return ret; +} + +static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { + .init = apple_pcie_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static const struct of_device_id apple_pcie_of_match[] = { + { .compatible = "apple,pcie", .data = &apple_pcie_cfg_ecam_ops }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_pcie_of_match); + +static struct platform_driver apple_pcie_driver = { + .probe = apple_pcie_probe, + .driver = { + .name = "pcie-apple", + .of_match_table = apple_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(apple_pcie_driver); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index 08bc788d9422..1fc7bd49a7ad 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -145,7 +145,7 @@ #define BRCM_INT_PCI_MSI_LEGACY_NR 8 #define BRCM_INT_PCI_MSI_SHIFT 0 -/* MSI target adresses */ +/* MSI target addresses */ #define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL #define BRCM_MSI_TARGET_ADDR_GT_4GB 0xffffffffcULL @@ -476,7 +476,7 @@ static struct msi_domain_info brcm_msi_domain_info = { static void brcm_pcie_msi_isr(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long status, virq; + unsigned long status; struct brcm_msi *msi; struct device *dev; u32 bit; @@ -489,10 +489,9 @@ static void brcm_pcie_msi_isr(struct irq_desc *desc) status >>= msi->legacy_shift; for_each_set_bit(bit, &status, msi->nr) { - virq = irq_find_mapping(msi->inner_domain, bit); - if (virq) - generic_handle_irq(virq); - else + int ret; + ret = generic_handle_domain_irq(msi->inner_domain, bit); + if (ret) dev_dbg(dev, "unexpected MSI\n"); } diff --git a/drivers/pci/controller/pcie-iproc-bcma.c b/drivers/pci/controller/pcie-iproc-bcma.c index 56b8ee7bf330..f918c713afb0 100644 --- a/drivers/pci/controller/pcie-iproc-bcma.c +++ b/drivers/pci/controller/pcie-iproc-bcma.c @@ -35,7 +35,6 @@ static int iproc_pcie_bcma_probe(struct bcma_device *bdev) { struct device *dev = &bdev->dev; struct iproc_pcie *pcie; - LIST_HEAD(resources); struct pci_host_bridge *bridge; int ret; @@ -60,19 +59,16 @@ static int iproc_pcie_bcma_probe(struct bcma_device *bdev) pcie->mem.end = bdev->addr_s[0] + SZ_128M - 1; pcie->mem.name = "PCIe MEM space"; pcie->mem.flags = IORESOURCE_MEM; - pci_add_resource(&resources, &pcie->mem); + pci_add_resource(&bridge->windows, &pcie->mem); + ret = devm_request_pci_bus_resources(dev, &bridge->windows); + if (ret) + return ret; pcie->map_irq = iproc_pcie_bcma_map_irq; - ret = iproc_pcie_setup(pcie, &resources); - if (ret) { - dev_err(dev, "PCIe controller setup failed\n"); - pci_free_resource_list(&resources); - return ret; - } - bcma_set_drvdata(bdev, pcie); - return 0; + + return iproc_pcie_setup(pcie, &bridge->windows); } static void iproc_pcie_bcma_remove(struct bcma_device *bdev) diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c index eede4e8f3f75..757b7fbcdc59 100644 --- a/drivers/pci/controller/pcie-iproc-msi.c +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -49,7 +49,7 @@ enum iproc_msi_reg { struct iproc_msi; /** - * iProc MSI group + * struct iproc_msi_grp - iProc MSI group * * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI * event queue. @@ -65,7 +65,7 @@ struct iproc_msi_grp { }; /** - * iProc event queue based MSI + * struct iproc_msi - iProc event queue based MSI * * Only meant to be used on platforms without MSI support integrated into the * GIC. @@ -171,7 +171,7 @@ static struct irq_chip iproc_msi_irq_chip = { static struct msi_domain_info iproc_msi_domain_info = { .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + MSI_FLAG_PCI_MSIX, .chip = &iproc_msi_irq_chip, }; @@ -250,20 +250,23 @@ static int iproc_msi_irq_domain_alloc(struct irq_domain *domain, struct iproc_msi *msi = domain->host_data; int hwirq, i; + if (msi->nr_cpus > 1 && nr_irqs > 1) + return -EINVAL; + mutex_lock(&msi->bitmap_lock); - /* Allocate 'nr_cpus' number of MSI vectors each time */ - hwirq = bitmap_find_next_zero_area(msi->bitmap, msi->nr_msi_vecs, 0, - msi->nr_cpus, 0); - if (hwirq < msi->nr_msi_vecs) { - bitmap_set(msi->bitmap, hwirq, msi->nr_cpus); - } else { - mutex_unlock(&msi->bitmap_lock); - return -ENOSPC; - } + /* + * Allocate 'nr_irqs' multiplied by 'nr_cpus' number of MSI vectors + * each time + */ + hwirq = bitmap_find_free_region(msi->bitmap, msi->nr_msi_vecs, + order_base_2(msi->nr_cpus * nr_irqs)); mutex_unlock(&msi->bitmap_lock); + if (hwirq < 0) + return -ENOSPC; + for (i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, hwirq + i, &iproc_msi_bottom_irq_chip, @@ -284,7 +287,8 @@ static void iproc_msi_irq_domain_free(struct irq_domain *domain, mutex_lock(&msi->bitmap_lock); hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq); - bitmap_clear(msi->bitmap, hwirq, msi->nr_cpus); + bitmap_release_region(msi->bitmap, hwirq, + order_base_2(msi->nr_cpus * nr_irqs)); mutex_unlock(&msi->bitmap_lock); @@ -322,7 +326,6 @@ static void iproc_msi_handler(struct irq_desc *desc) struct iproc_msi *msi; u32 eq, head, tail, nr_events; unsigned long hwirq; - int virq; chained_irq_enter(chip, desc); @@ -358,8 +361,7 @@ static void iproc_msi_handler(struct irq_desc *desc) /* process all outstanding events */ while (nr_events--) { hwirq = decode_msi_hwirq(msi, eq, head); - virq = irq_find_mapping(msi->inner_domain, hwirq); - generic_handle_irq(virq); + generic_handle_domain_irq(msi->inner_domain, hwirq); head++; head %= EQ_LEN; @@ -539,6 +541,9 @@ int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node) mutex_init(&msi->bitmap_lock); msi->nr_cpus = num_possible_cpus(); + if (msi->nr_cpus == 1) + iproc_msi_domain_info.flags |= MSI_FLAG_MULTI_PCI_MSI; + msi->nr_irqs = of_irq_count(node); if (!msi->nr_irqs) { dev_err(pcie->dev, "found no MSI GIC interrupt\n"); diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 02e52f698eeb..36b9d2c46cfa 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -89,8 +89,8 @@ #define IPROC_PCIE_REG_INVALID 0xffff /** - * iProc PCIe outbound mapping controller specific parameters - * + * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific + * parameters * @window_sizes: list of supported outbound mapping window sizes in MB * @nr_sizes: number of supported outbound mapping window sizes */ @@ -136,22 +136,20 @@ static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { }; /** - * iProc PCIe inbound mapping type + * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type + * @IPROC_PCIE_IB_MAP_MEM: DDR memory + * @IPROC_PCIE_IB_MAP_IO: device I/O memory + * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused */ enum iproc_pcie_ib_map_type { - /* for DDR memory */ IPROC_PCIE_IB_MAP_MEM = 0, - - /* for device I/O memory */ IPROC_PCIE_IB_MAP_IO, - - /* invalid or unused */ IPROC_PCIE_IB_MAP_INVALID }; /** - * iProc PCIe inbound mapping controller specific parameters - * + * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific + * parameters * @type: inbound mapping region type * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or * SZ_1G @@ -251,7 +249,7 @@ enum iproc_pcie_reg { /* * To hold the address of the register where the MSI writes are - * programed. When ARM GICv3 ITS is used, this should be programmed + * programmed. When ARM GICv3 ITS is used, this should be programmed * with the address of the GITS_TRANSLATER register. */ IPROC_PCIE_MSI_ADDR_LO, @@ -437,7 +435,7 @@ static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, writel(val, pcie->base + offset); } -/** +/* * APB error forwarding can be disabled during access of configuration * registers of the endpoint device, to prevent unsupported requests * (typically seen during enumeration with multi-function devices) from @@ -619,7 +617,7 @@ static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, return PCIBIOS_SUCCESSFUL; } -/** +/* * Note access to the configuration registers are protected at the higher layer * by 'pci_lock' in drivers/pci/access.c */ @@ -897,7 +895,7 @@ static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, return 0; } -/** +/* * Some iProc SoCs require the SW to configure the outbound address mapping * * Outbound address translation: diff --git a/drivers/pci/controller/pcie-iproc.h b/drivers/pci/controller/pcie-iproc.h index c2676e442f55..dcca315897c8 100644 --- a/drivers/pci/controller/pcie-iproc.h +++ b/drivers/pci/controller/pcie-iproc.h @@ -7,7 +7,13 @@ #define _PCIE_IPROC_H /** - * iProc PCIe interface type + * enum iproc_pcie_type - iProc PCIe interface type + * @IPROC_PCIE_PAXB_BCMA: BCMA-based host controllers + * @IPROC_PCIE_PAXB: PAXB-based host controllers for + * NS, NSP, Cygnus, NS2, and Pegasus SOCs + * @IPROC_PCIE_PAXB_V2: PAXB-based host controllers for Stingray SoCs + * @IPROC_PCIE_PAXC: PAXC-based host controllers + * @IPROC_PCIE_PAXC_V2: PAXC-based host controllers (second generation) * * PAXB is the wrapper used in root complex that can be connected to an * external endpoint device. @@ -24,7 +30,7 @@ enum iproc_pcie_type { }; /** - * iProc PCIe outbound mapping + * struct iproc_pcie_ob - iProc PCIe outbound mapping * @axi_offset: offset from the AXI address to the internal address used by * the iProc PCIe core * @nr_windows: total number of supported outbound mapping windows @@ -35,7 +41,7 @@ struct iproc_pcie_ob { }; /** - * iProc PCIe inbound mapping + * struct iproc_pcie_ib - iProc PCIe inbound mapping * @nr_regions: total number of supported inbound mapping regions */ struct iproc_pcie_ib { @@ -47,13 +53,13 @@ struct iproc_pcie_ib_map; struct iproc_msi; /** - * iProc PCIe device - * + * struct iproc_pcie - iProc PCIe device * @dev: pointer to device data structure * @type: iProc PCIe interface type * @reg_offsets: register offsets * @base: PCIe host controller I/O register base * @base_addr: PCIe host controller register base physical address + * @mem: host bridge memory window resource * @phy: optional PHY device that controls the Serdes * @map_irq: function callback to map interrupts * @ep_is_internal: indicates an internal emulated endpoint device is connected diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index 3c5b97716d40..17c59b0d6978 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -645,7 +645,6 @@ static void mtk_pcie_msi_handler(struct mtk_pcie_port *port, int set_idx) { struct mtk_msi_set *msi_set = &port->msi_sets[set_idx]; unsigned long msi_enable, msi_status; - unsigned int virq; irq_hw_number_t bit, hwirq; msi_enable = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); @@ -659,8 +658,7 @@ static void mtk_pcie_msi_handler(struct mtk_pcie_port *port, int set_idx) for_each_set_bit(bit, &msi_status, PCIE_MSI_IRQS_PER_SET) { hwirq = bit + set_idx * PCIE_MSI_IRQS_PER_SET; - virq = irq_find_mapping(port->msi_bottom_domain, hwirq); - generic_handle_irq(virq); + generic_handle_domain_irq(port->msi_bottom_domain, hwirq); } } while (true); } @@ -670,18 +668,15 @@ static void mtk_pcie_irq_handler(struct irq_desc *desc) struct mtk_pcie_port *port = irq_desc_get_handler_data(desc); struct irq_chip *irqchip = irq_desc_get_chip(desc); unsigned long status; - unsigned int virq; irq_hw_number_t irq_bit = PCIE_INTX_SHIFT; chained_irq_enter(irqchip, desc); status = readl_relaxed(port->base + PCIE_INT_STATUS_REG); for_each_set_bit_from(irq_bit, &status, PCI_NUM_INTX + - PCIE_INTX_SHIFT) { - virq = irq_find_mapping(port->intx_domain, - irq_bit - PCIE_INTX_SHIFT); - generic_handle_irq(virq); - } + PCIE_INTX_SHIFT) + generic_handle_domain_irq(port->intx_domain, + irq_bit - PCIE_INTX_SHIFT); irq_bit = PCIE_MSI_SHIFT; for_each_set_bit_from(irq_bit, &status, PCIE_MSI_SET_NUM + @@ -1012,6 +1007,7 @@ static const struct of_device_id mtk_pcie_of_match[] = { { .compatible = "mediatek,mt8192-pcie" }, {}, }; +MODULE_DEVICE_TABLE(of, mtk_pcie_of_match); static struct platform_driver mtk_pcie_driver = { .probe = mtk_pcie_probe, diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 62a042e75d9a..2f3f974977a3 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -14,6 +14,7 @@ #include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> +#include <linux/mfd/syscon.h> #include <linux/msi.h> #include <linux/module.h> #include <linux/of_address.h> @@ -23,6 +24,7 @@ #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> +#include <linux/regmap.h> #include <linux/reset.h> #include "../pci.h" @@ -207,6 +209,7 @@ struct mtk_pcie_port { * struct mtk_pcie - PCIe host information * @dev: pointer to PCIe device * @base: IO mapped register base + * @cfg: IO mapped register map for PCIe config * @free_ck: free-run reference clock * @mem: non-prefetchable memory resource * @ports: pointer to PCIe port information @@ -215,6 +218,7 @@ struct mtk_pcie_port { struct mtk_pcie { struct device *dev; void __iomem *base; + struct regmap *cfg; struct clk *free_ck; struct list_head ports; @@ -602,7 +606,6 @@ static void mtk_pcie_intr_handler(struct irq_desc *desc) struct mtk_pcie_port *port = irq_desc_get_handler_data(desc); struct irq_chip *irqchip = irq_desc_get_chip(desc); unsigned long status; - u32 virq; u32 bit = INTX_SHIFT; chained_irq_enter(irqchip, desc); @@ -612,9 +615,8 @@ static void mtk_pcie_intr_handler(struct irq_desc *desc) for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) { /* Clear the INTx */ writel(1 << bit, port->base + PCIE_INT_STATUS); - virq = irq_find_mapping(port->irq_domain, - bit - INTX_SHIFT); - generic_handle_irq(virq); + generic_handle_domain_irq(port->irq_domain, + bit - INTX_SHIFT); } } @@ -623,10 +625,8 @@ static void mtk_pcie_intr_handler(struct irq_desc *desc) unsigned long imsi_status; while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) { - for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) { - virq = irq_find_mapping(port->inner_domain, bit); - generic_handle_irq(virq); - } + for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) + generic_handle_domain_irq(port->inner_domain, bit); } /* Clear MSI interrupt status */ writel(MSI_STATUS, port->base + PCIE_INT_STATUS); @@ -650,7 +650,11 @@ static int mtk_pcie_setup_irq(struct mtk_pcie_port *port, return err; } - port->irq = platform_get_irq(pdev, port->slot); + if (of_find_property(dev->of_node, "interrupt-names", NULL)) + port->irq = platform_get_irq_byname(pdev, "pcie_irq"); + else + port->irq = platform_get_irq(pdev, port->slot); + if (port->irq < 0) return port->irq; @@ -682,6 +686,10 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port) val |= PCIE_CSR_LTSSM_EN(port->slot) | PCIE_CSR_ASPM_L1_EN(port->slot); writel(val, pcie->base + PCIE_SYS_CFG_V2); + } else if (pcie->cfg) { + val = PCIE_CSR_LTSSM_EN(port->slot) | + PCIE_CSR_ASPM_L1_EN(port->slot); + regmap_update_bits(pcie->cfg, PCIE_SYS_CFG_V2, val, val); } /* Assert all reset signals */ @@ -985,16 +993,23 @@ static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie) struct device *dev = pcie->dev; struct platform_device *pdev = to_platform_device(dev); struct resource *regs; + struct device_node *cfg_node; int err; /* get shared registers, which are optional */ regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys"); if (regs) { pcie->base = devm_ioremap_resource(dev, regs); - if (IS_ERR(pcie->base)) { - dev_err(dev, "failed to map shared register\n"); + if (IS_ERR(pcie->base)) return PTR_ERR(pcie->base); - } + } + + cfg_node = of_find_compatible_node(NULL, NULL, + "mediatek,generic-pciecfg"); + if (cfg_node) { + pcie->cfg = syscon_node_to_regmap(cfg_node); + if (IS_ERR(pcie->cfg)) + return PTR_ERR(pcie->cfg); } pcie->free_ck = devm_clk_get(dev, "free_ck"); @@ -1029,22 +1044,27 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie) struct device *dev = pcie->dev; struct device_node *node = dev->of_node, *child; struct mtk_pcie_port *port, *tmp; - int err; + int err, slot; + + slot = of_get_pci_domain_nr(dev->of_node); + if (slot < 0) { + for_each_available_child_of_node(node, child) { + err = of_pci_get_devfn(child); + if (err < 0) { + dev_err(dev, "failed to get devfn: %d\n", err); + goto error_put_node; + } - for_each_available_child_of_node(node, child) { - int slot; + slot = PCI_SLOT(err); - err = of_pci_get_devfn(child); - if (err < 0) { - dev_err(dev, "failed to parse devfn: %d\n", err); - goto error_put_node; + err = mtk_pcie_parse_port(pcie, child, slot); + if (err) + goto error_put_node; } - - slot = PCI_SLOT(err); - - err = mtk_pcie_parse_port(pcie, child, slot); + } else { + err = mtk_pcie_parse_port(pcie, node, slot); if (err) - goto error_put_node; + return err; } err = mtk_pcie_subsys_powerup(pcie); diff --git a/drivers/pci/controller/pcie-microchip-host.c b/drivers/pci/controller/pcie-microchip-host.c index 89c68c56d93b..329f930d17aa 100644 --- a/drivers/pci/controller/pcie-microchip-host.c +++ b/drivers/pci/controller/pcie-microchip-host.c @@ -341,7 +341,7 @@ static struct event_map local_status_to_event[] = { LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_SYS_ERR), }; -struct { +static struct { u32 base; u32 offset; u32 mask; @@ -412,16 +412,14 @@ static void mc_handle_msi(struct irq_desc *desc) port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; unsigned long status; u32 bit; - u32 virq; + int ret; status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); if (status & PM_MSI_INT_MSI_MASK) { status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); for_each_set_bit(bit, &status, msi->num_vectors) { - virq = irq_find_mapping(msi->dev_domain, bit); - if (virq) - generic_handle_irq(virq); - else + ret = generic_handle_domain_irq(msi->dev_domain, bit); + if (ret) dev_err_ratelimited(dev, "bad MSI IRQ %d\n", bit); } @@ -570,17 +568,15 @@ static void mc_handle_intx(struct irq_desc *desc) port->axi_base_addr + MC_PCIE_BRIDGE_ADDR; unsigned long status; u32 bit; - u32 virq; + int ret; status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); if (status & PM_MSI_INT_INTX_MASK) { status &= PM_MSI_INT_INTX_MASK; status >>= PM_MSI_INT_INTX_SHIFT; for_each_set_bit(bit, &status, PCI_NUM_INTX) { - virq = irq_find_mapping(port->intx_domain, bit); - if (virq) - generic_handle_irq(virq); - else + ret = generic_handle_domain_irq(port->intx_domain, bit); + if (ret) dev_err_ratelimited(dev, "bad INTx IRQ %d\n", bit); } @@ -745,7 +741,7 @@ static void mc_handle_event(struct irq_desc *desc) events = get_events(port); for_each_set_bit(bit, &events, NUM_EVENTS) - generic_handle_irq(irq_find_mapping(port->event_domain, bit)); + generic_handle_domain_irq(port->event_domain, bit); chained_irq_exit(chip, desc); } diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c new file mode 100644 index 000000000000..b60dfb45ef7b --- /dev/null +++ b/drivers/pci/controller/pcie-mt7621.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * BRIEF MODULE DESCRIPTION + * PCI init for Ralink RT2880 solution + * + * Copyright 2007 Ralink Inc. (bruce_chang@ralinktech.com.tw) + * + * May 2007 Bruce Chang + * Initial Release + * + * May 2009 Bruce Chang + * support RT2880/RT3883 PCIe + * + * May 2011 Bruce Chang + * support RT6855/MT7620 PCIe + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/sys_soc.h> + +/* MediaTek-specific configuration registers */ +#define PCIE_FTS_NUM 0x70c +#define PCIE_FTS_NUM_MASK GENMASK(15, 8) +#define PCIE_FTS_NUM_L0(x) (((x) & 0xff) << 8) + +/* Host-PCI bridge registers */ +#define RALINK_PCI_PCICFG_ADDR 0x0000 +#define RALINK_PCI_PCIMSK_ADDR 0x000c +#define RALINK_PCI_CONFIG_ADDR 0x0020 +#define RALINK_PCI_CONFIG_DATA 0x0024 +#define RALINK_PCI_MEMBASE 0x0028 +#define RALINK_PCI_IOBASE 0x002c + +/* PCIe RC control registers */ +#define RALINK_PCI_ID 0x0030 +#define RALINK_PCI_CLASS 0x0034 +#define RALINK_PCI_SUBID 0x0038 +#define RALINK_PCI_STATUS 0x0050 + +/* Some definition values */ +#define PCIE_REVISION_ID BIT(0) +#define PCIE_CLASS_CODE (0x60400 << 8) +#define PCIE_BAR_MAP_MAX GENMASK(30, 16) +#define PCIE_BAR_ENABLE BIT(0) +#define PCIE_PORT_INT_EN(x) BIT(20 + (x)) +#define PCIE_PORT_LINKUP BIT(0) +#define PCIE_PORT_CNT 3 + +#define PERST_DELAY_MS 100 + +/** + * struct mt7621_pcie_port - PCIe port information + * @base: I/O mapped register base + * @list: port list + * @pcie: pointer to PCIe host info + * @clk: pointer to the port clock gate + * @phy: pointer to PHY control block + * @pcie_rst: pointer to port reset control + * @gpio_rst: gpio reset + * @slot: port slot + * @enabled: indicates if port is enabled + */ +struct mt7621_pcie_port { + void __iomem *base; + struct list_head list; + struct mt7621_pcie *pcie; + struct clk *clk; + struct phy *phy; + struct reset_control *pcie_rst; + struct gpio_desc *gpio_rst; + u32 slot; + bool enabled; +}; + +/** + * struct mt7621_pcie - PCIe host information + * @base: IO Mapped Register Base + * @dev: Pointer to PCIe device + * @ports: pointer to PCIe port information + * @resets_inverted: depends on chip revision + * reset lines are inverted. + */ +struct mt7621_pcie { + void __iomem *base; + struct device *dev; + struct list_head ports; + bool resets_inverted; +}; + +static inline u32 pcie_read(struct mt7621_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->base + reg); +} + +static inline void pcie_write(struct mt7621_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->base + reg); +} + +static inline void pcie_rmw(struct mt7621_pcie *pcie, u32 reg, u32 clr, u32 set) +{ + u32 val = readl_relaxed(pcie->base + reg); + + val &= ~clr; + val |= set; + writel_relaxed(val, pcie->base + reg); +} + +static inline u32 pcie_port_read(struct mt7621_pcie_port *port, u32 reg) +{ + return readl_relaxed(port->base + reg); +} + +static inline void pcie_port_write(struct mt7621_pcie_port *port, + u32 val, u32 reg) +{ + writel_relaxed(val, port->base + reg); +} + +static inline u32 mt7621_pci_get_cfgaddr(unsigned int bus, unsigned int slot, + unsigned int func, unsigned int where) +{ + return (((where & 0xf00) >> 8) << 24) | (bus << 16) | (slot << 11) | + (func << 8) | (where & 0xfc) | 0x80000000; +} + +static void __iomem *mt7621_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct mt7621_pcie *pcie = bus->sysdata; + u32 address = mt7621_pci_get_cfgaddr(bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), where); + + writel_relaxed(address, pcie->base + RALINK_PCI_CONFIG_ADDR); + + return pcie->base + RALINK_PCI_CONFIG_DATA + (where & 3); +} + +struct pci_ops mt7621_pci_ops = { + .map_bus = mt7621_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static u32 read_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg) +{ + u32 address = mt7621_pci_get_cfgaddr(0, dev, 0, reg); + + pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); + return pcie_read(pcie, RALINK_PCI_CONFIG_DATA); +} + +static void write_config(struct mt7621_pcie *pcie, unsigned int dev, + u32 reg, u32 val) +{ + u32 address = mt7621_pci_get_cfgaddr(0, dev, 0, reg); + + pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); + pcie_write(pcie, val, RALINK_PCI_CONFIG_DATA); +} + +static inline void mt7621_rst_gpio_pcie_assert(struct mt7621_pcie_port *port) +{ + if (port->gpio_rst) + gpiod_set_value(port->gpio_rst, 1); +} + +static inline void mt7621_rst_gpio_pcie_deassert(struct mt7621_pcie_port *port) +{ + if (port->gpio_rst) + gpiod_set_value(port->gpio_rst, 0); +} + +static inline bool mt7621_pcie_port_is_linkup(struct mt7621_pcie_port *port) +{ + return (pcie_port_read(port, RALINK_PCI_STATUS) & PCIE_PORT_LINKUP) != 0; +} + +static inline void mt7621_control_assert(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + + if (pcie->resets_inverted) + reset_control_assert(port->pcie_rst); + else + reset_control_deassert(port->pcie_rst); +} + +static inline void mt7621_control_deassert(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + + if (pcie->resets_inverted) + reset_control_deassert(port->pcie_rst); + else + reset_control_assert(port->pcie_rst); +} + +static int setup_cm_memory_region(struct pci_host_bridge *host) +{ + struct mt7621_pcie *pcie = pci_host_bridge_priv(host); + struct device *dev = pcie->dev; + struct resource_entry *entry; + resource_size_t mask; + + entry = resource_list_first_type(&host->windows, IORESOURCE_MEM); + if (!entry) { + dev_err(dev, "cannot get memory resource\n"); + return -EINVAL; + } + + if (mips_cps_numiocu(0)) { + /* + * FIXME: hardware doesn't accept mask values with 1s after + * 0s (e.g. 0xffef), so it would be great to warn if that's + * about to happen + */ + mask = ~(entry->res->end - entry->res->start); + + write_gcr_reg1_base(entry->res->start); + write_gcr_reg1_mask(mask | CM_GCR_REGn_MASK_CMTGT_IOCU0); + dev_info(dev, "PCI coherence region base: 0x%08llx, mask/settings: 0x%08llx\n", + (unsigned long long)read_gcr_reg1_base(), + (unsigned long long)read_gcr_reg1_mask()); + } + + return 0; +} + +static int mt7621_pcie_parse_port(struct mt7621_pcie *pcie, + struct device_node *node, + int slot) +{ + struct mt7621_pcie_port *port; + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + char name[10]; + int err; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->base = devm_platform_ioremap_resource(pdev, slot + 1); + if (IS_ERR(port->base)) + return PTR_ERR(port->base); + + port->clk = devm_get_clk_from_child(dev, node, NULL); + if (IS_ERR(port->clk)) { + dev_err(dev, "failed to get pcie%d clock\n", slot); + return PTR_ERR(port->clk); + } + + port->pcie_rst = of_reset_control_get_exclusive(node, NULL); + if (PTR_ERR(port->pcie_rst) == -EPROBE_DEFER) { + dev_err(dev, "failed to get pcie%d reset control\n", slot); + return PTR_ERR(port->pcie_rst); + } + + snprintf(name, sizeof(name), "pcie-phy%d", slot); + port->phy = devm_of_phy_get(dev, node, name); + if (IS_ERR(port->phy)) { + dev_err(dev, "failed to get pcie-phy%d\n", slot); + err = PTR_ERR(port->phy); + goto remove_reset; + } + + port->gpio_rst = devm_gpiod_get_index_optional(dev, "reset", slot, + GPIOD_OUT_LOW); + if (IS_ERR(port->gpio_rst)) { + dev_err(dev, "failed to get GPIO for PCIe%d\n", slot); + err = PTR_ERR(port->gpio_rst); + goto remove_reset; + } + + port->slot = slot; + port->pcie = pcie; + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + return 0; + +remove_reset: + reset_control_put(port->pcie_rst); + return err; +} + +static int mt7621_pcie_parse_dt(struct mt7621_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *node = dev->of_node, *child; + int err; + + pcie->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + for_each_available_child_of_node(node, child) { + int slot; + + err = of_pci_get_devfn(child); + if (err < 0) { + of_node_put(child); + dev_err(dev, "failed to parse devfn: %d\n", err); + return err; + } + + slot = PCI_SLOT(err); + + err = mt7621_pcie_parse_port(pcie, child, slot); + if (err) { + of_node_put(child); + return err; + } + } + + return 0; +} + +static int mt7621_pcie_init_port(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + u32 slot = port->slot; + int err; + + err = phy_init(port->phy); + if (err) { + dev_err(dev, "failed to initialize port%d phy\n", slot); + return err; + } + + err = phy_power_on(port->phy); + if (err) { + dev_err(dev, "failed to power on port%d phy\n", slot); + phy_exit(port->phy); + return err; + } + + port->enabled = true; + + return 0; +} + +static void mt7621_pcie_reset_assert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) { + /* PCIe RC reset assert */ + mt7621_control_assert(port); + + /* PCIe EP reset assert */ + mt7621_rst_gpio_pcie_assert(port); + } + + msleep(PERST_DELAY_MS); +} + +static void mt7621_pcie_reset_rc_deassert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + mt7621_control_deassert(port); +} + +static void mt7621_pcie_reset_ep_deassert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + mt7621_rst_gpio_pcie_deassert(port); + + msleep(PERST_DELAY_MS); +} + +static int mt7621_pcie_init_ports(struct mt7621_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct mt7621_pcie_port *port, *tmp; + u8 num_disabled = 0; + int err; + + mt7621_pcie_reset_assert(pcie); + mt7621_pcie_reset_rc_deassert(pcie); + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + u32 slot = port->slot; + + if (slot == 1) { + port->enabled = true; + continue; + } + + err = mt7621_pcie_init_port(port); + if (err) { + dev_err(dev, "initializing port %d failed\n", slot); + list_del(&port->list); + } + } + + mt7621_pcie_reset_ep_deassert(pcie); + + tmp = NULL; + list_for_each_entry(port, &pcie->ports, list) { + u32 slot = port->slot; + + if (!mt7621_pcie_port_is_linkup(port)) { + dev_err(dev, "pcie%d no card, disable it (RST & CLK)\n", + slot); + mt7621_control_assert(port); + port->enabled = false; + num_disabled++; + + if (slot == 0) { + tmp = port; + continue; + } + + if (slot == 1 && tmp && !tmp->enabled) + phy_power_off(tmp->phy); + } + } + + return (num_disabled != PCIE_PORT_CNT) ? 0 : -ENODEV; +} + +static void mt7621_pcie_enable_port(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + u32 slot = port->slot; + u32 val; + + /* enable pcie interrupt */ + val = pcie_read(pcie, RALINK_PCI_PCIMSK_ADDR); + val |= PCIE_PORT_INT_EN(slot); + pcie_write(pcie, val, RALINK_PCI_PCIMSK_ADDR); + + /* map 2G DDR region */ + pcie_port_write(port, PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE, + PCI_BASE_ADDRESS_0); + + /* configure class code and revision ID */ + pcie_port_write(port, PCIE_CLASS_CODE | PCIE_REVISION_ID, + RALINK_PCI_CLASS); + + /* configure RC FTS number to 250 when it leaves L0s */ + val = read_config(pcie, slot, PCIE_FTS_NUM); + val &= ~PCIE_FTS_NUM_MASK; + val |= PCIE_FTS_NUM_L0(0x50); + write_config(pcie, slot, PCIE_FTS_NUM, val); +} + +static int mt7621_pcie_enable_ports(struct pci_host_bridge *host) +{ + struct mt7621_pcie *pcie = pci_host_bridge_priv(host); + struct device *dev = pcie->dev; + struct mt7621_pcie_port *port; + struct resource_entry *entry; + int err; + + entry = resource_list_first_type(&host->windows, IORESOURCE_IO); + if (!entry) { + dev_err(dev, "cannot get io resource\n"); + return -EINVAL; + } + + /* Setup MEMWIN and IOWIN */ + pcie_write(pcie, 0xffffffff, RALINK_PCI_MEMBASE); + pcie_write(pcie, entry->res->start - entry->offset, RALINK_PCI_IOBASE); + + list_for_each_entry(port, &pcie->ports, list) { + if (port->enabled) { + err = clk_prepare_enable(port->clk); + if (err) { + dev_err(dev, "enabling clk pcie%d\n", + port->slot); + return err; + } + + mt7621_pcie_enable_port(port); + dev_info(dev, "PCIE%d enabled\n", port->slot); + } + } + + return 0; +} + +static int mt7621_pcie_register_host(struct pci_host_bridge *host) +{ + struct mt7621_pcie *pcie = pci_host_bridge_priv(host); + + host->ops = &mt7621_pci_ops; + host->sysdata = pcie; + return pci_host_probe(host); +} + +static const struct soc_device_attribute mt7621_pci_quirks_match[] = { + { .soc_id = "mt7621", .revision = "E2" } +}; + +static int mt7621_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct soc_device_attribute *attr; + struct mt7621_pcie_port *port; + struct mt7621_pcie *pcie; + struct pci_host_bridge *bridge; + int err; + + if (!dev->of_node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->dev = dev; + platform_set_drvdata(pdev, pcie); + INIT_LIST_HEAD(&pcie->ports); + + attr = soc_device_match(mt7621_pci_quirks_match); + if (attr) + pcie->resets_inverted = true; + + err = mt7621_pcie_parse_dt(pcie); + if (err) { + dev_err(dev, "parsing DT failed\n"); + return err; + } + + err = mt7621_pcie_init_ports(pcie); + if (err) { + dev_err(dev, "nothing connected in virtual bridges\n"); + return 0; + } + + err = mt7621_pcie_enable_ports(bridge); + if (err) { + dev_err(dev, "error enabling pcie ports\n"); + goto remove_resets; + } + + err = setup_cm_memory_region(bridge); + if (err) { + dev_err(dev, "error setting up iocu mem regions\n"); + goto remove_resets; + } + + return mt7621_pcie_register_host(bridge); + +remove_resets: + list_for_each_entry(port, &pcie->ports, list) + reset_control_put(port->pcie_rst); + + return err; +} + +static int mt7621_pci_remove(struct platform_device *pdev) +{ + struct mt7621_pcie *pcie = platform_get_drvdata(pdev); + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + reset_control_put(port->pcie_rst); + + return 0; +} + +static const struct of_device_id mt7621_pci_ids[] = { + { .compatible = "mediatek,mt7621-pci" }, + {}, +}; +MODULE_DEVICE_TABLE(of, mt7621_pci_ids); + +static struct platform_driver mt7621_pci_driver = { + .probe = mt7621_pci_probe, + .remove = mt7621_pci_remove, + .driver = { + .name = "mt7621-pci", + .of_match_table = of_match_ptr(mt7621_pci_ids), + }, +}; +builtin_platform_driver(mt7621_pci_driver); diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c index b4a288e24aaf..f9682df1da61 100644 --- a/drivers/pci/controller/pcie-rcar-ep.c +++ b/drivers/pci/controller/pcie-rcar-ep.c @@ -6,16 +6,13 @@ * Author: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> */ -#include <linux/clk.h> #include <linux/delay.h> #include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_pci.h> #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/pci-epc.h> -#include <linux/phy/phy.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include "pcie-rcar.h" @@ -159,7 +156,7 @@ static int rcar_pcie_ep_get_pdata(struct rcar_pcie_endpoint *ep, return 0; } -static int rcar_pcie_ep_write_header(struct pci_epc *epc, u8 fn, +static int rcar_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_header *hdr) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); @@ -195,7 +192,7 @@ static int rcar_pcie_ep_write_header(struct pci_epc *epc, u8 fn, return 0; } -static int rcar_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, +static int rcar_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { int flags = epf_bar->flags | LAR_ENABLE | LAM_64BIT; @@ -246,7 +243,7 @@ static int rcar_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, return 0; } -static void rcar_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, +static void rcar_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_bar *epf_bar) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); @@ -259,7 +256,8 @@ static void rcar_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, clear_bit(atu_index + 1, ep->ib_window_map); } -static int rcar_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 interrupts) +static int rcar_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, + u8 interrupts) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); struct rcar_pcie *pcie = &ep->pcie; @@ -272,7 +270,7 @@ static int rcar_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 interrupts) return 0; } -static int rcar_pcie_ep_get_msi(struct pci_epc *epc, u8 fn) +static int rcar_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); struct rcar_pcie *pcie = &ep->pcie; @@ -285,7 +283,7 @@ static int rcar_pcie_ep_get_msi(struct pci_epc *epc, u8 fn) return ((flags & MSICAP0_MMESE_MASK) >> MSICAP0_MMESE_OFFSET); } -static int rcar_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, +static int rcar_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr, u64 pci_addr, size_t size) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); @@ -322,7 +320,7 @@ static int rcar_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, return 0; } -static void rcar_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, +static void rcar_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr) { struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); @@ -403,7 +401,7 @@ static int rcar_pcie_ep_assert_msi(struct rcar_pcie *pcie, return 0; } -static int rcar_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, +static int rcar_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, enum pci_epc_irq_type type, u16 interrupt_num) { @@ -451,7 +449,7 @@ static const struct pci_epc_features rcar_pcie_epc_features = { }; static const struct pci_epc_features* -rcar_pcie_ep_get_features(struct pci_epc *epc, u8 func_no) +rcar_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { return &rcar_pcie_epc_features; } @@ -492,9 +490,9 @@ static int rcar_pcie_ep_probe(struct platform_device *pdev) pcie->dev = dev; pm_runtime_enable(dev); - err = pm_runtime_get_sync(dev); + err = pm_runtime_resume_and_get(dev); if (err < 0) { - dev_err(dev, "pm_runtime_get_sync failed\n"); + dev_err(dev, "pm_runtime_resume_and_get failed\n"); goto err_pm_disable; } diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c index 765cf2b45e24..e12c2d8be05a 100644 --- a/drivers/pci/controller/pcie-rcar-host.c +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -13,22 +13,22 @@ #include <linux/bitops.h> #include <linux/clk.h> +#include <linux/clk-provider.h> #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> #include <linux/init.h> +#include <linux/iopoll.h> #include <linux/msi.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_pci.h> #include <linux/of_platform.h> #include <linux/pci.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> -#include <linux/slab.h> #include "pcie-rcar.h" @@ -41,6 +41,21 @@ struct rcar_msi { int irq2; }; +#ifdef CONFIG_ARM +/* + * Here we keep a static copy of the remapped PCIe controller address. + * This is only used on aarch32 systems, all of which have one single + * PCIe controller, to provide quick access to the PCIe controller in + * the L1 link state fixup function, called from the ARM fault handler. + */ +static void __iomem *pcie_base; +/* + * Static copy of bus clock pointer, so we can check whether the clock + * is enabled or not. + */ +static struct clk *pcie_bus_clk; +#endif + /* Structure representing the PCIe interface */ struct rcar_pcie_host { struct rcar_pcie pcie; @@ -486,12 +501,10 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) while (reg) { unsigned int index = find_first_bit(®, 32); - unsigned int msi_irq; + int ret; - msi_irq = irq_find_mapping(msi->domain->parent, index); - if (msi_irq) { - generic_handle_irq(msi_irq); - } else { + ret = generic_handle_domain_irq(msi->domain->parent, index); + if (ret) { /* Unknown MSI, just clear it */ dev_dbg(dev, "unexpected MSI\n"); rcar_pci_write_reg(pcie, BIT(index), PCIEMSIFR); @@ -776,6 +789,12 @@ static int rcar_pcie_get_resources(struct rcar_pcie_host *host) } host->msi.irq2 = i; +#ifdef CONFIG_ARM + /* Cache static copy for L1 link state fixup hook on aarch32 */ + pcie_base = pcie->base; + pcie_bus_clk = host->bus_clk; +#endif + return 0; err_irq2: @@ -1031,4 +1050,67 @@ static struct platform_driver rcar_pcie_driver = { }, .probe = rcar_pcie_probe, }; + +#ifdef CONFIG_ARM +static DEFINE_SPINLOCK(pmsr_lock); +static int rcar_pcie_aarch32_abort_handler(unsigned long addr, + unsigned int fsr, struct pt_regs *regs) +{ + unsigned long flags; + u32 pmsr, val; + int ret = 0; + + spin_lock_irqsave(&pmsr_lock, flags); + + if (!pcie_base || !__clk_is_enabled(pcie_bus_clk)) { + ret = 1; + goto unlock_exit; + } + + pmsr = readl(pcie_base + PMSR); + + /* + * Test if the PCIe controller received PM_ENTER_L1 DLLP and + * the PCIe controller is not in L1 link state. If true, apply + * fix, which will put the controller into L1 link state, from + * which it can return to L0s/L0 on its own. + */ + if ((pmsr & PMEL1RX) && ((pmsr & PMSTATE) != PMSTATE_L1)) { + writel(L1IATN, pcie_base + PMCTLR); + ret = readl_poll_timeout_atomic(pcie_base + PMSR, val, + val & L1FAEG, 10, 1000); + WARN(ret, "Timeout waiting for L1 link state, ret=%d\n", ret); + writel(L1FAEG | PMEL1RX, pcie_base + PMSR); + } + +unlock_exit: + spin_unlock_irqrestore(&pmsr_lock, flags); + return ret; +} + +static const struct of_device_id rcar_pcie_abort_handler_of_match[] __initconst = { + { .compatible = "renesas,pcie-r8a7779" }, + { .compatible = "renesas,pcie-r8a7790" }, + { .compatible = "renesas,pcie-r8a7791" }, + { .compatible = "renesas,pcie-rcar-gen2" }, + {}, +}; + +static int __init rcar_pcie_init(void) +{ + if (of_find_matching_node(NULL, rcar_pcie_abort_handler_of_match)) { +#ifdef CONFIG_ARM_LPAE + hook_fault_code(17, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, + "asynchronous external abort"); +#else + hook_fault_code(22, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, + "imprecise external abort"); +#endif + } + + return platform_driver_register(&rcar_pcie_driver); +} +device_initcall(rcar_pcie_init); +#else builtin_platform_driver(rcar_pcie_driver); +#endif diff --git a/drivers/pci/controller/pcie-rcar.h b/drivers/pci/controller/pcie-rcar.h index d4c698b5f821..9bb125db85c6 100644 --- a/drivers/pci/controller/pcie-rcar.h +++ b/drivers/pci/controller/pcie-rcar.h @@ -85,6 +85,13 @@ #define LTSMDIS BIT(31) #define MACCTLR_INIT_VAL (LTSMDIS | MACCTLR_NFTS_MASK) #define PMSR 0x01105c +#define L1FAEG BIT(31) +#define PMEL1RX BIT(23) +#define PMSTATE GENMASK(18, 16) +#define PMSTATE_L1 (3 << 16) +#define PMCTLR 0x011060 +#define L1IATN BIT(31) + #define MACS2R 0x011078 #define MACCGSPSETR 0x011084 #define SPCNGRSN BIT(31) diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c index 7631dc3961c1..5fb9ce6e536e 100644 --- a/drivers/pci/controller/pcie-rockchip-ep.c +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -122,7 +122,7 @@ static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, ROCKCHIP_PCIE_AT_OB_REGION_CPU_ADDR1(r)); } -static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, +static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_header *hdr) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); @@ -159,7 +159,7 @@ static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, return 0; } -static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, +static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_bar *epf_bar) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); @@ -227,7 +227,7 @@ static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, return 0; } -static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, +static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, struct pci_epf_bar *epf_bar) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); @@ -256,7 +256,7 @@ static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); } -static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, +static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr, u64 pci_addr, size_t size) { @@ -284,7 +284,7 @@ static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, return 0; } -static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, +static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, phys_addr_t addr) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); @@ -308,7 +308,7 @@ static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, clear_bit(r, &ep->ob_region_map); } -static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, +static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 multi_msg_cap) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); @@ -329,7 +329,7 @@ static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, return 0; } -static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn) +static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) { struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); struct rockchip_pcie *rockchip = &ep->rockchip; @@ -471,7 +471,7 @@ static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, return 0; } -static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, +static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, enum pci_epc_irq_type type, u16 interrupt_num) { @@ -510,7 +510,7 @@ static const struct pci_epc_features rockchip_pcie_epc_features = { }; static const struct pci_epc_features* -rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no) +rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { return &rockchip_pcie_epc_features; } diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c index f1d08a1b1591..c52316d0bfd2 100644 --- a/drivers/pci/controller/pcie-rockchip-host.c +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -517,7 +517,7 @@ static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) struct device *dev = rockchip->dev; u32 reg; u32 hwirq; - u32 virq; + int ret; chained_irq_enter(chip, desc); @@ -528,10 +528,8 @@ static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) hwirq = ffs(reg) - 1; reg &= ~BIT(hwirq); - virq = irq_find_mapping(rockchip->irq_domain, hwirq); - if (virq) - generic_handle_irq(virq); - else + ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); + if (ret) dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); } @@ -592,10 +590,6 @@ static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) if (err) return err; - err = rockchip_pcie_setup_irq(rockchip); - if (err) - return err; - rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); if (IS_ERR(rockchip->vpcie12v)) { if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) @@ -973,8 +967,6 @@ static int rockchip_pcie_probe(struct platform_device *pdev) if (err) goto err_vpcie; - rockchip_pcie_enable_interrupts(rockchip); - err = rockchip_pcie_init_irq_domain(rockchip); if (err < 0) goto err_deinit_port; @@ -992,6 +984,12 @@ static int rockchip_pcie_probe(struct platform_device *pdev) bridge->sysdata = rockchip; bridge->ops = &rockchip_pcie_ops; + err = rockchip_pcie_setup_irq(rockchip); + if (err) + goto err_remove_irq_domain; + + rockchip_pcie_enable_interrupts(rockchip); + err = pci_host_probe(bridge); if (err < 0) goto err_remove_irq_domain; diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c index 67937facd90c..95426df03200 100644 --- a/drivers/pci/controller/pcie-xilinx-cpm.c +++ b/drivers/pci/controller/pcie-xilinx-cpm.c @@ -222,7 +222,7 @@ static void xilinx_cpm_pcie_intx_flow(struct irq_desc *desc) pcie_read(port, XILINX_CPM_PCIE_REG_IDRN)); for_each_set_bit(i, &val, PCI_NUM_INTX) - generic_handle_irq(irq_find_mapping(port->intx_domain, i)); + generic_handle_domain_irq(port->intx_domain, i); chained_irq_exit(chip, desc); } @@ -282,7 +282,7 @@ static void xilinx_cpm_pcie_event_flow(struct irq_desc *desc) val = pcie_read(port, XILINX_CPM_PCIE_REG_IDR); val &= pcie_read(port, XILINX_CPM_PCIE_REG_IMR); for_each_set_bit(i, &val, 32) - generic_handle_irq(irq_find_mapping(port->cpm_domain, i)); + generic_handle_domain_irq(port->cpm_domain, i); pcie_write(port, val, XILINX_CPM_PCIE_REG_IDR); /* diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c index 8689311c5ef6..a72b4f9a2b00 100644 --- a/drivers/pci/controller/pcie-xilinx-nwl.c +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -6,6 +6,7 @@ * (C) Copyright 2014 - 2015, Xilinx, Inc. */ +#include <linux/clk.h> #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/irq.h> @@ -169,6 +170,7 @@ struct nwl_pcie { u8 last_busno; struct nwl_msi msi; struct irq_domain *legacy_irq_domain; + struct clk *clk; raw_spinlock_t leg_mask_lock; }; @@ -318,18 +320,14 @@ static void nwl_pcie_leg_handler(struct irq_desc *desc) struct nwl_pcie *pcie; unsigned long status; u32 bit; - u32 virq; chained_irq_enter(chip, desc); pcie = irq_desc_get_handler_data(desc); while ((status = nwl_bridge_readl(pcie, MSGF_LEG_STATUS) & MSGF_LEG_SR_MASKALL) != 0) { - for_each_set_bit(bit, &status, PCI_NUM_INTX) { - virq = irq_find_mapping(pcie->legacy_irq_domain, bit); - if (virq) - generic_handle_irq(virq); - } + for_each_set_bit(bit, &status, PCI_NUM_INTX) + generic_handle_domain_irq(pcie->legacy_irq_domain, bit); } chained_irq_exit(chip, desc); @@ -340,16 +338,13 @@ static void nwl_pcie_handle_msi_irq(struct nwl_pcie *pcie, u32 status_reg) struct nwl_msi *msi; unsigned long status; u32 bit; - u32 virq; msi = &pcie->msi; while ((status = nwl_bridge_readl(pcie, status_reg)) != 0) { for_each_set_bit(bit, &status, 32) { nwl_bridge_writel(pcie, 1 << bit, status_reg); - virq = irq_find_mapping(msi->dev_domain, bit); - if (virq) - generic_handle_irq(virq); + generic_handle_domain_irq(msi->dev_domain, bit); } } } @@ -823,6 +818,16 @@ static int nwl_pcie_probe(struct platform_device *pdev) return err; } + pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + err = clk_prepare_enable(pcie->clk); + if (err) { + dev_err(dev, "can't enable PCIe ref clock\n"); + return err; + } + err = nwl_pcie_bridge_init(pcie); if (err) { dev_err(dev, "HW Initialization failed\n"); diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 14001febf59a..aa9bdcebc838 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -385,7 +385,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data) } if (status & (XILINX_PCIE_INTR_INTX | XILINX_PCIE_INTR_MSI)) { - unsigned int irq; + struct irq_domain *domain; val = pcie_read(port, XILINX_PCIE_REG_RPIFR1); @@ -399,19 +399,18 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data) if (val & XILINX_PCIE_RPIFR1_MSI_INTR) { val = pcie_read(port, XILINX_PCIE_REG_RPIFR2) & XILINX_PCIE_RPIFR2_MSG_DATA; - irq = irq_find_mapping(port->msi_domain->parent, val); + domain = port->msi_domain->parent; } else { val = (val & XILINX_PCIE_RPIFR1_INTR_MASK) >> XILINX_PCIE_RPIFR1_INTR_SHIFT; - irq = irq_find_mapping(port->leg_domain, val); + domain = port->leg_domain; } /* Clear interrupt FIFO register 1 */ pcie_write(port, XILINX_PCIE_RPIFR1_ALL_MASK, XILINX_PCIE_REG_RPIFR1); - if (irq) - generic_handle_irq(irq); + generic_handle_domain_irq(domain, val); } if (status & XILINX_PCIE_INTR_SLV_UNSUPP) diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index e3fcdfec58b3..a45e8e59d3d4 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -6,19 +6,19 @@ #include <linux/device.h> #include <linux/interrupt.h> +#include <linux/iommu.h> #include <linux/irq.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/msi.h> #include <linux/pci.h> +#include <linux/pci-acpi.h> #include <linux/pci-ecam.h> #include <linux/srcu.h> #include <linux/rculist.h> #include <linux/rcupdate.h> #include <asm/irqdomain.h> -#include <asm/device.h> -#include <asm/msi.h> #define VMD_CFGBAR 0 #define VMD_MEMBAR1 2 @@ -69,6 +69,8 @@ enum vmd_features { VMD_FEAT_CAN_BYPASS_MSI_REMAP = (1 << 4), }; +static DEFINE_IDA(vmd_instance_ida); + /* * Lock for manipulating VMD IRQ lists. */ @@ -119,6 +121,8 @@ struct vmd_dev { struct pci_bus *bus; u8 busn_start; u8 first_vec; + char *name; + int instance; }; static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus) @@ -447,6 +451,56 @@ static struct pci_ops vmd_ops = { .write = vmd_pci_write, }; +#ifdef CONFIG_ACPI +static struct acpi_device *vmd_acpi_find_companion(struct pci_dev *pci_dev) +{ + struct pci_host_bridge *bridge; + u32 busnr, addr; + + if (pci_dev->bus->ops != &vmd_ops) + return NULL; + + bridge = pci_find_host_bridge(pci_dev->bus); + busnr = pci_dev->bus->number - bridge->bus->number; + /* + * The address computation below is only applicable to relative bus + * numbers below 32. + */ + if (busnr > 31) + return NULL; + + addr = (busnr << 24) | ((u32)pci_dev->devfn << 16) | 0x8000FFFFU; + + dev_dbg(&pci_dev->dev, "Looking for ACPI companion (address 0x%x)\n", + addr); + + return acpi_find_child_device(ACPI_COMPANION(bridge->dev.parent), addr, + false); +} + +static bool hook_installed; + +static void vmd_acpi_begin(void) +{ + if (pci_acpi_set_companion_lookup_hook(vmd_acpi_find_companion)) + return; + + hook_installed = true; +} + +static void vmd_acpi_end(void) +{ + if (!hook_installed) + return; + + pci_acpi_clear_companion_lookup_hook(); + hook_installed = false; +} +#else +static inline void vmd_acpi_begin(void) { } +static inline void vmd_acpi_end(void) { } +#endif /* CONFIG_ACPI */ + static void vmd_attach_resources(struct vmd_dev *vmd) { vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1]; @@ -599,7 +653,7 @@ static int vmd_alloc_irqs(struct vmd_dev *vmd) INIT_LIST_HEAD(&vmd->irqs[i].irq_list); err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i), vmd_irq, IRQF_NO_THREAD, - "vmd", &vmd->irqs[i]); + vmd->name, &vmd->irqs[i]); if (err) return err; } @@ -710,7 +764,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) * acceptable because the guest is usually CPU-limited and MSI * remapping doesn't become a performance bottleneck. */ - if (!(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) || + if (iommu_capable(vmd->dev->dev.bus, IOMMU_CAP_INTR_REMAP) || + !(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) || offset[0] || offset[1]) { ret = vmd_alloc_irqs(vmd); if (ret) @@ -747,6 +802,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) if (vmd->irq_domain) dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); + vmd_acpi_begin(); + pci_scan_child_bus(vmd->bus); pci_assign_unassigned_bus_resources(vmd->bus); @@ -760,6 +817,8 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) pci_bus_add_devices(vmd->bus); + vmd_acpi_end(); + WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj, "domain"), "Can't create symlink to domain\n"); return 0; @@ -779,18 +838,32 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) return -ENOMEM; vmd->dev = dev; + vmd->instance = ida_simple_get(&vmd_instance_ida, 0, 0, GFP_KERNEL); + if (vmd->instance < 0) + return vmd->instance; + + vmd->name = kasprintf(GFP_KERNEL, "vmd%d", vmd->instance); + if (!vmd->name) { + err = -ENOMEM; + goto out_release_instance; + } + err = pcim_enable_device(dev); if (err < 0) - return err; + goto out_release_instance; vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0); - if (!vmd->cfgbar) - return -ENOMEM; + if (!vmd->cfgbar) { + err = -ENOMEM; + goto out_release_instance; + } pci_set_master(dev); if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) && - dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) - return -ENODEV; + dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) { + err = -ENODEV; + goto out_release_instance; + } if (features & VMD_FEAT_OFFSET_FIRST_VECTOR) vmd->first_vec = 1; @@ -799,11 +872,16 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) pci_set_drvdata(dev, vmd); err = vmd_enable_domain(vmd, features); if (err) - return err; + goto out_release_instance; dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n", vmd->sysdata.domain); return 0; + + out_release_instance: + ida_simple_remove(&vmd_instance_ida, vmd->instance); + kfree(vmd->name); + return err; } static void vmd_cleanup_srcu(struct vmd_dev *vmd) @@ -824,6 +902,8 @@ static void vmd_remove(struct pci_dev *dev) vmd_cleanup_srcu(vmd); vmd_detach_resources(vmd); vmd_remove_irq_domain(vmd); + ida_simple_remove(&vmd_instance_ida, vmd->instance); + kfree(vmd->name); } #ifdef CONFIG_PM_SLEEP @@ -848,7 +928,7 @@ static int vmd_resume(struct device *dev) for (i = 0; i < vmd->msix_count; i++) { err = devm_request_irq(dev, pci_irq_vector(pdev, i), vmd_irq, IRQF_NO_THREAD, - "vmd", &vmd->irqs[i]); + vmd->name, &vmd->irqs[i]); if (err) return err; } diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index d2a1920bb055..1c40d2506aef 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -32,7 +32,7 @@ struct pci_config_window *pci_ecam_create(struct device *dev, struct pci_config_window *cfg; unsigned int bus_range, bus_range_max, bsz; struct resource *conflict; - int i, err; + int err; if (busr->start > busr->end) return ERR_PTR(-EINVAL); @@ -50,6 +50,7 @@ struct pci_config_window *pci_ecam_create(struct device *dev, cfg->busr.start = busr->start; cfg->busr.end = busr->end; cfg->busr.flags = IORESOURCE_BUS; + cfg->bus_shift = bus_shift; bus_range = resource_size(&cfg->busr); bus_range_max = resource_size(cfgres) >> bus_shift; if (bus_range > bus_range_max) { @@ -77,13 +78,6 @@ struct pci_config_window *pci_ecam_create(struct device *dev, cfg->winp = kcalloc(bus_range, sizeof(*cfg->winp), GFP_KERNEL); if (!cfg->winp) goto err_exit_malloc; - for (i = 0; i < bus_range; i++) { - cfg->winp[i] = - pci_remap_cfgspace(cfgres->start + i * bsz, - bsz); - if (!cfg->winp[i]) - goto err_exit_iomap; - } } else { cfg->win = pci_remap_cfgspace(cfgres->start, bus_range * bsz); if (!cfg->win) @@ -129,6 +123,44 @@ void pci_ecam_free(struct pci_config_window *cfg) } EXPORT_SYMBOL_GPL(pci_ecam_free); +static int pci_ecam_add_bus(struct pci_bus *bus) +{ + struct pci_config_window *cfg = bus->sysdata; + unsigned int bsz = 1 << cfg->bus_shift; + unsigned int busn = bus->number; + phys_addr_t start; + + if (!per_bus_mapping) + return 0; + + if (busn < cfg->busr.start || busn > cfg->busr.end) + return -EINVAL; + + busn -= cfg->busr.start; + start = cfg->res.start + busn * bsz; + + cfg->winp[busn] = pci_remap_cfgspace(start, bsz); + if (!cfg->winp[busn]) + return -ENOMEM; + + return 0; +} + +static void pci_ecam_remove_bus(struct pci_bus *bus) +{ + struct pci_config_window *cfg = bus->sysdata; + unsigned int busn = bus->number; + + if (!per_bus_mapping || busn < cfg->busr.start || busn > cfg->busr.end) + return; + + busn -= cfg->busr.start; + if (cfg->winp[busn]) { + iounmap(cfg->winp[busn]); + cfg->winp[busn] = NULL; + } +} + /* * Function to implement the pci_ops ->map_bus method */ @@ -167,6 +199,8 @@ EXPORT_SYMBOL_GPL(pci_ecam_map_bus); /* ECAM ops */ const struct pci_ecam_ops pci_generic_ecam_ops = { .pci_ops = { + .add_bus = pci_ecam_add_bus, + .remove_bus = pci_ecam_remove_bus, .map_bus = pci_ecam_map_bus, .read = pci_generic_config_read, .write = pci_generic_config_write, @@ -178,6 +212,8 @@ EXPORT_SYMBOL_GPL(pci_generic_ecam_ops); /* ECAM ops for 32-bit access only (non-compliant) */ const struct pci_ecam_ops pci_32b_ops = { .pci_ops = { + .add_bus = pci_ecam_add_bus, + .remove_bus = pci_ecam_remove_bus, .map_bus = pci_ecam_map_bus, .read = pci_generic_config_read32, .write = pci_generic_config_write32, @@ -187,6 +223,8 @@ const struct pci_ecam_ops pci_32b_ops = { /* ECAM ops for 32-bit read only (non-compliant) */ const struct pci_ecam_ops pci_32b_read_ops = { .pci_ops = { + .add_bus = pci_ecam_add_bus, + .remove_bus = pci_ecam_remove_bus, .map_bus = pci_ecam_map_bus, .read = pci_generic_config_read32, .write = pci_generic_config_write, diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index bce274d02dcf..5a03401f4571 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -87,6 +87,7 @@ struct epf_ntb { struct epf_ntb_epc { u8 func_no; + u8 vfunc_no; bool linkup; bool is_msix; int msix_bar; @@ -143,14 +144,15 @@ static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) struct epf_ntb_epc *ntb_epc; struct epf_ntb_ctrl *ctrl; struct pci_epc *epc; + u8 func_no, vfunc_no; bool is_msix; - u8 func_no; int ret; for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { ntb_epc = ntb->epc[type]; epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; is_msix = ntb_epc->is_msix; ctrl = ntb_epc->reg; if (link_up) @@ -158,7 +160,7 @@ static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) else ctrl->link_status &= ~LINK_STATUS_UP; irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; - ret = pci_epc_raise_irq(epc, func_no, irq_type, 1); + ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1); if (ret) { dev_err(&epc->dev, "%s intf: Failed to raise Link Up IRQ\n", @@ -238,10 +240,10 @@ static int epf_ntb_configure_mw(struct epf_ntb *ntb, enum pci_barno peer_barno; struct epf_ntb_ctrl *ctrl; phys_addr_t phys_addr; + u8 func_no, vfunc_no; struct pci_epc *epc; u64 addr, size; int ret = 0; - u8 func_no; ntb_epc = ntb->epc[type]; epc = ntb_epc->epc; @@ -267,8 +269,9 @@ static int epf_ntb_configure_mw(struct epf_ntb *ntb, } func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; - ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size); + ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size); if (ret) dev_err(&epc->dev, "%s intf: Failed to map memory window %d address\n", @@ -296,8 +299,8 @@ static void epf_ntb_teardown_mw(struct epf_ntb *ntb, enum pci_barno peer_barno; struct epf_ntb_ctrl *ctrl; phys_addr_t phys_addr; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; ntb_epc = ntb->epc[type]; epc = ntb_epc->epc; @@ -311,8 +314,9 @@ static void epf_ntb_teardown_mw(struct epf_ntb *ntb, if (mw + NTB_MW_OFFSET == BAR_DB_MW1) phys_addr += ctrl->mw1_offset; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; - pci_epc_unmap_addr(epc, func_no, phys_addr); + pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr); } /** @@ -385,8 +389,8 @@ static int epf_ntb_configure_msi(struct epf_ntb *ntb, struct epf_ntb_ctrl *peer_ctrl; enum pci_barno peer_barno; phys_addr_t phys_addr; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; int ret, i; ntb_epc = ntb->epc[type]; @@ -400,8 +404,9 @@ static int epf_ntb_configure_msi(struct epf_ntb *ntb, phys_addr = peer_epf_bar->phys_addr; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; - ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count, + ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count, db_entry_size, &db_data, &db_offset); if (ret) { dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", @@ -491,10 +496,10 @@ static int epf_ntb_configure_msix(struct epf_ntb *ntb, u32 db_entry_size, msg_data; enum pci_barno peer_barno; phys_addr_t phys_addr; + u8 func_no, vfunc_no; struct pci_epc *epc; size_t align; u64 msg_addr; - u8 func_no; int ret, i; ntb_epc = ntb->epc[type]; @@ -512,12 +517,13 @@ static int epf_ntb_configure_msix(struct epf_ntb *ntb, align = epc_features->align; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; db_entry_size = peer_ctrl->db_entry_size; for (i = 0; i < db_count; i++) { msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); msg_data = msix_tbl[i].msg_data; - ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr, + ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr, db_entry_size); if (ret) { dev_err(&epc->dev, @@ -586,8 +592,8 @@ epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) struct pci_epf_bar *peer_epf_bar; enum pci_barno peer_barno; phys_addr_t phys_addr; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; ntb_epc = ntb->epc[type]; epc = ntb_epc->epc; @@ -597,8 +603,9 @@ epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; phys_addr = peer_epf_bar->phys_addr; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; - pci_epc_unmap_addr(epc, func_no, phys_addr); + pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr); } /** @@ -728,14 +735,15 @@ static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) { struct pci_epf_bar *epf_bar; enum pci_barno barno; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; epf_bar = &ntb_epc->epf_bar[barno]; - pci_epc_clear_bar(epc, func_no, epf_bar); + pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); } /** @@ -775,9 +783,9 @@ static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, struct pci_epf_bar *peer_epf_bar, *epf_bar; enum pci_barno peer_barno, barno; u32 peer_spad_offset; + u8 func_no, vfunc_no; struct pci_epc *epc; struct device *dev; - u8 func_no; int ret; dev = &ntb->epf->dev; @@ -790,6 +798,7 @@ static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; epf_bar = &ntb_epc->epf_bar[barno]; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; epc = ntb_epc->epc; peer_spad_offset = peer_ntb_epc->reg->spad_offset; @@ -798,7 +807,7 @@ static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, epf_bar->barno = barno; epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; - ret = pci_epc_set_bar(epc, func_no, epf_bar); + ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); if (ret) { dev_err(dev, "%s intf: peer SPAD BAR set failed\n", pci_epc_interface_string(type)); @@ -842,14 +851,15 @@ static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) { struct pci_epf_bar *epf_bar; enum pci_barno barno; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; epf_bar = &ntb_epc->epf_bar[barno]; - pci_epc_clear_bar(epc, func_no, epf_bar); + pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); } /** @@ -886,10 +896,10 @@ static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) { struct pci_epf_bar *epf_bar; enum pci_barno barno; + u8 func_no, vfunc_no; struct epf_ntb *ntb; struct pci_epc *epc; struct device *dev; - u8 func_no; int ret; ntb = ntb_epc->epf_ntb; @@ -897,10 +907,11 @@ static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; epf_bar = &ntb_epc->epf_bar[barno]; - ret = pci_epc_set_bar(epc, func_no, epf_bar); + ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); if (ret) { dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", pci_epc_interface_string(ntb_epc->type)); @@ -1214,17 +1225,18 @@ static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) struct pci_epf_bar *epf_bar; enum epf_ntb_bar bar; enum pci_barno barno; + u8 func_no, vfunc_no; struct pci_epc *epc; - u8 func_no; epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { barno = ntb_epc->epf_ntb_bar[bar]; epf_bar = &ntb_epc->epf_bar[barno]; - pci_epc_clear_bar(epc, func_no, epf_bar); + pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar); } } @@ -1263,10 +1275,10 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, const struct pci_epc_features *epc_features; bool msix_capable, msi_capable; struct epf_ntb_epc *ntb_epc; + u8 func_no, vfunc_no; struct pci_epc *epc; struct device *dev; u32 db_count; - u8 func_no; int ret; ntb_epc = ntb->epc[type]; @@ -1282,6 +1294,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, } func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; db_count = ntb->db_count; if (db_count > MAX_DB_COUNT) { @@ -1293,7 +1306,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, epc = ntb_epc->epc; if (msi_capable) { - ret = pci_epc_set_msi(epc, func_no, db_count); + ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count); if (ret) { dev_err(dev, "%s intf: MSI configuration failed\n", pci_epc_interface_string(type)); @@ -1302,7 +1315,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, } if (msix_capable) { - ret = pci_epc_set_msix(epc, func_no, db_count, + ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count, ntb_epc->msix_bar, ntb_epc->msix_table_offset); if (ret) { @@ -1423,11 +1436,11 @@ static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, u32 num_mws, db_count; enum epf_ntb_bar bar; enum pci_barno barno; + u8 func_no, vfunc_no; struct pci_epc *epc; struct device *dev; size_t align; int ret, i; - u8 func_no; u64 size; ntb_epc = ntb->epc[type]; @@ -1437,6 +1450,7 @@ static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, epc_features = ntb_epc->epc_features; align = epc_features->align; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; epc = ntb_epc->epc; num_mws = ntb->num_mws; db_count = ntb->db_count; @@ -1464,7 +1478,7 @@ static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, barno = ntb_epc->epf_ntb_bar[bar]; epf_bar = &ntb_epc->epf_bar[barno]; - ret = pci_epc_set_bar(epc, func_no, epf_bar); + ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar); if (ret) { dev_err(dev, "%s intf: DoorBell BAR set failed\n", pci_epc_interface_string(type)); @@ -1536,9 +1550,9 @@ static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, const struct pci_epc_features *epc_features; struct pci_epf_bar *epf_bar; struct epf_ntb_epc *ntb_epc; + u8 func_no, vfunc_no; struct pci_epf *epf; struct device *dev; - u8 func_no; dev = &ntb->epf->dev; @@ -1547,6 +1561,7 @@ static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, return -ENOMEM; epf = ntb->epf; + vfunc_no = epf->vfunc_no; if (type == PRIMARY_INTERFACE) { func_no = epf->func_no; epf_bar = epf->bar; @@ -1558,11 +1573,12 @@ static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, ntb_epc->linkup = false; ntb_epc->epc = epc; ntb_epc->func_no = func_no; + ntb_epc->vfunc_no = vfunc_no; ntb_epc->type = type; ntb_epc->epf_bar = epf_bar; ntb_epc->epf_ntb = ntb; - epc_features = pci_epc_get_features(epc, func_no); + epc_features = pci_epc_get_features(epc, func_no, vfunc_no); if (!epc_features) return -EINVAL; ntb_epc->epc_features = epc_features; @@ -1702,10 +1718,10 @@ static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, enum pci_epc_interface_type type) { struct epf_ntb_epc *ntb_epc; + u8 func_no, vfunc_no; struct pci_epc *epc; struct pci_epf *epf; struct device *dev; - u8 func_no; int ret; ntb_epc = ntb->epc[type]; @@ -1713,6 +1729,7 @@ static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, dev = &epf->dev; epc = ntb_epc->epc; func_no = ntb_epc->func_no; + vfunc_no = ntb_epc->vfunc_no; ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); if (ret) { @@ -1742,11 +1759,13 @@ static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, goto err_db_mw_bar_init; } - ret = pci_epc_write_header(epc, func_no, epf->header); - if (ret) { - dev_err(dev, "%s intf: Configuration header write failed\n", - pci_epc_interface_string(type)); - goto err_write_header; + if (vfunc_no <= 1) { + ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); + if (ret) { + dev_err(dev, "%s intf: Configuration header write failed\n", + pci_epc_interface_string(type)); + goto err_write_header; + } } INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); @@ -1918,7 +1937,7 @@ static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ struct config_group *group = to_config_group(item); \ struct epf_ntb *ntb = to_epf_ntb(group); \ \ - return sprintf(page, "%d\n", ntb->_name); \ + return sysfs_emit(page, "%d\n", ntb->_name); \ } #define EPF_NTB_W(_name) \ @@ -1928,11 +1947,9 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ struct config_group *group = to_config_group(item); \ struct epf_ntb *ntb = to_epf_ntb(group); \ u32 val; \ - int ret; \ \ - ret = kstrtou32(page, 0, &val); \ - if (ret) \ - return ret; \ + if (kstrtou32(page, 0, &val) < 0) \ + return -EINVAL; \ \ ntb->_name = val; \ \ @@ -1949,7 +1966,7 @@ static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ \ sscanf(#_name, "mw%d", &win_no); \ \ - return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ + return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]); \ } #define EPF_NTB_MW_W(_name) \ @@ -1961,11 +1978,9 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ struct device *dev = &ntb->epf->dev; \ int win_no; \ u64 val; \ - int ret; \ \ - ret = kstrtou64(page, 0, &val); \ - if (ret) \ - return ret; \ + if (kstrtou64(page, 0, &val) < 0) \ + return -EINVAL; \ \ if (sscanf(#_name, "mw%d", &win_no) != 1) \ return -EINVAL; \ @@ -1986,11 +2001,9 @@ static ssize_t epf_ntb_num_mws_store(struct config_item *item, struct config_group *group = to_config_group(item); struct epf_ntb *ntb = to_epf_ntb(group); u32 val; - int ret; - ret = kstrtou32(page, 0, &val); - if (ret) - return ret; + if (kstrtou32(page, 0, &val) < 0) + return -EINVAL; if (val > MAX_MW) return -EINVAL; diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index d2708ca4bece..90d84d3bc868 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -247,8 +247,8 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test) goto err; } - ret = pci_epc_map_addr(epc, epf->func_no, src_phys_addr, reg->src_addr, - reg->size); + ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr, + reg->src_addr, reg->size); if (ret) { dev_err(dev, "Failed to map source address\n"); reg->status = STATUS_SRC_ADDR_INVALID; @@ -263,8 +263,8 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test) goto err_src_map_addr; } - ret = pci_epc_map_addr(epc, epf->func_no, dst_phys_addr, reg->dst_addr, - reg->size); + ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr, + reg->dst_addr, reg->size); if (ret) { dev_err(dev, "Failed to map destination address\n"); reg->status = STATUS_DST_ADDR_INVALID; @@ -291,13 +291,13 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test) pci_epf_test_print_rate("COPY", reg->size, &start, &end, use_dma); err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, dst_phys_addr); + pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr); err_dst_addr: pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size); err_src_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, src_phys_addr); + pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr); err_src_addr: pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size); @@ -331,8 +331,8 @@ static int pci_epf_test_read(struct pci_epf_test *epf_test) goto err; } - ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->src_addr, - reg->size); + ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, + reg->src_addr, reg->size); if (ret) { dev_err(dev, "Failed to map address\n"); reg->status = STATUS_SRC_ADDR_INVALID; @@ -386,7 +386,7 @@ err_dma_map: kfree(buf); err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, phys_addr); + pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); err_addr: pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size); @@ -419,8 +419,8 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test) goto err; } - ret = pci_epc_map_addr(epc, epf->func_no, phys_addr, reg->dst_addr, - reg->size); + ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr, + reg->dst_addr, reg->size); if (ret) { dev_err(dev, "Failed to map address\n"); reg->status = STATUS_DST_ADDR_INVALID; @@ -479,7 +479,7 @@ err_dma_map: kfree(buf); err_map_addr: - pci_epc_unmap_addr(epc, epf->func_no, phys_addr); + pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr); err_addr: pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size); @@ -501,13 +501,16 @@ static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, u8 irq_type, switch (irq_type) { case IRQ_TYPE_LEGACY: - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_LEGACY, 0); break; case IRQ_TYPE_MSI: - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI, irq); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_MSI, irq); break; case IRQ_TYPE_MSIX: - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX, irq); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_MSIX, irq); break; default: dev_err(dev, "Failed to raise IRQ, unknown type\n"); @@ -542,7 +545,8 @@ static void pci_epf_test_cmd_handler(struct work_struct *work) if (command & COMMAND_RAISE_LEGACY_IRQ) { reg->status = STATUS_IRQ_RAISED; - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_LEGACY, 0); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_LEGACY, 0); goto reset_handler; } @@ -580,22 +584,22 @@ static void pci_epf_test_cmd_handler(struct work_struct *work) } if (command & COMMAND_RAISE_MSI_IRQ) { - count = pci_epc_get_msi(epc, epf->func_no); + count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no); if (reg->irq_number > count || count <= 0) goto reset_handler; reg->status = STATUS_IRQ_RAISED; - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSI, - reg->irq_number); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_MSI, reg->irq_number); goto reset_handler; } if (command & COMMAND_RAISE_MSIX_IRQ) { - count = pci_epc_get_msix(epc, epf->func_no); + count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no); if (reg->irq_number > count || count <= 0) goto reset_handler; reg->status = STATUS_IRQ_RAISED; - pci_epc_raise_irq(epc, epf->func_no, PCI_EPC_IRQ_MSIX, - reg->irq_number); + pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, + PCI_EPC_IRQ_MSIX, reg->irq_number); goto reset_handler; } @@ -618,7 +622,8 @@ static void pci_epf_test_unbind(struct pci_epf *epf) epf_bar = &epf->bar[bar]; if (epf_test->reg[bar]) { - pci_epc_clear_bar(epc, epf->func_no, epf_bar); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, + epf_bar); pci_epf_free_space(epf, epf_test->reg[bar], bar, PRIMARY_INTERFACE); } @@ -650,7 +655,8 @@ static int pci_epf_test_set_bar(struct pci_epf *epf) if (!!(epc_features->reserved_bar & (1 << bar))) continue; - ret = pci_epc_set_bar(epc, epf->func_no, epf_bar); + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, + epf_bar); if (ret) { pci_epf_free_space(epf, epf_test->reg[bar], bar, PRIMARY_INTERFACE); @@ -674,16 +680,18 @@ static int pci_epf_test_core_init(struct pci_epf *epf) bool msi_capable = true; int ret; - epc_features = pci_epc_get_features(epc, epf->func_no); + epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); if (epc_features) { msix_capable = epc_features->msix_capable; msi_capable = epc_features->msi_capable; } - ret = pci_epc_write_header(epc, epf->func_no, header); - if (ret) { - dev_err(dev, "Configuration header write failed\n"); - return ret; + if (epf->vfunc_no <= 1) { + ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); + if (ret) { + dev_err(dev, "Configuration header write failed\n"); + return ret; + } } ret = pci_epf_test_set_bar(epf); @@ -691,7 +699,8 @@ static int pci_epf_test_core_init(struct pci_epf *epf) return ret; if (msi_capable) { - ret = pci_epc_set_msi(epc, epf->func_no, epf->msi_interrupts); + ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no, + epf->msi_interrupts); if (ret) { dev_err(dev, "MSI configuration failed\n"); return ret; @@ -699,7 +708,8 @@ static int pci_epf_test_core_init(struct pci_epf *epf) } if (msix_capable) { - ret = pci_epc_set_msix(epc, epf->func_no, epf->msix_interrupts, + ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no, + epf->msix_interrupts, epf_test->test_reg_bar, epf_test->msix_table_offset); if (ret) { @@ -832,7 +842,7 @@ static int pci_epf_test_bind(struct pci_epf *epf) if (WARN_ON_ONCE(!epc)) return -EINVAL; - epc_features = pci_epc_get_features(epc, epf->func_no); + epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); if (!epc_features) { dev_err(&epf->dev, "epc_features not implemented\n"); return -EOPNOTSUPP; diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index f3a8b833b479..d4850bdd837f 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -175,9 +175,8 @@ static ssize_t pci_epc_start_store(struct config_item *item, const char *page, epc = epc_group->epc; - ret = kstrtobool(page, &start); - if (ret) - return ret; + if (kstrtobool(page, &start) < 0) + return -EINVAL; if (!start) { pci_epc_stop(epc); @@ -198,8 +197,7 @@ static ssize_t pci_epc_start_store(struct config_item *item, const char *page, static ssize_t pci_epc_start_show(struct config_item *item, char *page) { - return sprintf(page, "%d\n", - to_pci_epc_group(item)->start); + return sysfs_emit(page, "%d\n", to_pci_epc_group(item)->start); } CONFIGFS_ATTR(pci_epc_, start); @@ -321,7 +319,7 @@ static ssize_t pci_epf_##_name##_show(struct config_item *item, char *page) \ struct pci_epf *epf = to_pci_epf_group(item)->epf; \ if (WARN_ON_ONCE(!epf->header)) \ return -EINVAL; \ - return sprintf(page, "0x%04x\n", epf->header->_name); \ + return sysfs_emit(page, "0x%04x\n", epf->header->_name); \ } #define PCI_EPF_HEADER_W_u32(_name) \ @@ -329,13 +327,11 @@ static ssize_t pci_epf_##_name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ u32 val; \ - int ret; \ struct pci_epf *epf = to_pci_epf_group(item)->epf; \ if (WARN_ON_ONCE(!epf->header)) \ return -EINVAL; \ - ret = kstrtou32(page, 0, &val); \ - if (ret) \ - return ret; \ + if (kstrtou32(page, 0, &val) < 0) \ + return -EINVAL; \ epf->header->_name = val; \ return len; \ } @@ -345,13 +341,11 @@ static ssize_t pci_epf_##_name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ u16 val; \ - int ret; \ struct pci_epf *epf = to_pci_epf_group(item)->epf; \ if (WARN_ON_ONCE(!epf->header)) \ return -EINVAL; \ - ret = kstrtou16(page, 0, &val); \ - if (ret) \ - return ret; \ + if (kstrtou16(page, 0, &val) < 0) \ + return -EINVAL; \ epf->header->_name = val; \ return len; \ } @@ -361,13 +355,11 @@ static ssize_t pci_epf_##_name##_store(struct config_item *item, \ const char *page, size_t len) \ { \ u8 val; \ - int ret; \ struct pci_epf *epf = to_pci_epf_group(item)->epf; \ if (WARN_ON_ONCE(!epf->header)) \ return -EINVAL; \ - ret = kstrtou8(page, 0, &val); \ - if (ret) \ - return ret; \ + if (kstrtou8(page, 0, &val) < 0) \ + return -EINVAL; \ epf->header->_name = val; \ return len; \ } @@ -376,11 +368,9 @@ static ssize_t pci_epf_msi_interrupts_store(struct config_item *item, const char *page, size_t len) { u8 val; - int ret; - ret = kstrtou8(page, 0, &val); - if (ret) - return ret; + if (kstrtou8(page, 0, &val) < 0) + return -EINVAL; to_pci_epf_group(item)->epf->msi_interrupts = val; @@ -390,19 +380,17 @@ static ssize_t pci_epf_msi_interrupts_store(struct config_item *item, static ssize_t pci_epf_msi_interrupts_show(struct config_item *item, char *page) { - return sprintf(page, "%d\n", - to_pci_epf_group(item)->epf->msi_interrupts); + return sysfs_emit(page, "%d\n", + to_pci_epf_group(item)->epf->msi_interrupts); } static ssize_t pci_epf_msix_interrupts_store(struct config_item *item, const char *page, size_t len) { u16 val; - int ret; - ret = kstrtou16(page, 0, &val); - if (ret) - return ret; + if (kstrtou16(page, 0, &val) < 0) + return -EINVAL; to_pci_epf_group(item)->epf->msix_interrupts = val; @@ -412,8 +400,8 @@ static ssize_t pci_epf_msix_interrupts_store(struct config_item *item, static ssize_t pci_epf_msix_interrupts_show(struct config_item *item, char *page) { - return sprintf(page, "%d\n", - to_pci_epf_group(item)->epf->msix_interrupts); + return sysfs_emit(page, "%d\n", + to_pci_epf_group(item)->epf->msix_interrupts); } PCI_EPF_HEADER_R(vendorid) @@ -475,6 +463,28 @@ static struct configfs_attribute *pci_epf_attrs[] = { NULL, }; +static int pci_epf_vepf_link(struct config_item *epf_pf_item, + struct config_item *epf_vf_item) +{ + struct pci_epf_group *epf_vf_group = to_pci_epf_group(epf_vf_item); + struct pci_epf_group *epf_pf_group = to_pci_epf_group(epf_pf_item); + struct pci_epf *epf_pf = epf_pf_group->epf; + struct pci_epf *epf_vf = epf_vf_group->epf; + + return pci_epf_add_vepf(epf_pf, epf_vf); +} + +static void pci_epf_vepf_unlink(struct config_item *epf_pf_item, + struct config_item *epf_vf_item) +{ + struct pci_epf_group *epf_vf_group = to_pci_epf_group(epf_vf_item); + struct pci_epf_group *epf_pf_group = to_pci_epf_group(epf_pf_item); + struct pci_epf *epf_pf = epf_pf_group->epf; + struct pci_epf *epf_vf = epf_vf_group->epf; + + pci_epf_remove_vepf(epf_pf, epf_vf); +} + static void pci_epf_release(struct config_item *item) { struct pci_epf_group *epf_group = to_pci_epf_group(item); @@ -487,6 +497,8 @@ static void pci_epf_release(struct config_item *item) } static struct configfs_item_operations pci_epf_ops = { + .allow_link = pci_epf_vepf_link, + .drop_link = pci_epf_vepf_unlink, .release = pci_epf_release, }; diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index adec9bee72cf..38621558d397 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -137,24 +137,29 @@ EXPORT_SYMBOL_GPL(pci_epc_get_next_free_bar); * @epc: the features supported by *this* EPC device will be returned * @func_no: the features supported by the EPC device specific to the * endpoint function with func_no will be returned + * @vfunc_no: the features supported by the EPC device specific to the + * virtual endpoint function with vfunc_no will be returned * * Invoke to get the features provided by the EPC which may be * specific to an endpoint function. Returns pci_epc_features on success * and NULL for any failures. */ const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, - u8 func_no) + u8 func_no, u8 vfunc_no) { const struct pci_epc_features *epc_features; if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return NULL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return NULL; + if (!epc->ops->get_features) return NULL; mutex_lock(&epc->lock); - epc_features = epc->ops->get_features(epc, func_no); + epc_features = epc->ops->get_features(epc, func_no, vfunc_no); mutex_unlock(&epc->lock); return epc_features; @@ -205,13 +210,14 @@ EXPORT_SYMBOL_GPL(pci_epc_start); /** * pci_epc_raise_irq() - interrupt the host system * @epc: the EPC device which has to interrupt the host - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @type: specify the type of interrupt; legacy, MSI or MSI-X * @interrupt_num: the MSI or MSI-X interrupt number * * Invoke to raise an legacy, MSI or MSI-X interrupt */ -int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, +int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, enum pci_epc_irq_type type, u16 interrupt_num) { int ret; @@ -219,11 +225,14 @@ int pci_epc_raise_irq(struct pci_epc *epc, u8 func_no, if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->raise_irq) return 0; mutex_lock(&epc->lock); - ret = epc->ops->raise_irq(epc, func_no, type, interrupt_num); + ret = epc->ops->raise_irq(epc, func_no, vfunc_no, type, interrupt_num); mutex_unlock(&epc->lock); return ret; @@ -235,6 +244,7 @@ EXPORT_SYMBOL_GPL(pci_epc_raise_irq); * MSI data * @epc: the EPC device which has the MSI capability * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @phys_addr: the physical address of the outbound region * @interrupt_num: the MSI interrupt number * @entry_size: Size of Outbound address region for each interrupt @@ -250,21 +260,25 @@ EXPORT_SYMBOL_GPL(pci_epc_raise_irq); * physical address (in outbound region) of the other interface to ring * doorbell. */ -int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, phys_addr_t phys_addr, - u8 interrupt_num, u32 entry_size, u32 *msi_data, - u32 *msi_addr_offset) +int pci_epc_map_msi_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + phys_addr_t phys_addr, u8 interrupt_num, u32 entry_size, + u32 *msi_data, u32 *msi_addr_offset) { int ret; if (IS_ERR_OR_NULL(epc)) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->map_msi_irq) return -EINVAL; mutex_lock(&epc->lock); - ret = epc->ops->map_msi_irq(epc, func_no, phys_addr, interrupt_num, - entry_size, msi_data, msi_addr_offset); + ret = epc->ops->map_msi_irq(epc, func_no, vfunc_no, phys_addr, + interrupt_num, entry_size, msi_data, + msi_addr_offset); mutex_unlock(&epc->lock); return ret; @@ -274,22 +288,26 @@ EXPORT_SYMBOL_GPL(pci_epc_map_msi_irq); /** * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated * @epc: the EPC device to which MSI interrupts was requested - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * * Invoke to get the number of MSI interrupts allocated by the RC */ -int pci_epc_get_msi(struct pci_epc *epc, u8 func_no) +int pci_epc_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { int interrupt; if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return 0; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return 0; + if (!epc->ops->get_msi) return 0; mutex_lock(&epc->lock); - interrupt = epc->ops->get_msi(epc, func_no); + interrupt = epc->ops->get_msi(epc, func_no, vfunc_no); mutex_unlock(&epc->lock); if (interrupt < 0) @@ -304,12 +322,13 @@ EXPORT_SYMBOL_GPL(pci_epc_get_msi); /** * pci_epc_set_msi() - set the number of MSI interrupt numbers required * @epc: the EPC device on which MSI has to be configured - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @interrupts: number of MSI interrupts required by the EPF * * Invoke to set the required number of MSI interrupts. */ -int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) +int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, u8 interrupts) { int ret; u8 encode_int; @@ -318,13 +337,16 @@ int pci_epc_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) interrupts > 32) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->set_msi) return 0; encode_int = order_base_2(interrupts); mutex_lock(&epc->lock); - ret = epc->ops->set_msi(epc, func_no, encode_int); + ret = epc->ops->set_msi(epc, func_no, vfunc_no, encode_int); mutex_unlock(&epc->lock); return ret; @@ -334,22 +356,26 @@ EXPORT_SYMBOL_GPL(pci_epc_set_msi); /** * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated * @epc: the EPC device to which MSI-X interrupts was requested - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * * Invoke to get the number of MSI-X interrupts allocated by the RC */ -int pci_epc_get_msix(struct pci_epc *epc, u8 func_no) +int pci_epc_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) { int interrupt; if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return 0; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return 0; + if (!epc->ops->get_msix) return 0; mutex_lock(&epc->lock); - interrupt = epc->ops->get_msix(epc, func_no); + interrupt = epc->ops->get_msix(epc, func_no, vfunc_no); mutex_unlock(&epc->lock); if (interrupt < 0) @@ -362,15 +388,16 @@ EXPORT_SYMBOL_GPL(pci_epc_get_msix); /** * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required * @epc: the EPC device on which MSI-X has to be configured - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @interrupts: number of MSI-X interrupts required by the EPF * @bir: BAR where the MSI-X table resides * @offset: Offset pointing to the start of MSI-X table * * Invoke to set the required number of MSI-X interrupts. */ -int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, - enum pci_barno bir, u32 offset) +int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u16 interrupts, enum pci_barno bir, u32 offset) { int ret; @@ -378,11 +405,15 @@ int pci_epc_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts, interrupts < 1 || interrupts > 2048) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->set_msix) return 0; mutex_lock(&epc->lock); - ret = epc->ops->set_msix(epc, func_no, interrupts - 1, bir, offset); + ret = epc->ops->set_msix(epc, func_no, vfunc_no, interrupts - 1, bir, + offset); mutex_unlock(&epc->lock); return ret; @@ -392,22 +423,26 @@ EXPORT_SYMBOL_GPL(pci_epc_set_msix); /** * pci_epc_unmap_addr() - unmap CPU address from PCI address * @epc: the EPC device on which address is allocated - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @phys_addr: physical address of the local system * * Invoke to unmap the CPU address from PCI address. */ -void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no, +void pci_epc_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t phys_addr) { if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return; + if (!epc->ops->unmap_addr) return; mutex_lock(&epc->lock); - epc->ops->unmap_addr(epc, func_no, phys_addr); + epc->ops->unmap_addr(epc, func_no, vfunc_no, phys_addr); mutex_unlock(&epc->lock); } EXPORT_SYMBOL_GPL(pci_epc_unmap_addr); @@ -415,14 +450,15 @@ EXPORT_SYMBOL_GPL(pci_epc_unmap_addr); /** * pci_epc_map_addr() - map CPU address to PCI address * @epc: the EPC device on which address is allocated - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @phys_addr: physical address of the local system * @pci_addr: PCI address to which the physical address should be mapped * @size: the size of the allocation * * Invoke to map CPU address with PCI address. */ -int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, +int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, phys_addr_t phys_addr, u64 pci_addr, size_t size) { int ret; @@ -430,11 +466,15 @@ int pci_epc_map_addr(struct pci_epc *epc, u8 func_no, if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->map_addr) return 0; mutex_lock(&epc->lock); - ret = epc->ops->map_addr(epc, func_no, phys_addr, pci_addr, size); + ret = epc->ops->map_addr(epc, func_no, vfunc_no, phys_addr, pci_addr, + size); mutex_unlock(&epc->lock); return ret; @@ -444,12 +484,13 @@ EXPORT_SYMBOL_GPL(pci_epc_map_addr); /** * pci_epc_clear_bar() - reset the BAR * @epc: the EPC device for which the BAR has to be cleared - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @epf_bar: the struct epf_bar that contains the BAR information * * Invoke to reset the BAR of the endpoint device. */ -void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, +void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions || @@ -457,11 +498,14 @@ void pci_epc_clear_bar(struct pci_epc *epc, u8 func_no, epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) return; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return; + if (!epc->ops->clear_bar) return; mutex_lock(&epc->lock); - epc->ops->clear_bar(epc, func_no, epf_bar); + epc->ops->clear_bar(epc, func_no, vfunc_no, epf_bar); mutex_unlock(&epc->lock); } EXPORT_SYMBOL_GPL(pci_epc_clear_bar); @@ -469,12 +513,13 @@ EXPORT_SYMBOL_GPL(pci_epc_clear_bar); /** * pci_epc_set_bar() - configure BAR in order for host to assign PCI addr space * @epc: the EPC device on which BAR has to be configured - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @epf_bar: the struct epf_bar that contains the BAR information * * Invoke to configure the BAR of the endpoint device. */ -int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, +int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_bar *epf_bar) { int ret; @@ -489,11 +534,14 @@ int pci_epc_set_bar(struct pci_epc *epc, u8 func_no, !(flags & PCI_BASE_ADDRESS_MEM_TYPE_64))) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + if (!epc->ops->set_bar) return 0; mutex_lock(&epc->lock); - ret = epc->ops->set_bar(epc, func_no, epf_bar); + ret = epc->ops->set_bar(epc, func_no, vfunc_no, epf_bar); mutex_unlock(&epc->lock); return ret; @@ -503,7 +551,8 @@ EXPORT_SYMBOL_GPL(pci_epc_set_bar); /** * pci_epc_write_header() - write standard configuration header * @epc: the EPC device to which the configuration header should be written - * @func_no: the endpoint function number in the EPC device + * @func_no: the physical endpoint function number in the EPC device + * @vfunc_no: the virtual endpoint function number in the physical function * @header: standard configuration header fields * * Invoke to write the configuration header to the endpoint controller. Every @@ -511,7 +560,7 @@ EXPORT_SYMBOL_GPL(pci_epc_set_bar); * configuration header would be written. The callback function should write * the header fields to this dedicated location. */ -int pci_epc_write_header(struct pci_epc *epc, u8 func_no, +int pci_epc_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *header) { int ret; @@ -519,11 +568,18 @@ int pci_epc_write_header(struct pci_epc *epc, u8 func_no, if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions) return -EINVAL; + if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no])) + return -EINVAL; + + /* Only Virtual Function #1 has deviceID */ + if (vfunc_no > 1) + return -EINVAL; + if (!epc->ops->write_header) return 0; mutex_lock(&epc->lock); - ret = epc->ops->write_header(epc, func_no, header); + ret = epc->ops->write_header(epc, func_no, vfunc_no, header); mutex_unlock(&epc->lock); return ret; @@ -548,7 +604,7 @@ int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf, u32 func_no; int ret = 0; - if (IS_ERR_OR_NULL(epc)) + if (IS_ERR_OR_NULL(epc) || epf->is_vf) return -EINVAL; if (type == PRIMARY_INTERFACE && epf->epc) @@ -644,7 +700,7 @@ EXPORT_SYMBOL_GPL(pci_epc_linkup); /** * pci_epc_init_notify() - Notify the EPF device that EPC device's core * initialization is completed. - * @epc: the EPC device whose core initialization is completeds + * @epc: the EPC device whose core initialization is completed * * Invoke to Notify the EPF device that the EPC device's initialization * is completed. diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c index e9289d10f822..9ed556936f48 100644 --- a/drivers/pci/endpoint/pci-epf-core.c +++ b/drivers/pci/endpoint/pci-epf-core.c @@ -62,13 +62,20 @@ EXPORT_SYMBOL_GPL(pci_epf_type_add_cfs); */ void pci_epf_unbind(struct pci_epf *epf) { + struct pci_epf *epf_vf; + if (!epf->driver) { dev_WARN(&epf->dev, "epf device not bound to driver\n"); return; } mutex_lock(&epf->lock); - epf->driver->ops->unbind(epf); + list_for_each_entry(epf_vf, &epf->pci_vepf, list) { + if (epf_vf->is_bound) + epf_vf->driver->ops->unbind(epf_vf); + } + if (epf->is_bound) + epf->driver->ops->unbind(epf); mutex_unlock(&epf->lock); module_put(epf->driver->owner); } @@ -83,10 +90,14 @@ EXPORT_SYMBOL_GPL(pci_epf_unbind); */ int pci_epf_bind(struct pci_epf *epf) { + struct device *dev = &epf->dev; + struct pci_epf *epf_vf; + u8 func_no, vfunc_no; + struct pci_epc *epc; int ret; if (!epf->driver) { - dev_WARN(&epf->dev, "epf device not bound to driver\n"); + dev_WARN(dev, "epf device not bound to driver\n"); return -EINVAL; } @@ -94,14 +105,141 @@ int pci_epf_bind(struct pci_epf *epf) return -EAGAIN; mutex_lock(&epf->lock); + list_for_each_entry(epf_vf, &epf->pci_vepf, list) { + vfunc_no = epf_vf->vfunc_no; + + if (vfunc_no < 1) { + dev_err(dev, "Invalid virtual function number\n"); + ret = -EINVAL; + goto ret; + } + + epc = epf->epc; + func_no = epf->func_no; + if (!IS_ERR_OR_NULL(epc)) { + if (!epc->max_vfs) { + dev_err(dev, "No support for virt function\n"); + ret = -EINVAL; + goto ret; + } + + if (vfunc_no > epc->max_vfs[func_no]) { + dev_err(dev, "PF%d: Exceeds max vfunc number\n", + func_no); + ret = -EINVAL; + goto ret; + } + } + + epc = epf->sec_epc; + func_no = epf->sec_epc_func_no; + if (!IS_ERR_OR_NULL(epc)) { + if (!epc->max_vfs) { + dev_err(dev, "No support for virt function\n"); + ret = -EINVAL; + goto ret; + } + + if (vfunc_no > epc->max_vfs[func_no]) { + dev_err(dev, "PF%d: Exceeds max vfunc number\n", + func_no); + ret = -EINVAL; + goto ret; + } + } + + epf_vf->func_no = epf->func_no; + epf_vf->sec_epc_func_no = epf->sec_epc_func_no; + epf_vf->epc = epf->epc; + epf_vf->sec_epc = epf->sec_epc; + ret = epf_vf->driver->ops->bind(epf_vf); + if (ret) + goto ret; + epf_vf->is_bound = true; + } + ret = epf->driver->ops->bind(epf); + if (ret) + goto ret; + epf->is_bound = true; + + mutex_unlock(&epf->lock); + return 0; + +ret: mutex_unlock(&epf->lock); + pci_epf_unbind(epf); return ret; } EXPORT_SYMBOL_GPL(pci_epf_bind); /** + * pci_epf_add_vepf() - associate virtual EP function to physical EP function + * @epf_pf: the physical EP function to which the virtual EP function should be + * associated + * @epf_vf: the virtual EP function to be added + * + * A physical endpoint function can be associated with multiple virtual + * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint + * function to a physical PCI endpoint function. + */ +int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) +{ + u32 vfunc_no; + + if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf)) + return -EINVAL; + + if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf) + return -EBUSY; + + if (epf_pf->sec_epc || epf_vf->sec_epc) + return -EBUSY; + + mutex_lock(&epf_pf->lock); + vfunc_no = find_first_zero_bit(&epf_pf->vfunction_num_map, + BITS_PER_LONG); + if (vfunc_no >= BITS_PER_LONG) { + mutex_unlock(&epf_pf->lock); + return -EINVAL; + } + + set_bit(vfunc_no, &epf_pf->vfunction_num_map); + epf_vf->vfunc_no = vfunc_no; + + epf_vf->epf_pf = epf_pf; + epf_vf->is_vf = true; + + list_add_tail(&epf_vf->list, &epf_pf->pci_vepf); + mutex_unlock(&epf_pf->lock); + + return 0; +} +EXPORT_SYMBOL_GPL(pci_epf_add_vepf); + +/** + * pci_epf_remove_vepf() - remove virtual EP function from physical EP function + * @epf_pf: the physical EP function from which the virtual EP function should + * be removed + * @epf_vf: the virtual EP function to be removed + * + * Invoke to remove a virtual endpoint function from the physical endpoint + * function. + */ +void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) +{ + if (IS_ERR_OR_NULL(epf_pf) || IS_ERR_OR_NULL(epf_vf)) + return; + + mutex_lock(&epf_pf->lock); + clear_bit(epf_vf->vfunc_no, &epf_pf->vfunction_num_map); + list_del(&epf_vf->list); + mutex_unlock(&epf_pf->lock); +} +EXPORT_SYMBOL_GPL(pci_epf_remove_vepf); + +/** * pci_epf_free_space() - free the allocated PCI EPF register space * @epf: the EPF device from whom to free the memory * @addr: the virtual address of the PCI EPF register space @@ -294,7 +432,7 @@ EXPORT_SYMBOL_GPL(pci_epf_destroy); /** * pci_epf_create() - create a new PCI EPF device * @name: the name of the PCI EPF device. This name will be used to bind the - * the EPF device to a EPF driver + * EPF device to a EPF driver * * Invoke to create a new PCI EPF device by providing the name of the function * device. @@ -317,6 +455,10 @@ struct pci_epf *pci_epf_create(const char *name) return ERR_PTR(-ENOMEM); } + /* VFs are numbered starting with 1. So set BIT(0) by default */ + epf->vfunction_num_map = 1; + INIT_LIST_HEAD(&epf->pci_vepf); + dev = &epf->dev; device_initialize(dev); dev->bus = &pci_epf_bus_type; @@ -387,17 +529,14 @@ static int pci_epf_device_probe(struct device *dev) return driver->probe(epf); } -static int pci_epf_device_remove(struct device *dev) +static void pci_epf_device_remove(struct device *dev) { - int ret = 0; struct pci_epf *epf = to_pci_epf(dev); struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver); if (driver->remove) - ret = driver->remove(epf); + driver->remove(epf); epf->driver = NULL; - - return ret; } static struct bus_type pci_epf_bus_type = { diff --git a/drivers/pci/host-bridge.c b/drivers/pci/host-bridge.c index e01d53f5b32f..afa50b446567 100644 --- a/drivers/pci/host-bridge.c +++ b/drivers/pci/host-bridge.c @@ -23,6 +23,7 @@ struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus) return to_pci_host_bridge(root_bus->bridge); } +EXPORT_SYMBOL_GPL(pci_find_host_bridge); struct device *pci_get_host_bridge_device(struct pci_dev *dev) { diff --git a/drivers/pci/hotplug/TODO b/drivers/pci/hotplug/TODO index a32070be5adf..cc6194aa24c1 100644 --- a/drivers/pci/hotplug/TODO +++ b/drivers/pci/hotplug/TODO @@ -40,9 +40,6 @@ ibmphp: * The return value of pci_hp_register() is not checked. -* iounmap(io_mem) is called in the error path of ebda_rsrc_controller() - and once more in the error path of its caller ibmphp_access_ebda(). - * The various slot data structures are difficult to follow and need to be simplified. A lot of functions are too large and too complex, they need to be broken up into smaller, manageable pieces. Negative examples are diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c index f031302ad401..12f4b351be67 100644 --- a/drivers/pci/hotplug/acpiphp_glue.c +++ b/drivers/pci/hotplug/acpiphp_glue.c @@ -22,7 +22,7 @@ * when the bridge is scanned and it loses a refcount when the bridge * is removed. * - When a P2P bridge is present, we elevate the refcount on the subordinate - * bus. It loses the refcount when the the driver unloads. + * bus. It loses the refcount when the driver unloads. */ #define pr_fmt(fmt) "acpiphp_glue: " fmt diff --git a/drivers/pci/hotplug/cpci_hotplug.h b/drivers/pci/hotplug/cpci_hotplug.h index f33ff2bca414..3fdd1b9bd8c3 100644 --- a/drivers/pci/hotplug/cpci_hotplug.h +++ b/drivers/pci/hotplug/cpci_hotplug.h @@ -75,6 +75,9 @@ int cpci_hp_unregister_bus(struct pci_bus *bus); int cpci_hp_start(void); int cpci_hp_stop(void); +/* Global variables */ +extern int cpci_debug; + /* * Internal function prototypes, these functions should not be used by * board/chassis drivers. diff --git a/drivers/pci/hotplug/cpci_hotplug_pci.c b/drivers/pci/hotplug/cpci_hotplug_pci.c index 2c16adb7f4ec..6c48066acb44 100644 --- a/drivers/pci/hotplug/cpci_hotplug_pci.c +++ b/drivers/pci/hotplug/cpci_hotplug_pci.c @@ -19,8 +19,6 @@ #define MY_NAME "cpci_hotplug" -extern int cpci_debug; - #define dbg(format, arg...) \ do { \ if (cpci_debug) \ diff --git a/drivers/pci/hotplug/cpqphp.h b/drivers/pci/hotplug/cpqphp.h index 77e4e0142fbc..2f7b49ea96e2 100644 --- a/drivers/pci/hotplug/cpqphp.h +++ b/drivers/pci/hotplug/cpqphp.h @@ -15,7 +15,7 @@ #define _CPQPHP_H #include <linux/interrupt.h> -#include <asm/io.h> /* for read? and write? functions */ +#include <linux/io.h> /* for read? and write? functions */ #include <linux/delay.h> /* for delays */ #include <linux/mutex.h> #include <linux/sched/signal.h> /* for signal_pending() */ diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c index b8aacb41a83c..f99a7927e5a8 100644 --- a/drivers/pci/hotplug/cpqphp_core.c +++ b/drivers/pci/hotplug/cpqphp_core.c @@ -296,9 +296,10 @@ static int ctrl_slot_cleanup(struct controller *ctrl) * * Won't work for more than one PCI-PCI bridge in a slot. * - * @bus_num - bus number of PCI device - * @dev_num - device number of PCI device - * @slot - Pointer to u8 where slot number will be returned + * @bus: pointer to the PCI bus structure + * @bus_num: bus number of PCI device + * @dev_num: device number of PCI device + * @slot: Pointer to u8 where slot number will be returned * * Output: SUCCESS or FAILURE */ diff --git a/drivers/pci/hotplug/cpqphp_ctrl.c b/drivers/pci/hotplug/cpqphp_ctrl.c index 68de958a9be8..ed7b58eb64d2 100644 --- a/drivers/pci/hotplug/cpqphp_ctrl.c +++ b/drivers/pci/hotplug/cpqphp_ctrl.c @@ -519,7 +519,7 @@ error: * @head: list to search * @size: size of node to find, must be a power of two. * - * Description: This function sorts the resource list by size and then returns + * Description: This function sorts the resource list by size and then * returns the first node of "size" length that is not in the ISA aliasing * window. If it finds a node larger than "size" it will split it up. */ @@ -1202,7 +1202,7 @@ static u8 set_controller_speed(struct controller *ctrl, u8 adapter_speed, u8 hp_ mdelay(5); - /* Reenable interrupts */ + /* Re-enable interrupts */ writel(0, ctrl->hpc_reg + INT_MASK); pci_write_config_byte(ctrl->pci_dev, 0x41, reg); @@ -1877,7 +1877,7 @@ static void interrupt_event_handler(struct controller *ctrl) /** * cpqhp_pushbutton_thread - handle pushbutton events - * @slot: target slot (struct) + * @t: pointer to struct timer_list which holds all timer-related callbacks * * Scheduled procedure to handle blocking stuff for the pushbuttons. * Handles all pending events and exits. diff --git a/drivers/pci/hotplug/cpqphp_pci.c b/drivers/pci/hotplug/cpqphp_pci.c index 1b2b3f3b648b..9038039ad6db 100644 --- a/drivers/pci/hotplug/cpqphp_pci.c +++ b/drivers/pci/hotplug/cpqphp_pci.c @@ -189,8 +189,10 @@ int cpqhp_set_irq(u8 bus_num, u8 dev_num, u8 int_pin, u8 irq_num) /* This should only be for x86 as it sets the Edge Level * Control Register */ - outb((u8) (temp_word & 0xFF), 0x4d0); outb((u8) ((temp_word & - 0xFF00) >> 8), 0x4d1); rc = 0; } + outb((u8)(temp_word & 0xFF), 0x4d0); + outb((u8)((temp_word & 0xFF00) >> 8), 0x4d1); + rc = 0; + } return rc; } diff --git a/drivers/pci/hotplug/ibmphp.h b/drivers/pci/hotplug/ibmphp.h index e90a4ebf6550..0399c60d2ec1 100644 --- a/drivers/pci/hotplug/ibmphp.h +++ b/drivers/pci/hotplug/ibmphp.h @@ -352,7 +352,7 @@ struct resource_node { u32 len; int type; /* MEM, IO, PFMEM */ u8 fromMem; /* this is to indicate that the range is from - * from the Memory bucket rather than from PFMem */ + * the Memory bucket rather than from PFMem */ struct resource_node *next; struct resource_node *nextRange; /* for the other mem range on bus */ }; @@ -736,7 +736,7 @@ struct controller { int ibmphp_init_devno(struct slot **); /* This function is called from EBDA, so we need it not be static */ int ibmphp_do_disable_slot(struct slot *slot_cur); -int ibmphp_update_slot_info(struct slot *); /* This function is called from HPC, so we need it to not be be static */ +int ibmphp_update_slot_info(struct slot *); /* This function is called from HPC, so we need it to not be static */ int ibmphp_configure_card(struct pci_func *, u8); int ibmphp_unconfigure_card(struct slot **, int); extern const struct hotplug_slot_ops ibmphp_hotplug_slot_ops; diff --git a/drivers/pci/hotplug/ibmphp_ebda.c b/drivers/pci/hotplug/ibmphp_ebda.c index 11a2661dc062..7fb75401ad8a 100644 --- a/drivers/pci/hotplug/ibmphp_ebda.c +++ b/drivers/pci/hotplug/ibmphp_ebda.c @@ -714,8 +714,7 @@ static int __init ebda_rsrc_controller(void) /* init hpc structure */ hpc_ptr = alloc_ebda_hpc(slot_num, bus_num); if (!hpc_ptr) { - rc = -ENOMEM; - goto error_no_hpc; + return -ENOMEM; } hpc_ptr->ctlr_id = ctlr_id; hpc_ptr->ctlr_relative_id = ctlr; @@ -910,8 +909,6 @@ error: kfree(tmp_slot); error_no_slot: free_ebda_hpc(hpc_ptr); -error_no_hpc: - iounmap(io_mem); return rc; } diff --git a/drivers/pci/hotplug/pci_hotplug_core.c b/drivers/pci/hotplug/pci_hotplug_core.c index 5ac31f683b85..058d5937d8a9 100644 --- a/drivers/pci/hotplug/pci_hotplug_core.c +++ b/drivers/pci/hotplug/pci_hotplug_core.c @@ -73,7 +73,7 @@ static ssize_t power_read_file(struct pci_slot *pci_slot, char *buf) if (retval) return retval; - return sprintf(buf, "%d\n", value); + return sysfs_emit(buf, "%d\n", value); } static ssize_t power_write_file(struct pci_slot *pci_slot, const char *buf, @@ -130,7 +130,7 @@ static ssize_t attention_read_file(struct pci_slot *pci_slot, char *buf) if (retval) return retval; - return sprintf(buf, "%d\n", value); + return sysfs_emit(buf, "%d\n", value); } static ssize_t attention_write_file(struct pci_slot *pci_slot, const char *buf, @@ -175,7 +175,7 @@ static ssize_t latch_read_file(struct pci_slot *pci_slot, char *buf) if (retval) return retval; - return sprintf(buf, "%d\n", value); + return sysfs_emit(buf, "%d\n", value); } static struct pci_slot_attribute hotplug_slot_attr_latch = { @@ -192,7 +192,7 @@ static ssize_t presence_read_file(struct pci_slot *pci_slot, char *buf) if (retval) return retval; - return sprintf(buf, "%d\n", value); + return sysfs_emit(buf, "%d\n", value); } static struct pci_slot_attribute hotplug_slot_attr_presence = { diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h index 4fd200d8b0a9..918dccbc74b6 100644 --- a/drivers/pci/hotplug/pciehp.h +++ b/drivers/pci/hotplug/pciehp.h @@ -47,6 +47,9 @@ extern int pciehp_poll_time; * struct controller - PCIe hotplug controller * @pcie: pointer to the controller's PCIe port service device * @slot_cap: cached copy of the Slot Capabilities register + * @inband_presence_disabled: In-Band Presence Detect Disable supported by + * controller and disabled per spec recommendation (PCIe r5.0, appendix I + * implementation note) * @slot_ctrl: cached copy of the Slot Control register * @ctrl_lock: serializes writes to the Slot Control register * @cmd_started: jiffies when the Slot Control register was last written; @@ -181,11 +184,13 @@ void pciehp_release_ctrl(struct controller *ctrl); int pciehp_sysfs_enable_slot(struct hotplug_slot *hotplug_slot); int pciehp_sysfs_disable_slot(struct hotplug_slot *hotplug_slot); -int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe); +int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe); int pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status); int pciehp_set_raw_indicator_status(struct hotplug_slot *h_slot, u8 status); int pciehp_get_raw_indicator_status(struct hotplug_slot *h_slot, u8 *status); +int pciehp_slot_reset(struct pcie_device *dev); + static inline const char *slot_name(struct controller *ctrl) { return hotplug_slot_name(&ctrl->hotplug_slot); diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c index ad3393930ecb..f34114d45259 100644 --- a/drivers/pci/hotplug/pciehp_core.c +++ b/drivers/pci/hotplug/pciehp_core.c @@ -351,6 +351,8 @@ static struct pcie_port_service_driver hpdriver_portdrv = { .runtime_suspend = pciehp_runtime_suspend, .runtime_resume = pciehp_runtime_resume, #endif /* PM */ + + .slot_reset = pciehp_slot_reset, }; int __init pcie_hp_init(void) diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c index fb3840e222ad..83a0fa119cae 100644 --- a/drivers/pci/hotplug/pciehp_hpc.c +++ b/drivers/pci/hotplug/pciehp_hpc.c @@ -563,6 +563,32 @@ void pciehp_power_off_slot(struct controller *ctrl) PCI_EXP_SLTCTL_PWR_OFF); } +static void pciehp_ignore_dpc_link_change(struct controller *ctrl, + struct pci_dev *pdev, int irq) +{ + /* + * Ignore link changes which occurred while waiting for DPC recovery. + * Could be several if DPC triggered multiple times consecutively. + */ + synchronize_hardirq(irq); + atomic_and(~PCI_EXP_SLTSTA_DLLSC, &ctrl->pending_events); + if (pciehp_poll_mode) + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_DLLSC); + ctrl_info(ctrl, "Slot(%s): Link Down/Up ignored (recovered by DPC)\n", + slot_name(ctrl)); + + /* + * If the link is unexpectedly down after successful recovery, + * the corresponding link change may have been ignored above. + * Synthesize it to ensure that it is acted on. + */ + down_read(&ctrl->reset_lock); + if (!pciehp_check_link_active(ctrl)) + pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC); + up_read(&ctrl->reset_lock); +} + static irqreturn_t pciehp_isr(int irq, void *dev_id) { struct controller *ctrl = (struct controller *)dev_id; @@ -707,6 +733,16 @@ static irqreturn_t pciehp_ist(int irq, void *dev_id) } /* + * Ignore Link Down/Up events caused by Downstream Port Containment + * if recovery from the error succeeded. + */ + if ((events & PCI_EXP_SLTSTA_DLLSC) && pci_dpc_recovered(pdev) && + ctrl->state == ON_STATE) { + events &= ~PCI_EXP_SLTSTA_DLLSC; + pciehp_ignore_dpc_link_change(ctrl, pdev, irq); + } + + /* * Disable requests have higher priority than Presence Detect Changed * or Data Link Layer State Changed events. */ @@ -826,6 +862,32 @@ void pcie_disable_interrupt(struct controller *ctrl) pcie_write_cmd(ctrl, 0, mask); } +/** + * pciehp_slot_reset() - ignore link event caused by error-induced hot reset + * @dev: PCI Express port service device + * + * Called from pcie_portdrv_slot_reset() after AER or DPC initiated a reset + * further up in the hierarchy to recover from an error. The reset was + * propagated down to this hotplug port. Ignore the resulting link flap. + * If the link failed to retrain successfully, synthesize the ignored event. + * Surprise removal during reset is detected through Presence Detect Changed. + */ +int pciehp_slot_reset(struct pcie_device *dev) +{ + struct controller *ctrl = get_service_data(dev); + + if (ctrl->state != ON_STATE) + return 0; + + pcie_capability_write_word(dev->port, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_DLLSC); + + if (!pciehp_check_link_active(ctrl)) + pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC); + + return 0; +} + /* * pciehp has a 1:1 bus:slot relationship so we ultimately want a secondary * bus reset of the bridge, but at the same time we want to ensure that it is @@ -834,7 +896,7 @@ void pcie_disable_interrupt(struct controller *ctrl) * momentarily, if we see that they could interfere. Also, clear any spurious * events after. */ -int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe) +int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe) { struct controller *ctrl = to_ctrl(hotplug_slot); struct pci_dev *pdev = ctrl_dev(ctrl); diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index 04565162a449..f4c2e6e01be0 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -526,7 +526,7 @@ scan: return 0; } -static int pnv_php_reset_slot(struct hotplug_slot *slot, int probe) +static int pnv_php_reset_slot(struct hotplug_slot *slot, bool probe) { struct pnv_php_slot *php_slot = to_pnv_php_slot(slot); struct pci_dev *bridge = php_slot->pdev; diff --git a/drivers/pci/hotplug/rpadlpar_sysfs.c b/drivers/pci/hotplug/rpadlpar_sysfs.c index dbfa0b55d31a..068b7810a574 100644 --- a/drivers/pci/hotplug/rpadlpar_sysfs.c +++ b/drivers/pci/hotplug/rpadlpar_sysfs.c @@ -50,7 +50,7 @@ static ssize_t add_slot_store(struct kobject *kobj, struct kobj_attribute *attr, static ssize_t add_slot_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sprintf(buf, "0\n"); + return sysfs_emit(buf, "0\n"); } static ssize_t remove_slot_store(struct kobject *kobj, @@ -80,7 +80,7 @@ static ssize_t remove_slot_store(struct kobject *kobj, static ssize_t remove_slot_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sprintf(buf, "0\n"); + return sysfs_emit(buf, "0\n"); } static struct kobj_attribute add_slot_attr = diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index 014868752cd4..a89b7de72dcf 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -57,19 +57,35 @@ static int disable_slot(struct hotplug_slot *hotplug_slot) return zpci_deconfigure_device(zdev); } +static int reset_slot(struct hotplug_slot *hotplug_slot, bool probe) +{ + struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, + hotplug_slot); + + if (zdev->state != ZPCI_FN_STATE_CONFIGURED) + return -EIO; + /* + * We can't take the zdev->lock as reset_slot may be called during + * probing and/or device removal which already happens under the + * zdev->lock. Instead the user should use the higher level + * pci_reset_function() or pci_bus_reset() which hold the PCI device + * lock preventing concurrent removal. If not using these functions + * holding the PCI device lock is required. + */ + + /* As long as the function is configured we can reset */ + if (probe) + return 0; + + return zpci_hot_reset_device(zdev); +} + static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) { struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev, hotplug_slot); - switch (zdev->state) { - case ZPCI_FN_STATE_STANDBY: - *value = 0; - break; - default: - *value = 1; - break; - } + *value = zpci_is_device_configured(zdev) ? 1 : 0; return 0; } @@ -83,6 +99,7 @@ static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value) static const struct hotplug_slot_ops s390_hotplug_slot_ops = { .enable_slot = enable_slot, .disable_slot = disable_slot, + .reset_slot = reset_slot, .get_power_status = get_power_status, .get_adapter_status = get_adapter_status, }; diff --git a/drivers/pci/hotplug/shpchp_hpc.c b/drivers/pci/hotplug/shpchp_hpc.c index 9e3b27744305..bd7557ca4910 100644 --- a/drivers/pci/hotplug/shpchp_hpc.c +++ b/drivers/pci/hotplug/shpchp_hpc.c @@ -295,7 +295,7 @@ static int shpc_write_cmd(struct slot *slot, u8 t_slot, u8 cmd) mutex_lock(&slot->ctrl->cmd_lock); if (!shpc_poll_ctrl_busy(ctrl)) { - /* After 1 sec and and the controller is still busy */ + /* After 1 sec and the controller is still busy */ ctrl_err(ctrl, "Controller is still busy after 1 sec\n"); retval = -EBUSY; goto out; diff --git a/drivers/pci/hotplug/shpchp_sysfs.c b/drivers/pci/hotplug/shpchp_sysfs.c index 45658bb5c554..64beed7a26be 100644 --- a/drivers/pci/hotplug/shpchp_sysfs.c +++ b/drivers/pci/hotplug/shpchp_sysfs.c @@ -24,50 +24,54 @@ static ssize_t show_ctrl(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev; - char *out = buf; int index, busnr; struct resource *res; struct pci_bus *bus; + size_t len = 0; pdev = to_pci_dev(dev); bus = pdev->subordinate; - out += sprintf(buf, "Free resources: memory\n"); + len += sysfs_emit_at(buf, len, "Free resources: memory\n"); pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_MEM) && !(res->flags & IORESOURCE_PREFETCH)) { - out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)resource_size(res)); + len += sysfs_emit_at(buf, len, + "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } - out += sprintf(out, "Free resources: prefetchable memory\n"); + len += sysfs_emit_at(buf, len, "Free resources: prefetchable memory\n"); pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_MEM) && (res->flags & IORESOURCE_PREFETCH)) { - out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)resource_size(res)); + len += sysfs_emit_at(buf, len, + "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } - out += sprintf(out, "Free resources: IO\n"); + len += sysfs_emit_at(buf, len, "Free resources: IO\n"); pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_IO)) { - out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)resource_size(res)); + len += sysfs_emit_at(buf, len, + "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } - out += sprintf(out, "Free resources: bus numbers\n"); + len += sysfs_emit_at(buf, len, "Free resources: bus numbers\n"); for (busnr = bus->busn_res.start; busnr <= bus->busn_res.end; busnr++) { if (!pci_find_bus(pci_domain_nr(bus), busnr)) break; } if (busnr < bus->busn_res.end) - out += sprintf(out, "start = %8.8x, length = %8.8x\n", - busnr, (int)(bus->busn_res.end - busnr)); + len += sysfs_emit_at(buf, len, + "start = %8.8x, length = %8.8x\n", + busnr, (int)(bus->busn_res.end - busnr)); - return out - buf; + return len; } static DEVICE_ATTR(ctrl, S_IRUGO, show_ctrl, NULL); diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index afc06e6ce115..0267977c9f17 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -183,11 +183,10 @@ static ssize_t sriov_vf_msix_count_store(struct device *dev, { struct pci_dev *vf_dev = to_pci_dev(dev); struct pci_dev *pdev = pci_physfn(vf_dev); - int val, ret; + int val, ret = 0; - ret = kstrtoint(buf, 0, &val); - if (ret) - return ret; + if (kstrtoint(buf, 0, &val) < 0) + return -EINVAL; if (val < 0) return -EINVAL; @@ -346,7 +345,7 @@ static ssize_t sriov_totalvfs_show(struct device *dev, { struct pci_dev *pdev = to_pci_dev(dev); - return sprintf(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); + return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); } static ssize_t sriov_numvfs_show(struct device *dev, @@ -361,7 +360,7 @@ static ssize_t sriov_numvfs_show(struct device *dev, num_vfs = pdev->sriov->num_VFs; device_unlock(&pdev->dev); - return sprintf(buf, "%u\n", num_vfs); + return sysfs_emit(buf, "%u\n", num_vfs); } /* @@ -376,12 +375,11 @@ static ssize_t sriov_numvfs_store(struct device *dev, const char *buf, size_t count) { struct pci_dev *pdev = to_pci_dev(dev); - int ret; + int ret = 0; u16 num_vfs; - ret = kstrtou16(buf, 0, &num_vfs); - if (ret < 0) - return ret; + if (kstrtou16(buf, 0, &num_vfs) < 0) + return -EINVAL; if (num_vfs > pci_sriov_get_totalvfs(pdev)) return -ERANGE; @@ -391,9 +389,16 @@ static ssize_t sriov_numvfs_store(struct device *dev, if (num_vfs == pdev->sriov->num_VFs) goto exit; + /* is PF driver loaded */ + if (!pdev->driver) { + pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); + ret = -ENOENT; + goto exit; + } + /* is PF driver loaded w/callback */ - if (!pdev->driver || !pdev->driver->sriov_configure) { - pci_info(pdev, "Driver does not support SRIOV configuration via sysfs\n"); + if (!pdev->driver->sriov_configure) { + pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); ret = -ENOENT; goto exit; } @@ -435,7 +440,7 @@ static ssize_t sriov_offset_show(struct device *dev, { struct pci_dev *pdev = to_pci_dev(dev); - return sprintf(buf, "%u\n", pdev->sriov->offset); + return sysfs_emit(buf, "%u\n", pdev->sriov->offset); } static ssize_t sriov_stride_show(struct device *dev, @@ -444,7 +449,7 @@ static ssize_t sriov_stride_show(struct device *dev, { struct pci_dev *pdev = to_pci_dev(dev); - return sprintf(buf, "%u\n", pdev->sriov->stride); + return sysfs_emit(buf, "%u\n", pdev->sriov->stride); } static ssize_t sriov_vf_device_show(struct device *dev, @@ -453,7 +458,7 @@ static ssize_t sriov_vf_device_show(struct device *dev, { struct pci_dev *pdev = to_pci_dev(dev); - return sprintf(buf, "%x\n", pdev->sriov->vf_device); + return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); } static ssize_t sriov_drivers_autoprobe_show(struct device *dev, @@ -462,7 +467,7 @@ static ssize_t sriov_drivers_autoprobe_show(struct device *dev, { struct pci_dev *pdev = to_pci_dev(dev); - return sprintf(buf, "%u\n", pdev->sriov->drivers_autoprobe); + return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); } static ssize_t sriov_drivers_autoprobe_store(struct device *dev, diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 217dc9f0231f..d84cf30bb279 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -129,93 +129,93 @@ void __weak arch_restore_msi_irqs(struct pci_dev *dev) return default_restore_msi_irqs(dev); } -static inline __attribute_const__ u32 msi_mask(unsigned x) -{ - /* Don't shift by >= width of type */ - if (x >= 5) - return 0xffffffff; - return (1 << (1 << x)) - 1; -} - /* * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to * mask all MSI interrupts by clearing the MSI enable bit does not work * reliably as devices without an INTx disable bit will then generate a * level IRQ which will never be cleared. */ -u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) +static inline __attribute_const__ u32 msi_multi_mask(struct msi_desc *desc) { - u32 mask_bits = desc->masked; + /* Don't shift by >= width of type */ + if (desc->msi_attrib.multi_cap >= 5) + return 0xffffffff; + return (1 << (1 << desc->msi_attrib.multi_cap)) - 1; +} - if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit) - return 0; +static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) +{ + raw_spinlock_t *lock = &desc->dev->msi_lock; + unsigned long flags; - mask_bits &= ~mask; - mask_bits |= flag; - pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, - mask_bits); + if (!desc->msi_attrib.can_mask) + return; - return mask_bits; + raw_spin_lock_irqsave(lock, flags); + desc->msi_mask &= ~clear; + desc->msi_mask |= set; + pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, + desc->msi_mask); + raw_spin_unlock_irqrestore(lock, flags); } -static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) +static inline void pci_msi_mask(struct msi_desc *desc, u32 mask) { - desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag); + pci_msi_update_mask(desc, 0, mask); } -static void __iomem *pci_msix_desc_addr(struct msi_desc *desc) +static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask) { - if (desc->msi_attrib.is_virtual) - return NULL; + pci_msi_update_mask(desc, mask, 0); +} - return desc->mask_base + - desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; +static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) +{ + return desc->mask_base + desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; } /* - * This internal function does not flush PCI writes to the device. - * All users must ensure that they read from the device before either - * assuming that the device state is up to date, or returning out of this - * file. This saves a few milliseconds when initialising devices with lots - * of MSI-X interrupts. + * This internal function does not flush PCI writes to the device. All + * users must ensure that they read from the device before either assuming + * that the device state is up to date, or returning out of this file. + * It does not affect the msi_desc::msix_ctrl cache either. Use with care! */ -u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag) +static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { - u32 mask_bits = desc->masked; - void __iomem *desc_addr; + void __iomem *desc_addr = pci_msix_desc_addr(desc); - if (pci_msi_ignore_mask) - return 0; - - desc_addr = pci_msix_desc_addr(desc); - if (!desc_addr) - return 0; - - mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; - if (flag & PCI_MSIX_ENTRY_CTRL_MASKBIT) - mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT; - - writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); + if (desc->msi_attrib.can_mask) + writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); +} - return mask_bits; +static inline void pci_msix_mask(struct msi_desc *desc) +{ + desc->msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); + /* Flush write to device */ + readl(desc->mask_base); } -static void msix_mask_irq(struct msi_desc *desc, u32 flag) +static inline void pci_msix_unmask(struct msi_desc *desc) { - desc->masked = __pci_msix_desc_mask_irq(desc, flag); + desc->msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; + pci_msix_write_vector_ctrl(desc, desc->msix_ctrl); } -static void msi_set_mask_bit(struct irq_data *data, u32 flag) +static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) { - struct msi_desc *desc = irq_data_get_msi_desc(data); + if (desc->msi_attrib.is_msix) + pci_msix_mask(desc); + else + pci_msi_mask(desc, mask); +} - if (desc->msi_attrib.is_msix) { - msix_mask_irq(desc, flag); - readl(desc->mask_base); /* Flush write to device */ - } else { - unsigned offset = data->irq - desc->irq; - msi_mask_irq(desc, 1 << offset, flag << offset); - } +static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask) +{ + if (desc->msi_attrib.is_msix) + pci_msix_unmask(desc); + else + pci_msi_unmask(desc, mask); } /** @@ -224,7 +224,9 @@ static void msi_set_mask_bit(struct irq_data *data, u32 flag) */ void pci_msi_mask_irq(struct irq_data *data) { - msi_set_mask_bit(data, 1); + struct msi_desc *desc = irq_data_get_msi_desc(data); + + __pci_msi_mask_desc(desc, BIT(data->irq - desc->irq)); } EXPORT_SYMBOL_GPL(pci_msi_mask_irq); @@ -234,7 +236,9 @@ EXPORT_SYMBOL_GPL(pci_msi_mask_irq); */ void pci_msi_unmask_irq(struct irq_data *data) { - msi_set_mask_bit(data, 0); + struct msi_desc *desc = irq_data_get_msi_desc(data); + + __pci_msi_unmask_desc(desc, BIT(data->irq - desc->irq)); } EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); @@ -255,10 +259,8 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (entry->msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); - if (!base) { - WARN_ON(1); + if (WARN_ON_ONCE(entry->msi_attrib.is_virtual)) return; - } msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); @@ -289,13 +291,32 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) /* Don't touch the hardware now */ } else if (entry->msi_attrib.is_msix) { void __iomem *base = pci_msix_desc_addr(entry); + u32 ctrl = entry->msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - if (!base) + if (entry->msi_attrib.is_virtual) goto skip; + /* + * The specification mandates that the entry is masked + * when the message is modified: + * + * "If software changes the Address or Data value of an + * entry while the entry is unmasked, the result is + * undefined." + */ + if (unmasked) + pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); + writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); writel(msg->data, base + PCI_MSIX_ENTRY_DATA); + + if (unmasked) + pci_msix_write_vector_ctrl(entry, ctrl); + + /* Ensure that the writes are visible in the device */ + readl(base + PCI_MSIX_ENTRY_DATA); } else { int pos = dev->msi_cap; u16 msgctl; @@ -316,6 +337,8 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) pci_write_config_word(dev, pos + PCI_MSI_DATA_32, msg->data); } + /* Ensure that the writes are visible in the device */ + pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); } skip: @@ -338,15 +361,18 @@ static void free_msi_irqs(struct pci_dev *dev) { struct list_head *msi_list = dev_to_msi_list(&dev->dev); struct msi_desc *entry, *tmp; - struct attribute **msi_attrs; - struct device_attribute *dev_attr; - int i, count = 0; + int i; for_each_pci_msi_entry(entry, dev) if (entry->irq) for (i = 0; i < entry->nvec_used; i++) BUG_ON(irq_has_action(entry->irq + i)); + if (dev->msi_irq_groups) { + msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups); + dev->msi_irq_groups = NULL; + } + pci_msi_teardown_msi_irqs(dev); list_for_each_entry_safe(entry, tmp, msi_list, list) { @@ -358,22 +384,6 @@ static void free_msi_irqs(struct pci_dev *dev) list_del(&entry->list); free_msi_entry(entry); } - - if (dev->msi_irq_groups) { - sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups); - msi_attrs = dev->msi_irq_groups[0]->attrs; - while (msi_attrs[count]) { - dev_attr = container_of(msi_attrs[count], - struct device_attribute, attr); - kfree(dev_attr->attr.name); - kfree(dev_attr); - ++count; - } - kfree(msi_attrs); - kfree(dev->msi_irq_groups[0]); - kfree(dev->msi_irq_groups); - dev->msi_irq_groups = NULL; - } } static void pci_intx_for_msi(struct pci_dev *dev, int enable) @@ -408,8 +418,7 @@ static void __pci_restore_msi_state(struct pci_dev *dev) arch_restore_msi_irqs(dev); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); - msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap), - entry->masked); + pci_msi_update_mask(entry, 0, 0); control &= ~PCI_MSI_FLAGS_QSIZE; control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); @@ -440,7 +449,7 @@ static void __pci_restore_msix_state(struct pci_dev *dev) arch_restore_msi_irqs(dev); for_each_pci_msi_entry(entry, dev) - msix_mask_irq(entry, entry->masked); + pci_msix_write_vector_ctrl(entry, entry->msix_ctrl); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } @@ -452,102 +461,6 @@ void pci_restore_msi_state(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_restore_msi_state); -static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct msi_desc *entry; - unsigned long irq; - int retval; - - retval = kstrtoul(attr->attr.name, 10, &irq); - if (retval) - return retval; - - entry = irq_get_msi_desc(irq); - if (entry) - return sprintf(buf, "%s\n", - entry->msi_attrib.is_msix ? "msix" : "msi"); - - return -ENODEV; -} - -static int populate_msi_sysfs(struct pci_dev *pdev) -{ - struct attribute **msi_attrs; - struct attribute *msi_attr; - struct device_attribute *msi_dev_attr; - struct attribute_group *msi_irq_group; - const struct attribute_group **msi_irq_groups; - struct msi_desc *entry; - int ret = -ENOMEM; - int num_msi = 0; - int count = 0; - int i; - - /* Determine how many msi entries we have */ - for_each_pci_msi_entry(entry, pdev) - num_msi += entry->nvec_used; - if (!num_msi) - return 0; - - /* Dynamically create the MSI attributes for the PCI device */ - msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL); - if (!msi_attrs) - return -ENOMEM; - for_each_pci_msi_entry(entry, pdev) { - for (i = 0; i < entry->nvec_used; i++) { - msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); - if (!msi_dev_attr) - goto error_attrs; - msi_attrs[count] = &msi_dev_attr->attr; - - sysfs_attr_init(&msi_dev_attr->attr); - msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", - entry->irq + i); - if (!msi_dev_attr->attr.name) - goto error_attrs; - msi_dev_attr->attr.mode = S_IRUGO; - msi_dev_attr->show = msi_mode_show; - ++count; - } - } - - msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); - if (!msi_irq_group) - goto error_attrs; - msi_irq_group->name = "msi_irqs"; - msi_irq_group->attrs = msi_attrs; - - msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL); - if (!msi_irq_groups) - goto error_irq_group; - msi_irq_groups[0] = msi_irq_group; - - ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups); - if (ret) - goto error_irq_groups; - pdev->msi_irq_groups = msi_irq_groups; - - return 0; - -error_irq_groups: - kfree(msi_irq_groups); -error_irq_group: - kfree(msi_irq_group); -error_attrs: - count = 0; - msi_attr = msi_attrs[count]; - while (msi_attr) { - msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); - kfree(msi_attr->name); - kfree(msi_dev_attr); - ++count; - msi_attr = msi_attrs[count]; - } - kfree(msi_attrs); - return ret; -} - static struct msi_desc * msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) { @@ -564,12 +477,16 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) goto out; pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + /* Lies, damned lies, and MSIs */ + if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING) + control |= PCI_MSI_FLAGS_MASKBIT; entry->msi_attrib.is_msix = 0; entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); entry->msi_attrib.is_virtual = 0; entry->msi_attrib.entry_nr = 0; - entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); + entry->msi_attrib.can_mask = !pci_msi_ignore_mask && + !!(control & PCI_MSI_FLAGS_MASKBIT); entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); @@ -580,8 +497,8 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd) entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; /* Save the initial mask status */ - if (entry->msi_attrib.maskbit) - pci_read_config_dword(dev, entry->mask_pos, &entry->masked); + if (entry->msi_attrib.can_mask) + pci_read_config_dword(dev, entry->mask_pos, &entry->msi_mask); out: kfree(masks); @@ -592,8 +509,11 @@ static int msi_verify_entries(struct pci_dev *dev) { struct msi_desc *entry; + if (!dev->no_64bit_msi) + return 0; + for_each_pci_msi_entry(entry, dev) { - if (entry->msg.address_hi && dev->no_64bit_msi) { + if (entry->msg.address_hi) { pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n", entry->msg.address_hi, entry->msg.address_lo); return -EIO; @@ -617,9 +537,9 @@ static int msi_verify_entries(struct pci_dev *dev) static int msi_capability_init(struct pci_dev *dev, int nvec, struct irq_affinity *affd) { + const struct attribute_group **groups; struct msi_desc *entry; int ret; - unsigned mask; pci_msi_set_enable(dev, 0); /* Disable MSI during set up */ @@ -628,33 +548,27 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, return -ENOMEM; /* All MSIs are unmasked by default; mask them all */ - mask = msi_mask(entry->msi_attrib.multi_cap); - msi_mask_irq(entry, mask, mask); + pci_msi_mask(entry, msi_multi_mask(entry)); list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); /* Configure MSI capability structure */ ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); - if (ret) { - msi_mask_irq(entry, mask, ~mask); - free_msi_irqs(dev); - return ret; - } + if (ret) + goto err; ret = msi_verify_entries(dev); - if (ret) { - msi_mask_irq(entry, mask, ~mask); - free_msi_irqs(dev); - return ret; - } + if (ret) + goto err; - ret = populate_msi_sysfs(dev); - if (ret) { - msi_mask_irq(entry, mask, ~mask); - free_msi_irqs(dev); - return ret; + groups = msi_populate_sysfs(&dev->dev); + if (IS_ERR(groups)) { + ret = PTR_ERR(groups); + goto err; } + dev->msi_irq_groups = groups; + /* Set MSI enabled bits */ pci_intx_for_msi(dev, 0); pci_msi_set_enable(dev, 1); @@ -663,9 +577,15 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, pcibios_free_irq(dev); dev->irq = entry->irq; return 0; + +err: + pci_msi_unmask(entry, msi_multi_mask(entry)); + free_msi_irqs(dev); + return ret; } -static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries) +static void __iomem *msix_map_region(struct pci_dev *dev, + unsigned int nr_entries) { resource_size_t phys_addr; u32 table_offset; @@ -691,6 +611,7 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, { struct irq_affinity_desc *curmsk, *masks = NULL; struct msi_desc *entry; + void __iomem *addr; int ret, i; int vec_count = pci_msix_vec_count(dev); @@ -711,6 +632,7 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, entry->msi_attrib.is_msix = 1; entry->msi_attrib.is_64 = 1; + if (entries) entry->msi_attrib.entry_nr = entries[i].entry; else @@ -719,9 +641,17 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, entry->msi_attrib.is_virtual = entry->msi_attrib.entry_nr >= vec_count; + entry->msi_attrib.can_mask = !pci_msi_ignore_mask && + !entry->msi_attrib.is_virtual; + entry->msi_attrib.default_irq = dev->irq; entry->mask_base = base; + if (entry->msi_attrib.can_mask) { + addr = pci_msix_desc_addr(entry); + entry->msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); + } + list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); if (masks) curmsk++; @@ -732,26 +662,28 @@ out: return ret; } -static void msix_program_entries(struct pci_dev *dev, - struct msix_entry *entries) +static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries) { struct msi_desc *entry; - int i = 0; - void __iomem *desc_addr; for_each_pci_msi_entry(entry, dev) { - if (entries) - entries[i++].vector = entry->irq; + if (entries) { + entries->vector = entry->irq; + entries++; + } + } +} - desc_addr = pci_msix_desc_addr(entry); - if (desc_addr) - entry->masked = readl(desc_addr + - PCI_MSIX_ENTRY_VECTOR_CTRL); - else - entry->masked = 0; +static void msix_mask_all(void __iomem *base, int tsize) +{ + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + int i; - msix_mask_irq(entry, 1); - } + if (pci_msi_ignore_mask) + return; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) + writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL); } /** @@ -768,22 +700,31 @@ static void msix_program_entries(struct pci_dev *dev, static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { - int ret; - u16 control; + const struct attribute_group **groups; void __iomem *base; + int ret, tsize; + u16 control; - /* Ensure MSI-X is disabled while it is set up */ - pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); + /* + * Some devices require MSI-X to be enabled before the MSI-X + * registers can be accessed. Mask all the vectors to prevent + * interrupts coming in before they're fully set up. + */ + pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_MASKALL | + PCI_MSIX_FLAGS_ENABLE); pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* Request & Map MSI-X table region */ - base = msix_map_region(dev, msix_table_size(control)); - if (!base) - return -ENOMEM; + tsize = msix_table_size(control); + base = msix_map_region(dev, tsize); + if (!base) { + ret = -ENOMEM; + goto out_disable; + } ret = msix_setup_entries(dev, base, entries, nvec, affd); if (ret) - return ret; + goto out_disable; ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); if (ret) @@ -794,23 +735,29 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, if (ret) goto out_free; - /* - * Some devices require MSI-X to be enabled before we can touch the - * MSI-X registers. We need to mask all the vectors to prevent - * interrupts coming in before they're fully set up. - */ - pci_msix_clear_and_set_ctrl(dev, 0, - PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE); + msix_update_entries(dev, entries); - msix_program_entries(dev, entries); - - ret = populate_msi_sysfs(dev); - if (ret) + groups = msi_populate_sysfs(&dev->dev); + if (IS_ERR(groups)) { + ret = PTR_ERR(groups); goto out_free; + } + + dev->msi_irq_groups = groups; /* Set MSI-X enabled bits and unmask the function */ pci_intx_for_msi(dev, 0); dev->msix_enabled = 1; + + /* + * Ensure that all table entries are masked to prevent + * stale entries from firing in a crash kernel. + * + * Done late to deal with a broken Marvell NVME device + * which takes the MSI-X mask bits into account even + * when MSI-X is disabled, which prevents MSI delivery. + */ + msix_mask_all(base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); pcibios_free_irq(dev); @@ -836,6 +783,9 @@ out_avail: out_free: free_msi_irqs(dev); +out_disable: + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); + return ret; } @@ -916,7 +866,6 @@ EXPORT_SYMBOL(pci_msi_vec_count); static void pci_msi_shutdown(struct pci_dev *dev) { struct msi_desc *desc; - u32 mask; if (!pci_msi_enable || !dev || !dev->msi_enabled) return; @@ -929,9 +878,7 @@ static void pci_msi_shutdown(struct pci_dev *dev) dev->msi_enabled = 0; /* Return the device with MSI unmasked as initial states */ - mask = msi_mask(desc->msi_attrib.multi_cap); - /* Keep cached state to be restored */ - __pci_msi_desc_mask_irq(desc, mask, ~mask); + pci_msi_unmask(desc, msi_multi_mask(desc)); /* Restore dev->irq to its default pin-assertion IRQ */ dev->irq = desc->msi_attrib.default_irq; @@ -1016,10 +963,8 @@ static void pci_msix_shutdown(struct pci_dev *dev) } /* Return the device with MSI-X masked as initial states */ - for_each_pci_msi_entry(entry, dev) { - /* Keep cached states to be restored */ - __pci_msix_desc_mask_irq(entry, 1); - } + for_each_pci_msi_entry(entry, dev) + pci_msix_mask(entry); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); pci_intx_for_msi(dev, 1); diff --git a/drivers/pci/of.c b/drivers/pci/of.c index a143b02b2dcd..0b1237cff239 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -310,7 +310,7 @@ static int devm_of_pci_get_host_bridge_resources(struct device *dev, /* Check for ranges property */ err = of_pci_range_parser_init(&parser, dev_node); if (err) - goto failed; + return 0; dev_dbg(dev, "Parsing ranges property...\n"); for_each_of_pci_range(&parser, &range) { @@ -423,7 +423,7 @@ failed: */ static int of_irq_parse_pci(const struct pci_dev *pdev, struct of_phandle_args *out_irq) { - struct device_node *dn, *ppnode; + struct device_node *dn, *ppnode = NULL; struct pci_dev *ppdev; __be32 laddr[3]; u8 pin; @@ -452,8 +452,14 @@ static int of_irq_parse_pci(const struct pci_dev *pdev, struct of_phandle_args * if (pin == 0) return -ENODEV; + /* Local interrupt-map in the device node? Use it! */ + if (of_get_property(dn, "interrupt-map", NULL)) { + pin = pci_swizzle_interrupt_pin(pdev, pin); + ppnode = dn; + } + /* Now we walk up the PCI tree */ - for (;;) { + while (!ppnode) { /* Get the pci_dev of our parent */ ppdev = pdev->bus->self; diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 196382630363..8d47cb7218d1 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -48,12 +48,16 @@ static ssize_t size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev = to_pci_dev(dev); + struct pci_p2pdma *p2pdma; size_t size = 0; - if (pdev->p2pdma->pool) - size = gen_pool_size(pdev->p2pdma->pool); + rcu_read_lock(); + p2pdma = rcu_dereference(pdev->p2pdma); + if (p2pdma && p2pdma->pool) + size = gen_pool_size(p2pdma->pool); + rcu_read_unlock(); - return scnprintf(buf, PAGE_SIZE, "%zd\n", size); + return sysfs_emit(buf, "%zd\n", size); } static DEVICE_ATTR_RO(size); @@ -61,12 +65,16 @@ static ssize_t available_show(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev = to_pci_dev(dev); + struct pci_p2pdma *p2pdma; size_t avail = 0; - if (pdev->p2pdma->pool) - avail = gen_pool_avail(pdev->p2pdma->pool); + rcu_read_lock(); + p2pdma = rcu_dereference(pdev->p2pdma); + if (p2pdma && p2pdma->pool) + avail = gen_pool_avail(p2pdma->pool); + rcu_read_unlock(); - return scnprintf(buf, PAGE_SIZE, "%zd\n", avail); + return sysfs_emit(buf, "%zd\n", avail); } static DEVICE_ATTR_RO(available); @@ -74,9 +82,16 @@ static ssize_t published_show(struct device *dev, struct device_attribute *attr, char *buf) { struct pci_dev *pdev = to_pci_dev(dev); + struct pci_p2pdma *p2pdma; + bool published = false; - return scnprintf(buf, PAGE_SIZE, "%d\n", - pdev->p2pdma->p2pmem_published); + rcu_read_lock(); + p2pdma = rcu_dereference(pdev->p2pdma); + if (p2pdma) + published = p2pdma->p2pmem_published; + rcu_read_unlock(); + + return sysfs_emit(buf, "%d\n", published); } static DEVICE_ATTR_RO(published); @@ -95,8 +110,9 @@ static const struct attribute_group p2pmem_group = { static void pci_p2pdma_release(void *data) { struct pci_dev *pdev = data; - struct pci_p2pdma *p2pdma = pdev->p2pdma; + struct pci_p2pdma *p2pdma; + p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); if (!p2pdma) return; @@ -128,16 +144,14 @@ static int pci_p2pdma_setup(struct pci_dev *pdev) if (error) goto out_pool_destroy; - pdev->p2pdma = p2p; - error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group); if (error) goto out_pool_destroy; + rcu_assign_pointer(pdev->p2pdma, p2p); return 0; out_pool_destroy: - pdev->p2pdma = NULL; gen_pool_destroy(p2p->pool); out: devm_kfree(&pdev->dev, p2p); @@ -159,6 +173,7 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, { struct pci_p2pdma_pagemap *p2p_pgmap; struct dev_pagemap *pgmap; + struct pci_p2pdma *p2pdma; void *addr; int error; @@ -200,7 +215,8 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, goto pgmap_free; } - error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr, + p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); + error = gen_pool_add_owner(p2pdma->pool, (unsigned long)addr, pci_bus_address(pdev, bar) + offset, range_len(&pgmap->range), dev_to_node(&pdev->dev), pgmap->ref); @@ -308,10 +324,41 @@ static const struct pci_p2pdma_whitelist_entry { {} }; +/* + * This lookup function tries to find the PCI device corresponding to a given + * host bridge. + * + * It assumes the host bridge device is the first PCI device in the + * bus->devices list and that the devfn is 00.0. These assumptions should hold + * for all the devices in the whitelist above. + * + * This function is equivalent to pci_get_slot(host->bus, 0), however it does + * not take the pci_bus_sem lock seeing __host_bridge_whitelist() must not + * sleep. + * + * For this to be safe, the caller should hold a reference to a device on the + * bridge, which should ensure the host_bridge device will not be freed + * or removed from the head of the devices list. + */ +static struct pci_dev *pci_host_bridge_dev(struct pci_host_bridge *host) +{ + struct pci_dev *root; + + root = list_first_entry_or_null(&host->bus->devices, + struct pci_dev, bus_list); + + if (!root) + return NULL; + if (root->devfn != PCI_DEVFN(0, 0)) + return NULL; + + return root; +} + static bool __host_bridge_whitelist(struct pci_host_bridge *host, - bool same_host_bridge) + bool same_host_bridge, bool warn) { - struct pci_dev *root = pci_get_slot(host->bus, PCI_DEVFN(0, 0)); + struct pci_dev *root = pci_host_bridge_dev(host); const struct pci_p2pdma_whitelist_entry *entry; unsigned short vendor, device; @@ -320,7 +367,6 @@ static bool __host_bridge_whitelist(struct pci_host_bridge *host, vendor = root->vendor; device = root->device; - pci_dev_put(root); for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) { if (vendor != entry->vendor || device != entry->device) @@ -331,6 +377,10 @@ static bool __host_bridge_whitelist(struct pci_host_bridge *host, return true; } + if (warn) + pci_warn(root, "Host bridge not in P2PDMA whitelist: %04x:%04x\n", + vendor, device); + return false; } @@ -338,44 +388,90 @@ static bool __host_bridge_whitelist(struct pci_host_bridge *host, * If we can't find a common upstream bridge take a look at the root * complex and compare it to a whitelist of known good hardware. */ -static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b) +static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b, + bool warn) { struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus); struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus); if (host_a == host_b) - return __host_bridge_whitelist(host_a, true); + return __host_bridge_whitelist(host_a, true, warn); - if (__host_bridge_whitelist(host_a, false) && - __host_bridge_whitelist(host_b, false)) + if (__host_bridge_whitelist(host_a, false, warn) && + __host_bridge_whitelist(host_b, false, warn)) return true; return false; } +static unsigned long map_types_idx(struct pci_dev *client) +{ + return (pci_domain_nr(client->bus) << 16) | + (client->bus->number << 8) | client->devfn; +} + +/* + * Calculate the P2PDMA mapping type and distance between two PCI devices. + * + * If the two devices are the same PCI function, return + * PCI_P2PDMA_MAP_BUS_ADDR and a distance of 0. + * + * If they are two functions of the same device, return + * PCI_P2PDMA_MAP_BUS_ADDR and a distance of 2 (one hop up to the bridge, + * then one hop back down to another function of the same device). + * + * In the case where two devices are connected to the same PCIe switch, + * return a distance of 4. This corresponds to the following PCI tree: + * + * -+ Root Port + * \+ Switch Upstream Port + * +-+ Switch Downstream Port 0 + * + \- Device A + * \-+ Switch Downstream Port 1 + * \- Device B + * + * The distance is 4 because we traverse from Device A to Downstream Port 0 + * to the common Switch Upstream Port, back down to Downstream Port 1 and + * then to Device B. The mapping type returned depends on the ACS + * redirection setting of the ports along the path. + * + * If ACS redirect is set on any port in the path, traffic between the + * devices will go through the host bridge, so return + * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; otherwise return + * PCI_P2PDMA_MAP_BUS_ADDR. + * + * Any two devices that have a data path that goes through the host bridge + * will consult a whitelist. If the host bridge is in the whitelist, return + * PCI_P2PDMA_MAP_THRU_HOST_BRIDGE with the distance set to the number of + * ports per above. If the device is not in the whitelist, return + * PCI_P2PDMA_MAP_NOT_SUPPORTED. + */ static enum pci_p2pdma_map_type -__upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client, - int *dist, bool *acs_redirects, struct seq_buf *acs_list) +calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client, + int *dist, bool verbose) { + enum pci_p2pdma_map_type map_type = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; struct pci_dev *a = provider, *b = client, *bb; + bool acs_redirects = false; + struct pci_p2pdma *p2pdma; + struct seq_buf acs_list; + int acs_cnt = 0; int dist_a = 0; int dist_b = 0; - int acs_cnt = 0; + char buf[128]; - if (acs_redirects) - *acs_redirects = false; + seq_buf_init(&acs_list, buf, sizeof(buf)); /* * Note, we don't need to take references to devices returned by * pci_upstream_bridge() seeing we hold a reference to a child * device which will already hold a reference to the upstream bridge. */ - while (a) { dist_b = 0; if (pci_bridge_has_acs_redir(a)) { - seq_buf_print_bus_devfn(acs_list, a); + seq_buf_print_bus_devfn(&acs_list, a); acs_cnt++; } @@ -393,10 +489,8 @@ __upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client, dist_a++; } - if (dist) - *dist = dist_a + dist_b; - - return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; + *dist = dist_a + dist_b; + goto map_through_host_bridge; check_b_path_acs: bb = b; @@ -406,124 +500,45 @@ check_b_path_acs: break; if (pci_bridge_has_acs_redir(bb)) { - seq_buf_print_bus_devfn(acs_list, bb); + seq_buf_print_bus_devfn(&acs_list, bb); acs_cnt++; } bb = pci_upstream_bridge(bb); } - if (dist) - *dist = dist_a + dist_b; - - if (acs_cnt) { - if (acs_redirects) - *acs_redirects = true; - - return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; - } - - return PCI_P2PDMA_MAP_BUS_ADDR; -} - -static unsigned long map_types_idx(struct pci_dev *client) -{ - return (pci_domain_nr(client->bus) << 16) | - (client->bus->number << 8) | client->devfn; -} - -/* - * Find the distance through the nearest common upstream bridge between - * two PCI devices. - * - * If the two devices are the same device then 0 will be returned. - * - * If there are two virtual functions of the same device behind the same - * bridge port then 2 will be returned (one step down to the PCIe switch, - * then one step back to the same device). - * - * In the case where two devices are connected to the same PCIe switch, the - * value 4 will be returned. This corresponds to the following PCI tree: - * - * -+ Root Port - * \+ Switch Upstream Port - * +-+ Switch Downstream Port - * + \- Device A - * \-+ Switch Downstream Port - * \- Device B - * - * The distance is 4 because we traverse from Device A through the downstream - * port of the switch, to the common upstream port, back up to the second - * downstream port and then to Device B. - * - * Any two devices that cannot communicate using p2pdma will return - * PCI_P2PDMA_MAP_NOT_SUPPORTED. - * - * Any two devices that have a data path that goes through the host bridge - * will consult a whitelist. If the host bridges are on the whitelist, - * this function will return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE. - * - * If either bridge is not on the whitelist this function returns - * PCI_P2PDMA_MAP_NOT_SUPPORTED. - * - * If a bridge which has any ACS redirection bits set is in the path, - * acs_redirects will be set to true. In this case, a list of all infringing - * bridge addresses will be populated in acs_list (assuming it's non-null) - * for printk purposes. - */ -static enum pci_p2pdma_map_type -upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client, - int *dist, bool *acs_redirects, struct seq_buf *acs_list) -{ - enum pci_p2pdma_map_type map_type; - - map_type = __upstream_bridge_distance(provider, client, dist, - acs_redirects, acs_list); + *dist = dist_a + dist_b; - if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) { - if (!cpu_supports_p2pdma() && - !host_bridge_whitelist(provider, client)) - map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED; + if (!acs_cnt) { + map_type = PCI_P2PDMA_MAP_BUS_ADDR; + goto done; } - if (provider->p2pdma) - xa_store(&provider->p2pdma->map_types, map_types_idx(client), - xa_mk_value(map_type), GFP_KERNEL); - - return map_type; -} - -static enum pci_p2pdma_map_type -upstream_bridge_distance_warn(struct pci_dev *provider, struct pci_dev *client, - int *dist) -{ - struct seq_buf acs_list; - bool acs_redirects; - int ret; - - seq_buf_init(&acs_list, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE); - if (!acs_list.buffer) - return -ENOMEM; - - ret = upstream_bridge_distance(provider, client, dist, &acs_redirects, - &acs_list); - if (acs_redirects) { + if (verbose) { + acs_list.buffer[acs_list.len-1] = 0; /* drop final semicolon */ pci_warn(client, "ACS redirect is set between the client and provider (%s)\n", pci_name(provider)); - /* Drop final semicolon */ - acs_list.buffer[acs_list.len-1] = 0; pci_warn(client, "to disable ACS redirect for this path, add the kernel parameter: pci=disable_acs_redir=%s\n", acs_list.buffer); } + acs_redirects = true; - if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED) { - pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge or whitelisted host bridge\n", - pci_name(provider)); +map_through_host_bridge: + if (!cpu_supports_p2pdma() && + !host_bridge_whitelist(provider, client, acs_redirects)) { + if (verbose) + pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge or whitelisted host bridge\n", + pci_name(provider)); + map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED; } - - kfree(acs_list.buffer); - - return ret; +done: + rcu_read_lock(); + p2pdma = rcu_dereference(provider->p2pdma); + if (p2pdma) + xa_store(&p2pdma->map_types, map_types_idx(client), + xa_mk_value(map_type), GFP_KERNEL); + rcu_read_unlock(); + return map_type; } /** @@ -546,11 +561,11 @@ upstream_bridge_distance_warn(struct pci_dev *provider, struct pci_dev *client, int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose) { + enum pci_p2pdma_map_type map; bool not_supported = false; struct pci_dev *pci_client; int total_dist = 0; - int distance; - int i, ret; + int i, distance; if (num_clients == 0) return -1; @@ -564,16 +579,12 @@ int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, return -1; } - if (verbose) - ret = upstream_bridge_distance_warn(provider, - pci_client, &distance); - else - ret = upstream_bridge_distance(provider, pci_client, - &distance, NULL, NULL); + map = calc_map_type_and_dist(provider, pci_client, &distance, + verbose); pci_dev_put(pci_client); - if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED) + if (map == PCI_P2PDMA_MAP_NOT_SUPPORTED) not_supported = true; if (not_supported && !verbose) @@ -595,7 +606,15 @@ EXPORT_SYMBOL_GPL(pci_p2pdma_distance_many); */ bool pci_has_p2pmem(struct pci_dev *pdev) { - return pdev->p2pdma && pdev->p2pdma->p2pmem_published; + struct pci_p2pdma *p2pdma; + bool res; + + rcu_read_lock(); + p2pdma = rcu_dereference(pdev->p2pdma); + res = p2pdma && p2pdma->p2pmem_published; + rcu_read_unlock(); + + return res; } EXPORT_SYMBOL_GPL(pci_has_p2pmem); @@ -675,6 +694,7 @@ void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size) { void *ret = NULL; struct percpu_ref *ref; + struct pci_p2pdma *p2pdma; /* * Pairs with synchronize_rcu() in pci_p2pdma_release() to @@ -682,16 +702,16 @@ void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size) * read-lock. */ rcu_read_lock(); - if (unlikely(!pdev->p2pdma)) + p2pdma = rcu_dereference(pdev->p2pdma); + if (unlikely(!p2pdma)) goto out; - ret = (void *)gen_pool_alloc_owner(pdev->p2pdma->pool, size, - (void **) &ref); + ret = (void *)gen_pool_alloc_owner(p2pdma->pool, size, (void **) &ref); if (!ret) goto out; if (unlikely(!percpu_ref_tryget_live(ref))) { - gen_pool_free(pdev->p2pdma->pool, (unsigned long) ret, size); + gen_pool_free(p2pdma->pool, (unsigned long) ret, size); ret = NULL; goto out; } @@ -710,8 +730,9 @@ EXPORT_SYMBOL_GPL(pci_alloc_p2pmem); void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size) { struct percpu_ref *ref; + struct pci_p2pdma *p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); - gen_pool_free_owner(pdev->p2pdma->pool, (uintptr_t)addr, size, + gen_pool_free_owner(p2pdma->pool, (uintptr_t)addr, size, (void **) &ref); percpu_ref_put(ref); } @@ -725,9 +746,13 @@ EXPORT_SYMBOL_GPL(pci_free_p2pmem); */ pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr) { + struct pci_p2pdma *p2pdma; + if (!addr) return 0; - if (!pdev->p2pdma) + + p2pdma = rcu_dereference_protected(pdev->p2pdma, 1); + if (!p2pdma) return 0; /* @@ -735,7 +760,7 @@ pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr) * bus address as the physical address. So gen_pool_virt_to_phys() * actually returns the bus address despite the misleading name. */ - return gen_pool_virt_to_phys(pdev->p2pdma->pool, (unsigned long)addr); + return gen_pool_virt_to_phys(p2pdma->pool, (unsigned long)addr); } EXPORT_SYMBOL_GPL(pci_p2pmem_virt_to_bus); @@ -806,19 +831,40 @@ EXPORT_SYMBOL_GPL(pci_p2pmem_free_sgl); */ void pci_p2pmem_publish(struct pci_dev *pdev, bool publish) { - if (pdev->p2pdma) - pdev->p2pdma->p2pmem_published = publish; + struct pci_p2pdma *p2pdma; + + rcu_read_lock(); + p2pdma = rcu_dereference(pdev->p2pdma); + if (p2pdma) + p2pdma->p2pmem_published = publish; + rcu_read_unlock(); } EXPORT_SYMBOL_GPL(pci_p2pmem_publish); -static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct pci_dev *provider, - struct pci_dev *client) +static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct dev_pagemap *pgmap, + struct device *dev) { + enum pci_p2pdma_map_type type = PCI_P2PDMA_MAP_NOT_SUPPORTED; + struct pci_dev *provider = to_p2p_pgmap(pgmap)->provider; + struct pci_dev *client; + struct pci_p2pdma *p2pdma; + if (!provider->p2pdma) return PCI_P2PDMA_MAP_NOT_SUPPORTED; - return xa_to_value(xa_load(&provider->p2pdma->map_types, - map_types_idx(client))); + if (!dev_is_pci(dev)) + return PCI_P2PDMA_MAP_NOT_SUPPORTED; + + client = to_pci_dev(dev); + + rcu_read_lock(); + p2pdma = rcu_dereference(provider->p2pdma); + + if (p2pdma) + type = xa_to_value(xa_load(&p2pdma->map_types, + map_types_idx(client))); + rcu_read_unlock(); + return type; } static int __pci_p2pdma_map_sg(struct pci_p2pdma_pagemap *p2p_pgmap, @@ -828,7 +874,7 @@ static int __pci_p2pdma_map_sg(struct pci_p2pdma_pagemap *p2p_pgmap, int i; for_each_sg(sg, s, nents, i) { - s->dma_address = sg_phys(s) - p2p_pgmap->bus_offset; + s->dma_address = sg_phys(s) + p2p_pgmap->bus_offset; sg_dma_len(s) = s->length; } @@ -853,14 +899,8 @@ int pci_p2pdma_map_sg_attrs(struct device *dev, struct scatterlist *sg, { struct pci_p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(sg_page(sg)->pgmap); - struct pci_dev *client; - - if (WARN_ON_ONCE(!dev_is_pci(dev))) - return 0; - client = to_pci_dev(dev); - - switch (pci_p2pdma_map_type(p2p_pgmap->provider, client)) { + switch (pci_p2pdma_map_type(sg_page(sg)->pgmap, dev)) { case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: return dma_map_sg_attrs(dev, sg, nents, dir, attrs); case PCI_P2PDMA_MAP_BUS_ADDR: @@ -884,17 +924,9 @@ EXPORT_SYMBOL_GPL(pci_p2pdma_map_sg_attrs); void pci_p2pdma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs) { - struct pci_p2pdma_pagemap *p2p_pgmap = - to_p2p_pgmap(sg_page(sg)->pgmap); enum pci_p2pdma_map_type map_type; - struct pci_dev *client; - - if (WARN_ON_ONCE(!dev_is_pci(dev))) - return; - - client = to_pci_dev(dev); - map_type = pci_p2pdma_map_type(p2p_pgmap->provider, client); + map_type = pci_p2pdma_map_type(sg_page(sg)->pgmap, dev); if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) dma_unmap_sg_attrs(dev, sg, nents, dir, attrs); @@ -911,7 +943,7 @@ EXPORT_SYMBOL_GPL(pci_p2pdma_unmap_sg_attrs); * * Parses an attribute value to decide whether to enable p2pdma. * The value can select a PCI device (using its full BDF device - * name) or a boolean (in any format strtobool() accepts). A false + * name) or a boolean (in any format kstrtobool() accepts). A false * value disables p2pdma, a true value expects the caller * to automatically find a compatible device and specifying a PCI device * expects the caller to use the specific provider. @@ -943,11 +975,11 @@ int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev, } else if ((page[0] == '0' || page[0] == '1') && !iscntrl(page[1])) { /* * If the user enters a PCI device that doesn't exist - * like "0000:01:00.1", we don't want strtobool to think + * like "0000:01:00.1", we don't want kstrtobool to think * it's a '0' when it's clearly not what the user wanted. * So we require 0's and 1's to be exactly one character. */ - } else if (!strtobool(page, use_p2pdma)) { + } else if (!kstrtobool(page, use_p2pdma)) { return 0; } diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 36bc23e21759..a42dbf448860 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -17,6 +17,7 @@ #include <linux/pci-acpi.h> #include <linux/pm_runtime.h> #include <linux/pm_qos.h> +#include <linux/rwsem.h> #include "pci.h" /* @@ -905,7 +906,7 @@ acpi_status pci_acpi_add_pm_notifier(struct acpi_device *dev, * choose highest power _SxD or any lower power */ -static pci_power_t acpi_pci_choose_state(struct pci_dev *pdev) +pci_power_t acpi_pci_choose_state(struct pci_dev *pdev) { int acpi_state, d_max; @@ -934,61 +935,78 @@ static pci_power_t acpi_pci_choose_state(struct pci_dev *pdev) static struct acpi_device *acpi_pci_find_companion(struct device *dev); -static bool acpi_pci_bridge_d3(struct pci_dev *dev) +void pci_set_acpi_fwnode(struct pci_dev *dev) { - const struct fwnode_handle *fwnode; + if (!dev_fwnode(&dev->dev) && !pci_dev_is_added(dev)) + ACPI_COMPANION_SET(&dev->dev, + acpi_pci_find_companion(&dev->dev)); +} + +/** + * pci_dev_acpi_reset - do a function level reset using _RST method + * @dev: device to reset + * @probe: if true, return 0 if device supports _RST + */ +int pci_dev_acpi_reset(struct pci_dev *dev, bool probe) +{ + acpi_handle handle = ACPI_HANDLE(&dev->dev); + + if (!handle || !acpi_has_method(handle, "_RST")) + return -ENOTTY; + + if (probe) + return 0; + + if (ACPI_FAILURE(acpi_evaluate_object(handle, "_RST", NULL, NULL))) { + pci_warn(dev, "ACPI _RST failed\n"); + return -ENOTTY; + } + + return 0; +} + +bool acpi_pci_power_manageable(struct pci_dev *dev) +{ + struct acpi_device *adev = ACPI_COMPANION(&dev->dev); + + return adev && acpi_device_power_manageable(adev); +} + +bool acpi_pci_bridge_d3(struct pci_dev *dev) +{ + const union acpi_object *obj; struct acpi_device *adev; - struct pci_dev *root; - u8 val; + struct pci_dev *rpdev; - if (!dev->is_hotplug_bridge) + if (acpi_pci_disabled || !dev->is_hotplug_bridge) return false; /* Assume D3 support if the bridge is power-manageable by ACPI. */ - adev = ACPI_COMPANION(&dev->dev); - if (!adev && !pci_dev_is_added(dev)) { - adev = acpi_pci_find_companion(&dev->dev); - ACPI_COMPANION_SET(&dev->dev, adev); - } - - if (adev && acpi_device_power_manageable(adev)) + if (acpi_pci_power_manageable(dev)) return true; /* - * Look for a special _DSD property for the root port and if it - * is set we know the hierarchy behind it supports D3 just fine. + * The ACPI firmware will provide the device-specific properties through + * _DSD configuration object. Look for the 'HotPlugSupportInD3' property + * for the root port and if it is set we know the hierarchy behind it + * supports D3 just fine. */ - root = pcie_find_root_port(dev); - if (!root) + rpdev = pcie_find_root_port(dev); + if (!rpdev) return false; - adev = ACPI_COMPANION(&root->dev); - if (root == dev) { - /* - * It is possible that the ACPI companion is not yet bound - * for the root port so look it up manually here. - */ - if (!adev && !pci_dev_is_added(root)) - adev = acpi_pci_find_companion(&root->dev); - } - + adev = ACPI_COMPANION(&rpdev->dev); if (!adev) return false; - fwnode = acpi_fwnode_handle(adev); - if (fwnode_property_read_u8(fwnode, "HotPlugSupportInD3", &val)) + if (acpi_dev_get_property(adev, "HotPlugSupportInD3", + ACPI_TYPE_INTEGER, &obj) < 0) return false; - return val == 1; -} - -static bool acpi_pci_power_manageable(struct pci_dev *dev) -{ - struct acpi_device *adev = ACPI_COMPANION(&dev->dev); - return adev ? acpi_device_power_manageable(adev) : false; + return obj->integer.value == 1; } -static int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state) +int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state) { struct acpi_device *adev = ACPI_COMPANION(&dev->dev); static const u8 state_conv[] = { @@ -1026,7 +1044,7 @@ static int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state) return error; } -static pci_power_t acpi_pci_get_power_state(struct pci_dev *dev) +pci_power_t acpi_pci_get_power_state(struct pci_dev *dev) { struct acpi_device *adev = ACPI_COMPANION(&dev->dev); static const pci_power_t state_conv[] = { @@ -1048,7 +1066,7 @@ static pci_power_t acpi_pci_get_power_state(struct pci_dev *dev) return state_conv[state]; } -static void acpi_pci_refresh_power_state(struct pci_dev *dev) +void acpi_pci_refresh_power_state(struct pci_dev *dev) { struct acpi_device *adev = ACPI_COMPANION(&dev->dev); @@ -1073,17 +1091,23 @@ static int acpi_pci_propagate_wakeup(struct pci_bus *bus, bool enable) return 0; } -static int acpi_pci_wakeup(struct pci_dev *dev, bool enable) +int acpi_pci_wakeup(struct pci_dev *dev, bool enable) { + if (acpi_pci_disabled) + return 0; + if (acpi_pm_device_can_wakeup(&dev->dev)) return acpi_pm_set_device_wakeup(&dev->dev, enable); return acpi_pci_propagate_wakeup(dev->bus, enable); } -static bool acpi_pci_need_resume(struct pci_dev *dev) +bool acpi_pci_need_resume(struct pci_dev *dev) { - struct acpi_device *adev = ACPI_COMPANION(&dev->dev); + struct acpi_device *adev; + + if (acpi_pci_disabled) + return false; /* * In some cases (eg. Samsung 305V4A) leaving a bridge in suspend over @@ -1095,6 +1119,7 @@ static bool acpi_pci_need_resume(struct pci_dev *dev) if (pci_is_bridge(dev) && acpi_target_system_state() != ACPI_STATE_S0) return true; + adev = ACPI_COMPANION(&dev->dev); if (!adev || !acpi_device_power_manageable(adev)) return false; @@ -1108,17 +1133,6 @@ static bool acpi_pci_need_resume(struct pci_dev *dev) return !!adev->power.flags.dsw_present; } -static const struct pci_platform_pm_ops acpi_pci_platform_pm = { - .bridge_d3 = acpi_pci_bridge_d3, - .is_manageable = acpi_pci_power_manageable, - .set_state = acpi_pci_set_power_state, - .get_state = acpi_pci_get_power_state, - .refresh_state = acpi_pci_refresh_power_state, - .choose_state = acpi_pci_choose_state, - .set_wakeup = acpi_pci_wakeup, - .need_resume = acpi_pci_need_resume, -}; - void acpi_pci_add_bus(struct pci_bus *bus) { union acpi_object *obj; @@ -1159,6 +1173,69 @@ void acpi_pci_remove_bus(struct pci_bus *bus) } /* ACPI bus type */ + + +static DECLARE_RWSEM(pci_acpi_companion_lookup_sem); +static struct acpi_device *(*pci_acpi_find_companion_hook)(struct pci_dev *); + +/** + * pci_acpi_set_companion_lookup_hook - Set ACPI companion lookup callback. + * @func: ACPI companion lookup callback pointer or NULL. + * + * Set a special ACPI companion lookup callback for PCI devices whose companion + * objects in the ACPI namespace have _ADR with non-standard bus-device-function + * encodings. + * + * Return 0 on success or a negative error code on failure (in which case no + * changes are made). + * + * The caller is responsible for the appropriate ordering of the invocations of + * this function with respect to the enumeration of the PCI devices needing the + * callback installed by it. + */ +int pci_acpi_set_companion_lookup_hook(struct acpi_device *(*func)(struct pci_dev *)) +{ + int ret; + + if (!func) + return -EINVAL; + + down_write(&pci_acpi_companion_lookup_sem); + + if (pci_acpi_find_companion_hook) { + ret = -EBUSY; + } else { + pci_acpi_find_companion_hook = func; + ret = 0; + } + + up_write(&pci_acpi_companion_lookup_sem); + + return ret; +} +EXPORT_SYMBOL_GPL(pci_acpi_set_companion_lookup_hook); + +/** + * pci_acpi_clear_companion_lookup_hook - Clear ACPI companion lookup callback. + * + * Clear the special ACPI companion lookup callback previously set by + * pci_acpi_set_companion_lookup_hook(). Block until the last running instance + * of the callback returns before clearing it. + * + * The caller is responsible for the appropriate ordering of the invocations of + * this function with respect to the enumeration of the PCI devices needing the + * callback cleared by it. + */ +void pci_acpi_clear_companion_lookup_hook(void) +{ + down_write(&pci_acpi_companion_lookup_sem); + + pci_acpi_find_companion_hook = NULL; + + up_write(&pci_acpi_companion_lookup_sem); +} +EXPORT_SYMBOL_GPL(pci_acpi_clear_companion_lookup_hook); + static struct acpi_device *acpi_pci_find_companion(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); @@ -1166,6 +1243,19 @@ static struct acpi_device *acpi_pci_find_companion(struct device *dev) bool check_children; u64 addr; + if (!dev->parent) + return NULL; + + down_read(&pci_acpi_companion_lookup_sem); + + adev = pci_acpi_find_companion_hook ? + pci_acpi_find_companion_hook(pci_dev) : NULL; + + up_read(&pci_acpi_companion_lookup_sem); + + if (adev) + return adev; + check_children = pci_is_bridge(pci_dev); /* Please ref to ACPI spec for the syntax of _ADR */ addr = (PCI_SLOT(pci_dev->devfn) << 16) | PCI_FUNC(pci_dev->devfn); @@ -1260,13 +1350,9 @@ static void pci_acpi_set_external_facing(struct pci_dev *dev) dev->external_facing = 1; } -static void pci_acpi_setup(struct device *dev) +void pci_acpi_setup(struct device *dev, struct acpi_device *adev) { struct pci_dev *pci_dev = to_pci_dev(dev); - struct acpi_device *adev = ACPI_COMPANION(dev); - - if (!adev) - return; pci_acpi_optimize_delay(pci_dev, adev->handle); pci_acpi_set_external_facing(pci_dev); @@ -1290,14 +1376,10 @@ static void pci_acpi_setup(struct device *dev) acpi_device_power_add_dependent(adev, dev); } -static void pci_acpi_cleanup(struct device *dev) +void pci_acpi_cleanup(struct device *dev, struct acpi_device *adev) { - struct acpi_device *adev = ACPI_COMPANION(dev); struct pci_dev *pci_dev = to_pci_dev(dev); - if (!adev) - return; - pci_acpi_remove_edr_notifier(pci_dev); pci_acpi_remove_pm_notifier(adev); if (adev->wakeup.flags.valid) { @@ -1309,20 +1391,6 @@ static void pci_acpi_cleanup(struct device *dev) } } -static bool pci_acpi_bus_match(struct device *dev) -{ - return dev_is_pci(dev); -} - -static struct acpi_bus_type acpi_pci_bus = { - .name = "PCI", - .match = pci_acpi_bus_match, - .find_companion = acpi_pci_find_companion, - .setup = pci_acpi_setup, - .cleanup = pci_acpi_cleanup, -}; - - static struct fwnode_handle *(*pci_msi_get_fwnode_cb)(struct device *dev); /** @@ -1364,8 +1432,6 @@ struct irq_domain *pci_host_bridge_acpi_msi_domain(struct pci_bus *bus) static int __init acpi_pci_init(void) { - int ret; - if (acpi_gbl_FADT.boot_flags & ACPI_FADT_NO_MSI) { pr_info("ACPI FADT declares the system doesn't support MSI, so disable it\n"); pci_no_msi(); @@ -1376,11 +1442,9 @@ static int __init acpi_pci_init(void) pcie_no_aspm(); } - ret = register_acpi_bus_type(&acpi_pci_bus); - if (ret) + if (acpi_pci_disabled) return 0; - pci_set_platform_pm(&acpi_pci_platform_pm); acpi_pci_slot_init(); acpiphp_init(); diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index fdaf86a888b7..db97cddfc85e 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -431,8 +431,21 @@ int pci_bridge_emul_conf_write(struct pci_bridge_emul *bridge, int where, /* Clear the W1C bits */ new &= ~((value << shift) & (behavior[reg / 4].w1c & mask)); + /* Save the new value with the cleared W1C bits into the cfgspace */ cfgspace[reg / 4] = cpu_to_le32(new); + /* + * Clear the W1C bits not specified by the write mask, so that the + * write_op() does not clear them. + */ + new &= ~(behavior[reg / 4].w1c & ~mask); + + /* + * Set the W1C bits specified by the write mask, so that write_op() + * knows about that they are to be cleared. + */ + new |= (value << shift) & (behavior[reg / 4].w1c & mask); + if (write_op) write_op(bridge, reg, old, new, mask); diff --git a/drivers/pci/pci-bridge-emul.h b/drivers/pci/pci-bridge-emul.h index b31883022a8e..49bbd37ee318 100644 --- a/drivers/pci/pci-bridge-emul.h +++ b/drivers/pci/pci-bridge-emul.h @@ -54,7 +54,7 @@ struct pci_bridge_emul_pcie_conf { __le16 slotctl; __le16 slotsta; __le16 rootctl; - __le16 rsvd; + __le16 rootcap; __le32 rootsta; __le32 devcap2; __le16 devctl2; diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index ec44a79e951a..588588cfda48 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -136,7 +136,7 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, struct pci_dev *dev) { struct pci_dynid *dynid; - const struct pci_device_id *found_id = NULL; + const struct pci_device_id *found_id = NULL, *ids; /* When driver_override is set, only bind to the matching driver */ if (dev->driver_override && strcmp(dev->driver_override, drv->name)) @@ -152,14 +152,28 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, } spin_unlock(&drv->dynids.lock); - if (!found_id) - found_id = pci_match_id(drv->id_table, dev); + if (found_id) + return found_id; - /* driver_override will always match, send a dummy id */ - if (!found_id && dev->driver_override) - found_id = &pci_device_id_any; + for (ids = drv->id_table; (found_id = pci_match_id(ids, dev)); + ids = found_id + 1) { + /* + * The match table is split based on driver_override. + * In case override_only was set, enforce driver_override + * matching. + */ + if (found_id->override_only) { + if (dev->driver_override) + return found_id; + } else { + return found_id; + } + } - return found_id; + /* driver_override will always match, send a dummy id */ + if (dev->driver_override) + return &pci_device_id_any; + return NULL; } /** @@ -383,7 +397,7 @@ static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) const struct pci_device_id *id; int error = 0; - if (!pci_dev->driver && drv->probe) { + if (drv->probe) { error = -ENODEV; id = pci_match_device(drv, pci_dev); @@ -440,21 +454,19 @@ static int pci_device_probe(struct device *dev) return error; } -static int pci_device_remove(struct device *dev) +static void pci_device_remove(struct device *dev) { struct pci_dev *pci_dev = to_pci_dev(dev); struct pci_driver *drv = pci_dev->driver; - if (drv) { - if (drv->remove) { - pm_runtime_get_sync(dev); - drv->remove(pci_dev); - pm_runtime_put_noidle(dev); - } - pcibios_free_irq(pci_dev); - pci_dev->driver = NULL; - pci_iov_remove(pci_dev); + if (drv->remove) { + pm_runtime_get_sync(dev); + drv->remove(pci_dev); + pm_runtime_put_noidle(dev); } + pcibios_free_irq(pci_dev); + pci_dev->driver = NULL; + pci_iov_remove(pci_dev); /* Undo the runtime PM settings in local_pci_probe() */ pm_runtime_put_sync(dev); @@ -476,7 +488,6 @@ static int pci_device_remove(struct device *dev) */ pci_dev_put(pci_dev); - return 0; } static void pci_device_shutdown(struct device *dev) @@ -563,7 +574,7 @@ static int pci_pm_reenable_device(struct pci_dev *pci_dev) { int retval; - /* if the device was enabled before suspend, reenable */ + /* if the device was enabled before suspend, re-enable */ retval = pci_reenable_device(pci_dev); /* * if the device was busmaster before the suspend, make it busmaster @@ -1385,6 +1396,7 @@ int __pci_register_driver(struct pci_driver *drv, struct module *owner, drv->driver.owner = owner; drv->driver.mod_name = mod_name; drv->driver.groups = drv->groups; + drv->driver.dev_groups = drv->dev_groups; spin_lock_init(&drv->dynids.lock); INIT_LIST_HEAD(&drv->dynids.list); @@ -1528,7 +1540,7 @@ static int pci_uevent(struct device *dev, struct kobj_uevent_env *env) return 0; } -#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH) +#if defined(CONFIG_PCIEAER) || defined(CONFIG_EEH) /** * pci_uevent_ers - emit a uevent during recovery path of PCI device * @pdev: PCI device undergoing error recovery diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c index c32f3b7540e8..0c6446519640 100644 --- a/drivers/pci/pci-label.c +++ b/drivers/pci/pci-label.c @@ -139,14 +139,17 @@ enum acpi_attr_enum { ACPI_ATTR_INDEX_SHOW, }; -static void dsm_label_utf16s_to_utf8s(union acpi_object *obj, char *buf) +static int dsm_label_utf16s_to_utf8s(union acpi_object *obj, char *buf) { int len; + len = utf16s_to_utf8s((const wchar_t *)obj->buffer.pointer, obj->buffer.length, UTF16_LITTLE_ENDIAN, - buf, PAGE_SIZE); - buf[len] = '\n'; + buf, PAGE_SIZE - 1); + buf[len++] = '\n'; + + return len; } static int dsm_get_label(struct device *dev, char *buf, @@ -154,7 +157,7 @@ static int dsm_get_label(struct device *dev, char *buf, { acpi_handle handle = ACPI_HANDLE(dev); union acpi_object *obj, *tmp; - int len = -1; + int len = 0; if (!handle) return -1; @@ -175,20 +178,19 @@ static int dsm_get_label(struct device *dev, char *buf, * this entry must return a null string. */ if (attr == ACPI_ATTR_INDEX_SHOW) { - scnprintf(buf, PAGE_SIZE, "%llu\n", tmp->integer.value); + len = sysfs_emit(buf, "%llu\n", tmp->integer.value); } else if (attr == ACPI_ATTR_LABEL_SHOW) { if (tmp[1].type == ACPI_TYPE_STRING) - scnprintf(buf, PAGE_SIZE, "%s\n", - tmp[1].string.pointer); + len = sysfs_emit(buf, "%s\n", + tmp[1].string.pointer); else if (tmp[1].type == ACPI_TYPE_BUFFER) - dsm_label_utf16s_to_utf8s(tmp + 1, buf); + len = dsm_label_utf16s_to_utf8s(tmp + 1, buf); } - len = strlen(buf) > 0 ? strlen(buf) : -1; } ACPI_FREE(obj); - return len; + return len > 0 ? len : -1; } static ssize_t label_show(struct device *dev, struct device_attribute *attr, diff --git a/drivers/pci/pci-mid.c b/drivers/pci/pci-mid.c index aafd58da3a89..fbfd78127123 100644 --- a/drivers/pci/pci-mid.c +++ b/drivers/pci/pci-mid.c @@ -16,45 +16,23 @@ #include "pci.h" -static bool mid_pci_power_manageable(struct pci_dev *dev) +static bool pci_mid_pm_enabled __read_mostly; + +bool pci_use_mid_pm(void) { - return true; + return pci_mid_pm_enabled; } -static int mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state) +int mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state) { return intel_mid_pci_set_power_state(pdev, state); } -static pci_power_t mid_pci_get_power_state(struct pci_dev *pdev) +pci_power_t mid_pci_get_power_state(struct pci_dev *pdev) { return intel_mid_pci_get_power_state(pdev); } -static pci_power_t mid_pci_choose_state(struct pci_dev *pdev) -{ - return PCI_D3hot; -} - -static int mid_pci_wakeup(struct pci_dev *dev, bool enable) -{ - return 0; -} - -static bool mid_pci_need_resume(struct pci_dev *dev) -{ - return false; -} - -static const struct pci_platform_pm_ops mid_pci_platform_pm = { - .is_manageable = mid_pci_power_manageable, - .set_state = mid_pci_set_power_state, - .get_state = mid_pci_get_power_state, - .choose_state = mid_pci_choose_state, - .set_wakeup = mid_pci_wakeup, - .need_resume = mid_pci_need_resume, -}; - /* * This table should be in sync with the one in * arch/x86/platform/intel-mid/pwr.c. @@ -71,7 +49,8 @@ static int __init mid_pci_init(void) id = x86_match_cpu(lpss_cpu_ids); if (id) - pci_set_platform_pm(&mid_pci_platform_pm); + pci_mid_pm_enabled = true; + return 0; } arch_initcall(mid_pci_init); diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index beb8d1f4fafe..cfe2f85af09e 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -26,6 +26,7 @@ #include <linux/slab.h> #include <linux/vgaarb.h> #include <linux/pm_runtime.h> +#include <linux/msi.h> #include <linux/of.h> #include "pci.h" @@ -49,7 +50,28 @@ pci_config_attr(subsystem_vendor, "0x%04x\n"); pci_config_attr(subsystem_device, "0x%04x\n"); pci_config_attr(revision, "0x%02x\n"); pci_config_attr(class, "0x%06x\n"); -pci_config_attr(irq, "%u\n"); + +static ssize_t irq_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + +#ifdef CONFIG_PCI_MSI + /* + * For MSI, show the first MSI IRQ; for all other cases including + * MSI-X, show the legacy INTx IRQ. + */ + if (pdev->msi_enabled) { + struct msi_desc *desc = first_pci_msi_entry(pdev); + + return sysfs_emit(buf, "%u\n", desc->irq); + } +#endif + + return sysfs_emit(buf, "%u\n", pdev->irq); +} +static DEVICE_ATTR_RO(irq); static ssize_t broken_parity_status_show(struct device *dev, struct device_attribute *attr, @@ -81,8 +103,10 @@ static ssize_t pci_dev_show_local_cpu(struct device *dev, bool list, const struct cpumask *mask; #ifdef CONFIG_NUMA - mask = (dev_to_node(dev) == -1) ? cpu_online_mask : - cpumask_of_node(dev_to_node(dev)); + if (dev_to_node(dev) == NUMA_NO_NODE) + mask = cpu_online_mask; + else + mask = cpumask_of_node(dev_to_node(dev)); #else mask = cpumask_of_pcibus(to_pci_dev(dev)->bus); #endif @@ -273,15 +297,15 @@ static ssize_t enable_store(struct device *dev, struct device_attribute *attr, { struct pci_dev *pdev = to_pci_dev(dev); unsigned long val; - ssize_t result = kstrtoul(buf, 0, &val); - - if (result < 0) - return result; + ssize_t result = 0; /* this can crash the machine when done on the "wrong" device */ if (!capable(CAP_SYS_ADMIN)) return -EPERM; + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + device_lock(dev); if (dev->driver) result = -EBUSY; @@ -312,14 +336,13 @@ static ssize_t numa_node_store(struct device *dev, size_t count) { struct pci_dev *pdev = to_pci_dev(dev); - int node, ret; + int node; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - ret = kstrtoint(buf, 0, &node); - if (ret) - return ret; + if (kstrtoint(buf, 0, &node) < 0) + return -EINVAL; if ((node < 0 && node != NUMA_NO_NODE) || node >= MAX_NUMNODES) return -EINVAL; @@ -378,12 +401,12 @@ static ssize_t msi_bus_store(struct device *dev, struct device_attribute *attr, struct pci_bus *subordinate = pdev->subordinate; unsigned long val; - if (kstrtoul(buf, 0, &val) < 0) - return -EINVAL; - if (!capable(CAP_SYS_ADMIN)) return -EPERM; + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + /* * "no_msi" and "bus_flags" only affect what happens when a driver * requests MSI or MSI-X. They don't affect any drivers that have @@ -537,7 +560,7 @@ static ssize_t devspec_show(struct device *dev, if (np == NULL) return 0; - return sysfs_emit(buf, "%pOF", np); + return sysfs_emit(buf, "%pOF\n", np); } static DEVICE_ATTR_RO(devspec); #endif @@ -965,7 +988,7 @@ void pci_create_legacy_files(struct pci_bus *b) b->legacy_io->read = pci_read_legacy_io; b->legacy_io->write = pci_write_legacy_io; b->legacy_io->mmap = pci_mmap_legacy_io; - b->legacy_io->mapping = iomem_get_mapping(); + b->legacy_io->f_mapping = iomem_get_mapping; pci_adjust_legacy_attr(b, pci_mmap_io); error = device_create_bin_file(&b->dev, b->legacy_io); if (error) @@ -978,7 +1001,7 @@ void pci_create_legacy_files(struct pci_bus *b) b->legacy_mem->size = 1024*1024; b->legacy_mem->attr.mode = 0600; b->legacy_mem->mmap = pci_mmap_legacy_mem; - b->legacy_io->mapping = iomem_get_mapping(); + b->legacy_mem->f_mapping = iomem_get_mapping; pci_adjust_legacy_attr(b, pci_mmap_mem); error = device_create_bin_file(&b->dev, b->legacy_mem); if (error) @@ -1195,7 +1218,7 @@ static int pci_create_attr(struct pci_dev *pdev, int num, int write_combine) } } if (res_attr->mmap) - res_attr->mapping = iomem_get_mapping(); + res_attr->f_mapping = iomem_get_mapping; res_attr->attr.name = res_attr_name; res_attr->attr.mode = 0600; res_attr->size = pci_resource_len(pdev, num); @@ -1339,10 +1362,10 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, { struct pci_dev *pdev = to_pci_dev(dev); unsigned long val; - ssize_t result = kstrtoul(buf, 0, &val); + ssize_t result; - if (result < 0) - return result; + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; if (val != 1) return -EINVAL; @@ -1367,7 +1390,7 @@ static umode_t pci_dev_reset_attr_is_visible(struct kobject *kobj, { struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); - if (!pdev->reset_fn) + if (!pci_reset_supported(pdev)) return 0; return a->mode; @@ -1491,6 +1514,7 @@ const struct attribute_group *pci_dev_groups[] = { &pci_dev_config_attr_group, &pci_dev_rom_attr_group, &pci_dev_reset_attr_group, + &pci_dev_reset_method_attr_group, &pci_dev_vpd_attr_group, #ifdef CONFIG_DMI &pci_dev_smbios_attr_group, diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8d4ebe095d0c..3d2fb394986a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -31,6 +31,7 @@ #include <linux/vmalloc.h> #include <asm/dma.h> #include <linux/aer.h> +#include <linux/bitfield.h> #include "pci.h" DEFINE_MUTEX(pci_slot_mutex); @@ -72,6 +73,11 @@ static void pci_dev_d3_sleep(struct pci_dev *dev) msleep(delay); } +bool pci_reset_supported(struct pci_dev *dev) +{ + return dev->reset_methods[0] != 0; +} + #ifdef CONFIG_PCI_DOMAINS int pci_domains_supported = 1; #endif @@ -206,32 +212,36 @@ int pci_status_get_and_clear_errors(struct pci_dev *pdev) EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors); #ifdef CONFIG_HAS_IOMEM -void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) +static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar, + bool write_combine) { struct resource *res = &pdev->resource[bar]; + resource_size_t start = res->start; + resource_size_t size = resource_size(res); /* * Make sure the BAR is actually a memory resource, not an IO resource */ if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { - pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res); + pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res); return NULL; } - return ioremap(res->start, resource_size(res)); + + if (write_combine) + return ioremap_wc(start, size); + + return ioremap(start, size); +} + +void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) +{ + return __pci_ioremap_resource(pdev, bar, false); } EXPORT_SYMBOL_GPL(pci_ioremap_bar); void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) { - /* - * Make sure the BAR is actually a memory resource, not an IO resource - */ - if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { - WARN_ON(1); - return NULL; - } - return ioremap_wc(pci_resource_start(pdev, bar), - pci_resource_len(pdev, bar)); + return __pci_ioremap_resource(pdev, bar, true); } EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); #endif @@ -259,13 +269,13 @@ static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, const char **endptr) { int ret; - int seg, bus, slot, func; + unsigned int seg, bus, slot, func; char *wpath, *p; char end; *endptr = strchrnul(path, ';'); - wpath = kmemdup_nul(path, *endptr - path, GFP_KERNEL); + wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC); if (!wpath) return -ENOMEM; @@ -723,6 +733,38 @@ u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap) EXPORT_SYMBOL_GPL(pci_find_vsec_capability); /** + * pci_find_dvsec_capability - Find DVSEC for vendor + * @dev: PCI device to query + * @vendor: Vendor ID to match for the DVSEC + * @dvsec: Designated Vendor-specific capability ID + * + * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability + * offset in config space; otherwise return 0. + */ +u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec) +{ + int pos; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC); + if (!pos) + return 0; + + while (pos) { + u16 v, id; + + pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v); + pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id); + if (vendor == v && dvsec == id) + return pos; + + pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC); + } + + return 0; +} +EXPORT_SYMBOL_GPL(pci_find_dvsec_capability); + +/** * pci_find_parent_resource - return resource region of parent bus of given * region * @dev: PCI device structure contains resources to be searched @@ -915,8 +957,8 @@ static void pci_std_enable_acs(struct pci_dev *dev) /* Upstream Forwarding */ ctrl |= (cap & PCI_ACS_UF); - /* Enable Translation Blocking for external devices */ - if (dev->external_facing || dev->untrusted) + /* Enable Translation Blocking for external devices and noats */ + if (pci_ats_disabled() || dev->external_facing || dev->untrusted) ctrl |= (cap & PCI_ACS_TB); pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); @@ -962,61 +1004,67 @@ static void pci_restore_bars(struct pci_dev *dev) pci_update_resource(dev, i); } -static const struct pci_platform_pm_ops *pci_platform_pm; - -int pci_set_platform_pm(const struct pci_platform_pm_ops *ops) -{ - if (!ops->is_manageable || !ops->set_state || !ops->get_state || - !ops->choose_state || !ops->set_wakeup || !ops->need_resume) - return -EINVAL; - pci_platform_pm = ops; - return 0; -} - static inline bool platform_pci_power_manageable(struct pci_dev *dev) { - return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false; + if (pci_use_mid_pm()) + return true; + + return acpi_pci_power_manageable(dev); } static inline int platform_pci_set_power_state(struct pci_dev *dev, pci_power_t t) { - return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS; + if (pci_use_mid_pm()) + return mid_pci_set_power_state(dev, t); + + return acpi_pci_set_power_state(dev, t); } static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) { - return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN; + if (pci_use_mid_pm()) + return mid_pci_get_power_state(dev); + + return acpi_pci_get_power_state(dev); } static inline void platform_pci_refresh_power_state(struct pci_dev *dev) { - if (pci_platform_pm && pci_platform_pm->refresh_state) - pci_platform_pm->refresh_state(dev); + if (!pci_use_mid_pm()) + acpi_pci_refresh_power_state(dev); } static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) { - return pci_platform_pm ? - pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR; + if (pci_use_mid_pm()) + return PCI_POWER_ERROR; + + return acpi_pci_choose_state(dev); } static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) { - return pci_platform_pm ? - pci_platform_pm->set_wakeup(dev, enable) : -ENODEV; + if (pci_use_mid_pm()) + return PCI_POWER_ERROR; + + return acpi_pci_wakeup(dev, enable); } static inline bool platform_pci_need_resume(struct pci_dev *dev) { - return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false; + if (pci_use_mid_pm()) + return false; + + return acpi_pci_need_resume(dev); } static inline bool platform_pci_bridge_d3(struct pci_dev *dev) { - if (pci_platform_pm && pci_platform_pm->bridge_d3) - return pci_platform_pm->bridge_d3(dev); - return false; + if (pci_use_mid_pm()) + return false; + + return acpi_pci_bridge_d3(dev); } /** @@ -1175,9 +1223,7 @@ void pci_update_current_state(struct pci_dev *dev, pci_power_t state) */ void pci_refresh_power_state(struct pci_dev *dev) { - if (platform_pci_power_manageable(dev)) - platform_pci_refresh_power_state(dev); - + platform_pci_refresh_power_state(dev); pci_update_current_state(dev, dev->current_state); } @@ -1190,14 +1236,10 @@ int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) { int error; - if (platform_pci_power_manageable(dev)) { - error = platform_pci_set_power_state(dev, state); - if (!error) - pci_update_current_state(dev, state); - } else - error = -ENODEV; - - if (error && !dev->pm_cap) /* Fall back to PCI_D0 */ + error = platform_pci_set_power_state(dev, state); + if (!error) + pci_update_current_state(dev, state); + else if (!dev->pm_cap) /* Fall back to PCI_D0 */ dev->current_state = PCI_D0; return error; @@ -1378,44 +1420,6 @@ int pci_set_power_state(struct pci_dev *dev, pci_power_t state) } EXPORT_SYMBOL(pci_set_power_state); -/** - * pci_choose_state - Choose the power state of a PCI device - * @dev: PCI device to be suspended - * @state: target sleep state for the whole system. This is the value - * that is passed to suspend() function. - * - * Returns PCI power state suitable for given device and given system - * message. - */ -pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) -{ - pci_power_t ret; - - if (!dev->pm_cap) - return PCI_D0; - - ret = platform_pci_choose_state(dev); - if (ret != PCI_POWER_ERROR) - return ret; - - switch (state.event) { - case PM_EVENT_ON: - return PCI_D0; - case PM_EVENT_FREEZE: - case PM_EVENT_PRETHAW: - /* REVISIT both freeze and pre-thaw "should" use D0 */ - case PM_EVENT_SUSPEND: - case PM_EVENT_HIBERNATE: - return PCI_D3hot; - default: - pci_info(dev, "unrecognized suspend event %d\n", - state.event); - BUG(); - } - return PCI_D0; -} -EXPORT_SYMBOL(pci_choose_state); - #define PCI_EXP_SAVE_REGS 7 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, @@ -1467,6 +1471,24 @@ static int pci_save_pcie_state(struct pci_dev *dev) return 0; } +void pci_bridge_reconfigure_ltr(struct pci_dev *dev) +{ +#ifdef CONFIG_PCIEASPM + struct pci_dev *bridge; + u32 ctl; + + bridge = pci_upstream_bridge(dev); + if (bridge && bridge->ltr_path) { + pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl); + if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) { + pci_dbg(bridge, "re-enabling LTR\n"); + pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); + } + } +#endif +} + static void pci_restore_pcie_state(struct pci_dev *dev) { int i = 0; @@ -1477,6 +1499,13 @@ static void pci_restore_pcie_state(struct pci_dev *dev) if (!save_state) return; + /* + * Downstream ports reset the LTR enable bit when link goes down. + * Check and re-configure the bit here before restoring device. + * PCIe r5.0, sec 7.5.3.16. + */ + pci_bridge_reconfigure_ltr(dev); + cap = (u16 *)&save_state->cap.data[0]; pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); @@ -1906,11 +1935,7 @@ static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) * so that things like MSI message writing will behave as expected * (e.g. if the device really is in D0 at enable time). */ - if (dev->pm_cap) { - u16 pmcsr; - pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); - dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); - } + pci_update_current_state(dev, dev->current_state); if (atomic_inc_return(&dev->enable_cnt) > 1) return 0; /* already enabled */ @@ -2085,14 +2110,14 @@ void pcim_pin_device(struct pci_dev *pdev) EXPORT_SYMBOL(pcim_pin_device); /* - * pcibios_add_device - provide arch specific hooks when adding device dev + * pcibios_device_add - provide arch specific hooks when adding device dev * @dev: the PCI device being added * * Permits the platform to provide architecture specific functionality when * devices are added. This is the default implementation. Architecture * implementations can override this. */ -int __weak pcibios_add_device(struct pci_dev *dev) +int __weak pcibios_device_add(struct pci_dev *dev) { return 0; } @@ -2212,6 +2237,7 @@ int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) } EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); +#ifdef CONFIG_PCIEAER void pcie_clear_device_status(struct pci_dev *dev) { u16 sta; @@ -2219,6 +2245,7 @@ void pcie_clear_device_status(struct pci_dev *dev) pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); } +#endif /** * pcie_clear_root_pme_status - Clear root port PME interrupt status. @@ -2495,7 +2522,14 @@ static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable if (enable) { int error; - if (pci_pme_capable(dev, state)) + /* + * Enable PME signaling if the device can signal PME from + * D3cold regardless of whether or not it can signal PME from + * the current target state, because that will allow it to + * signal PME when the hierarchy above it goes into D3cold and + * the device itself ends up in D3cold as a result of that. + */ + if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold)) pci_pme_active(dev, true); else ret = 1; @@ -2564,8 +2598,6 @@ EXPORT_SYMBOL(pci_wake_from_d3); */ static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) { - pci_power_t target_state = PCI_D3hot; - if (platform_pci_power_manageable(dev)) { /* * Call the platform to find the target state for the device. @@ -2575,43 +2607,44 @@ static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) switch (state) { case PCI_POWER_ERROR: case PCI_UNKNOWN: - break; + return PCI_D3hot; + case PCI_D1: case PCI_D2: if (pci_no_d1d2(dev)) - break; - fallthrough; - default: - target_state = state; + return PCI_D3hot; } - return target_state; + return state; } - if (!dev->pm_cap) - target_state = PCI_D0; - /* * If the device is in D3cold even though it's not power-manageable by * the platform, it may have been powered down by non-standard means. * Best to let it slumber. */ if (dev->current_state == PCI_D3cold) - target_state = PCI_D3cold; + return PCI_D3cold; + else if (!dev->pm_cap) + return PCI_D0; + + if (wakeup && dev->pme_support) { + pci_power_t state = PCI_D3hot; - if (wakeup) { /* * Find the deepest state from which the device can generate * PME#. */ - if (dev->pme_support) { - while (target_state - && !(dev->pme_support & (1 << target_state))) - target_state--; - } + while (state && !(dev->pme_support & (1 << state))) + state--; + + if (state) + return state; + else if (dev->pme_support & 1) + return PCI_D0; } - return target_state; + return PCI_D3hot; } /** @@ -2664,8 +2697,13 @@ EXPORT_SYMBOL(pci_prepare_to_sleep); */ int pci_back_from_sleep(struct pci_dev *dev) { + int ret = pci_set_power_state(dev, PCI_D0); + + if (ret) + return ret; + pci_enable_wake(dev, PCI_D0, false); - return pci_set_power_state(dev, PCI_D0); + return 0; } EXPORT_SYMBOL(pci_back_from_sleep); @@ -2825,6 +2863,22 @@ void pci_dev_complete_resume(struct pci_dev *pci_dev) spin_unlock_irq(&dev->power.lock); } +/** + * pci_choose_state - Choose the power state of a PCI device. + * @dev: Target PCI device. + * @state: Target state for the whole system. + * + * Returns PCI power state suitable for @dev and @state. + */ +pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) +{ + if (state.event == PM_EVENT_ON) + return PCI_D0; + + return pci_target_state(dev, false); +} +EXPORT_SYMBOL(pci_choose_state); + void pci_config_pm_runtime_get(struct pci_dev *pdev) { struct device *dev = &pdev->dev; @@ -3702,6 +3756,14 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) struct pci_dev *bridge; u32 cap, ctl2; + /* + * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit + * in Device Control 2 is reserved in VFs and the PF value applies + * to all associated VFs. + */ + if (dev->is_virtfn) + return -EINVAL; + if (!pci_is_pcie(dev)) return -EINVAL; @@ -4106,6 +4168,7 @@ unsigned long __weak pci_address_to_pio(phys_addr_t address) * architectures that have memory mapped IO functions defined (and the * PCI_IOBASE value defined) should call this function. */ +#ifndef pci_remap_iospace int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) { #if defined(PCI_IOBASE) && defined(CONFIG_MMU) @@ -4129,6 +4192,7 @@ int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) #endif } EXPORT_SYMBOL(pci_remap_iospace); +#endif /** * pci_unmap_iospace - Unmap the memory mapped I/O space @@ -4622,31 +4686,11 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev) EXPORT_SYMBOL(pci_wait_for_pending_transaction); /** - * pcie_has_flr - check if a device supports function level resets - * @dev: device to check - * - * Returns true if the device advertises support for PCIe function level - * resets. - */ -bool pcie_has_flr(struct pci_dev *dev) -{ - u32 cap; - - if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) - return false; - - pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); - return cap & PCI_EXP_DEVCAP_FLR; -} -EXPORT_SYMBOL_GPL(pcie_has_flr); - -/** * pcie_flr - initiate a PCIe function level reset * @dev: device to reset * - * Initiate a function level reset on @dev. The caller should ensure the - * device supports FLR before calling this function, e.g. by using the - * pcie_has_flr() helper. + * Initiate a function level reset unconditionally on @dev without + * checking any flags and DEVCAP */ int pcie_flr(struct pci_dev *dev) { @@ -4669,7 +4713,29 @@ int pcie_flr(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pcie_flr); -static int pci_af_flr(struct pci_dev *dev, int probe) +/** + * pcie_reset_flr - initiate a PCIe function level reset + * @dev: device to reset + * @probe: if true, return 0 if device can be reset this way + * + * Initiate a function level reset on @dev. + */ +int pcie_reset_flr(struct pci_dev *dev, bool probe) +{ + if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) + return -ENOTTY; + + if (!(dev->devcap & PCI_EXP_DEVCAP_FLR)) + return -ENOTTY; + + if (probe) + return 0; + + return pcie_flr(dev); +} +EXPORT_SYMBOL_GPL(pcie_reset_flr); + +static int pci_af_flr(struct pci_dev *dev, bool probe) { int pos; u8 cap; @@ -4716,7 +4782,7 @@ static int pci_af_flr(struct pci_dev *dev, int probe) /** * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. * @dev: Device to reset. - * @probe: If set, only check if the device can be reset this way. + * @probe: if true, return 0 if the device can be reset this way. * * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is * unset, it will be reinitialized internally when going from PCI_D3hot to @@ -4728,7 +4794,7 @@ static int pci_af_flr(struct pci_dev *dev, int probe) * by default (i.e. unless the @dev's d3hot_delay field has a different value). * Moreover, only devices in D0 can be reset by this function. */ -static int pci_pm_reset(struct pci_dev *dev, int probe) +static int pci_pm_reset(struct pci_dev *dev, bool probe) { u16 csr; @@ -4988,7 +5054,7 @@ int pci_bridge_secondary_bus_reset(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); -static int pci_parent_bus_reset(struct pci_dev *dev, int probe) +static int pci_parent_bus_reset(struct pci_dev *dev, bool probe) { struct pci_dev *pdev; @@ -5006,7 +5072,7 @@ static int pci_parent_bus_reset(struct pci_dev *dev, int probe) return pci_bridge_secondary_bus_reset(dev->bus->self); } -static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe) +static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) { int rc = -ENOTTY; @@ -5021,7 +5087,7 @@ static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe) return rc; } -static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) +static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) { if (dev->multifunction || dev->subordinate || !dev->slot || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) @@ -5030,15 +5096,26 @@ static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) return pci_reset_hotplug_slot(dev->slot->hotplug, probe); } -static void pci_dev_lock(struct pci_dev *dev) +static int pci_reset_bus_function(struct pci_dev *dev, bool probe) +{ + int rc; + + rc = pci_dev_reset_slot_function(dev, probe); + if (rc != -ENOTTY) + return rc; + return pci_parent_bus_reset(dev, probe); +} + +void pci_dev_lock(struct pci_dev *dev) { pci_cfg_access_lock(dev); /* block PM suspend, driver probe, etc. */ device_lock(&dev->dev); } +EXPORT_SYMBOL_GPL(pci_dev_lock); /* Return 1 on successful lock, 0 on contention */ -static int pci_dev_trylock(struct pci_dev *dev) +int pci_dev_trylock(struct pci_dev *dev) { if (pci_cfg_access_trylock(dev)) { if (device_trylock(&dev->dev)) @@ -5048,12 +5125,14 @@ static int pci_dev_trylock(struct pci_dev *dev) return 0; } +EXPORT_SYMBOL_GPL(pci_dev_trylock); -static void pci_dev_unlock(struct pci_dev *dev) +void pci_dev_unlock(struct pci_dev *dev) { device_unlock(&dev->dev); pci_cfg_access_unlock(dev); } +EXPORT_SYMBOL_GPL(pci_dev_unlock); static void pci_dev_save_and_disable(struct pci_dev *dev) { @@ -5102,6 +5181,139 @@ static void pci_dev_restore(struct pci_dev *dev) err_handler->reset_done(dev); } +/* dev->reset_methods[] is a 0-terminated list of indices into this array */ +static const struct pci_reset_fn_method pci_reset_fn_methods[] = { + { }, + { pci_dev_specific_reset, .name = "device_specific" }, + { pci_dev_acpi_reset, .name = "acpi" }, + { pcie_reset_flr, .name = "flr" }, + { pci_af_flr, .name = "af_flr" }, + { pci_pm_reset, .name = "pm" }, + { pci_reset_bus_function, .name = "bus" }, +}; + +static ssize_t reset_method_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + ssize_t len = 0; + int i, m; + + for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { + m = pdev->reset_methods[i]; + if (!m) + break; + + len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "", + pci_reset_fn_methods[m].name); + } + + if (len) + len += sysfs_emit_at(buf, len, "\n"); + + return len; +} + +static int reset_method_lookup(const char *name) +{ + int m; + + for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { + if (sysfs_streq(name, pci_reset_fn_methods[m].name)) + return m; + } + + return 0; /* not found */ +} + +static ssize_t reset_method_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct pci_dev *pdev = to_pci_dev(dev); + char *options, *name; + int m, n; + u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 }; + + if (sysfs_streq(buf, "")) { + pdev->reset_methods[0] = 0; + pci_warn(pdev, "All device reset methods disabled by user"); + return count; + } + + if (sysfs_streq(buf, "default")) { + pci_init_reset_methods(pdev); + return count; + } + + options = kstrndup(buf, count, GFP_KERNEL); + if (!options) + return -ENOMEM; + + n = 0; + while ((name = strsep(&options, " ")) != NULL) { + if (sysfs_streq(name, "")) + continue; + + name = strim(name); + + m = reset_method_lookup(name); + if (!m) { + pci_err(pdev, "Invalid reset method '%s'", name); + goto error; + } + + if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) { + pci_err(pdev, "Unsupported reset method '%s'", name); + goto error; + } + + if (n == PCI_NUM_RESET_METHODS - 1) { + pci_err(pdev, "Too many reset methods\n"); + goto error; + } + + reset_methods[n++] = m; + } + + reset_methods[n] = 0; + + /* Warn if dev-specific supported but not highest priority */ + if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 && + reset_methods[0] != 1) + pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user"); + memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods)); + kfree(options); + return count; + +error: + /* Leave previous methods unchanged */ + kfree(options); + return -EINVAL; +} +static DEVICE_ATTR_RW(reset_method); + +static struct attribute *pci_dev_reset_method_attrs[] = { + &dev_attr_reset_method.attr, + NULL, +}; + +static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj, + struct attribute *a, int n) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + + if (!pci_reset_supported(pdev)) + return 0; + + return a->mode; +} + +const struct attribute_group pci_dev_reset_method_attr_group = { + .attrs = pci_dev_reset_method_attrs, + .is_visible = pci_dev_reset_method_attr_is_visible, +}; + /** * __pci_reset_function_locked - reset a PCI device function while holding * the @dev mutex lock. @@ -5124,72 +5336,64 @@ static void pci_dev_restore(struct pci_dev *dev) */ int __pci_reset_function_locked(struct pci_dev *dev) { - int rc; + int i, m, rc; might_sleep(); /* - * A reset method returns -ENOTTY if it doesn't support this device - * and we should try the next method. + * A reset method returns -ENOTTY if it doesn't support this device and + * we should try the next method. * - * If it returns 0 (success), we're finished. If it returns any - * other error, we're also finished: this indicates that further - * reset mechanisms might be broken on the device. + * If it returns 0 (success), we're finished. If it returns any other + * error, we're also finished: this indicates that further reset + * mechanisms might be broken on the device. */ - rc = pci_dev_specific_reset(dev, 0); - if (rc != -ENOTTY) - return rc; - if (pcie_has_flr(dev)) { - rc = pcie_flr(dev); + for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { + m = dev->reset_methods[i]; + if (!m) + return -ENOTTY; + + rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET); + if (!rc) + return 0; if (rc != -ENOTTY) return rc; } - rc = pci_af_flr(dev, 0); - if (rc != -ENOTTY) - return rc; - rc = pci_pm_reset(dev, 0); - if (rc != -ENOTTY) - return rc; - rc = pci_dev_reset_slot_function(dev, 0); - if (rc != -ENOTTY) - return rc; - return pci_parent_bus_reset(dev, 0); + + return -ENOTTY; } EXPORT_SYMBOL_GPL(__pci_reset_function_locked); /** - * pci_probe_reset_function - check whether the device can be safely reset - * @dev: PCI device to reset + * pci_init_reset_methods - check whether device can be safely reset + * and store supported reset mechanisms. + * @dev: PCI device to check for reset mechanisms * * Some devices allow an individual function to be reset without affecting - * other functions in the same device. The PCI device must be responsive - * to PCI config space in order to use this function. + * other functions in the same device. The PCI device must be in D0-D3hot + * state. * - * Returns 0 if the device function can be reset or negative if the - * device doesn't support resetting a single function. + * Stores reset mechanisms supported by device in reset_methods byte array + * which is a member of struct pci_dev. */ -int pci_probe_reset_function(struct pci_dev *dev) +void pci_init_reset_methods(struct pci_dev *dev) { - int rc; + int m, i, rc; + + BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS); might_sleep(); - rc = pci_dev_specific_reset(dev, 1); - if (rc != -ENOTTY) - return rc; - if (pcie_has_flr(dev)) - return 0; - rc = pci_af_flr(dev, 1); - if (rc != -ENOTTY) - return rc; - rc = pci_pm_reset(dev, 1); - if (rc != -ENOTTY) - return rc; - rc = pci_dev_reset_slot_function(dev, 1); - if (rc != -ENOTTY) - return rc; + i = 0; + for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { + rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE); + if (!rc) + dev->reset_methods[i++] = m; + else if (rc != -ENOTTY) + break; + } - return pci_parent_bus_reset(dev, 1); + dev->reset_methods[i] = 0; } /** @@ -5212,7 +5416,7 @@ int pci_reset_function(struct pci_dev *dev) { int rc; - if (!dev->reset_fn) + if (!pci_reset_supported(dev)) return -ENOTTY; pci_dev_lock(dev); @@ -5248,7 +5452,7 @@ int pci_reset_function_locked(struct pci_dev *dev) { int rc; - if (!dev->reset_fn) + if (!pci_reset_supported(dev)) return -ENOTTY; pci_dev_save_and_disable(dev); @@ -5271,7 +5475,7 @@ int pci_try_reset_function(struct pci_dev *dev) { int rc; - if (!dev->reset_fn) + if (!pci_reset_supported(dev)) return -ENOTTY; if (!pci_dev_trylock(dev)) @@ -5499,7 +5703,7 @@ static void pci_slot_restore_locked(struct pci_slot *slot) } } -static int pci_slot_reset(struct pci_slot *slot, int probe) +static int pci_slot_reset(struct pci_slot *slot, bool probe) { int rc; @@ -5527,7 +5731,7 @@ static int pci_slot_reset(struct pci_slot *slot, int probe) */ int pci_probe_reset_slot(struct pci_slot *slot) { - return pci_slot_reset(slot, 1); + return pci_slot_reset(slot, PCI_RESET_PROBE); } EXPORT_SYMBOL_GPL(pci_probe_reset_slot); @@ -5550,14 +5754,14 @@ static int __pci_reset_slot(struct pci_slot *slot) { int rc; - rc = pci_slot_reset(slot, 1); + rc = pci_slot_reset(slot, PCI_RESET_PROBE); if (rc) return rc; if (pci_slot_trylock(slot)) { pci_slot_save_and_disable_locked(slot); might_sleep(); - rc = pci_reset_hotplug_slot(slot->hotplug, 0); + rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET); pci_slot_restore_locked(slot); pci_slot_unlock(slot); } else @@ -5566,7 +5770,7 @@ static int __pci_reset_slot(struct pci_slot *slot) return rc; } -static int pci_bus_reset(struct pci_bus *bus, int probe) +static int pci_bus_reset(struct pci_bus *bus, bool probe) { int ret; @@ -5612,14 +5816,14 @@ int pci_bus_error_reset(struct pci_dev *bridge) goto bus_reset; list_for_each_entry(slot, &bus->slots, list) - if (pci_slot_reset(slot, 0)) + if (pci_slot_reset(slot, PCI_RESET_DO_RESET)) goto bus_reset; mutex_unlock(&pci_slot_mutex); return 0; bus_reset: mutex_unlock(&pci_slot_mutex); - return pci_bus_reset(bridge->subordinate, 0); + return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET); } /** @@ -5630,7 +5834,7 @@ bus_reset: */ int pci_probe_reset_bus(struct pci_bus *bus) { - return pci_bus_reset(bus, 1); + return pci_bus_reset(bus, PCI_RESET_PROBE); } EXPORT_SYMBOL_GPL(pci_probe_reset_bus); @@ -5644,7 +5848,7 @@ static int __pci_reset_bus(struct pci_bus *bus) { int rc; - rc = pci_bus_reset(bus, 1); + rc = pci_bus_reset(bus, PCI_RESET_PROBE); if (rc) return rc; @@ -6168,11 +6372,12 @@ EXPORT_SYMBOL_GPL(pci_pr3_present); * cannot be left as a userspace activity). DMA aliases should therefore * be configured via quirks, such as the PCI fixup header quirk. */ -void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, unsigned nr_devfns) +void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, + unsigned int nr_devfns) { int devfn_to; - nr_devfns = min(nr_devfns, (unsigned) MAX_NR_DEVFNS - devfn_from); + nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from); devfn_to = devfn_from + nr_devfns - 1; if (!dev->dma_alias_mask) @@ -6449,34 +6654,40 @@ static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) spin_lock(&resource_alignment_lock); if (resource_alignment_param) - count = scnprintf(buf, PAGE_SIZE, "%s", resource_alignment_param); + count = sysfs_emit(buf, "%s\n", resource_alignment_param); spin_unlock(&resource_alignment_lock); - /* - * When set by the command line, resource_alignment_param will not - * have a trailing line feed, which is ugly. So conditionally add - * it here. - */ - if (count >= 2 && buf[count - 2] != '\n' && count < PAGE_SIZE - 1) { - buf[count - 1] = '\n'; - buf[count++] = 0; - } - return count; } static ssize_t resource_alignment_store(struct bus_type *bus, const char *buf, size_t count) { - char *param = kstrndup(buf, count, GFP_KERNEL); + char *param, *old, *end; + if (count >= (PAGE_SIZE - 1)) + return -EINVAL; + + param = kstrndup(buf, count, GFP_KERNEL); if (!param) return -ENOMEM; + end = strchr(param, '\n'); + if (end) + *end = '\0'; + spin_lock(&resource_alignment_lock); - kfree(resource_alignment_param); - resource_alignment_param = param; + old = resource_alignment_param; + if (strlen(param)) { + resource_alignment_param = param; + } else { + kfree(param); + resource_alignment_param = NULL; + } spin_unlock(&resource_alignment_lock); + + kfree(old); + return count; } diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 37c913bbc6e1..3d60cabde1a1 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -33,53 +33,36 @@ enum pci_mmap_api { int pci_mmap_fits(struct pci_dev *pdev, int resno, struct vm_area_struct *vmai, enum pci_mmap_api mmap_api); -int pci_probe_reset_function(struct pci_dev *dev); +bool pci_reset_supported(struct pci_dev *dev); +void pci_init_reset_methods(struct pci_dev *dev); int pci_bridge_secondary_bus_reset(struct pci_dev *dev); int pci_bus_error_reset(struct pci_dev *dev); +struct pci_cap_saved_data { + u16 cap_nr; + bool cap_extended; + unsigned int size; + u32 data[]; +}; + +struct pci_cap_saved_state { + struct hlist_node next; + struct pci_cap_saved_data cap; +}; + +void pci_allocate_cap_save_buffers(struct pci_dev *dev); +void pci_free_cap_save_buffers(struct pci_dev *dev); +int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size); +int pci_add_ext_cap_save_buffer(struct pci_dev *dev, + u16 cap, unsigned int size); +struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap); +struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, + u16 cap); + #define PCI_PM_D2_DELAY 200 /* usec; see PCIe r4.0, sec 5.9.1 */ #define PCI_PM_D3HOT_WAIT 10 /* msec */ #define PCI_PM_D3COLD_WAIT 100 /* msec */ -/** - * struct pci_platform_pm_ops - Firmware PM callbacks - * - * @bridge_d3: Does the bridge allow entering into D3 - * - * @is_manageable: returns 'true' if given device is power manageable by the - * platform firmware - * - * @set_state: invokes the platform firmware to set the device's power state - * - * @get_state: queries the platform firmware for a device's current power state - * - * @refresh_state: asks the platform to refresh the device's power state data - * - * @choose_state: returns PCI power state of given device preferred by the - * platform; to be used during system-wide transitions from a - * sleeping state to the working state and vice versa - * - * @set_wakeup: enables/disables wakeup capability for the device - * - * @need_resume: returns 'true' if the given device (which is currently - * suspended) needs to be resumed to be configured for system - * wakeup. - * - * If given platform is generally capable of power managing PCI devices, all of - * these callbacks are mandatory. - */ -struct pci_platform_pm_ops { - bool (*bridge_d3)(struct pci_dev *dev); - bool (*is_manageable)(struct pci_dev *dev); - int (*set_state)(struct pci_dev *dev, pci_power_t state); - pci_power_t (*get_state)(struct pci_dev *dev); - void (*refresh_state)(struct pci_dev *dev); - pci_power_t (*choose_state)(struct pci_dev *dev); - int (*set_wakeup)(struct pci_dev *dev, bool enable); - bool (*need_resume)(struct pci_dev *dev); -}; - -int pci_set_platform_pm(const struct pci_platform_pm_ops *ops); void pci_update_current_state(struct pci_dev *dev, pci_power_t state); void pci_refresh_power_state(struct pci_dev *dev); int pci_power_up(struct pci_dev *dev); @@ -100,11 +83,10 @@ void pci_pm_init(struct pci_dev *dev); void pci_ea_init(struct pci_dev *dev); void pci_msi_init(struct pci_dev *dev); void pci_msix_init(struct pci_dev *dev); -void pci_allocate_cap_save_buffers(struct pci_dev *dev); -void pci_free_cap_save_buffers(struct pci_dev *dev); bool pci_bridge_d3_possible(struct pci_dev *dev); void pci_bridge_d3_update(struct pci_dev *dev); void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev); +void pci_bridge_reconfigure_ltr(struct pci_dev *dev); static inline void pci_wakeup_event(struct pci_dev *dev) { @@ -324,8 +306,8 @@ struct pci_sriov { /** * pci_dev_set_io_state - Set the new error state if possible. * - * @dev - pci device to set new error_state - * @new - the state we want dev to be in + * @dev: PCI device to set new error_state + * @new: the state we want dev to be in * * Must be called with device_lock held. * @@ -385,6 +367,8 @@ static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) /* pci_dev priv_flags */ #define PCI_DEV_ADDED 0 +#define PCI_DPC_RECOVERED 1 +#define PCI_DPC_RECOVERING 2 static inline void pci_dev_assign_added(struct pci_dev *dev, bool added) { @@ -439,10 +423,12 @@ void pci_restore_dpc_state(struct pci_dev *dev); void pci_dpc_init(struct pci_dev *pdev); void dpc_process_error(struct pci_dev *pdev); pci_ers_result_t dpc_reset_link(struct pci_dev *pdev); +bool pci_dpc_recovered(struct pci_dev *pdev); #else static inline void pci_save_dpc_state(struct pci_dev *dev) {} static inline void pci_restore_dpc_state(struct pci_dev *dev) {} static inline void pci_dpc_init(struct pci_dev *pdev) {} +static inline bool pci_dpc_recovered(struct pci_dev *pdev) { return false; } #endif #ifdef CONFIG_PCIEPORTBUS @@ -593,23 +579,25 @@ static inline void pcie_ecrc_get_policy(char *str) { } #ifdef CONFIG_PCIE_PTM void pci_ptm_init(struct pci_dev *dev); -int pci_enable_ptm(struct pci_dev *dev, u8 *granularity); #else static inline void pci_ptm_init(struct pci_dev *dev) { } -static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity) -{ return -EINVAL; } #endif struct pci_dev_reset_methods { u16 vendor; u16 device; - int (*reset)(struct pci_dev *dev, int probe); + int (*reset)(struct pci_dev *dev, bool probe); +}; + +struct pci_reset_fn_method { + int (*reset_fn)(struct pci_dev *pdev, bool probe); + char *name; }; #ifdef CONFIG_PCI_QUIRKS -int pci_dev_specific_reset(struct pci_dev *dev, int probe); +int pci_dev_specific_reset(struct pci_dev *dev, bool probe); #else -static inline int pci_dev_specific_reset(struct pci_dev *dev, int probe) +static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe) { return -ENOTTY; } @@ -697,15 +685,80 @@ static inline int pci_aer_raw_clear_status(struct pci_dev *dev) { return -EINVAL #ifdef CONFIG_ACPI int pci_acpi_program_hp_params(struct pci_dev *dev); extern const struct attribute_group pci_dev_acpi_attr_group; +void pci_set_acpi_fwnode(struct pci_dev *dev); +int pci_dev_acpi_reset(struct pci_dev *dev, bool probe); +bool acpi_pci_power_manageable(struct pci_dev *dev); +bool acpi_pci_bridge_d3(struct pci_dev *dev); +int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state); +pci_power_t acpi_pci_get_power_state(struct pci_dev *dev); +void acpi_pci_refresh_power_state(struct pci_dev *dev); +int acpi_pci_wakeup(struct pci_dev *dev, bool enable); +bool acpi_pci_need_resume(struct pci_dev *dev); +pci_power_t acpi_pci_choose_state(struct pci_dev *pdev); #else +static inline int pci_dev_acpi_reset(struct pci_dev *dev, bool probe) +{ + return -ENOTTY; +} +static inline void pci_set_acpi_fwnode(struct pci_dev *dev) {} static inline int pci_acpi_program_hp_params(struct pci_dev *dev) { return -ENODEV; } +static inline bool acpi_pci_power_manageable(struct pci_dev *dev) +{ + return false; +} +static inline bool acpi_pci_bridge_d3(struct pci_dev *dev) +{ + return false; +} +static inline int acpi_pci_set_power_state(struct pci_dev *dev, pci_power_t state) +{ + return -ENODEV; +} +static inline pci_power_t acpi_pci_get_power_state(struct pci_dev *dev) +{ + return PCI_UNKNOWN; +} +static inline void acpi_pci_refresh_power_state(struct pci_dev *dev) {} +static inline int acpi_pci_wakeup(struct pci_dev *dev, bool enable) +{ + return -ENODEV; +} +static inline bool acpi_pci_need_resume(struct pci_dev *dev) +{ + return false; +} +static inline pci_power_t acpi_pci_choose_state(struct pci_dev *pdev) +{ + return PCI_POWER_ERROR; +} #endif #ifdef CONFIG_PCIEASPM extern const struct attribute_group aspm_ctrl_attr_group; #endif +extern const struct attribute_group pci_dev_reset_method_attr_group; + +#ifdef CONFIG_X86_INTEL_MID +bool pci_use_mid_pm(void); +int mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state); +pci_power_t mid_pci_get_power_state(struct pci_dev *pdev); +#else +static inline bool pci_use_mid_pm(void) +{ + return false; +} +static inline int mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state) +{ + return -ENODEV; +} +static inline pci_power_t mid_pci_get_power_state(struct pci_dev *pdev) +{ + return PCI_UNKNOWN; +} +#endif + #endif /* DRIVERS_PCI_H */ diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index b2980db88cc0..5783a2f79e6a 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -2,12 +2,12 @@ # # Makefile for PCI Express features and port driver -pcieportdrv-y := portdrv_core.o portdrv_pci.o err.o rcec.o +pcieportdrv-y := portdrv_core.o portdrv_pci.o rcec.o obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o obj-$(CONFIG_PCIEASPM) += aspm.o -obj-$(CONFIG_PCIEAER) += aer.o +obj-$(CONFIG_PCIEAER) += aer.o err.o obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o obj-$(CONFIG_PCIE_PME) += pme.o obj-$(CONFIG_PCIE_DPC) += dpc.o diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index ec943cee5ecc..9fa1f97e5b27 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -57,7 +57,7 @@ struct aer_stats { * "as seen by this device". Note that this may mean that if an * end point is causing problems, the AER counters may increment * at its link partner (e.g. root port) because the errors will be - * "seen" by the link partner and not the the problematic end point + * "seen" by the link partner and not the problematic end point * itself (which may report all counters as 0 as it never saw any * problems). */ @@ -529,21 +529,23 @@ static const char *aer_agent_string[] = { char *buf) \ { \ unsigned int i; \ - char *str = buf; \ struct pci_dev *pdev = to_pci_dev(dev); \ u64 *stats = pdev->aer_stats->stats_array; \ + size_t len = 0; \ \ for (i = 0; i < ARRAY_SIZE(strings_array); i++) { \ if (strings_array[i]) \ - str += sprintf(str, "%s %llu\n", \ - strings_array[i], stats[i]); \ + len += sysfs_emit_at(buf, len, "%s %llu\n", \ + strings_array[i], \ + stats[i]); \ else if (stats[i]) \ - str += sprintf(str, #stats_array "_bit[%d] %llu\n",\ - i, stats[i]); \ + len += sysfs_emit_at(buf, len, \ + #stats_array "_bit[%d] %llu\n",\ + i, stats[i]); \ } \ - str += sprintf(str, "TOTAL_%s %llu\n", total_string, \ - pdev->aer_stats->total_field); \ - return str-buf; \ + len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ + pdev->aer_stats->total_field); \ + return len; \ } \ static DEVICE_ATTR_RO(name) @@ -563,7 +565,7 @@ aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, char *buf) \ { \ struct pci_dev *pdev = to_pci_dev(dev); \ - return sprintf(buf, "%llu\n", pdev->aer_stats->field); \ + return sysfs_emit(buf, "%llu\n", pdev->aer_stats->field); \ } \ static DEVICE_ATTR_RO(name) @@ -983,7 +985,7 @@ static void aer_recover_work_func(struct work_struct *work) pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, entry.devfn); if (!pdev) { - pr_err("AER recover: Can not find pci_dev for %04x:%02x:%02x:%x\n", + pr_err("no pci_dev for %04x:%02x:%02x.%x\n", entry.domain, entry.bus, PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); continue; @@ -1022,7 +1024,7 @@ void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, &aer_recover_ring_lock)) schedule_work(&aer_recover_work); else - pr_err("AER recover: Buffer overflow when recovering AER for %04x:%02x:%02x:%x\n", + pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); } EXPORT_SYMBOL_GPL(aer_recover_queue); @@ -1405,13 +1407,11 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) } if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { - if (pcie_has_flr(dev)) { - rc = pcie_flr(dev); - pci_info(dev, "has been reset (%d)\n", rc); - } else { - pci_info(dev, "not reset (no FLR support)\n"); - rc = -ENOTTY; - } + rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); + if (!rc) + pci_info(dev, "has been reset\n"); + else + pci_info(dev, "not reset (no FLR support: %d)\n", rc); } else { rc = pci_bus_error_reset(dev); pci_info(dev, "%s Port link has been reset (%d)\n", diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index ac0557a305af..52c74682601a 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1208,7 +1208,7 @@ static ssize_t aspm_attr_show_common(struct device *dev, struct pci_dev *pdev = to_pci_dev(dev); struct pcie_link_state *link = pcie_aspm_get_link(pdev); - return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); + return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); } static ssize_t aspm_attr_store_common(struct device *dev, @@ -1219,7 +1219,7 @@ static ssize_t aspm_attr_store_common(struct device *dev, struct pcie_link_state *link = pcie_aspm_get_link(pdev); bool state_enable; - if (strtobool(buf, &state_enable) < 0) + if (kstrtobool(buf, &state_enable) < 0) return -EINVAL; down_read(&pci_bus_sem); @@ -1265,7 +1265,7 @@ static ssize_t clkpm_show(struct device *dev, struct pci_dev *pdev = to_pci_dev(dev); struct pcie_link_state *link = pcie_aspm_get_link(pdev); - return sprintf(buf, "%d\n", link->clkpm_enabled); + return sysfs_emit(buf, "%d\n", link->clkpm_enabled); } static ssize_t clkpm_store(struct device *dev, @@ -1276,7 +1276,7 @@ static ssize_t clkpm_store(struct device *dev, struct pcie_link_state *link = pcie_aspm_get_link(pdev); bool state_enable; - if (strtobool(buf, &state_enable) < 0) + if (kstrtobool(buf, &state_enable) < 0) return -EINVAL; down_read(&pci_bus_sem); diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index e05aba86a317..c556e7beafe3 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -71,6 +71,58 @@ void pci_restore_dpc_state(struct pci_dev *dev) pci_write_config_word(dev, dev->dpc_cap + PCI_EXP_DPC_CTL, *cap); } +static DECLARE_WAIT_QUEUE_HEAD(dpc_completed_waitqueue); + +#ifdef CONFIG_HOTPLUG_PCI_PCIE +static bool dpc_completed(struct pci_dev *pdev) +{ + u16 status; + + pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_STATUS, &status); + if ((status != 0xffff) && (status & PCI_EXP_DPC_STATUS_TRIGGER)) + return false; + + if (test_bit(PCI_DPC_RECOVERING, &pdev->priv_flags)) + return false; + + return true; +} + +/** + * pci_dpc_recovered - whether DPC triggered and has recovered successfully + * @pdev: PCI device + * + * Return true if DPC was triggered for @pdev and has recovered successfully. + * Wait for recovery if it hasn't completed yet. Called from the PCIe hotplug + * driver to recognize and ignore Link Down/Up events caused by DPC. + */ +bool pci_dpc_recovered(struct pci_dev *pdev) +{ + struct pci_host_bridge *host; + + if (!pdev->dpc_cap) + return false; + + /* + * Synchronization between hotplug and DPC is not supported + * if DPC is owned by firmware and EDR is not enabled. + */ + host = pci_find_host_bridge(pdev->bus); + if (!host->native_dpc && !IS_ENABLED(CONFIG_PCIE_EDR)) + return false; + + /* + * Need a timeout in case DPC never completes due to failure of + * dpc_wait_rp_inactive(). The spec doesn't mandate a time limit, + * but reports indicate that DPC completes within 4 seconds. + */ + wait_event_timeout(dpc_completed_waitqueue, dpc_completed(pdev), + msecs_to_jiffies(4000)); + + return test_and_clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags); +} +#endif /* CONFIG_HOTPLUG_PCI_PCIE */ + static int dpc_wait_rp_inactive(struct pci_dev *pdev) { unsigned long timeout = jiffies + HZ; @@ -91,8 +143,11 @@ static int dpc_wait_rp_inactive(struct pci_dev *pdev) pci_ers_result_t dpc_reset_link(struct pci_dev *pdev) { + pci_ers_result_t ret; u16 cap; + set_bit(PCI_DPC_RECOVERING, &pdev->priv_flags); + /* * DPC disables the Link automatically in hardware, so it has * already been reset by the time we get here. @@ -106,18 +161,27 @@ pci_ers_result_t dpc_reset_link(struct pci_dev *pdev) if (!pcie_wait_for_link(pdev, false)) pci_info(pdev, "Data Link Layer Link Active not cleared in 1000 msec\n"); - if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev)) - return PCI_ERS_RESULT_DISCONNECT; + if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev)) { + clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags); + ret = PCI_ERS_RESULT_DISCONNECT; + goto out; + } pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS, PCI_EXP_DPC_STATUS_TRIGGER); if (!pcie_wait_for_link(pdev, true)) { pci_info(pdev, "Data Link Layer Link Active not set in 1000 msec\n"); - return PCI_ERS_RESULT_DISCONNECT; + clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags); + ret = PCI_ERS_RESULT_DISCONNECT; + } else { + set_bit(PCI_DPC_RECOVERED, &pdev->priv_flags); + ret = PCI_ERS_RESULT_RECOVERED; } - - return PCI_ERS_RESULT_RECOVERED; +out: + clear_bit(PCI_DPC_RECOVERING, &pdev->priv_flags); + wake_up_all(&dpc_completed_waitqueue); + return ret; } static void dpc_process_rp_pio_error(struct pci_dev *pdev) diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index b576aa890c76..0c5a143025af 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -49,14 +49,16 @@ static int report_error_detected(struct pci_dev *dev, pci_channel_state_t state, enum pci_ers_result *result) { + struct pci_driver *pdrv; pci_ers_result_t vote; const struct pci_error_handlers *err_handler; device_lock(&dev->dev); + pdrv = dev->driver; if (!pci_dev_set_io_state(dev, state) || - !dev->driver || - !dev->driver->err_handler || - !dev->driver->err_handler->error_detected) { + !pdrv || + !pdrv->err_handler || + !pdrv->err_handler->error_detected) { /* * If any device in the subtree does not have an error_detected * callback, PCI_ERS_RESULT_NO_AER_DRIVER prevents subsequent @@ -70,7 +72,7 @@ static int report_error_detected(struct pci_dev *dev, vote = PCI_ERS_RESULT_NONE; } } else { - err_handler = dev->driver->err_handler; + err_handler = pdrv->err_handler; vote = err_handler->error_detected(dev, state); } pci_uevent_ers(dev, vote); @@ -91,16 +93,18 @@ static int report_normal_detected(struct pci_dev *dev, void *data) static int report_mmio_enabled(struct pci_dev *dev, void *data) { + struct pci_driver *pdrv; pci_ers_result_t vote, *result = data; const struct pci_error_handlers *err_handler; device_lock(&dev->dev); - if (!dev->driver || - !dev->driver->err_handler || - !dev->driver->err_handler->mmio_enabled) + pdrv = dev->driver; + if (!pdrv || + !pdrv->err_handler || + !pdrv->err_handler->mmio_enabled) goto out; - err_handler = dev->driver->err_handler; + err_handler = pdrv->err_handler; vote = err_handler->mmio_enabled(dev); *result = merge_result(*result, vote); out: @@ -110,16 +114,18 @@ out: static int report_slot_reset(struct pci_dev *dev, void *data) { + struct pci_driver *pdrv; pci_ers_result_t vote, *result = data; const struct pci_error_handlers *err_handler; device_lock(&dev->dev); - if (!dev->driver || - !dev->driver->err_handler || - !dev->driver->err_handler->slot_reset) + pdrv = dev->driver; + if (!pdrv || + !pdrv->err_handler || + !pdrv->err_handler->slot_reset) goto out; - err_handler = dev->driver->err_handler; + err_handler = pdrv->err_handler; vote = err_handler->slot_reset(dev); *result = merge_result(*result, vote); out: @@ -129,16 +135,18 @@ out: static int report_resume(struct pci_dev *dev, void *data) { + struct pci_driver *pdrv; const struct pci_error_handlers *err_handler; device_lock(&dev->dev); + pdrv = dev->driver; if (!pci_dev_set_io_state(dev, pci_channel_io_normal) || - !dev->driver || - !dev->driver->err_handler || - !dev->driver->err_handler->resume) + !pdrv || + !pdrv->err_handler || + !pdrv->err_handler->resume) goto out; - err_handler = dev->driver->err_handler; + err_handler = pdrv->err_handler; err_handler->resume(dev); out: pci_uevent_ers(dev, PCI_ERS_RESULT_RECOVERED); diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index 2ff5724b8f13..0ef4bf5f811d 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -85,8 +85,7 @@ struct pcie_port_service_driver { int (*runtime_suspend)(struct pcie_device *dev); int (*runtime_resume)(struct pcie_device *dev); - /* Device driver may resume normal operations */ - void (*error_resume)(struct pci_dev *dev); + int (*slot_reset)(struct pcie_device *dev); int port_type; /* Type of the port this driver can handle */ u32 service; /* Port service this device represents */ @@ -110,6 +109,7 @@ void pcie_port_service_unregister(struct pcie_port_service_driver *new); extern struct bus_type pcie_port_bus_type; int pcie_port_device_register(struct pci_dev *dev); +int pcie_port_device_iter(struct device *dev, void *data); #ifdef CONFIG_PM int pcie_port_device_suspend(struct device *dev); int pcie_port_device_resume_noirq(struct device *dev); @@ -118,8 +118,6 @@ int pcie_port_device_runtime_suspend(struct device *dev); int pcie_port_device_runtime_resume(struct device *dev); #endif void pcie_port_device_remove(struct pci_dev *dev); -int __must_check pcie_port_bus_register(void); -void pcie_port_bus_unregister(void); struct pci_dev; diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c index e1fed6649c41..bda630889f95 100644 --- a/drivers/pci/pcie/portdrv_core.c +++ b/drivers/pci/pcie/portdrv_core.c @@ -166,9 +166,6 @@ static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask) { int ret, i; - for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) - irqs[i] = -1; - /* * If we support PME but can't use MSI/MSI-X for it, we have to * fall back to INTx or other interrupts, e.g., a system shared @@ -257,8 +254,13 @@ static int get_port_device_capability(struct pci_dev *dev) services |= PCIE_PORT_SERVICE_DPC; if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM || - pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) - services |= PCIE_PORT_SERVICE_BWNOTIF; + pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) { + u32 linkcap; + + pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &linkcap); + if (linkcap & PCI_EXP_LNKCAP_LBNC) + services |= PCIE_PORT_SERVICE_BWNOTIF; + } return services; } @@ -312,8 +314,10 @@ static int pcie_device_init(struct pci_dev *pdev, int service, int irq) */ int pcie_port_device_register(struct pci_dev *dev) { - int status, capabilities, i, nr_service; - int irqs[PCIE_PORT_DEVICE_MAXSERVICES]; + int status, capabilities, irq_services, i, nr_service; + int irqs[PCIE_PORT_DEVICE_MAXSERVICES] = { + [0 ... PCIE_PORT_DEVICE_MAXSERVICES-1] = -1 + }; /* Enable PCI Express port device */ status = pci_enable_device(dev); @@ -326,18 +330,32 @@ int pcie_port_device_register(struct pci_dev *dev) return 0; pci_set_master(dev); - /* - * Initialize service irqs. Don't use service devices that - * require interrupts if there is no way to generate them. - * However, some drivers may have a polling mode (e.g. pciehp_poll_mode) - * that can be used in the absence of irqs. Allow them to determine - * if that is to be used. - */ - status = pcie_init_service_irqs(dev, irqs, capabilities); - if (status) { - capabilities &= PCIE_PORT_SERVICE_HP; - if (!capabilities) - goto error_disable; + + irq_services = 0; + if (IS_ENABLED(CONFIG_PCIE_PME)) + irq_services |= PCIE_PORT_SERVICE_PME; + if (IS_ENABLED(CONFIG_PCIEAER)) + irq_services |= PCIE_PORT_SERVICE_AER; + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) + irq_services |= PCIE_PORT_SERVICE_HP; + if (IS_ENABLED(CONFIG_PCIE_DPC)) + irq_services |= PCIE_PORT_SERVICE_DPC; + irq_services &= capabilities; + + if (irq_services) { + /* + * Initialize service IRQs. Don't use service devices that + * require interrupts if there is no way to generate them. + * However, some drivers may have a polling mode (e.g. + * pciehp_poll_mode) that can be used in the absence of IRQs. + * Allow them to determine if that is to be used. + */ + status = pcie_init_service_irqs(dev, irqs, irq_services); + if (status) { + irq_services &= PCIE_PORT_SERVICE_HP; + if (!irq_services) + goto error_disable; + } } /* Allocate child services if any */ @@ -362,24 +380,24 @@ error_disable: return status; } -#ifdef CONFIG_PM -typedef int (*pcie_pm_callback_t)(struct pcie_device *); +typedef int (*pcie_callback_t)(struct pcie_device *); -static int pm_iter(struct device *dev, void *data) +int pcie_port_device_iter(struct device *dev, void *data) { struct pcie_port_service_driver *service_driver; size_t offset = *(size_t *)data; - pcie_pm_callback_t cb; + pcie_callback_t cb; if ((dev->bus == &pcie_port_bus_type) && dev->driver) { service_driver = to_service_driver(dev->driver); - cb = *(pcie_pm_callback_t *)((void *)service_driver + offset); + cb = *(pcie_callback_t *)((void *)service_driver + offset); if (cb) return cb(to_pcie_device(dev)); } return 0; } +#ifdef CONFIG_PM /** * pcie_port_device_suspend - suspend port services associated with a PCIe port * @dev: PCI Express port to handle @@ -387,13 +405,13 @@ static int pm_iter(struct device *dev, void *data) int pcie_port_device_suspend(struct device *dev) { size_t off = offsetof(struct pcie_port_service_driver, suspend); - return device_for_each_child(dev, &off, pm_iter); + return device_for_each_child(dev, &off, pcie_port_device_iter); } int pcie_port_device_resume_noirq(struct device *dev) { size_t off = offsetof(struct pcie_port_service_driver, resume_noirq); - return device_for_each_child(dev, &off, pm_iter); + return device_for_each_child(dev, &off, pcie_port_device_iter); } /** @@ -403,7 +421,7 @@ int pcie_port_device_resume_noirq(struct device *dev) int pcie_port_device_resume(struct device *dev) { size_t off = offsetof(struct pcie_port_service_driver, resume); - return device_for_each_child(dev, &off, pm_iter); + return device_for_each_child(dev, &off, pcie_port_device_iter); } /** @@ -413,7 +431,7 @@ int pcie_port_device_resume(struct device *dev) int pcie_port_device_runtime_suspend(struct device *dev) { size_t off = offsetof(struct pcie_port_service_driver, runtime_suspend); - return device_for_each_child(dev, &off, pm_iter); + return device_for_each_child(dev, &off, pcie_port_device_iter); } /** @@ -423,7 +441,7 @@ int pcie_port_device_runtime_suspend(struct device *dev) int pcie_port_device_runtime_resume(struct device *dev) { size_t off = offsetof(struct pcie_port_service_driver, runtime_resume); - return device_for_each_child(dev, &off, pm_iter); + return device_for_each_child(dev, &off, pcie_port_device_iter); } #endif /* PM */ diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index c7ff1eea225a..35eca6277a96 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -160,6 +160,9 @@ static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev) { + size_t off = offsetof(struct pcie_port_service_driver, slot_reset); + device_for_each_child(&dev->dev, &off, pcie_port_device_iter); + pci_restore_state(dev); pci_save_state(dev); return PCI_ERS_RESULT_RECOVERED; @@ -170,29 +173,6 @@ static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev) return PCI_ERS_RESULT_RECOVERED; } -static int resume_iter(struct device *device, void *data) -{ - struct pcie_device *pcie_device; - struct pcie_port_service_driver *driver; - - if (device->bus == &pcie_port_bus_type && device->driver) { - driver = to_service_driver(device->driver); - if (driver && driver->error_resume) { - pcie_device = to_pcie_device(device); - - /* Forward error message to service drivers */ - driver->error_resume(pcie_device->port); - } - } - - return 0; -} - -static void pcie_portdrv_err_resume(struct pci_dev *dev) -{ - device_for_each_child(&dev->dev, NULL, resume_iter); -} - /* * LINUX Device Driver Model */ @@ -210,7 +190,6 @@ static const struct pci_error_handlers pcie_portdrv_err_handler = { .error_detected = pcie_portdrv_error_detected, .slot_reset = pcie_portdrv_slot_reset, .mmio_enabled = pcie_portdrv_mmio_enabled, - .resume = pcie_portdrv_err_resume, }; static struct pci_driver pcie_portdriver = { diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index 95d4eef2c9e8..368a254e3124 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -60,10 +60,8 @@ void pci_save_ptm_state(struct pci_dev *dev) return; save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_PTM); - if (!save_state) { - pci_err(dev, "no suspend buffer for PTM\n"); + if (!save_state) return; - } cap = (u16 *)&save_state->cap.data[0]; pci_read_config_word(dev, ptm + PCI_PTM_CTRL, cap); @@ -204,3 +202,12 @@ int pci_enable_ptm(struct pci_dev *dev, u8 *granularity) return 0; } EXPORT_SYMBOL(pci_enable_ptm); + +bool pcie_ptm_enabled(struct pci_dev *dev) +{ + if (!dev) + return false; + + return dev->ptm_enabled; +} +EXPORT_SYMBOL(pcie_ptm_enabled); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 275204646c68..087d3658f75c 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -19,6 +19,7 @@ #include <linux/hypervisor.h> #include <linux/irqdomain.h> #include <linux/pm_runtime.h> +#include <linux/bitfield.h> #include "pci.h" #define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */ @@ -594,6 +595,7 @@ static void pci_init_host_bridge(struct pci_host_bridge *bridge) bridge->native_pme = 1; bridge->native_ltr = 1; bridge->native_dpc = 1; + bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET; device_initialize(&bridge->dev); } @@ -828,11 +830,15 @@ static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus) { struct irq_domain *d; + /* If the host bridge driver sets a MSI domain of the bridge, use it */ + d = dev_get_msi_domain(bus->bridge); + /* * Any firmware interface that can resolve the msi_domain * should be called from here. */ - d = pci_host_bridge_of_msi_domain(bus); + if (!d) + d = pci_host_bridge_of_msi_domain(bus); if (!d) d = pci_host_bridge_acpi_msi_domain(bus); @@ -877,11 +883,11 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus) static int pci_register_host_bridge(struct pci_host_bridge *bridge) { struct device *parent = bridge->dev.parent; - struct resource_entry *window, *n; + struct resource_entry *window, *next, *n; struct pci_bus *bus, *b; - resource_size_t offset; + resource_size_t offset, next_offset; LIST_HEAD(resources); - struct resource *res; + struct resource *res, *next_res; char addr[64], *fmt; const char *name; int err; @@ -898,7 +904,10 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) bus->ops = bridge->ops; bus->number = bus->busn_res.start = bridge->busnr; #ifdef CONFIG_PCI_DOMAINS_GENERIC - bus->domain_nr = pci_bus_find_domain_nr(bus, parent); + if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET) + bus->domain_nr = pci_bus_find_domain_nr(bus, parent); + else + bus->domain_nr = bridge->domain_nr; #endif b = pci_find_bus(pci_domain_nr(bus), bridge->busnr); @@ -961,11 +970,34 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE) dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n"); + /* Coalesce contiguous windows */ + resource_list_for_each_entry_safe(window, n, &resources) { + if (list_is_last(&window->node, &resources)) + break; + + next = list_next_entry(window, node); + offset = window->offset; + res = window->res; + next_offset = next->offset; + next_res = next->res; + + if (res->flags != next_res->flags || offset != next_offset) + continue; + + if (res->end + 1 == next_res->start) { + next_res->start = res->start; + res->flags = res->start = res->end = 0; + } + } + /* Add initial resources to the bus */ resource_list_for_each_entry_safe(window, n, &resources) { - list_move_tail(&window->node, &bridge->windows); offset = window->offset; res = window->res; + if (!res->end) + continue; + + list_move_tail(&window->node, &bridge->windows); if (res->flags & IORESOURCE_BUS) pci_bus_insert_busn_res(bus, bus->number, res->end); @@ -1498,8 +1530,8 @@ void set_pcie_port_type(struct pci_dev *pdev) pdev->pcie_cap = pos; pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, ®16); pdev->pcie_flags_reg = reg16; - pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, ®16); - pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD; + pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->devcap); + pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap); parent = pci_upstream_bridge(pdev); if (!parent) @@ -1576,6 +1608,26 @@ static void set_pcie_untrusted(struct pci_dev *dev) dev->untrusted = true; } +static void pci_set_removable(struct pci_dev *dev) +{ + struct pci_dev *parent = pci_upstream_bridge(dev); + + /* + * We (only) consider everything downstream from an external_facing + * device to be removable by the user. We're mainly concerned with + * consumer platforms with user accessible thunderbolt ports that are + * vulnerable to DMA attacks, and we expect those ports to be marked by + * the firmware as external_facing. Devices in traditional hotplug + * slots can technically be removed, but the expectation is that unless + * the port is marked with external_facing, such devices are less + * accessible to user / may not be removed by end user, and thus not + * exposed as "removable" to userspace. + */ + if (parent && + (parent->external_facing || dev_is_removable(&parent->dev))) + dev_set_removable(&dev->dev, DEVICE_REMOVABLE); +} + /** * pci_ext_cfg_is_aliased - Is ext config space just an alias of std config? * @dev: PCI device @@ -1789,6 +1841,9 @@ int pci_setup_device(struct pci_dev *dev) dev->error_state = pci_channel_io_normal; set_pcie_port_type(dev); + pci_set_of_node(dev); + pci_set_acpi_fwnode(dev); + pci_dev_assign_slot(dev); /* @@ -1823,6 +1878,8 @@ int pci_setup_device(struct pci_dev *dev) /* Early fixups, before probing the BARs */ pci_fixup_device(pci_fixup_early, dev); + pci_set_removable(dev); + pci_info(dev, "[%04x:%04x] type %02x class %#08x\n", dev->vendor, dev->device, dev->hdr_type, dev->class); @@ -1924,6 +1981,7 @@ int pci_setup_device(struct pci_dev *dev) default: /* unknown header */ pci_err(dev, "unknown header type %02x, ignoring device\n", dev->hdr_type); + pci_release_of_node(dev); return -EIO; bad: @@ -2133,9 +2191,21 @@ static void pci_configure_ltr(struct pci_dev *dev) * Complex and all intermediate Switches indicate support for LTR. * PCIe r4.0, sec 6.18. */ - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || - ((bridge = pci_upstream_bridge(dev)) && - bridge->ltr_path)) { + if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) { + pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); + dev->ltr_path = 1; + return; + } + + /* + * If we're configuring a hot-added device, LTR was likely + * disabled in the upstream bridge, so re-enable it before enabling + * it in the new device. + */ + bridge = pci_upstream_bridge(dev); + if (bridge && bridge->ltr_path) { + pci_bridge_reconfigure_ltr(dev); pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_LTR_EN); dev->ltr_path = 1; @@ -2203,7 +2273,6 @@ static void pci_release_capabilities(struct pci_dev *dev) { pci_aer_exit(dev); pci_rcec_exit(dev); - pci_vpd_release(dev); pci_iov_release(dev); pci_free_cap_save_buffers(dev); } @@ -2227,6 +2296,7 @@ static void pci_release_dev(struct device *dev) pci_bus_put(pci_dev->bus); kfree(pci_dev->driver_override); bitmap_free(pci_dev->dma_alias_mask); + dev_dbg(dev, "device released\n"); kfree(pci_dev); } @@ -2351,10 +2421,7 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) dev->vendor = l & 0xffff; dev->device = (l >> 16) & 0xffff; - pci_set_of_node(dev); - if (pci_setup_device(dev)) { - pci_release_of_node(dev); pci_bus_put(dev->bus); kfree(dev); return NULL; @@ -2405,9 +2472,7 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_rcec_init(dev); /* Root Complex Event Collector */ pcie_report_downtraining(dev); - - if (pci_probe_reset_function(dev) == 0) - dev->reset_fn = 1; + pci_init_reset_methods(dev); } /* @@ -2420,7 +2485,7 @@ static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev) struct irq_domain *d; /* - * If a domain has been set through the pcibios_add_device() + * If a domain has been set through the pcibios_device_add() * callback, then this is the one (platform code knows best). */ d = dev_get_msi_domain(&dev->dev); @@ -2488,7 +2553,7 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) list_add_tail(&dev->bus_list, &bus->devices); up_write(&pci_bus_sem); - ret = pcibios_add_device(dev); + ret = pcibios_device_add(dev); WARN_ON(ret < 0); /* Set up MSI IRQ domain */ @@ -2520,11 +2585,12 @@ struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn) } EXPORT_SYMBOL(pci_scan_single_device); -static unsigned next_fn(struct pci_bus *bus, struct pci_dev *dev, unsigned fn) +static unsigned int next_fn(struct pci_bus *bus, struct pci_dev *dev, + unsigned int fn) { int pos; u16 cap = 0; - unsigned next_fn; + unsigned int next_fn; if (pci_ari_enabled(bus)) { if (!dev) @@ -2583,7 +2649,7 @@ static int only_one_child(struct pci_bus *bus) */ int pci_scan_slot(struct pci_bus *bus, int devfn) { - unsigned fn, nr = 0; + unsigned int fn, nr = 0; struct pci_dev *dev; if (only_one_child(bus) && (devfn > 0)) diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index 9bab07302bbf..cb18f8a13ab6 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -83,6 +83,7 @@ static ssize_t proc_bus_pci_read(struct file *file, char __user *buf, buf += 4; pos += 4; cnt -= 4; + cond_resched(); } if (cnt >= 2) { @@ -230,8 +231,8 @@ static long proc_bus_pci_ioctl(struct file *file, unsigned int cmd, break; } /* If arch decided it can't, fall through... */ -#endif /* HAVE_PCI_MMAP */ fallthrough; +#endif /* HAVE_PCI_MMAP */ default: ret = -EINVAL; break; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 22b2bb1109c9..003950c738d2 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -27,6 +27,7 @@ #include <linux/nvme.h> #include <linux/platform_data/x86/apple.h> #include <linux/pm_runtime.h> +#include <linux/suspend.h> #include <linux/switchtec.h> #include <asm/dma.h> /* isa_dma_bridge_buggy */ #include "pci.h" @@ -500,7 +501,7 @@ static void quirk_s3_64M(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_S3, PCI_DEVICE_ID_S3_868, quirk_s3_64M); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_S3, PCI_DEVICE_ID_S3_968, quirk_s3_64M); -static void quirk_io(struct pci_dev *dev, int pos, unsigned size, +static void quirk_io(struct pci_dev *dev, int pos, unsigned int size, const char *name) { u32 region; @@ -551,7 +552,7 @@ static void quirk_cs5536_vsa(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CS5536_ISA, quirk_cs5536_vsa); static void quirk_io_region(struct pci_dev *dev, int port, - unsigned size, int nr, const char *name) + unsigned int size, int nr, const char *name) { u16 region; struct pci_bus_region bus_region; @@ -665,7 +666,7 @@ static void piix4_io_quirk(struct pci_dev *dev, const char *name, unsigned int p base = devres & 0xffff; size = 16; for (;;) { - unsigned bit = size >> 1; + unsigned int bit = size >> 1; if ((bit & mask) == bit) break; size = bit; @@ -691,7 +692,7 @@ static void piix4_mem_quirk(struct pci_dev *dev, const char *name, unsigned int mask = (devres & 0x3f) << 16; size = 128 << 16; for (;;) { - unsigned bit = size >> 1; + unsigned int bit = size >> 1; if ((bit & mask) == bit) break; size = bit; @@ -805,7 +806,7 @@ static void ich6_lpc_acpi_gpio(struct pci_dev *dev) "ICH6 GPIO"); } -static void ich6_lpc_generic_decode(struct pci_dev *dev, unsigned reg, +static void ich6_lpc_generic_decode(struct pci_dev *dev, unsigned int reg, const char *name, int dynsize) { u32 val; @@ -849,7 +850,7 @@ static void quirk_ich6_lpc(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH6_0, quirk_ich6_lpc); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH6_1, quirk_ich6_lpc); -static void ich7_lpc_generic_decode(struct pci_dev *dev, unsigned reg, +static void ich7_lpc_generic_decode(struct pci_dev *dev, unsigned int reg, const char *name) { u32 val; @@ -1821,6 +1822,45 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_E7525_MCH, quir DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_HUAWEI, 0x1610, PCI_CLASS_BRIDGE_PCI, 8, quirk_pcie_mch); /* + * HiSilicon KunPeng920 and KunPeng930 have devices appear as PCI but are + * actually on the AMBA bus. These fake PCI devices can support SVA via + * SMMU stall feature, by setting dma-can-stall for ACPI platforms. + * + * Normally stalling must not be enabled for PCI devices, since it would + * break the PCI requirement for free-flowing writes and may lead to + * deadlock. We expect PCI devices to support ATS and PRI if they want to + * be fault-tolerant, so there's no ACPI binding to describe anything else, + * even when a "PCI" device turns out to be a regular old SoC device + * dressed up as a RCiEP and normal rules don't apply. + */ +static void quirk_huawei_pcie_sva(struct pci_dev *pdev) +{ + struct property_entry properties[] = { + PROPERTY_ENTRY_BOOL("dma-can-stall"), + {}, + }; + + if (pdev->revision != 0x21 && pdev->revision != 0x30) + return; + + pdev->pasid_no_tlp = 1; + + /* + * Set the dma-can-stall property on ACPI platforms. Device tree + * can set it directly. + */ + if (!pdev->dev.of_node && + device_add_properties(&pdev->dev, properties)) + pci_warn(pdev, "could not add stall property"); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa250, quirk_huawei_pcie_sva); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa251, quirk_huawei_pcie_sva); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa255, quirk_huawei_pcie_sva); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa256, quirk_huawei_pcie_sva); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa258, quirk_huawei_pcie_sva); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_HUAWEI, 0xa259, quirk_huawei_pcie_sva); + +/* * It's possible for the MSI to get corrupted if SHPC and ACPI are used * together on certain PXH-based systems. */ @@ -1899,6 +1939,7 @@ static void quirk_ryzen_xhci_d3hot(struct pci_dev *dev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15e0, quirk_ryzen_xhci_d3hot); DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15e1, quirk_ryzen_xhci_d3hot); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1639, quirk_ryzen_xhci_d3hot); #ifdef CONFIG_X86_IO_APIC static int dmi_disable_ioapicreroute(const struct dmi_system_id *d) @@ -2659,7 +2700,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, * then the device can't use INTx interrupts. Tegra's PCIe root ports don't * generate MSI interrupts for PME and AER events instead only INTx interrupts * are generated. Though Tegra's PCIe root ports can generate MSI interrupts - * for other events, since PCIe specificiation doesn't support using a mix of + * for other events, since PCIe specification doesn't support using a mix of * INTx and MSI/MSI-X, it is required to disable MSI interrupts to avoid port * service drivers registering their respective ISRs for MSIs. */ @@ -3233,12 +3274,13 @@ static void fixup_mpss_256(struct pci_dev *dev) { dev->pcie_mpss = 1; /* 256 bytes */ } -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SOLARFLARE, - PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0, fixup_mpss_256); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SOLARFLARE, - PCI_DEVICE_ID_SOLARFLARE_SFC4000A_1, fixup_mpss_256); -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SOLARFLARE, - PCI_DEVICE_ID_SOLARFLARE_SFC4000B, fixup_mpss_256); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SOLARFLARE, + PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0, fixup_mpss_256); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SOLARFLARE, + PCI_DEVICE_ID_SOLARFLARE_SFC4000A_1, fixup_mpss_256); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SOLARFLARE, + PCI_DEVICE_ID_SOLARFLARE_SFC4000B, fixup_mpss_256); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_ASMEDIA, 0x0612, fixup_mpss_256); /* * Intel 5000 and 5100 Memory controllers have an erratum with read completion @@ -3570,6 +3612,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0032, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003c, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0033, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0034, quirk_no_bus_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003e, quirk_no_bus_reset); /* * Root port on some Cavium CN8xxx chips do not successfully complete a bus @@ -3656,6 +3699,16 @@ static void quirk_apple_poweroff_thunderbolt(struct pci_dev *dev) return; if (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM) return; + + /* + * SXIO/SXFP/SXLF turns off power to the Thunderbolt controller. + * We don't know how to turn it back on again, but firmware does, + * so we can only use SXIO/SXFP/SXLF if we're suspending via + * firmware. + */ + if (!pm_suspend_via_firmware()) + return; + bridge = ACPI_HANDLE(&dev->dev); if (!bridge) return; @@ -3691,7 +3744,7 @@ DECLARE_PCI_FIXUP_SUSPEND_LATE(PCI_VENDOR_ID_INTEL, * reset a single function if other methods (e.g. FLR, PM D0->D3) are * not available. */ -static int reset_intel_82599_sfp_virtfn(struct pci_dev *dev, int probe) +static int reset_intel_82599_sfp_virtfn(struct pci_dev *dev, bool probe) { /* * http://www.intel.com/content/dam/doc/datasheet/82599-10-gbe-controller-datasheet.pdf @@ -3713,7 +3766,7 @@ static int reset_intel_82599_sfp_virtfn(struct pci_dev *dev, int probe) #define NSDE_PWR_STATE 0xd0100 #define IGD_OPERATION_TIMEOUT 10000 /* set timeout 10 seconds */ -static int reset_ivb_igd(struct pci_dev *dev, int probe) +static int reset_ivb_igd(struct pci_dev *dev, bool probe) { void __iomem *mmio_base; unsigned long timeout; @@ -3756,7 +3809,7 @@ reset_complete: } /* Device-specific reset method for Chelsio T4-based adapters */ -static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe) +static int reset_chelsio_generic_dev(struct pci_dev *dev, bool probe) { u16 old_command; u16 msix_flags; @@ -3834,14 +3887,14 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe) * Chapter 3: NVMe control registers * Chapter 7.3: Reset behavior */ -static int nvme_disable_and_flr(struct pci_dev *dev, int probe) +static int nvme_disable_and_flr(struct pci_dev *dev, bool probe) { void __iomem *bar; u16 cmd; u32 cfg; if (dev->class != PCI_CLASS_STORAGE_EXPRESS || - !pcie_has_flr(dev) || !pci_resource_start(dev, 0)) + pcie_reset_flr(dev, PCI_RESET_PROBE) || !pci_resource_start(dev, 0)) return -ENOTTY; if (probe) @@ -3908,15 +3961,12 @@ static int nvme_disable_and_flr(struct pci_dev *dev, int probe) * device too soon after FLR. A 250ms delay after FLR has heuristically * proven to produce reliably working results for device assignment cases. */ -static int delay_250ms_after_flr(struct pci_dev *dev, int probe) +static int delay_250ms_after_flr(struct pci_dev *dev, bool probe) { - if (!pcie_has_flr(dev)) - return -ENOTTY; - if (probe) - return 0; + return pcie_reset_flr(dev, PCI_RESET_PROBE); - pcie_flr(dev); + pcie_reset_flr(dev, PCI_RESET_DO_RESET); msleep(250); @@ -3931,7 +3981,7 @@ static int delay_250ms_after_flr(struct pci_dev *dev, int probe) #define HINIC_OPERATION_TIMEOUT 15000 /* 15 seconds */ /* Device-specific reset method for Huawei Intelligent NIC virtual functions */ -static int reset_hinic_vf_dev(struct pci_dev *pdev, int probe) +static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe) { unsigned long timeout; void __iomem *bar; @@ -4008,7 +4058,7 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = { * because when a host assigns a device to a guest VM, the host may need * to reset the device but probably doesn't have a driver for it. */ -int pci_dev_specific_reset(struct pci_dev *dev, int probe) +int pci_dev_specific_reset(struct pci_dev *dev, bool probe) { const struct pci_dev_reset_methods *i; @@ -4603,6 +4653,18 @@ static int pci_quirk_qcom_rp_acs(struct pci_dev *dev, u16 acs_flags) PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); } +/* + * Each of these NXP Root Ports is in a Root Complex with a unique segment + * number and does provide isolation features to disable peer transactions + * and validate bus numbers in requests, but does not provide an ACS + * capability. + */ +static int pci_quirk_nxp_rp_acs(struct pci_dev *dev, u16 acs_flags) +{ + return pci_acs_ctrl_enabled(acs_flags, + PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF); +} + static int pci_quirk_al_acs(struct pci_dev *dev, u16 acs_flags) { if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT) @@ -4829,6 +4891,10 @@ static const struct pci_dev_acs_enabled { { 0x10df, 0x720, pci_quirk_mf_endpoint_acs }, /* Emulex Skyhawk-R */ /* Cavium ThunderX */ { PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, pci_quirk_cavium_acs }, + /* Cavium multi-function devices */ + { PCI_VENDOR_ID_CAVIUM, 0xA026, pci_quirk_mf_endpoint_acs }, + { PCI_VENDOR_ID_CAVIUM, 0xA059, pci_quirk_mf_endpoint_acs }, + { PCI_VENDOR_ID_CAVIUM, 0xA060, pci_quirk_mf_endpoint_acs }, /* APM X-Gene */ { PCI_VENDOR_ID_AMCC, 0xE004, pci_quirk_xgene_acs }, /* Ampere Computing */ @@ -4849,6 +4915,39 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_ZHAOXIN, 0x3038, pci_quirk_mf_endpoint_acs }, { PCI_VENDOR_ID_ZHAOXIN, 0x3104, pci_quirk_mf_endpoint_acs }, { PCI_VENDOR_ID_ZHAOXIN, 0x9083, pci_quirk_mf_endpoint_acs }, + /* NXP root ports, xx=16, 12, or 08 cores */ + /* LX2xx0A : without security features + CAN-FD */ + { PCI_VENDOR_ID_NXP, 0x8d81, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8da1, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d83, pci_quirk_nxp_rp_acs }, + /* LX2xx0C : security features + CAN-FD */ + { PCI_VENDOR_ID_NXP, 0x8d80, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8da0, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d82, pci_quirk_nxp_rp_acs }, + /* LX2xx0E : security features + CAN */ + { PCI_VENDOR_ID_NXP, 0x8d90, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8db0, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d92, pci_quirk_nxp_rp_acs }, + /* LX2xx0N : without security features + CAN */ + { PCI_VENDOR_ID_NXP, 0x8d91, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8db1, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d93, pci_quirk_nxp_rp_acs }, + /* LX2xx2A : without security features + CAN-FD */ + { PCI_VENDOR_ID_NXP, 0x8d89, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8da9, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d8b, pci_quirk_nxp_rp_acs }, + /* LX2xx2C : security features + CAN-FD */ + { PCI_VENDOR_ID_NXP, 0x8d88, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8da8, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d8a, pci_quirk_nxp_rp_acs }, + /* LX2xx2E : security features + CAN */ + { PCI_VENDOR_ID_NXP, 0x8d98, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8db8, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d9a, pci_quirk_nxp_rp_acs }, + /* LX2xx2N : without security features + CAN */ + { PCI_VENDOR_ID_NXP, 0x8d99, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8db9, pci_quirk_nxp_rp_acs }, + { PCI_VENDOR_ID_NXP, 0x8d9b, pci_quirk_nxp_rp_acs }, /* Zhaoxin Root/Downstream Ports */ { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, { 0 } @@ -5020,7 +5119,7 @@ static int pci_quirk_enable_intel_spt_pch_acs(struct pci_dev *dev) ctrl |= (cap & PCI_ACS_CR); ctrl |= (cap & PCI_ACS_UF); - if (dev->external_facing || dev->untrusted) + if (pci_ats_disabled() || dev->external_facing || dev->untrusted) ctrl |= (cap & PCI_ACS_TB); pci_write_config_dword(dev, pos + INTEL_SPT_ACS_CTRL, ctrl); @@ -5337,7 +5436,7 @@ DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda); /* - * Create device link for NVIDIA GPU with integrated USB xHCI Host + * Create device link for GPUs with integrated USB xHCI Host * controller to VGA. */ static void quirk_gpu_usb(struct pci_dev *usb) @@ -5346,9 +5445,11 @@ static void quirk_gpu_usb(struct pci_dev *usb) } DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_CLASS_SERIAL_USB, 8, quirk_gpu_usb); +DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_ATI, PCI_ANY_ID, + PCI_CLASS_SERIAL_USB, 8, quirk_gpu_usb); /* - * Create device link for NVIDIA GPU with integrated Type-C UCSI controller + * Create device link for GPUs with integrated Type-C UCSI controller * to VGA. Currently there is no class code defined for UCSI device over PCI * so using UNKNOWN class for now and it will be updated when UCSI * over PCI gets a class code. @@ -5361,6 +5462,9 @@ static void quirk_gpu_usb_typec_ucsi(struct pci_dev *ucsi) DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_CLASS_SERIAL_UNKNOWN, 8, quirk_gpu_usb_typec_ucsi); +DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_ATI, PCI_ANY_ID, + PCI_CLASS_SERIAL_UNKNOWN, 8, + quirk_gpu_usb_typec_ucsi); /* * Enable the NVIDIA GPU integrated HDA controller if the BIOS left it @@ -5618,7 +5722,7 @@ static void quirk_reset_lenovo_thinkpad_p50_nvgpu(struct pci_dev *pdev) if (pdev->subsystem_vendor != PCI_VENDOR_ID_LENOVO || pdev->subsystem_device != 0x222e || - !pdev->reset_fn) + !pci_reset_supported(pdev)) return; if (pci_enable_device_mem(pdev)) @@ -5692,3 +5796,64 @@ static void apex_pci_fixup_class(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_CLASS_HEADER(0x1ac1, 0x089a, PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); + +/* + * Pericom PI7C9X2G404/PI7C9X2G304/PI7C9X2G303 switch erratum E5 - + * ACS P2P Request Redirect is not functional + * + * When ACS P2P Request Redirect is enabled and bandwidth is not balanced + * between upstream and downstream ports, packets are queued in an internal + * buffer until CPLD packet. The workaround is to use the switch in store and + * forward mode. + */ +#define PI7C9X2Gxxx_MODE_REG 0x74 +#define PI7C9X2Gxxx_STORE_FORWARD_MODE BIT(0) +static void pci_fixup_pericom_acs_store_forward(struct pci_dev *pdev) +{ + struct pci_dev *upstream; + u16 val; + + /* Downstream ports only */ + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) + return; + + /* Check for ACS P2P Request Redirect use */ + if (!pdev->acs_cap) + return; + pci_read_config_word(pdev, pdev->acs_cap + PCI_ACS_CTRL, &val); + if (!(val & PCI_ACS_RR)) + return; + + upstream = pci_upstream_bridge(pdev); + if (!upstream) + return; + + pci_read_config_word(upstream, PI7C9X2Gxxx_MODE_REG, &val); + if (!(val & PI7C9X2Gxxx_STORE_FORWARD_MODE)) { + pci_info(upstream, "Setting PI7C9X2Gxxx store-forward mode to avoid ACS erratum\n"); + pci_write_config_word(upstream, PI7C9X2Gxxx_MODE_REG, val | + PI7C9X2Gxxx_STORE_FORWARD_MODE); + } +} +/* + * Apply fixup on enable and on resume, in order to apply the fix up whenever + * ACS configuration changes or switch mode is reset + */ +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2304, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2304, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303, + pci_fixup_pericom_acs_store_forward); + +static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) +{ + pdev->dev_flags |= PCI_DEV_FLAGS_HAS_MSI_MASKING; +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0ab8, nvidia_ion_ahci_fixup); diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index dd12c2fcc7dc..4c54c75050dc 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -19,7 +19,6 @@ static void pci_stop_dev(struct pci_dev *dev) pci_pme_active(dev, false); if (pci_dev_is_added(dev)) { - dev->reset_fn = 0; device_release_driver(&dev->dev); pci_proc_detach_device(dev); diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c index 8fc9a4e911e3..e18d3a4383ba 100644 --- a/drivers/pci/rom.c +++ b/drivers/pci/rom.c @@ -85,7 +85,7 @@ static size_t pci_get_rom_size(struct pci_dev *pdev, void __iomem *rom, { void __iomem *image; int last_image; - unsigned length; + unsigned int length; image = rom; do { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 2ce636937c6e..547396ec50b5 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -1525,7 +1525,7 @@ static void pci_bridge_release_resources(struct pci_bus *bus, { struct pci_dev *dev = bus->self; struct resource *r; - unsigned old_flags = 0; + unsigned int old_flags = 0; struct resource *b_res; int idx = 1; diff --git a/drivers/pci/setup-irq.c b/drivers/pci/setup-irq.c index 7129494754dd..cc7d26b015f3 100644 --- a/drivers/pci/setup-irq.c +++ b/drivers/pci/setup-irq.c @@ -8,7 +8,6 @@ * David Miller (davem@redhat.com) */ - #include <linux/kernel.h> #include <linux/pci.h> #include <linux/errno.h> @@ -28,25 +27,26 @@ void pci_assign_irq(struct pci_dev *dev) return; } - /* If this device is not on the primary bus, we need to figure out - which interrupt pin it will come in on. We know which slot it - will come in on 'cos that slot is where the bridge is. Each - time the interrupt line passes through a PCI-PCI bridge we must - apply the swizzle function. */ - + /* + * If this device is not on the primary bus, we need to figure out + * which interrupt pin it will come in on. We know which slot it + * will come in on because that slot is where the bridge is. Each + * time the interrupt line passes through a PCI-PCI bridge we must + * apply the swizzle function. + */ pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); /* Cope with illegal. */ if (pin > 4) pin = 1; if (pin) { - /* Follow the chain of bridges, swizzling as we go. */ + /* Follow the chain of bridges, swizzling as we go. */ if (hbrg->swizzle_irq) slot = (*(hbrg->swizzle_irq))(dev, &pin); /* - * If a swizzling function is not used map_irq must - * ignore slot + * If a swizzling function is not used, map_irq() must + * ignore slot. */ irq = (*(hbrg->map_irq))(dev, slot, pin); if (irq == -1) @@ -56,7 +56,9 @@ void pci_assign_irq(struct pci_dev *dev) pci_dbg(dev, "assign IRQ: got %d\n", dev->irq); - /* Always tell the device, so the driver knows what is - the real IRQ to use; the device does not use it. */ + /* + * Always tell the device, so the driver knows what is the real IRQ + * to use; the device does not use it. + */ pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq); } diff --git a/drivers/pci/slot.c b/drivers/pci/slot.c index d627dd9179b4..751a26668e3a 100644 --- a/drivers/pci/slot.c +++ b/drivers/pci/slot.c @@ -39,19 +39,19 @@ static const struct sysfs_ops pci_slot_sysfs_ops = { static ssize_t address_read_file(struct pci_slot *slot, char *buf) { if (slot->number == 0xff) - return sprintf(buf, "%04x:%02x\n", - pci_domain_nr(slot->bus), - slot->bus->number); - else - return sprintf(buf, "%04x:%02x:%02x\n", - pci_domain_nr(slot->bus), - slot->bus->number, - slot->number); + return sysfs_emit(buf, "%04x:%02x\n", + pci_domain_nr(slot->bus), + slot->bus->number); + + return sysfs_emit(buf, "%04x:%02x:%02x\n", + pci_domain_nr(slot->bus), + slot->bus->number, + slot->number); } static ssize_t bus_speed_read(enum pci_bus_speed speed, char *buf) { - return sprintf(buf, "%s\n", pci_speed_string(speed)); + return sysfs_emit(buf, "%s\n", pci_speed_string(speed)); } static ssize_t max_speed_read_file(struct pci_slot *slot, char *buf) diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c index ba52459928f7..38c2b036fb8e 100644 --- a/drivers/pci/switch/switchtec.c +++ b/drivers/pci/switch/switchtec.c @@ -45,6 +45,7 @@ enum mrpc_state { MRPC_QUEUED, MRPC_RUNNING, MRPC_DONE, + MRPC_IO_ERROR, }; struct switchtec_user { @@ -66,6 +67,19 @@ struct switchtec_user { int event_cnt; }; +/* + * The MMIO reads to the device_id register should always return the device ID + * of the device, otherwise the firmware is probably stuck or unreachable + * due to a firmware reset which clears PCI state including the BARs and Memory + * Space Enable bits. + */ +static int is_firmware_running(struct switchtec_dev *stdev) +{ + u32 device = ioread32(&stdev->mmio_sys_info->device_id); + + return stdev->pdev->device == device; +} + static struct switchtec_user *stuser_create(struct switchtec_dev *stdev) { struct switchtec_user *stuser; @@ -113,6 +127,7 @@ static void stuser_set_state(struct switchtec_user *stuser, [MRPC_QUEUED] = "QUEUED", [MRPC_RUNNING] = "RUNNING", [MRPC_DONE] = "DONE", + [MRPC_IO_ERROR] = "IO_ERROR", }; stuser->state = state; @@ -184,9 +199,26 @@ static int mrpc_queue_cmd(struct switchtec_user *stuser) return 0; } +static void mrpc_cleanup_cmd(struct switchtec_dev *stdev) +{ + /* requires the mrpc_mutex to already be held when called */ + + struct switchtec_user *stuser = list_entry(stdev->mrpc_queue.next, + struct switchtec_user, list); + + stuser->cmd_done = true; + wake_up_interruptible(&stuser->cmd_comp); + list_del_init(&stuser->list); + stuser_put(stuser); + stdev->mrpc_busy = 0; + + mrpc_cmd_submit(stdev); +} + static void mrpc_complete_cmd(struct switchtec_dev *stdev) { /* requires the mrpc_mutex to already be held when called */ + struct switchtec_user *stuser; if (list_empty(&stdev->mrpc_queue)) @@ -206,7 +238,8 @@ static void mrpc_complete_cmd(struct switchtec_dev *stdev) stuser_set_state(stuser, MRPC_DONE); stuser->return_code = 0; - if (stuser->status != SWITCHTEC_MRPC_STATUS_DONE) + if (stuser->status != SWITCHTEC_MRPC_STATUS_DONE && + stuser->status != SWITCHTEC_MRPC_STATUS_ERROR) goto out; if (stdev->dma_mrpc) @@ -223,13 +256,7 @@ static void mrpc_complete_cmd(struct switchtec_dev *stdev) memcpy_fromio(stuser->data, &stdev->mmio_mrpc->output_data, stuser->read_len); out: - stuser->cmd_done = true; - wake_up_interruptible(&stuser->cmd_comp); - list_del_init(&stuser->list); - stuser_put(stuser); - stdev->mrpc_busy = 0; - - mrpc_cmd_submit(stdev); + mrpc_cleanup_cmd(stdev); } static void mrpc_event_work(struct work_struct *work) @@ -246,6 +273,23 @@ static void mrpc_event_work(struct work_struct *work) mutex_unlock(&stdev->mrpc_mutex); } +static void mrpc_error_complete_cmd(struct switchtec_dev *stdev) +{ + /* requires the mrpc_mutex to already be held when called */ + + struct switchtec_user *stuser; + + if (list_empty(&stdev->mrpc_queue)) + return; + + stuser = list_entry(stdev->mrpc_queue.next, + struct switchtec_user, list); + + stuser_set_state(stuser, MRPC_IO_ERROR); + + mrpc_cleanup_cmd(stdev); +} + static void mrpc_timeout_work(struct work_struct *work) { struct switchtec_dev *stdev; @@ -257,6 +301,11 @@ static void mrpc_timeout_work(struct work_struct *work) mutex_lock(&stdev->mrpc_mutex); + if (!is_firmware_running(stdev)) { + mrpc_error_complete_cmd(stdev); + goto out; + } + if (stdev->dma_mrpc) status = stdev->dma_mrpc->status; else @@ -280,7 +329,7 @@ static ssize_t device_version_show(struct device *dev, ver = ioread32(&stdev->mmio_sys_info->device_version); - return sprintf(buf, "%x\n", ver); + return sysfs_emit(buf, "%x\n", ver); } static DEVICE_ATTR_RO(device_version); @@ -292,7 +341,7 @@ static ssize_t fw_version_show(struct device *dev, ver = ioread32(&stdev->mmio_sys_info->firmware_version); - return sprintf(buf, "%08x\n", ver); + return sysfs_emit(buf, "%08x\n", ver); } static DEVICE_ATTR_RO(fw_version); @@ -327,7 +376,7 @@ static ssize_t field ## _show(struct device *dev, \ return io_string_show(buf, &si->gen4.field, \ sizeof(si->gen4.field)); \ else \ - return -ENOTSUPP; \ + return -EOPNOTSUPP; \ } \ \ static DEVICE_ATTR_RO(field) @@ -344,7 +393,7 @@ static ssize_t component_vendor_show(struct device *dev, /* component_vendor field not supported after gen3 */ if (stdev->gen != SWITCHTEC_GEN3) - return sprintf(buf, "none\n"); + return sysfs_emit(buf, "none\n"); return io_string_show(buf, &si->gen3.component_vendor, sizeof(si->gen3.component_vendor)); @@ -359,9 +408,9 @@ static ssize_t component_id_show(struct device *dev, /* component_id field not supported after gen3 */ if (stdev->gen != SWITCHTEC_GEN3) - return sprintf(buf, "none\n"); + return sysfs_emit(buf, "none\n"); - return sprintf(buf, "PM%04X\n", id); + return sysfs_emit(buf, "PM%04X\n", id); } static DEVICE_ATTR_RO(component_id); @@ -373,9 +422,9 @@ static ssize_t component_revision_show(struct device *dev, /* component_revision field not supported after gen3 */ if (stdev->gen != SWITCHTEC_GEN3) - return sprintf(buf, "255\n"); + return sysfs_emit(buf, "255\n"); - return sprintf(buf, "%d\n", rev); + return sysfs_emit(buf, "%d\n", rev); } static DEVICE_ATTR_RO(component_revision); @@ -384,7 +433,7 @@ static ssize_t partition_show(struct device *dev, { struct switchtec_dev *stdev = to_stdev(dev); - return sprintf(buf, "%d\n", stdev->partition); + return sysfs_emit(buf, "%d\n", stdev->partition); } static DEVICE_ATTR_RO(partition); @@ -393,7 +442,7 @@ static ssize_t partition_count_show(struct device *dev, { struct switchtec_dev *stdev = to_stdev(dev); - return sprintf(buf, "%d\n", stdev->partition_count); + return sysfs_emit(buf, "%d\n", stdev->partition_count); } static DEVICE_ATTR_RO(partition_count); @@ -544,6 +593,11 @@ static ssize_t switchtec_dev_read(struct file *filp, char __user *data, if (rc) return rc; + if (stuser->state == MRPC_IO_ERROR) { + mutex_unlock(&stdev->mrpc_mutex); + return -EIO; + } + if (stuser->state != MRPC_DONE) { mutex_unlock(&stdev->mrpc_mutex); return -EBADE; @@ -569,7 +623,8 @@ static ssize_t switchtec_dev_read(struct file *filp, char __user *data, out: mutex_unlock(&stdev->mrpc_mutex); - if (stuser->status == SWITCHTEC_MRPC_STATUS_DONE) + if (stuser->status == SWITCHTEC_MRPC_STATUS_DONE || + stuser->status == SWITCHTEC_MRPC_STATUS_ERROR) return size; else if (stuser->status == SWITCHTEC_MRPC_STATUS_INTERRUPTED) return -ENXIO; @@ -613,7 +668,7 @@ static int ioctl_flash_info(struct switchtec_dev *stdev, info.flash_length = ioread32(&fi->gen4.flash_length); info.num_partitions = SWITCHTEC_NUM_PARTITIONS_GEN4; } else { - return -ENOTSUPP; + return -EOPNOTSUPP; } if (copy_to_user(uinfo, &info, sizeof(info))) @@ -821,7 +876,7 @@ static int ioctl_flash_part_info(struct switchtec_dev *stdev, if (ret) return ret; } else { - return -ENOTSUPP; + return -EOPNOTSUPP; } if (copy_to_user(uinfo, &info, sizeof(info))) @@ -969,6 +1024,9 @@ static int event_ctl(struct switchtec_dev *stdev, return PTR_ERR(reg); hdr = ioread32(reg); + if (hdr & SWITCHTEC_EVENT_NOT_SUPP) + return -EOPNOTSUPP; + for (i = 0; i < ARRAY_SIZE(ctl->data); i++) ctl->data[i] = ioread32(®[i + 1]); @@ -1041,7 +1099,7 @@ static int ioctl_event_ctl(struct switchtec_dev *stdev, for (ctl.index = 0; ctl.index < nr_idxs; ctl.index++) { ctl.flags = event_flags; ret = event_ctl(stdev, &ctl); - if (ret < 0) + if (ret < 0 && ret != -EOPNOTSUPP) return ret; } } else { @@ -1078,7 +1136,7 @@ static int ioctl_pff_to_port(struct switchtec_dev *stdev, break; } - reg = ioread32(&pcfg->vep_pff_inst_id); + reg = ioread32(&pcfg->vep_pff_inst_id) & 0xFF; if (reg == p.pff) { p.port = SWITCHTEC_IOCTL_PFF_VEP; break; @@ -1124,7 +1182,7 @@ static int ioctl_port_to_pff(struct switchtec_dev *stdev, p.pff = ioread32(&pcfg->usp_pff_inst_id); break; case SWITCHTEC_IOCTL_PFF_VEP: - p.pff = ioread32(&pcfg->vep_pff_inst_id); + p.pff = ioread32(&pcfg->vep_pff_inst_id) & 0xFF; break; default: if (p.port > ARRAY_SIZE(pcfg->dsp_pff_inst_id)) @@ -1348,6 +1406,9 @@ static int mask_event(struct switchtec_dev *stdev, int eid, int idx) hdr_reg = event_regs[eid].map_reg(stdev, off, idx); hdr = ioread32(hdr_reg); + if (hdr & SWITCHTEC_EVENT_NOT_SUPP) + return 0; + if (!(hdr & SWITCHTEC_EVENT_OCCURRED && hdr & SWITCHTEC_EVENT_EN_IRQ)) return 0; @@ -1498,7 +1559,7 @@ static void init_pff(struct switchtec_dev *stdev) if (reg < stdev->pff_csr_count) stdev->pff_local[reg] = 1; - reg = ioread32(&pcfg->vep_pff_inst_id); + reg = ioread32(&pcfg->vep_pff_inst_id) & 0xFF; if (reg < stdev->pff_csr_count) stdev->pff_local[reg] = 1; @@ -1556,7 +1617,7 @@ static int switchtec_init_pci(struct switchtec_dev *stdev, else if (stdev->gen == SWITCHTEC_GEN4) part_id = &stdev->mmio_sys_info->gen4.partition_id; else - return -ENOTSUPP; + return -EOPNOTSUPP; stdev->partition = ioread8(part_id); stdev->partition_count = ioread8(&stdev->mmio_ntb->partition_count); diff --git a/drivers/pci/syscall.c b/drivers/pci/syscall.c index 8b003c890b87..61a6fe3cde21 100644 --- a/drivers/pci/syscall.c +++ b/drivers/pci/syscall.c @@ -19,11 +19,12 @@ SYSCALL_DEFINE5(pciconfig_read, unsigned long, bus, unsigned long, dfn, u8 byte; u16 word; u32 dword; - long err; - int cfg_ret; + int err, cfg_ret; + err = -EPERM; + dev = NULL; if (!capable(CAP_SYS_ADMIN)) - return -EPERM; + goto error; err = -ENODEV; dev = pci_get_domain_bus_and_slot(0, bus, dfn); diff --git a/drivers/pci/vpd.c b/drivers/pci/vpd.c index 26bf7c877de5..a4fc4d0690fe 100644 --- a/drivers/pci/vpd.c +++ b/drivers/pci/vpd.c @@ -9,116 +9,109 @@ #include <linux/delay.h> #include <linux/export.h> #include <linux/sched/signal.h> +#include <asm/unaligned.h> #include "pci.h" -/* VPD access through PCI 2.2+ VPD capability */ +#define PCI_VPD_LRDT_TAG_SIZE 3 +#define PCI_VPD_SRDT_LEN_MASK 0x07 +#define PCI_VPD_SRDT_TAG_SIZE 1 +#define PCI_VPD_STIN_END 0x0f +#define PCI_VPD_INFO_FLD_HDR_SIZE 3 -struct pci_vpd_ops { - ssize_t (*read)(struct pci_dev *dev, loff_t pos, size_t count, void *buf); - ssize_t (*write)(struct pci_dev *dev, loff_t pos, size_t count, const void *buf); -}; +static u16 pci_vpd_lrdt_size(const u8 *lrdt) +{ + return get_unaligned_le16(lrdt + 1); +} -struct pci_vpd { - const struct pci_vpd_ops *ops; - struct mutex lock; - unsigned int len; - u16 flag; - u8 cap; - unsigned int busy:1; - unsigned int valid:1; -}; +static u8 pci_vpd_srdt_tag(const u8 *srdt) +{ + return *srdt >> 3; +} -static struct pci_dev *pci_get_func0_dev(struct pci_dev *dev) +static u8 pci_vpd_srdt_size(const u8 *srdt) { - return pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); + return *srdt & PCI_VPD_SRDT_LEN_MASK; } -/** - * pci_read_vpd - Read one entry from Vital Product Data - * @dev: pci device struct - * @pos: offset in vpd space - * @count: number of bytes to read - * @buf: pointer to where to store result - */ -ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf) +static u8 pci_vpd_info_field_size(const u8 *info_field) { - if (!dev->vpd || !dev->vpd->ops) - return -ENODEV; - return dev->vpd->ops->read(dev, pos, count, buf); + return info_field[2]; } -EXPORT_SYMBOL(pci_read_vpd); -/** - * pci_write_vpd - Write entry to Vital Product Data - * @dev: pci device struct - * @pos: offset in vpd space - * @count: number of bytes to write - * @buf: buffer containing write data - */ -ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) +/* VPD access through PCI 2.2+ VPD capability */ + +static struct pci_dev *pci_get_func0_dev(struct pci_dev *dev) { - if (!dev->vpd || !dev->vpd->ops) - return -ENODEV; - return dev->vpd->ops->write(dev, pos, count, buf); + return pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); } -EXPORT_SYMBOL(pci_write_vpd); -#define PCI_VPD_MAX_SIZE (PCI_VPD_ADDR_MASK + 1) +#define PCI_VPD_MAX_SIZE (PCI_VPD_ADDR_MASK + 1) +#define PCI_VPD_SZ_INVALID UINT_MAX /** * pci_vpd_size - determine actual size of Vital Product Data * @dev: pci device struct - * @old_size: current assumed size, also maximum allowed size */ -static size_t pci_vpd_size(struct pci_dev *dev, size_t old_size) +static size_t pci_vpd_size(struct pci_dev *dev) { - size_t off = 0; - unsigned char header[1+2]; /* 1 byte tag, 2 bytes length */ + size_t off = 0, size; + unsigned char tag, header[1+2]; /* 1 byte tag, 2 bytes length */ - while (off < old_size && pci_read_vpd(dev, off, 1, header) == 1) { - unsigned char tag; + while (pci_read_vpd_any(dev, off, 1, header) == 1) { + size = 0; - if (!header[0] && !off) { - pci_info(dev, "Invalid VPD tag 00, assume missing optional VPD EPROM\n"); - return 0; - } + if (off == 0 && (header[0] == 0x00 || header[0] == 0xff)) + goto error; if (header[0] & PCI_VPD_LRDT) { /* Large Resource Data Type Tag */ - tag = pci_vpd_lrdt_tag(header); - /* Only read length from known tag items */ - if ((tag == PCI_VPD_LTIN_ID_STRING) || - (tag == PCI_VPD_LTIN_RO_DATA) || - (tag == PCI_VPD_LTIN_RW_DATA)) { - if (pci_read_vpd(dev, off+1, 2, - &header[1]) != 2) { - pci_warn(dev, "invalid large VPD tag %02x size at offset %zu", - tag, off + 1); - return 0; - } - off += PCI_VPD_LRDT_TAG_SIZE + - pci_vpd_lrdt_size(header); + if (pci_read_vpd_any(dev, off + 1, 2, &header[1]) != 2) { + pci_warn(dev, "failed VPD read at offset %zu\n", + off + 1); + return off ?: PCI_VPD_SZ_INVALID; } + size = pci_vpd_lrdt_size(header); + if (off + size > PCI_VPD_MAX_SIZE) + goto error; + + off += PCI_VPD_LRDT_TAG_SIZE + size; } else { /* Short Resource Data Type Tag */ - off += PCI_VPD_SRDT_TAG_SIZE + - pci_vpd_srdt_size(header); tag = pci_vpd_srdt_tag(header); + size = pci_vpd_srdt_size(header); + if (off + size > PCI_VPD_MAX_SIZE) + goto error; + + off += PCI_VPD_SRDT_TAG_SIZE + size; + if (tag == PCI_VPD_STIN_END) /* End tag descriptor */ + return off; } + } + return off; + +error: + pci_info(dev, "invalid VPD tag %#04x (size %zu) at offset %zu%s\n", + header[0], size, off, off == 0 ? + "; assume missing optional EEPROM" : ""); + return off ?: PCI_VPD_SZ_INVALID; +} - if (tag == PCI_VPD_STIN_END) /* End tag descriptor */ - return off; +static bool pci_vpd_available(struct pci_dev *dev, bool check_size) +{ + struct pci_vpd *vpd = &dev->vpd; - if ((tag != PCI_VPD_LTIN_ID_STRING) && - (tag != PCI_VPD_LTIN_RO_DATA) && - (tag != PCI_VPD_LTIN_RW_DATA)) { - pci_warn(dev, "invalid %s VPD tag %02x at offset %zu", - (header[0] & PCI_VPD_LRDT) ? "large" : "short", - tag, off); - return 0; + if (!vpd->cap) + return false; + + if (vpd->len == 0 && check_size) { + vpd->len = pci_vpd_size(dev); + if (vpd->len == PCI_VPD_SZ_INVALID) { + vpd->cap = 0; + return false; } } - return 0; + + return true; } /* @@ -126,33 +119,26 @@ static size_t pci_vpd_size(struct pci_dev *dev, size_t old_size) * This code has to spin since there is no other notification from the PCI * hardware. Since the VPD is often implemented by serial attachment to an * EEPROM, it may take many milliseconds to complete. + * @set: if true wait for flag to be set, else wait for it to be cleared * * Returns 0 on success, negative values indicate error. */ -static int pci_vpd_wait(struct pci_dev *dev) +static int pci_vpd_wait(struct pci_dev *dev, bool set) { - struct pci_vpd *vpd = dev->vpd; + struct pci_vpd *vpd = &dev->vpd; unsigned long timeout = jiffies + msecs_to_jiffies(125); unsigned long max_sleep = 16; u16 status; int ret; - if (!vpd->busy) - return 0; - do { ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR, &status); if (ret < 0) return ret; - if ((status & PCI_VPD_ADDR_F) == vpd->flag) { - vpd->busy = 0; + if (!!(status & PCI_VPD_ADDR_F) == set) return 0; - } - - if (fatal_signal_pending(current)) - return -EINTR; if (time_after(jiffies, timeout)) break; @@ -167,50 +153,47 @@ static int pci_vpd_wait(struct pci_dev *dev) } static ssize_t pci_vpd_read(struct pci_dev *dev, loff_t pos, size_t count, - void *arg) + void *arg, bool check_size) { - struct pci_vpd *vpd = dev->vpd; - int ret; + struct pci_vpd *vpd = &dev->vpd; + unsigned int max_len; + int ret = 0; loff_t end = pos + count; u8 *buf = arg; + if (!pci_vpd_available(dev, check_size)) + return -ENODEV; + if (pos < 0) return -EINVAL; - if (!vpd->valid) { - vpd->valid = 1; - vpd->len = pci_vpd_size(dev, vpd->len); - } + max_len = check_size ? vpd->len : PCI_VPD_MAX_SIZE; - if (vpd->len == 0) - return -EIO; - - if (pos > vpd->len) + if (pos >= max_len) return 0; - if (end > vpd->len) { - end = vpd->len; + if (end > max_len) { + end = max_len; count = end - pos; } if (mutex_lock_killable(&vpd->lock)) return -EINTR; - ret = pci_vpd_wait(dev); - if (ret < 0) - goto out; - while (pos < end) { u32 val; unsigned int i, skip; + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, pos & ~3); if (ret < 0) break; - vpd->busy = 1; - vpd->flag = PCI_VPD_ADDR_F; - ret = pci_vpd_wait(dev); + ret = pci_vpd_wait(dev, true); if (ret < 0) break; @@ -228,49 +211,37 @@ static ssize_t pci_vpd_read(struct pci_dev *dev, loff_t pos, size_t count, val >>= 8; } } -out: + mutex_unlock(&vpd->lock); return ret ? ret : count; } static ssize_t pci_vpd_write(struct pci_dev *dev, loff_t pos, size_t count, - const void *arg) + const void *arg, bool check_size) { - struct pci_vpd *vpd = dev->vpd; + struct pci_vpd *vpd = &dev->vpd; + unsigned int max_len; const u8 *buf = arg; loff_t end = pos + count; int ret = 0; + if (!pci_vpd_available(dev, check_size)) + return -ENODEV; + if (pos < 0 || (pos & 3) || (count & 3)) return -EINVAL; - if (!vpd->valid) { - vpd->valid = 1; - vpd->len = pci_vpd_size(dev, vpd->len); - } - - if (vpd->len == 0) - return -EIO; + max_len = check_size ? vpd->len : PCI_VPD_MAX_SIZE; - if (end > vpd->len) + if (end > max_len) return -EINVAL; if (mutex_lock_killable(&vpd->lock)) return -EINTR; - ret = pci_vpd_wait(dev); - if (ret < 0) - goto out; - while (pos < end) { - u32 val; - - val = *buf++; - val |= *buf++ << 8; - val |= *buf++ << 16; - val |= *buf++ << 24; - - ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val); + ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, + get_unaligned_le32(buf)); if (ret < 0) break; ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, @@ -278,85 +249,25 @@ static ssize_t pci_vpd_write(struct pci_dev *dev, loff_t pos, size_t count, if (ret < 0) break; - vpd->busy = 1; - vpd->flag = 0; - ret = pci_vpd_wait(dev); + ret = pci_vpd_wait(dev, false); if (ret < 0) break; + buf += sizeof(u32); pos += sizeof(u32); } -out: + mutex_unlock(&vpd->lock); return ret ? ret : count; } -static const struct pci_vpd_ops pci_vpd_ops = { - .read = pci_vpd_read, - .write = pci_vpd_write, -}; - -static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count, - void *arg) -{ - struct pci_dev *tdev = pci_get_func0_dev(dev); - ssize_t ret; - - if (!tdev) - return -ENODEV; - - ret = pci_read_vpd(tdev, pos, count, arg); - pci_dev_put(tdev); - return ret; -} - -static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count, - const void *arg) -{ - struct pci_dev *tdev = pci_get_func0_dev(dev); - ssize_t ret; - - if (!tdev) - return -ENODEV; - - ret = pci_write_vpd(tdev, pos, count, arg); - pci_dev_put(tdev); - return ret; -} - -static const struct pci_vpd_ops pci_vpd_f0_ops = { - .read = pci_vpd_f0_read, - .write = pci_vpd_f0_write, -}; - void pci_vpd_init(struct pci_dev *dev) { - struct pci_vpd *vpd; - u8 cap; - - cap = pci_find_capability(dev, PCI_CAP_ID_VPD); - if (!cap) - return; - - vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC); - if (!vpd) + if (dev->vpd.len == PCI_VPD_SZ_INVALID) return; - vpd->len = PCI_VPD_MAX_SIZE; - if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) - vpd->ops = &pci_vpd_f0_ops; - else - vpd->ops = &pci_vpd_ops; - mutex_init(&vpd->lock); - vpd->cap = cap; - vpd->busy = 0; - vpd->valid = 0; - dev->vpd = vpd; -} - -void pci_vpd_release(struct pci_dev *dev) -{ - kfree(dev->vpd); + dev->vpd.cap = pci_find_capability(dev, PCI_CAP_ID_VPD); + mutex_init(&dev->vpd.lock); } static ssize_t vpd_read(struct file *filp, struct kobject *kobj, @@ -388,7 +299,7 @@ static umode_t vpd_attr_is_visible(struct kobject *kobj, { struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); - if (!pdev->vpd) + if (!pdev->vpd.cap) return 0; return a->attr.mode; @@ -399,23 +310,64 @@ const struct attribute_group pci_dev_vpd_attr_group = { .is_bin_visible = vpd_attr_is_visible, }; -int pci_vpd_find_tag(const u8 *buf, unsigned int len, u8 rdt) +void *pci_vpd_alloc(struct pci_dev *dev, unsigned int *size) +{ + unsigned int len; + void *buf; + int cnt; + + if (!pci_vpd_available(dev, true)) + return ERR_PTR(-ENODEV); + + len = dev->vpd.len; + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + cnt = pci_read_vpd(dev, 0, len, buf); + if (cnt != len) { + kfree(buf); + return ERR_PTR(-EIO); + } + + if (size) + *size = len; + + return buf; +} +EXPORT_SYMBOL_GPL(pci_vpd_alloc); + +static int pci_vpd_find_tag(const u8 *buf, unsigned int len, u8 rdt, unsigned int *size) { int i = 0; /* look for LRDT tags only, end tag is the only SRDT tag */ while (i + PCI_VPD_LRDT_TAG_SIZE <= len && buf[i] & PCI_VPD_LRDT) { - if (buf[i] == rdt) + unsigned int lrdt_len = pci_vpd_lrdt_size(buf + i); + u8 tag = buf[i]; + + i += PCI_VPD_LRDT_TAG_SIZE; + if (tag == rdt) { + if (i + lrdt_len > len) + lrdt_len = len - i; + if (size) + *size = lrdt_len; return i; + } - i += PCI_VPD_LRDT_TAG_SIZE + pci_vpd_lrdt_size(buf + i); + i += lrdt_len; } return -ENOENT; } -EXPORT_SYMBOL_GPL(pci_vpd_find_tag); -int pci_vpd_find_info_keyword(const u8 *buf, unsigned int off, +int pci_vpd_find_id_string(const u8 *buf, unsigned int len, unsigned int *size) +{ + return pci_vpd_find_tag(buf, len, PCI_VPD_LRDT_ID_STRING, size); +} +EXPORT_SYMBOL_GPL(pci_vpd_find_id_string); + +static int pci_vpd_find_info_keyword(const u8 *buf, unsigned int off, unsigned int len, const char *kw) { int i; @@ -431,7 +383,132 @@ int pci_vpd_find_info_keyword(const u8 *buf, unsigned int off, return -ENOENT; } -EXPORT_SYMBOL_GPL(pci_vpd_find_info_keyword); + +static ssize_t __pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf, + bool check_size) +{ + ssize_t ret; + + if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) { + dev = pci_get_func0_dev(dev); + if (!dev) + return -ENODEV; + + ret = pci_vpd_read(dev, pos, count, buf, check_size); + pci_dev_put(dev); + return ret; + } + + return pci_vpd_read(dev, pos, count, buf, check_size); +} + +/** + * pci_read_vpd - Read one entry from Vital Product Data + * @dev: PCI device struct + * @pos: offset in VPD space + * @count: number of bytes to read + * @buf: pointer to where to store result + */ +ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf) +{ + return __pci_read_vpd(dev, pos, count, buf, true); +} +EXPORT_SYMBOL(pci_read_vpd); + +/* Same, but allow to access any address */ +ssize_t pci_read_vpd_any(struct pci_dev *dev, loff_t pos, size_t count, void *buf) +{ + return __pci_read_vpd(dev, pos, count, buf, false); +} +EXPORT_SYMBOL(pci_read_vpd_any); + +static ssize_t __pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, + const void *buf, bool check_size) +{ + ssize_t ret; + + if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) { + dev = pci_get_func0_dev(dev); + if (!dev) + return -ENODEV; + + ret = pci_vpd_write(dev, pos, count, buf, check_size); + pci_dev_put(dev); + return ret; + } + + return pci_vpd_write(dev, pos, count, buf, check_size); +} + +/** + * pci_write_vpd - Write entry to Vital Product Data + * @dev: PCI device struct + * @pos: offset in VPD space + * @count: number of bytes to write + * @buf: buffer containing write data + */ +ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) +{ + return __pci_write_vpd(dev, pos, count, buf, true); +} +EXPORT_SYMBOL(pci_write_vpd); + +/* Same, but allow to access any address */ +ssize_t pci_write_vpd_any(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) +{ + return __pci_write_vpd(dev, pos, count, buf, false); +} +EXPORT_SYMBOL(pci_write_vpd_any); + +int pci_vpd_find_ro_info_keyword(const void *buf, unsigned int len, + const char *kw, unsigned int *size) +{ + int ro_start, infokw_start; + unsigned int ro_len, infokw_size; + + ro_start = pci_vpd_find_tag(buf, len, PCI_VPD_LRDT_RO_DATA, &ro_len); + if (ro_start < 0) + return ro_start; + + infokw_start = pci_vpd_find_info_keyword(buf, ro_start, ro_len, kw); + if (infokw_start < 0) + return infokw_start; + + infokw_size = pci_vpd_info_field_size(buf + infokw_start); + infokw_start += PCI_VPD_INFO_FLD_HDR_SIZE; + + if (infokw_start + infokw_size > len) + return -EINVAL; + + if (size) + *size = infokw_size; + + return infokw_start; +} +EXPORT_SYMBOL_GPL(pci_vpd_find_ro_info_keyword); + +int pci_vpd_check_csum(const void *buf, unsigned int len) +{ + const u8 *vpd = buf; + unsigned int size; + u8 csum = 0; + int rv_start; + + rv_start = pci_vpd_find_ro_info_keyword(buf, len, PCI_VPD_RO_KEYWORD_CHKSUM, &size); + if (rv_start == -ENOENT) /* no checksum in VPD */ + return 1; + else if (rv_start < 0) + return rv_start; + + if (!size) + return -EINVAL; + + while (rv_start >= 0) + csum += vpd[rv_start--]; + + return csum ? -EILSEQ : 0; +} +EXPORT_SYMBOL_GPL(pci_vpd_check_csum); #ifdef CONFIG_PCI_QUIRKS /* @@ -450,7 +527,7 @@ static void quirk_f0_vpd_link(struct pci_dev *dev) if (!f0) return; - if (f0->vpd && dev->class == f0->class && + if (f0->vpd.cap && dev->class == f0->class && dev->vendor == f0->vendor && dev->device == f0->device) dev->dev_flags |= PCI_DEV_FLAGS_VPD_REF_F0; @@ -468,41 +545,27 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, */ static void quirk_blacklist_vpd(struct pci_dev *dev) { - if (dev->vpd) { - dev->vpd->len = 0; - pci_warn(dev, FW_BUG "disabling VPD access (can't determine size of non-standard VPD format)\n"); - } + dev->vpd.len = PCI_VPD_SZ_INVALID; + pci_warn(dev, FW_BUG "disabling VPD access (can't determine size of non-standard VPD format)\n"); } -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0060, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x007c, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0413, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0078, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0079, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0073, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x0071, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005b, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x002f, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005d, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LSI_LOGIC, 0x005f, quirk_blacklist_vpd); -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATTANSIC, PCI_ANY_ID, - quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0060, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x007c, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0413, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0078, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0079, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0073, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x0071, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x005b, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x002f, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x005d, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LSI_LOGIC, 0x005f, quirk_blacklist_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATTANSIC, PCI_ANY_ID, quirk_blacklist_vpd); /* * The Amazon Annapurna Labs 0x0031 device id is reused for other non Root Port * device types, so the quirk is registered for the PCI_CLASS_BRIDGE_PCI class. */ -DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, 0x0031, - PCI_CLASS_BRIDGE_PCI, 8, quirk_blacklist_vpd); - -static void pci_vpd_set_size(struct pci_dev *dev, size_t len) -{ - struct pci_vpd *vpd = dev->vpd; - - if (!vpd || len == 0 || len > PCI_VPD_MAX_SIZE) - return; - - vpd->valid = 1; - vpd->len = len; -} +DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, 0x0031, + PCI_CLASS_BRIDGE_PCI, 8, quirk_blacklist_vpd); static void quirk_chelsio_extend_vpd(struct pci_dev *dev) { @@ -522,12 +585,12 @@ static void quirk_chelsio_extend_vpd(struct pci_dev *dev) * limits. */ if (chip == 0x0 && prod >= 0x20) - pci_vpd_set_size(dev, 8192); + dev->vpd.len = 8192; else if (chip >= 0x4 && func < 0x8) - pci_vpd_set_size(dev, 2048); + dev->vpd.len = 2048; } -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID, - quirk_chelsio_extend_vpd); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID, + quirk_chelsio_extend_vpd); #endif diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index b7a8f3a1921f..d858d25b6cab 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -115,7 +115,7 @@ static int do_pci_op(struct pcifront_device *pdev, struct xen_pci_op *op) struct xen_pci_op *active_op = &pdev->sh_info->op; unsigned long irq_flags; evtchn_port_t port = pdev->evtchn; - unsigned irq = pdev->irq; + unsigned int irq = pdev->irq; s64 ns, ns_timeout; spin_lock_irqsave(&pdev->sh_info_lock, irq_flags); @@ -153,10 +153,10 @@ static int do_pci_op(struct pcifront_device *pdev, struct xen_pci_op *op) } /* - * We might lose backend service request since we - * reuse same evtchn with pci_conf backend response. So re-schedule - * aer pcifront service. - */ + * We might lose backend service request since we + * reuse same evtchn with pci_conf backend response. So re-schedule + * aer pcifront service. + */ if (test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags)) { dev_err(&pdev->xdev->dev, @@ -414,7 +414,8 @@ static int pcifront_scan_bus(struct pcifront_device *pdev, struct pci_dev *d; unsigned int devfn; - /* Scan the bus for functions and add. + /* + * Scan the bus for functions and add. * We omit handling of PCI bridge attachment because pciback prevents * bridges from being exported. */ @@ -492,8 +493,10 @@ static int pcifront_scan_root(struct pcifront_device *pdev, list_add(&bus_entry->list, &pdev->root_buses); - /* pci_scan_root_bus skips devices which do not have a - * devfn==0. The pcifront_scan_bus enumerates all devfn. */ + /* + * pci_scan_root_bus skips devices which do not have a + * devfn==0. The pcifront_scan_bus enumerates all devfn. + */ err = pcifront_scan_bus(pdev, domain, bus, b); /* Claim resources before going "live" with our devices */ @@ -585,61 +588,43 @@ static pci_ers_result_t pcifront_common_process(int cmd, struct pcifront_device *pdev, pci_channel_state_t state) { - pci_ers_result_t result; struct pci_driver *pdrv; int bus = pdev->sh_info->aer_op.bus; int devfn = pdev->sh_info->aer_op.devfn; int domain = pdev->sh_info->aer_op.domain; struct pci_dev *pcidev; - int flag = 0; dev_dbg(&pdev->xdev->dev, "pcifront AER process: cmd %x (bus:%x, devfn%x)", cmd, bus, devfn); - result = PCI_ERS_RESULT_NONE; pcidev = pci_get_domain_bus_and_slot(domain, bus, devfn); - if (!pcidev || !pcidev->driver) { + if (!pcidev || !pcidev->dev.driver) { dev_err(&pdev->xdev->dev, "device or AER driver is NULL\n"); pci_dev_put(pcidev); - return result; + return PCI_ERS_RESULT_NONE; } - pdrv = pcidev->driver; - - if (pdrv) { - if (pdrv->err_handler && pdrv->err_handler->error_detected) { - pci_dbg(pcidev, "trying to call AER service\n"); - if (pcidev) { - flag = 1; - switch (cmd) { - case XEN_PCI_OP_aer_detected: - result = pdrv->err_handler-> - error_detected(pcidev, state); - break; - case XEN_PCI_OP_aer_mmio: - result = pdrv->err_handler-> - mmio_enabled(pcidev); - break; - case XEN_PCI_OP_aer_slotreset: - result = pdrv->err_handler-> - slot_reset(pcidev); - break; - case XEN_PCI_OP_aer_resume: - pdrv->err_handler->resume(pcidev); - break; - default: - dev_err(&pdev->xdev->dev, - "bad request in aer recovery " - "operation!\n"); - - } - } + pdrv = to_pci_driver(pcidev->dev.driver); + + if (pdrv->err_handler && pdrv->err_handler->error_detected) { + pci_dbg(pcidev, "trying to call AER service\n"); + switch (cmd) { + case XEN_PCI_OP_aer_detected: + return pdrv->err_handler->error_detected(pcidev, state); + case XEN_PCI_OP_aer_mmio: + return pdrv->err_handler->mmio_enabled(pcidev); + case XEN_PCI_OP_aer_slotreset: + return pdrv->err_handler->slot_reset(pcidev); + case XEN_PCI_OP_aer_resume: + pdrv->err_handler->resume(pcidev); + return PCI_ERS_RESULT_NONE; + default: + dev_err(&pdev->xdev->dev, + "bad request in aer recovery operation!\n"); } } - if (!flag) - result = PCI_ERS_RESULT_NONE; - return result; + return PCI_ERS_RESULT_NONE; } @@ -651,8 +636,10 @@ static void pcifront_do_aer(struct work_struct *data) pci_channel_state_t state = (pci_channel_state_t)pdev->sh_info->aer_op.err; - /*If a pci_conf op is in progress, - we have to wait until it is done before service aer op*/ + /* + * If a pci_conf op is in progress, we have to wait until it is done + * before service aer op + */ dev_dbg(&pdev->xdev->dev, "pcifront service aer bus %x devfn %x\n", pdev->sh_info->aer_op.bus, pdev->sh_info->aer_op.devfn); @@ -676,6 +663,7 @@ static void pcifront_do_aer(struct work_struct *data) static irqreturn_t pcifront_handler_aer(int irq, void *dev) { struct pcifront_device *pdev = dev; + schedule_pcifront_aer_op(pdev); return IRQ_HANDLED; } @@ -693,7 +681,7 @@ static int pcifront_connect_and_init_dma(struct pcifront_device *pdev) spin_unlock(&pcifront_dev_lock); - if (!err && !is_swiotlb_active()) { + if (!err && !is_swiotlb_active(&pdev->xdev->dev)) { err = pci_xen_swiotlb_init_late(); if (err) dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n"); @@ -1027,6 +1015,7 @@ static int pcifront_detach_devices(struct pcifront_device *pdev) /* Find devices being detached and remove them. */ for (i = 0; i < num_devs; i++) { int l, state; + l = snprintf(str, sizeof(str), "state-%d", i); if (unlikely(l >= (sizeof(str) - 1))) { err = -ENOMEM; @@ -1078,7 +1067,7 @@ out: return err; } -static void __ref pcifront_backend_changed(struct xenbus_device *xdev, +static void pcifront_backend_changed(struct xenbus_device *xdev, enum xenbus_state be_state) { struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev); @@ -1137,6 +1126,7 @@ out: static int pcifront_xenbus_remove(struct xenbus_device *xdev) { struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev); + if (pdev) free_pdev(pdev); |